From 02574388a308b0fc1c4a54f1432e4d8f9552128e Mon Sep 17 00:00:00 2001
From: github-bot-metro <github-bot@metro.org>
Date: Wed, 6 Nov 2024 19:50:00 +0000
Subject: [PATCH] Deployed eb8322d9 to 1.4.0 with MkDocs 1.6.1 and mike 2.1.3

---
 1.4.0/101_guides_list/index.html              |  33 +-
 1.4.0/404.html                                |  33 +-
 1.4.0/AMIviaSpreadsheets/index.html           |  54 +-
 1.4.0/CODE_OF_CONDUCT/index.html              |  33 +-
 1.4.0/I7solrImporter/index.html               |  33 +-
 1.4.0/about/index.html                        |  33 +-
 1.4.0/acknowledgments/index.html              |  33 +-
 1.4.0/ami_index/index.html                    |  33 +-
 1.4.0/ami_lod_rec/index.html                  |  35 +-
 1.4.0/ami_spreadsheet_overview/index.html     |  33 +-
 1.4.0/ami_update/index.html                   |  33 +-
 1.4.0/annotations/index.html                  |  35 +-
 1.4.0/archifilepersistencestrategy/index.html |  33 +-
 1.4.0/archipelag-logo-usage/index.html        |  33 +-
 .../index.html                                |  33 +-
 .../index.html                                |  33 +-
 .../index.html                                |  33 +-
 .../index.html                                |  33 +-
 .../index.html                                |  33 +-
 .../index.html                                |  33 +-
 .../index.html                                |  33 +-
 .../index.html                                |  33 +-
 .../index.html                                |  33 +-
 .../index.html                                |  33 +-
 .../index.html                                |  33 +-
 1.4.0/archipelago-deployment-osx/index.html   |  35 +-
 .../archipelago-deployment-readme/index.html  |  33 +-
 .../archipelago-deployment-ubuntu/index.html  |  33 +-
 .../archipelago-deployment-windows/index.html |  33 +-
 1.4.0/archipelago-glossary/index.html         | 914 +++++++++++++++++-
 1.4.0/createdisplaymodes/index.html           |  33 +-
 1.4.0/customwebformelements/index.html        |  33 +-
 1.4.0/devops/index.html                       |  40 +-
 1.4.0/documentation_about/index.html          |  33 +-
 1.4.0/documentation_features/index.html       |  33 +-
 1.4.0/documentation_technical/index.html      |  33 +-
 1.4.0/documentation_template/index.html       |  33 +-
 1.4.0/documentation_workflow/index.html       |  33 +-
 1.4.0/drupal_core_update/index.html           |  33 +-
 1.4.0/embargo/index.html                      |  33 +-
 1.4.0/experimental_tools/index.html           |  33 +-
 1.4.0/export-to-csv/index.html                |  33 +-
 1.4.0/find_and_replace/index.html             |  33 +-
 .../index.html                                |  33 +-
 1.4.0/find_and_replace_action_text/index.html |  33 +-
 .../index.html                                |  33 +-
 1.4.0/firstobject/index.html                  |  33 +-
 1.4.0/fragaria/index.html                     |  33 +-
 1.4.0/generalqa_minio_logging/index.html      |  33 +-
 1.4.0/generalqa_smtp_configuration/index.html |  33 +-
 .../index.html                                |  33 +-
 1.4.0/giveortake/index.html                   |  33 +-
 1.4.0/googleapi/index.html                    |  35 +-
 1.4.0/iiif-content-search/index.html          |  33 +-
 1.4.0/iiif_server_settings/index.html         |  33 +-
 1.4.0/index.html                              |  33 +-
 1.4.0/inthewild/index.html                    |  40 +-
 1.4.0/metadata_display_preview/index.html     |  33 +-
 1.4.0/metadata_display_usage/index.html       |  33 +-
 1.4.0/metadatainarchipelago/index.html        |  33 +-
 1.4.0/metadatatwigs/index.html                |  33 +-
 .../index.html                                |  33 +-
 1.4.0/open_api/index.html                     |  33 +-
 1.4.0/ourtake/index.html                      |  33 +-
 1.4.0/presentations_events/index.html         |  33 +-
 1.4.0/search-within-collection/index.html     |  33 +-
 1.4.0/search/search_index.json                |   2 +-
 1.4.0/search_advanced/index.html              |  33 +-
 1.4.0/search_solr_index/index.html            |  33 +-
 1.4.0/security_bots/index.html                | 120 ++-
 1.4.0/sitemap.xml                             | 172 ++--
 1.4.0/sitemap.xml.gz                          | Bin 1029 -> 1030 bytes
 1.4.0/sslsetup/index.html                     |  33 +-
 .../strawberry_key_name_providers/index.html  |  33 +-
 1.4.0/strawberryfield-formatters/index.html   |  33 +-
 1.4.0/strawberryfields/index.html             |  33 +-
 1.4.0/strawberryrunners/index.html            |  33 +-
 1.4.0/strawberryrunners_pager_ocr/index.html  |  33 +-
 1.4.0/strawberryrunners_subtitle/index.html   |  33 +-
 .../strawberryrunners_wacz_binary/index.html  |  33 +-
 .../strawberryrunners_webpage_text/index.html |  33 +-
 1.4.0/traditional-install/index.html          |  35 +-
 1.4.0/twig_extensions/index.html              |  33 +-
 1.4.0/twig_recipe_cards/index.html            |  33 +-
 1.4.0/utility_scripts/index.html              |  35 +-
 1.4.0/webformLoDfromCSV/index.html            |  33 +-
 1.4.0/webforms/index.html                     |  33 +-
 1.4.0/webformsasinput/index.html              |  33 +-
 1.4.0/workingtwigs/index.html                 |  51 +-
 1.4.0/xdebug/index.html                       |  37 +-
 90 files changed, 3517 insertions(+), 565 deletions(-)

diff --git a/1.4.0/101_guides_list/index.html b/1.4.0/101_guides_list/index.html
index dcf64c09..b8d309e4 100644
--- a/1.4.0/101_guides_list/index.html
+++ b/1.4.0/101_guides_list/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -579,6 +579,8 @@
       
         
       
+        
+      
     
     
       
@@ -1144,6 +1146,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1164,10 +1187,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1178,8 +1201,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/404.html b/1.4.0/404.html
index c81e3a63..8ed22fac 100644
--- a/1.4.0/404.html
+++ b/1.4.0/404.html
@@ -16,7 +16,7 @@
       
       
       <link rel="icon" href="/1.4.0/images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -547,6 +547,8 @@
       
         
       
+        
+      
     
     
       
@@ -1112,6 +1114,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="/1.4.0/security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1132,10 +1155,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1146,8 +1169,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/AMIviaSpreadsheets/index.html b/1.4.0/AMIviaSpreadsheets/index.html
index 544c5ff1..bed80d6f 100644
--- a/1.4.0/AMIviaSpreadsheets/index.html
+++ b/1.4.0/AMIviaSpreadsheets/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
@@ -3518,15 +3541,16 @@ <h4 id="step-4-global-ado-mappings">Step 4: Global ADO Mappings</h4>
 </ul>
 <details class="info">
 <summary>Click to read more about Archipelago's default relationship mappings</summary>
-<pre><code>- `ismemberof` and/or `ispartof` (and/or whatever predicate corresponds with the relationship you are mapping)
-- these columns can be used to connect related objects using the object-to-object relationship that matches your needs
-- in default Archipelago configurations, `ismemberof` is used for Collection Membership and `ispartof` is used for Parent-Child Object Relationships (so a Child ADO would reference the Parent ADO in `ispartof`)
-- these columns can hold 3 types of values
-    - empty (no value)
-    - an integer to connect an object to another object's corresponding row in the same spreadsheet/CSV
-      * Ex: Row 2 corresponds to a Digital Object Collection; for a Digital Object corresponding to Row 3, the 'ismemberof' column contains a value of '2'. The Digital Object in Row 3 would be ingested as a member of the Digital Object Collection in Row 2.
-    - a UUID to connect with an already ingested object
-</code></pre>
+<ul>
+<li><code>ismemberof</code> and/or <code>ispartof</code> (and/or whatever predicate corresponds with the relationship you are mapping)</li>
+<li>these columns can be used to connect related objects using the object-to-object relationship that matches your needs</li>
+<li>in default Archipelago configurations, <code>ismemberof</code> is used for Collection Membership and <code>ispartof</code> is used for Parent-Child Object Relationships (so a Child ADO would reference the Parent ADO in <code>ispartof</code>)</li>
+<li>these columns can hold 3 types of values</li>
+<li>empty (no value)</li>
+<li>an integer to connect an object to another object's corresponding row in the same spreadsheet/CSV</li>
+<li>Ex: Row 2 corresponds to a Digital Object Collection; for a Digital Object corresponding to Row 3, the 'ismemberof' column contains a value of '2'. The Digital Object in Row 3 would be ingested as a member of the Digital Object Collection in Row 2.</li>
+<li>a UUID to connect with an already ingested object</li>
+</ul>
 </details>
 <ul>
 <li>By default, the option to automatically assigns UUIDs is selected. Keep 'Automatically assign UUID' checked unless your source data already contains UUIDs in a <code>node_uuid</code> column.</li>
@@ -3717,7 +3741,7 @@ <h4 id="step-5-10">Step 5-10:</h4>
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">August 11, 2023</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">November 6, 2024</span>
   </span>
 
     
diff --git a/1.4.0/CODE_OF_CONDUCT/index.html b/1.4.0/CODE_OF_CONDUCT/index.html
index 1a72561d..a97bc7a8 100644
--- a/1.4.0/CODE_OF_CONDUCT/index.html
+++ b/1.4.0/CODE_OF_CONDUCT/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -567,6 +567,8 @@
       
         
       
+        
+      
     
     
       
@@ -1132,6 +1134,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1152,10 +1175,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1166,8 +1189,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/I7solrImporter/index.html b/1.4.0/I7solrImporter/index.html
index 03c098b5..37fea4ce 100644
--- a/1.4.0/I7solrImporter/index.html
+++ b/1.4.0/I7solrImporter/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/about/index.html b/1.4.0/about/index.html
index 0bcaee60..464afd35 100644
--- a/1.4.0/about/index.html
+++ b/1.4.0/about/index.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -563,6 +563,8 @@
       
         
       
+        
+      
     
     
       
@@ -1128,6 +1130,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1148,10 +1171,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1162,8 +1185,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/acknowledgments/index.html b/1.4.0/acknowledgments/index.html
index df6304da..77bc5f97 100644
--- a/1.4.0/acknowledgments/index.html
+++ b/1.4.0/acknowledgments/index.html
@@ -20,7 +20,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -565,6 +565,8 @@
       
         
       
+        
+      
     
     
       
@@ -1130,6 +1132,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1150,10 +1173,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1164,8 +1187,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/ami_index/index.html b/1.4.0/ami_index/index.html
index c747b2b5..6ddb4971 100644
--- a/1.4.0/ami_index/index.html
+++ b/1.4.0/ami_index/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/ami_lod_rec/index.html b/1.4.0/ami_lod_rec/index.html
index 9a290c28..fb04ae9b 100644
--- a/1.4.0/ami_lod_rec/index.html
+++ b/1.4.0/ami_lod_rec/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
@@ -3386,7 +3409,7 @@ <h2 id="step-6-ami-set-review-and-twig-metadata-display-preparation">Step 6: AMI
 <li>'loc_rdftype_GenreForm' =&gt;  LoC MADS RDF by type: Genre Form</li>
 <li>'loc_rdftype_Geographic' =&gt; LoC MADS RDF by type: Geographic</li>
 <li>'loc_rdftype_Temporal' =&gt;  LoC MADS RDF by type: Temporal</li>
-<li>'loc_rdftype_ExtraterrestrialArea' =&gt; LoC MADS RDF by type: Extraterrestrial Area <img alt="👾" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f47e.svg" title=":space_invader:" /></li>
+<li>'loc_rdftype_ExtraterrestrialArea' =&gt; LoC MADS RDF by type: Extraterrestrial Area <img alt="👾" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f47e.svg" title=":space_invader:" /></li>
 <li>'viaf_subjects_thing' =&gt; VIAF</li>
 <li>'getty_aat_fuzzy' =&gt; Getty aat Fuzzy</li>
 <li>'getty_aat_terms' =&gt; Getty aat Terms</li>
diff --git a/1.4.0/ami_spreadsheet_overview/index.html b/1.4.0/ami_spreadsheet_overview/index.html
index 393ac2d5..4eba0a13 100644
--- a/1.4.0/ami_spreadsheet_overview/index.html
+++ b/1.4.0/ami_spreadsheet_overview/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/ami_update/index.html b/1.4.0/ami_update/index.html
index 1acaef81..a0fc7159 100644
--- a/1.4.0/ami_update/index.html
+++ b/1.4.0/ami_update/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/annotations/index.html b/1.4.0/annotations/index.html
index 087d15ff..902edcca 100644
--- a/1.4.0/annotations/index.html
+++ b/1.4.0/annotations/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
@@ -3191,7 +3214,7 @@ <h3 id="enabling-annotations">Enabling Annotations</h3>
 <li>Select the 'Update' button.</li>
 <li>Also <strong>Save</strong> your settings using the button at the bottom of the page.</li>
 </ol>
-<p><img alt="🎉" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f389.svg" title=":tada:" /> You are now ready to get started adding annotations! <img alt="🎉" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f389.svg" title=":tada:" /></p>
+<p><img alt="🎉" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f389.svg" title=":tada:" /> You are now ready to get started adding annotations! <img alt="🎉" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f389.svg" title=":tada:" /></p>
 <h3 id="adding-and-saving-annotations">Adding and Saving Annotations</h3>
 <ol>
 <li>Navigate to the image-based Digital Object you would like to apply annotations to.</li>
diff --git a/1.4.0/archifilepersistencestrategy/index.html b/1.4.0/archifilepersistencestrategy/index.html
index 51746646..21b8faa6 100644
--- a/1.4.0/archifilepersistencestrategy/index.html
+++ b/1.4.0/archifilepersistencestrategy/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -618,6 +618,8 @@
       
         
       
+        
+      
     
     
       
@@ -1183,6 +1185,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1203,10 +1226,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1217,8 +1240,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/archipelag-logo-usage/index.html b/1.4.0/archipelag-logo-usage/index.html
index 55b5a7d1..3a011506 100644
--- a/1.4.0/archipelag-logo-usage/index.html
+++ b/1.4.0/archipelag-logo-usage/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -567,6 +567,8 @@
       
         
       
+        
+      
     
     
       
@@ -1132,6 +1134,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1152,10 +1175,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1166,8 +1189,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/archipelago-deployment-UpgradeDrupalD9toD10/index.html b/1.4.0/archipelago-deployment-UpgradeDrupalD9toD10/index.html
index e4e3dc9c..cc997a8c 100644
--- a/1.4.0/archipelago-deployment-UpgradeDrupalD9toD10/index.html
+++ b/1.4.0/archipelago-deployment-UpgradeDrupalD9toD10/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1418,6 +1420,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1438,10 +1461,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1452,8 +1475,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/archipelago-deployment-democontent/index.html b/1.4.0/archipelago-deployment-democontent/index.html
index b1ccce8c..26d8b99b 100644
--- a/1.4.0/archipelago-deployment-democontent/index.html
+++ b/1.4.0/archipelago-deployment-democontent/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1277,6 +1279,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1297,10 +1320,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1311,8 +1334,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/archipelago-deployment-live-UpgradeDrupalD9toD10/index.html b/1.4.0/archipelago-deployment-live-UpgradeDrupalD9toD10/index.html
index f6685f5f..6731af84 100644
--- a/1.4.0/archipelago-deployment-live-UpgradeDrupalD9toD10/index.html
+++ b/1.4.0/archipelago-deployment-live-UpgradeDrupalD9toD10/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1427,6 +1429,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1447,10 +1470,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1461,8 +1484,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/archipelago-deployment-live-UpgradeFrom1dot3/index.html b/1.4.0/archipelago-deployment-live-UpgradeFrom1dot3/index.html
index 377e3699..3718051e 100644
--- a/1.4.0/archipelago-deployment-live-UpgradeFrom1dot3/index.html
+++ b/1.4.0/archipelago-deployment-live-UpgradeFrom1dot3/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1436,6 +1438,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1456,10 +1479,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1470,8 +1493,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/archipelago-deployment-live-gitworkflow/index.html b/1.4.0/archipelago-deployment-live-gitworkflow/index.html
index e9ea5875..cbabed8d 100644
--- a/1.4.0/archipelago-deployment-live-gitworkflow/index.html
+++ b/1.4.0/archipelago-deployment-live-gitworkflow/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1337,6 +1339,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1357,10 +1380,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1371,8 +1394,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/archipelago-deployment-live-moveToLive/index.html b/1.4.0/archipelago-deployment-live-moveToLive/index.html
index 3f3fdfa4..9a557906 100644
--- a/1.4.0/archipelago-deployment-live-moveToLive/index.html
+++ b/1.4.0/archipelago-deployment-live-moveToLive/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1397,6 +1399,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1417,10 +1440,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1431,8 +1454,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/archipelago-deployment-live-readme/index.html b/1.4.0/archipelago-deployment-live-readme/index.html
index e0fe4653..5b6743fa 100644
--- a/1.4.0/archipelago-deployment-live-readme/index.html
+++ b/1.4.0/archipelago-deployment-live-readme/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1466,6 +1468,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1486,10 +1509,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1500,8 +1523,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/archipelago-deployment-live-search_solr_index/index.html b/1.4.0/archipelago-deployment-live-search_solr_index/index.html
index bdf37754..254ed261 100644
--- a/1.4.0/archipelago-deployment-live-search_solr_index/index.html
+++ b/1.4.0/archipelago-deployment-live-search_solr_index/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1343,6 +1345,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1363,10 +1386,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1377,8 +1400,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/archipelago-deployment-live-updatingContainers/index.html b/1.4.0/archipelago-deployment-live-updatingContainers/index.html
index 249ce0d9..ea96049e 100644
--- a/1.4.0/archipelago-deployment-live-updatingContainers/index.html
+++ b/1.4.0/archipelago-deployment-live-updatingContainers/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1220,6 +1222,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1240,10 +1263,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1254,8 +1277,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/archipelago-deployment-live-upgradeFromD8ToD9/index.html b/1.4.0/archipelago-deployment-live-upgradeFromD8ToD9/index.html
index 9a8c034e..580a1994 100644
--- a/1.4.0/archipelago-deployment-live-upgradeFromD8ToD9/index.html
+++ b/1.4.0/archipelago-deployment-live-upgradeFromD8ToD9/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1373,6 +1375,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1393,10 +1416,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1407,8 +1430,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/archipelago-deployment-live-upgradeFromRC3/index.html b/1.4.0/archipelago-deployment-live-upgradeFromRC3/index.html
index 596f4481..8ff5c0a4 100644
--- a/1.4.0/archipelago-deployment-live-upgradeFromRC3/index.html
+++ b/1.4.0/archipelago-deployment-live-upgradeFromRC3/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1364,6 +1366,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1384,10 +1407,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1398,8 +1421,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/archipelago-deployment-osx/index.html b/1.4.0/archipelago-deployment-osx/index.html
index 6569751b..e0c033a5 100644
--- a/1.4.0/archipelago-deployment-osx/index.html
+++ b/1.4.0/archipelago-deployment-osx/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1307,6 +1309,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1327,10 +1350,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1341,8 +1364,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
@@ -3554,7 +3577,7 @@ <h2 id="license">License</h2>
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 18, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 24, 2024</span>
   </span>
 
     
diff --git a/1.4.0/archipelago-deployment-readme/index.html b/1.4.0/archipelago-deployment-readme/index.html
index 00739e1c..ac872360 100644
--- a/1.4.0/archipelago-deployment-readme/index.html
+++ b/1.4.0/archipelago-deployment-readme/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1148,6 +1150,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1168,10 +1191,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1182,8 +1205,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/archipelago-deployment-ubuntu/index.html b/1.4.0/archipelago-deployment-ubuntu/index.html
index ce374535..c5e86f83 100644
--- a/1.4.0/archipelago-deployment-ubuntu/index.html
+++ b/1.4.0/archipelago-deployment-ubuntu/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1331,6 +1333,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1351,10 +1374,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1365,8 +1388,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/archipelago-deployment-windows/index.html b/1.4.0/archipelago-deployment-windows/index.html
index c2c40361..1e74082c 100644
--- a/1.4.0/archipelago-deployment-windows/index.html
+++ b/1.4.0/archipelago-deployment-windows/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1253,6 +1255,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1273,10 +1296,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1287,8 +1310,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/archipelago-glossary/index.html b/1.4.0/archipelago-glossary/index.html
index 6868394b..20edfbb0 100644
--- a/1.4.0/archipelago-glossary/index.html
+++ b/1.4.0/archipelago-glossary/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -385,6 +385,17 @@
         
       
       
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+          
+  
+  <span class="md-ellipsis">
+    Archipelago Glossary
+  </span>
+  
+
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      
       <a href="./" class="md-nav__link md-nav__link--active">
         
   
@@ -395,6 +406,367 @@
 
       </a>
       
+        
+
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+    
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#ados-archipelago-digital-objects-and-collections" class="md-nav__link">
+    <span class="md-ellipsis">
+      ADOs : Archipelago Digital Objects and Collections
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#ado-to-view-mode-mapping" class="md-nav__link">
+    <span class="md-ellipsis">
+      ADO to View Mode Mapping
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#ami-archipelago-multi-importer" class="md-nav__link">
+    <span class="md-ellipsis">
+      AMI: Archipelago Multi Importer
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#ami-sets" class="md-nav__link">
+    <span class="md-ellipsis">
+      AMI Sets
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#ami-reports-reports-tab" class="md-nav__link">
+    <span class="md-ellipsis">
+      AMI Reports / Reports Tab
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#ami-update" class="md-nav__link">
+    <span class="md-ellipsis">
+      AMI Update
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#apentitymapping-json-key" class="md-nav__link">
+    <span class="md-ellipsis">
+      ap:entitymapping (JSON key)
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#asas_file_type-json-key" class="md-nav__link">
+    <span class="md-ellipsis">
+      as:{as_file_type} (JSON key)
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#compound-object" class="md-nav__link">
+    <span class="md-ellipsis">
+      Compound Object
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#creative-work-series" class="md-nav__link">
+    <span class="md-ellipsis">
+      Creative Work Series
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#data-transformation-selections" class="md-nav__link">
+    <span class="md-ellipsis">
+      Data Transformation Selections
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#digital-object" class="md-nav__link">
+    <span class="md-ellipsis">
+      Digital Object
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#digital-object-collection-compound-object-creative-work-series" class="md-nav__link">
+    <span class="md-ellipsis">
+      Digital Object Collection / Compound Object / Creative Work Series
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#display-mode-view-mode" class="md-nav__link">
+    <span class="md-ellipsis">
+      Display Mode / View Mode
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#find-and-replace" class="md-nav__link">
+    <span class="md-ellipsis">
+      Find and Replace
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#formatters-strawberryfield-formatters" class="md-nav__link">
+    <span class="md-ellipsis">
+      Formatters / Strawberryfield Formatters
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#fragaria-redirects" class="md-nav__link">
+    <span class="md-ellipsis">
+      Fragaria Redirects
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#ismemberof-json-key" class="md-nav__link">
+    <span class="md-ellipsis">
+      ismemberof (JSON key)
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#ispartof-json-key" class="md-nav__link">
+    <span class="md-ellipsis">
+      ispartof (JSON key)
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#label-json-key" class="md-nav__link">
+    <span class="md-ellipsis">
+      label (JSON key)
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#linked-data-reconciliation-lod-reconciliation" class="md-nav__link">
+    <span class="md-ellipsis">
+      Linked Data Reconciliation / LoD Reconciliation
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#metadata-api-module" class="md-nav__link">
+    <span class="md-ellipsis">
+      Metadata API Module
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#metadata-display-entities-twig-templates" class="md-nav__link">
+    <span class="md-ellipsis">
+      Metadata Display Entities / Twig Templates
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#metadata-display-preview" class="md-nav__link">
+    <span class="md-ellipsis">
+      Metadata Display Preview
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#metadata-display-usage" class="md-nav__link">
+    <span class="md-ellipsis">
+      Metadata Display Usage
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#process-tab-ami-set-processing" class="md-nav__link">
+    <span class="md-ellipsis">
+      Process Tab / AMI Set Processing
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#search-and-replace" class="md-nav__link">
+    <span class="md-ellipsis">
+      Search and Replace
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#search-and-solr" class="md-nav__link">
+    <span class="md-ellipsis">
+      Search and Solr
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#strawberryfield-or-strawberry-field" class="md-nav__link">
+    <span class="md-ellipsis">
+      Strawberryfield (or Strawberry Field)
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#strawberry-flavour" class="md-nav__link">
+    <span class="md-ellipsis">
+      Strawberry Flavour
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#strawberryfield-formatter" class="md-nav__link">
+    <span class="md-ellipsis">
+      Strawberryfield Formatter
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#strawberry-keyname-provider" class="md-nav__link">
+    <span class="md-ellipsis">
+      Strawberry Keyname Provider
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#strawberryfield-modules" class="md-nav__link">
+    <span class="md-ellipsis">
+      Strawberryfield Modules
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#strawberry-runners-sbr" class="md-nav__link">
+    <span class="md-ellipsis">
+      Strawberry Runners / SBR
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#twig-templates-metadata-display-entities" class="md-nav__link">
+    <span class="md-ellipsis">
+      Twig Templates / Metadata Display Entities
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#type-json-key" class="md-nav__link">
+    <span class="md-ellipsis">
+      type (JSON key)
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#view-mode-display-mode" class="md-nav__link">
+    <span class="md-ellipsis">
+      View Mode / Display Mode
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#webforms" class="md-nav__link">
+    <span class="md-ellipsis">
+      Webforms
+    </span>
+  </a>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+      
     </li>
   
 
@@ -579,6 +951,8 @@
       
         
       
+        
+      
     
     
       
@@ -1144,6 +1518,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1164,10 +1559,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1178,8 +1573,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
@@ -3064,6 +3459,356 @@
     
   
   
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#ados-archipelago-digital-objects-and-collections" class="md-nav__link">
+    <span class="md-ellipsis">
+      ADOs : Archipelago Digital Objects and Collections
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#ado-to-view-mode-mapping" class="md-nav__link">
+    <span class="md-ellipsis">
+      ADO to View Mode Mapping
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#ami-archipelago-multi-importer" class="md-nav__link">
+    <span class="md-ellipsis">
+      AMI: Archipelago Multi Importer
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#ami-sets" class="md-nav__link">
+    <span class="md-ellipsis">
+      AMI Sets
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#ami-reports-reports-tab" class="md-nav__link">
+    <span class="md-ellipsis">
+      AMI Reports / Reports Tab
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#ami-update" class="md-nav__link">
+    <span class="md-ellipsis">
+      AMI Update
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#apentitymapping-json-key" class="md-nav__link">
+    <span class="md-ellipsis">
+      ap:entitymapping (JSON key)
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#asas_file_type-json-key" class="md-nav__link">
+    <span class="md-ellipsis">
+      as:{as_file_type} (JSON key)
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#compound-object" class="md-nav__link">
+    <span class="md-ellipsis">
+      Compound Object
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#creative-work-series" class="md-nav__link">
+    <span class="md-ellipsis">
+      Creative Work Series
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#data-transformation-selections" class="md-nav__link">
+    <span class="md-ellipsis">
+      Data Transformation Selections
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#digital-object" class="md-nav__link">
+    <span class="md-ellipsis">
+      Digital Object
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#digital-object-collection-compound-object-creative-work-series" class="md-nav__link">
+    <span class="md-ellipsis">
+      Digital Object Collection / Compound Object / Creative Work Series
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#display-mode-view-mode" class="md-nav__link">
+    <span class="md-ellipsis">
+      Display Mode / View Mode
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#find-and-replace" class="md-nav__link">
+    <span class="md-ellipsis">
+      Find and Replace
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#formatters-strawberryfield-formatters" class="md-nav__link">
+    <span class="md-ellipsis">
+      Formatters / Strawberryfield Formatters
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#fragaria-redirects" class="md-nav__link">
+    <span class="md-ellipsis">
+      Fragaria Redirects
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#ismemberof-json-key" class="md-nav__link">
+    <span class="md-ellipsis">
+      ismemberof (JSON key)
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#ispartof-json-key" class="md-nav__link">
+    <span class="md-ellipsis">
+      ispartof (JSON key)
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#label-json-key" class="md-nav__link">
+    <span class="md-ellipsis">
+      label (JSON key)
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#linked-data-reconciliation-lod-reconciliation" class="md-nav__link">
+    <span class="md-ellipsis">
+      Linked Data Reconciliation / LoD Reconciliation
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#metadata-api-module" class="md-nav__link">
+    <span class="md-ellipsis">
+      Metadata API Module
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#metadata-display-entities-twig-templates" class="md-nav__link">
+    <span class="md-ellipsis">
+      Metadata Display Entities / Twig Templates
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#metadata-display-preview" class="md-nav__link">
+    <span class="md-ellipsis">
+      Metadata Display Preview
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#metadata-display-usage" class="md-nav__link">
+    <span class="md-ellipsis">
+      Metadata Display Usage
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#process-tab-ami-set-processing" class="md-nav__link">
+    <span class="md-ellipsis">
+      Process Tab / AMI Set Processing
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#search-and-replace" class="md-nav__link">
+    <span class="md-ellipsis">
+      Search and Replace
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#search-and-solr" class="md-nav__link">
+    <span class="md-ellipsis">
+      Search and Solr
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#strawberryfield-or-strawberry-field" class="md-nav__link">
+    <span class="md-ellipsis">
+      Strawberryfield (or Strawberry Field)
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#strawberry-flavour" class="md-nav__link">
+    <span class="md-ellipsis">
+      Strawberry Flavour
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#strawberryfield-formatter" class="md-nav__link">
+    <span class="md-ellipsis">
+      Strawberryfield Formatter
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#strawberry-keyname-provider" class="md-nav__link">
+    <span class="md-ellipsis">
+      Strawberry Keyname Provider
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#strawberryfield-modules" class="md-nav__link">
+    <span class="md-ellipsis">
+      Strawberryfield Modules
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#strawberry-runners-sbr" class="md-nav__link">
+    <span class="md-ellipsis">
+      Strawberry Runners / SBR
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#twig-templates-metadata-display-entities" class="md-nav__link">
+    <span class="md-ellipsis">
+      Twig Templates / Metadata Display Entities
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#type-json-key" class="md-nav__link">
+    <span class="md-ellipsis">
+      type (JSON key)
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#view-mode-display-mode" class="md-nav__link">
+    <span class="md-ellipsis">
+      View Mode / Display Mode
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#webforms" class="md-nav__link">
+    <span class="md-ellipsis">
+      Webforms
+    </span>
+  </a>
+  
+</li>
+      
+    </ul>
+  
 </nav>
                   </div>
                 </div>
@@ -3081,8 +3826,161 @@
 
 
 <h1 id="archipelago-glossary-of-terms">Archipelago Glossary of Terms</h1>
-<p>This documentation page is under construction, and will be availble soon. </p>
-<p>This page will provide a list for quick reference points for some of the most common Archipelago terminology you are likely to encounter.</p>
+<p>This list provides quick reference points for some of the most common Archipelago terminology you are likely to encounter.</p>
+<p>If you are not seeing a term listed here, try using the 'Search' found above to look for your term across all documentation pages.</p>
+<h3 id="ados-archipelago-digital-objects-and-collections">ADOs : Archipelago Digital Objects and Collections</h3>
+<p>Abbreviation of Archipelago Digital Object. Can be used to refer to ADO Collections/Compounds/Creative Work Series Parents as well.</p>
+<p>Essentially, any Content type that uses a 'Strawberryfield' (see reference below) is an ADO.</p>
+<h3 id="ado-to-view-mode-mapping">ADO to View Mode Mapping</h3>
+<p>This refers to the 'ADO to View Mode Mapping' form found at <code>/admin/config/archipelago/viewmode_mapping</code>. This form is used to determine the 'Display Mode' used for an ADO based on the object's <code>type</code>.</p>
+<p>See also the 'Display Mode' and <code>type</code> references below.</p>
+<h3 id="ami-archipelago-multi-importer">AMI: Archipelago Multi Importer</h3>
+<p>Archipelago's Module for module for batch/bulk/mass ingests of Archipelago digital objects (ADOs) and collections. AMI also enables you to perform batch administrative actions, such as updating, patching/revising, or deleting digital objects and collections.</p>
+<p>Related documentation: <a href="../ami_index/">Archipelago Multi-Importer (AMI)</a>.</p>
+<h3 id="ami-sets">AMI Sets</h3>
+<p>This refers to the special AMI custom entities that hold an Ingest or Update Strategy, a source data CSV with data/metadata, and possibly a CSV for Processed Data coming from Archipelago's <a href="../ami_lod_rec/">Linked Data Reconciliation</a>.</p>
+<p>Related documentation: <a href="../ami_index/#ami-set-entity">Archipelago Multi-Importer (AMI) - AMI Set Entity</a>.</p>
+<h3 id="ami-reports-reports-tab">AMI Reports / Reports Tab</h3>
+<p>The AMI Reports tab contains information related to the last Processing operation performed for an AMI Set.</p>
+<p>Related documentation: <a href="../AMIviaSpreadsheets/#step-10-review-your-newly-created-digital-objects-directly-or-via-ami-set-report">Step 10: Review your newly created Digital Objects directly or via AMI Set Report</a>.</p>
+<h3 id="ami-update">AMI Update</h3>
+<p>AMI's Update Operations can be used to Update, Replace, or Append metadata values or files for existing Digital Objects and Collections found in your Archipelago. You can prepare and use AMI Update Sets in different ways using one of three functional operations, depending on your update needs.</p>
+<p>Related documentation: <a href="../ami_update/">AMI Update Sets</a>.</p>
+<h3 id="apentitymapping-json-key"><code>ap:entitymapping</code> (JSON key)</h3>
+<p>This key is used to provide structural mapping hints for Archipelago, such as whether an ADO uses <code>images</code> or <code>ispartof</code>. </p>
+<p>Related documentation: <a href="../metadatainarchipelago/#the-apentitymapping-key">Metadata in Archipelago - The ap:entitymapping key</a></p>
+<h3 id="asas_file_type-json-key"><code>as:{as_file_type}</code> (JSON key)</h3>
+<p>This key is used to provide filetype mapping information for Archipelago, and will be automatically generated depending on the type of file(s) associated with an ADO. </p>
+<p>Related documentation: <a href="../metadatainarchipelago/#the-asas_file_type-keys">Metadata in Archipelago - The as:{as_file_type} key</a></p>
+<h3 id="compound-object">Compound Object</h3>
+<p>See 'Digital Object Collection / Compound Object / Creative Work Series' below.</p>
+<h3 id="creative-work-series">Creative Work Series</h3>
+<p>See 'Digital Object Collection / Compound Object / Creative Work Series' below.</p>
+<h3 id="data-transformation-selections">Data Transformation Selections</h3>
+<p>The data transformation approach used in an AMI Set that determines how your source data will be transformed into ADOs upon AMI Set Processing.</p>
+<p>Related documentation: <a href="../AMIviaSpreadsheets/#step-3-data-transformation-selections">Step 3. Data Transformation Selections</a>.</p>
+<h3 id="digital-object">Digital Object</h3>
+<p>This is the Drupal Content Type that refers to Digital Objects in Archipelago. </p>
+<p>This is the Content Type used for standalone/non-hierarchically structured or Child Objects in Parent-Child Relationships.</p>
+<p>For most Archipelago repositories, this is the most common Drupal Content Type used.</p>
+<p>More context can be found here: <a href="../metadatainarchipelago/#drupal-and-json">Metadata in Archipelago - Drupal and JSON</a></p>
+<h3 id="digital-object-collection-compound-object-creative-work-series">Digital Object Collection / Compound Object / Creative Work Series</h3>
+<p>This is the Drupal Content Type that refers to Digital Object Collections or Parent Compound Objects / Creative Work Series in Archipelago. </p>
+<p>This is Content Type used for Parent Objects in Parent-Child Relationships (such as Book to Page, where Book = Parent and Page = Child). </p>
+<p>More context can be found here: <a href="../metadatainarchipelago/#drupal-and-json">Metadata in Archipelago - Drupal and JSON</a></p>
+<h3 id="display-mode-view-mode">Display Mode / View Mode</h3>
+<p>A Display Mode is a configureable page layout, using Formatters and other options, for the general display of an ADO in Archipelago. You can define Display Modes for different <code>types</code> of Digital Objects and Collections. </p>
+<p>Display Mode and View Mode can be used interchangeably.</p>
+<p>Related documentation: <a href="../webformsasinput/">Primer on Display Modes</a>. </p>
+<h3 id="find-and-replace">Find and Replace</h3>
+<p>Archipelago's Advanced Batch Find and Replace functionality provides different ways for you to efficiently Find/Search and Replace metadata values found in the raw JSON of your Digital Objects and Collections.</p>
+<p>Related documentation: <a href="../find_and_replace/">Advanced Batch Find and Replace</a></p>
+<h3 id="formatters-strawberryfield-formatters">Formatters / Strawberryfield Formatters</h3>
+<p>See 'Strawberryfield Formatters' below.</p>
+<h3 id="fragaria-redirects">Fragaria Redirects</h3>
+<p>Archipelago's Fragaria Redirect Module is a special module that provides dynamic redirect routes matched against existing Search API field values.</p>
+<p>Related documentation: <a href="../fragaria/">Fragaria Redirects Module</a>. </p>
+<h3 id="ismemberof-json-key"><code>ismemberof</code> (JSON key)</h3>
+<p>This key, in default Archipelago configurations, is used for Collection Membership.</p>
+<p>This key can contain multiple values if appropriate for your repository's Collections structure.</p>
+<p>For ingested ADOs, this key will contain 'Drupal Node id' integers for corresponding Collection(s).</p>
+<p>In AMI sets, this key can contain integers to connect to an object's corresponding row in the same spreadsheet/CSV, or a UUID for an already ingested Collection object.</p>
+<p>This key should never contain string values for Collection labels/titles.</p>
+<p>See also <code>ap:entitymapping</code> key above.</p>
+<h3 id="ispartof-json-key"><code>ispartof</code> (JSON key)</h3>
+<p>This key, in default Archipelago configurations, is used for Parent-Child Object Relationships (so a Child ADO would reference the Parent ADO in <code>ispartof</code>).</p>
+<p>This key should contain single values only, due to the nature of the Parent-Child Object Relationship.</p>
+<p>For ingested ADOs, this key will contain 'Drupal Node id' integers for a corresponding Parent Object.</p>
+<p>In AMI sets, this key can contain integers to connect to an object's corresponding row in the same spreadsheet/CSV, or a UUID for an already ingested Parent object.</p>
+<p>This key should never contain string values for Parent Object labels/titles.</p>
+<p>See also <code>ap:entitymapping</code> key above.</p>
+<h3 id="label-json-key"><code>label</code> (JSON key)</h3>
+<p>This key is used for the title of the Digital Object or Collection. This key is <strong>required</strong> for all ADOs. </p>
+<p>Related documentation: <a href="../metadatainarchipelago/#the-label-key">Metadata in Archipelago - The label key</a></p>
+<h3 id="linked-data-reconciliation-lod-reconciliation">Linked Data Reconciliation / LoD Reconciliation</h3>
+<p>AMI's (Archipelago Multi Importer's) Linked Data Reconciliation tool can be used to enrich your metadata with Linked Data (LoD). Using this tool, you can map values from your topical/subject metadata elements to your preferred LoD vocabulary source. These mappings can then be transformed via a corresponding Metadata Display (Twig) template to process the values into JSON-formatted metadata for your specified AMI set.</p>
+<p>Related documentation: <a href="../ami_lod_rec/">Linked Data Reconciliation</a>.</p>
+<h3 id="metadata-api-module">Metadata API Module</h3>
+<p>Archipelago's Metadata API Module is a special Strawberryfield Formatter Module that implements Metadata API Endpoints and uses Views and Metadata Display Entities as a configuration option to provide dynamic APIs based on Metadata present in each Archipelago Digital Object (or Node) that contains a Strawberryfield type of field (JSON).</p>
+<p>Related documentation: <a href="../open_api/">Metadata API Module</a></p>
+<h3 id="metadata-display-entities-twig-templates">Metadata Display Entities / Twig Templates</h3>
+<p>Metadata Display Entities are particular Twig Templates used in Archipelago to transform or 'cast' JSON metadata/data into different displays and outputs.</p>
+<p>Related documentation: <a href="../metadatatwigs/">Twig Templates and Archipelago</a></p>
+<h3 id="metadata-display-preview">Metadata Display Preview</h3>
+<p>Archipelago's Metadata Display Preview is a very handy tool for your repository toolkit that enables you to preview the output of your Metadata Display (Twig) Templates (found at <code>/metadatadisplay/list</code>).</p>
+<p>Related documentation: <a href="../metadata_display_preview/">Metadata Display Preview</a></p>
+<h3 id="metadata-display-usage">Metadata Display Usage</h3>
+<p>Archipelago's Metadata Display Usage tab is an extension of Metadata Display Preview that provides a convenient way for you to check where a Metadata Display Template is currently being used across your Archipelago. </p>
+<p>Related documentation: <a href="../metadata_display_usage/">Metadata Display Usage</a></p>
+<h3 id="process-tab-ami-set-processing">Process Tab / AMI Set Processing</h3>
+<p>The Process Tab contains configuration options for selecting outcomes related to the Processing of your AMI Set.</p>
+<p>Related documentation: <a href="../AMIviaSpreadsheets/#step-7-ami-set-processings">Step 7: AMI Set Processing</a>.</p>
+<h3 id="search-and-replace">Search and Replace</h3>
+<p>See 'Find and Replace' above.</p>
+<h3 id="search-and-solr">Search and Solr</h3>
+<p>Refers to the integrations of related (Archipelago, Drupal) Search and Solr Index components in Archipelago.</p>
+<p>Related documentation: <a href="search-solr-index.md">Search &amp; Solr Overview</a></p>
+<h3 id="strawberryfield-or-strawberry-field">Strawberryfield (or Strawberry Field)</h3>
+<p>The flexible, extensible JSON blob Field attached to every Archipelago Digital Object / Collection / Compound Object / Creative Work Series.</p>
+<p>The Strawberryfield contains all of the data and metadata associated with ADOs in a single Field (per ADO).</p>
+<p>In default Archipelagos, can be viewed in the 'Raw JSON' display beneath every ADO when logged in as an authenticated user.</p>
+<p>Related documentation: <a href="../metadatainarchipelago/">Metadata in Archipelago</a>.</p>
+<h3 id="strawberry-flavour">Strawberry Flavour</h3>
+<p>A 'Strawberry flavour' refers to a post-processing related document, such as a generated HOCR solr document or new filetype created by Archipelago such as a WACZ file generated from an original WARC file.</p>
+<h3 id="strawberryfield-formatter">Strawberryfield Formatter</h3>
+<p>Strawberryfield Formatters are Archipelago specific Field Formatters that can transform Strawberryfield JSON data into different formats for display, including in IIIF Manifests and/or different Viewers.</p>
+<p>More information found at: <a href="../strawberryfield-formatters/">Strawberryfield Formatters</a></p>
+<h3 id="strawberry-keyname-provider">Strawberry Keyname Provider</h3>
+<p>Strawberry Keyname Providers feed the values from your desired JSON keys to Solr fields. They are a crucial part of the Search and Solr setup in Archipelago.</p>
+<p>Related Documentation: <a href="../search_solr_index/#2-strawberry-keyname-providers">Search &amp; Solr Overview - Strawberry Keyname Providers</a></p>
+<h3 id="strawberryfield-modules">Strawberryfield Modules</h3>
+<p>Strawberryfield Modules are the Archipelago specific modules at the heart of our system.</p>
+<ul>
+<li>Strawberryfield<ul>
+<li><a href="https://github.com/esmero/strawberryfield">Github Repository</a></li>
+<li><a href="../strawberryfields/">Related documentation</a></li>
+</ul>
+</li>
+<li>Format Strawberryfield<ul>
+<li><a href="https://github.com/esmero/format_strawberryfield">Github Repository</a></li>
+<li><a href="../strawberryfield-formatters/">Related documentation</a></li>
+</ul>
+</li>
+<li>Webform Strawberryfield<ul>
+<li><a href="https://github.com/esmero/webform_strawberryfield">Github Repository</a></li>
+<li><a href="../webforms/">Related documentation</a></li>
+</ul>
+</li>
+<li>Strawberry Runners<ul>
+<li><a href="https://github.com/esmero/strawberry_runners">Github Repository</a></li>
+<li><a href="../strawberryrunners/">Related documentation</a></li>
+</ul>
+</li>
+<li>Archipelago Multi-Importer (AMI)<ul>
+<li><a href="https://github.com/esmero/ami">Github Repository</a></li>
+<li><a href="../ami_index/">Related documentation</a></li>
+</ul>
+</li>
+<li>Fragaria (Redirects)<ul>
+<li><a href="https://github.com/esmero/fragaria">Github Repository</a></li>
+<li><a href="../fragaria/">Related Documentation</a></li>
+</ul>
+</li>
+</ul>
+<p>Related documentation: <a href="../strawberryfields/">Strawberryfields Forever</a>.</p>
+<h3 id="strawberry-runners-sbr">Strawberry Runners / SBR</h3>
+<p>Archipelago's Strawberry Runners (SBR) module provides provides a set of post-processing capabilities for the JSON based metadata, files and entities that comprise your Archipelago Digital Objects (ADOs). </p>
+<p>Related documentation: <a href="../strawberryrunners/">Strawberry Runners Post-Processing Configuration</a>.</p>
+<h3 id="twig-templates-metadata-display-entities">Twig Templates / Metadata Display Entities</h3>
+<p>See 'Metadata Display Entities' above.</p>
+<h3 id="type-json-key"><code>type</code> (JSON key)</h3>
+<p>This key is used for the Digital Object or Digital Object Collection/semantic Type, such as 'Photograph' or 'Collection'. This key is <strong>required</strong> for all ADOs.</p>
+<p>Related documentation: <a href="../metadatainarchipelago/#the-type-key">Metadata in Archipelago - The type key</a></p>
+<h3 id="view-mode-display-mode">View Mode / Display Mode</h3>
+<p>See 'Display Mode' entry above.</p>
+<h3 id="webforms">Webforms</h3>
+<p>The 'Webform Strawberryfield' module provides Drupal Webform ( == awesome piece of code) integrations for StrawberryField, so you can really have control over your Metadata ingests entered and edited via webforms.</p>
+<p>Related documentation: <a href="../webforms/">Webforms in Archipelago</a></p>
 <hr />
 <p>Thank you for reading! Please contact us on our <a href="https://groups.google.com/forum/#!forum/archipelago-commons">Archipelago Commons Google Group</a> with any questions or feedback.</p>
 <p>Return to the <a href="../">Archipelago Documentation main page</a>.</p>
@@ -3108,7 +4006,7 @@ <h1 id="archipelago-glossary-of-terms">Archipelago Glossary of Terms</h1>
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 17, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">November 6, 2024</span>
   </span>
 
     
diff --git a/1.4.0/createdisplaymodes/index.html b/1.4.0/createdisplaymodes/index.html
index 610a4e3e..525e2761 100644
--- a/1.4.0/createdisplaymodes/index.html
+++ b/1.4.0/createdisplaymodes/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/customwebformelements/index.html b/1.4.0/customwebformelements/index.html
index 05f4a178..725fcf94 100644
--- a/1.4.0/customwebformelements/index.html
+++ b/1.4.0/customwebformelements/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/devops/index.html b/1.4.0/devops/index.html
index 92cbc119..06f290ff 100644
--- a/1.4.0/devops/index.html
+++ b/1.4.0/devops/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -636,6 +636,8 @@
       
         
       
+        
+      
     
     
       
@@ -1201,6 +1203,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1221,10 +1244,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1235,8 +1258,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
@@ -3233,7 +3256,9 @@ <h3 id="under-the-hood-archipelagos-architecture-is">Under the hood, Archipelago
 </tr>
 </tbody>
 </table>
-<p>_Information related to non-Dockerized installation and configruation can be found here: <a href="../traditional-install/">Traditional Installation Notes</a></p>
+<ul>
+<li>Information related to non-Dockerized installation and configruation can be found here: <a href="../traditional-install/">Traditional Installation Notes</a></li>
+</ul>
 <h3 id="strawberryfield-modules-at-the-heart-of-every-archipelago">Strawberryfield Modules at the heart of every Archipelago:</h3>
 <ul>
 <li><a href="https://github.com/esmero/strawberryfield">Strawberryfield</a></li>
@@ -3241,6 +3266,7 @@ <h3 id="strawberryfield-modules-at-the-heart-of-every-archipelago">Strawberryfie
 <li><a href="https://github.com/esmero/webform_strawberryfield">Webform Strawberryfield</a></li>
 <li><a href="https://github.com/esmero/strawberry_runners">Strawberry Runners</a></li>
 <li><a href="https://github.com/esmero/ami">Archipelago Multi-Importer (AMI)</a></li>
+<li><a href="https://github.com/esmero/fragaria">Fragaria Redirects</a></li>
 </ul>
 <p>Documentation related to the Strawberryfield modules can be found here: <a href="../strawberryfields/">Strawberryfields Forever</a></p>
 <h3 id="archipelago-also-extends-these-powerful-tools">Archipelago also extends these powerful tools:</h3>
@@ -3281,7 +3307,7 @@ <h3 id="archipelago-also-extends-these-powerful-tools">Archipelago also extends
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 17, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">November 6, 2024</span>
   </span>
 
     
diff --git a/1.4.0/documentation_about/index.html b/1.4.0/documentation_about/index.html
index bbdbeee3..b1c6e07a 100644
--- a/1.4.0/documentation_about/index.html
+++ b/1.4.0/documentation_about/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -567,6 +567,8 @@
       
         
       
+        
+      
     
     
       
@@ -1132,6 +1134,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1152,10 +1175,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1166,8 +1189,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/documentation_features/index.html b/1.4.0/documentation_features/index.html
index 52cac211..3ef59493 100644
--- a/1.4.0/documentation_features/index.html
+++ b/1.4.0/documentation_features/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -567,6 +567,8 @@
       
         
       
+        
+      
     
     
       
@@ -1132,6 +1134,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1152,10 +1175,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1166,8 +1189,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/documentation_technical/index.html b/1.4.0/documentation_technical/index.html
index 8b5aae3d..cb57ba33 100644
--- a/1.4.0/documentation_technical/index.html
+++ b/1.4.0/documentation_technical/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -567,6 +567,8 @@
       
         
       
+        
+      
     
     
       
@@ -1132,6 +1134,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1152,10 +1175,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1166,8 +1189,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/documentation_template/index.html b/1.4.0/documentation_template/index.html
index a87383bb..2acdb275 100644
--- a/1.4.0/documentation_template/index.html
+++ b/1.4.0/documentation_template/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -567,6 +567,8 @@
       
         
       
+        
+      
     
     
       
@@ -1132,6 +1134,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1152,10 +1175,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1166,8 +1189,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/documentation_workflow/index.html b/1.4.0/documentation_workflow/index.html
index 030134fd..94cad7a8 100644
--- a/1.4.0/documentation_workflow/index.html
+++ b/1.4.0/documentation_workflow/index.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -563,6 +563,8 @@
       
         
       
+        
+      
     
     
       
@@ -1128,6 +1130,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1148,10 +1171,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1162,8 +1185,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/drupal_core_update/index.html b/1.4.0/drupal_core_update/index.html
index f61afa3f..346e5f2c 100644
--- a/1.4.0/drupal_core_update/index.html
+++ b/1.4.0/drupal_core_update/index.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -563,6 +563,8 @@
       
         
       
+        
+      
     
     
       
@@ -1128,6 +1130,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1148,10 +1171,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1162,8 +1185,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/embargo/index.html b/1.4.0/embargo/index.html
index 17e2cd13..87fdde98 100644
--- a/1.4.0/embargo/index.html
+++ b/1.4.0/embargo/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/experimental_tools/index.html b/1.4.0/experimental_tools/index.html
index 441a3ba4..b2697c65 100644
--- a/1.4.0/experimental_tools/index.html
+++ b/1.4.0/experimental_tools/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/export-to-csv/index.html b/1.4.0/export-to-csv/index.html
index c1a3ca48..9f773421 100644
--- a/1.4.0/export-to-csv/index.html
+++ b/1.4.0/export-to-csv/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/find_and_replace/index.html b/1.4.0/find_and_replace/index.html
index 567cbbc2..740f5bc6 100644
--- a/1.4.0/find_and_replace/index.html
+++ b/1.4.0/find_and_replace/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/find_and_replace_action_json_patch/index.html b/1.4.0/find_and_replace_action_json_patch/index.html
index 2ce43772..ab1d38ac 100644
--- a/1.4.0/find_and_replace_action_json_patch/index.html
+++ b/1.4.0/find_and_replace_action_json_patch/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/find_and_replace_action_text/index.html b/1.4.0/find_and_replace_action_text/index.html
index 565a952d..d0f0809b 100644
--- a/1.4.0/find_and_replace_action_text/index.html
+++ b/1.4.0/find_and_replace_action_text/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/find_and_replace_action_webform/index.html b/1.4.0/find_and_replace_action_webform/index.html
index 22c924c7..c944bc97 100644
--- a/1.4.0/find_and_replace_action_webform/index.html
+++ b/1.4.0/find_and_replace_action_webform/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/firstobject/index.html b/1.4.0/firstobject/index.html
index c3f4d7f3..e5a156f1 100644
--- a/1.4.0/firstobject/index.html
+++ b/1.4.0/firstobject/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/fragaria/index.html b/1.4.0/fragaria/index.html
index 23253c37..1d1ab553 100644
--- a/1.4.0/fragaria/index.html
+++ b/1.4.0/fragaria/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/generalqa_minio_logging/index.html b/1.4.0/generalqa_minio_logging/index.html
index a6fb0edd..5000db50 100644
--- a/1.4.0/generalqa_minio_logging/index.html
+++ b/1.4.0/generalqa_minio_logging/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1133,6 +1135,27 @@
               
                 
   
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
   
     
   
@@ -1158,10 +1181,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" checked>
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1172,8 +1195,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/generalqa_smtp_configuration/index.html b/1.4.0/generalqa_smtp_configuration/index.html
index 5490c9a7..13a3a181 100644
--- a/1.4.0/generalqa_smtp_configuration/index.html
+++ b/1.4.0/generalqa_smtp_configuration/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1133,6 +1135,27 @@
               
                 
   
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
   
     
   
@@ -1158,10 +1181,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" checked>
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1172,8 +1195,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/generalqa_twig_modules_configuration/index.html b/1.4.0/generalqa_twig_modules_configuration/index.html
index 81d0f4d5..b00e2209 100644
--- a/1.4.0/generalqa_twig_modules_configuration/index.html
+++ b/1.4.0/generalqa_twig_modules_configuration/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1133,6 +1135,27 @@
               
                 
   
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
   
     
   
@@ -1158,10 +1181,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" checked>
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1172,8 +1195,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/giveortake/index.html b/1.4.0/giveortake/index.html
index 74d7ccc0..3edf689e 100644
--- a/1.4.0/giveortake/index.html
+++ b/1.4.0/giveortake/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -567,6 +567,8 @@
       
         
       
+        
+      
     
     
       
@@ -1132,6 +1134,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1152,10 +1175,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1166,8 +1189,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/googleapi/index.html b/1.4.0/googleapi/index.html
index 4293a308..28237b95 100644
--- a/1.4.0/googleapi/index.html
+++ b/1.4.0/googleapi/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
@@ -3248,7 +3271,7 @@ <h3 id="generating-google-oauth2-credentials">Generating Google OAuth2 Credentia
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 27, 2021</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 18, 2021</span>
   </span>
 
     
diff --git a/1.4.0/iiif-content-search/index.html b/1.4.0/iiif-content-search/index.html
index b281791d..20beb82b 100644
--- a/1.4.0/iiif-content-search/index.html
+++ b/1.4.0/iiif-content-search/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/iiif_server_settings/index.html b/1.4.0/iiif_server_settings/index.html
index b943c6d2..2030ef00 100644
--- a/1.4.0/iiif_server_settings/index.html
+++ b/1.4.0/iiif_server_settings/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/index.html b/1.4.0/index.html
index 286cc14c..59225a3a 100644
--- a/1.4.0/index.html
+++ b/1.4.0/index.html
@@ -20,7 +20,7 @@
       
       
       <link rel="icon" href="images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -575,6 +575,8 @@
       
         
       
+        
+      
     
     
       
@@ -1140,6 +1142,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1160,10 +1183,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1174,8 +1197,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/inthewild/index.html b/1.4.0/inthewild/index.html
index 1067ce48..ebbf8719 100644
--- a/1.4.0/inthewild/index.html
+++ b/1.4.0/inthewild/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -567,6 +567,8 @@
       
         
       
+        
+      
     
     
       
@@ -1132,6 +1134,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1152,10 +1175,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1166,8 +1189,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
@@ -3218,10 +3241,7 @@ <h2 id="metro-archipelago">METRO + Archipelago</h2>
 </ul>
 </li>
 <li>
-<p><a href="https://www.archives.nysed.gov">New York State Archives</a> Finding Aids Discovery Portal</p>
-<ul>
-<li>*Migration and development kicked off Spring 2024</li>
-</ul>
+<p><a href="https://findingaids.nysed.gov">New York State Archives</a> Finding Aids Discovery Portal</p>
 </li>
 <li>
 <p><a href="https://www.nyspersonalhistory.com">New York State COVID-19 Personal History Initiative</a></p>
@@ -3306,7 +3326,7 @@ <h2 id="we-should-be-here">We should be here</h2>
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">August 20, 2024</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 28, 2024</span>
   </span>
 
     
diff --git a/1.4.0/metadata_display_preview/index.html b/1.4.0/metadata_display_preview/index.html
index f5dc5a40..32bb45ca 100644
--- a/1.4.0/metadata_display_preview/index.html
+++ b/1.4.0/metadata_display_preview/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/metadata_display_usage/index.html b/1.4.0/metadata_display_usage/index.html
index 0383bb2c..52b79170 100644
--- a/1.4.0/metadata_display_usage/index.html
+++ b/1.4.0/metadata_display_usage/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/metadatainarchipelago/index.html b/1.4.0/metadatainarchipelago/index.html
index b2e780f7..102248fb 100644
--- a/1.4.0/metadatainarchipelago/index.html
+++ b/1.4.0/metadatainarchipelago/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -774,6 +774,8 @@
       
         
       
+        
+      
     
     
       
@@ -1339,6 +1341,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1359,10 +1382,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1373,8 +1396,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/metadatatwigs/index.html b/1.4.0/metadatatwigs/index.html
index 9a355047..27279e08 100644
--- a/1.4.0/metadatatwigs/index.html
+++ b/1.4.0/metadatatwigs/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/modifyingfileextensionsinwebform/index.html b/1.4.0/modifyingfileextensionsinwebform/index.html
index 96bd6987..3e91fdec 100644
--- a/1.4.0/modifyingfileextensionsinwebform/index.html
+++ b/1.4.0/modifyingfileextensionsinwebform/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/open_api/index.html b/1.4.0/open_api/index.html
index 305ad6b1..56374e0c 100644
--- a/1.4.0/open_api/index.html
+++ b/1.4.0/open_api/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/ourtake/index.html b/1.4.0/ourtake/index.html
index 47831cba..56b3269e 100644
--- a/1.4.0/ourtake/index.html
+++ b/1.4.0/ourtake/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -618,6 +618,8 @@
       
         
       
+        
+      
     
     
       
@@ -1183,6 +1185,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1203,10 +1226,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1217,8 +1240,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/presentations_events/index.html b/1.4.0/presentations_events/index.html
index b35404b0..cf4ab0cc 100644
--- a/1.4.0/presentations_events/index.html
+++ b/1.4.0/presentations_events/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -567,6 +567,8 @@
       
         
       
+        
+      
     
     
       
@@ -1132,6 +1134,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1152,10 +1175,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1166,8 +1189,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/search-within-collection/index.html b/1.4.0/search-within-collection/index.html
index f676ca66..9a692e68 100644
--- a/1.4.0/search-within-collection/index.html
+++ b/1.4.0/search-within-collection/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/search/search_index.json b/1.4.0/search/search_index.json
index 7d3f2df8..345cabcf 100644
--- a/1.4.0/search/search_index.json
+++ b/1.4.0/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Archipelago Commons Intro","text":"<p>Archipelago Commons, or simply Archipelago, is an Open Source Digital Objects Repository / DAM Server Architecture based on the popular CMS <code>Drupal 9/10+</code> and released under <code>GLP V.3 License</code>. Archipelago is developed and supported at the Metropolitan New York Library Council (METRO).</p> <p>Archipelago is a mix of deeply integrated custom-coded Drupal modules (made with care by us, the Digital Services Team and METRO) and a curated and well-configured Drupal instance, running under a discrete and well-planned set of complementary additional service containers. You can learn more about the different Software Services used by Archipelago here, and Archipelago's unique approach to Metadata here.</p> <p>Archipelago's primary focus is to serve the greater <code>GLAM community</code> (libraries, archives, museums, universities and colleges, cultural heritage organizations) by providing a flexible, consistent, and unified way of describing, storing, linking, exposing metadata and media assets that make up rich repository collections all around our shared beautiful world. We respect identities and existing workflows, and we endeavor to design Archipelago in ways that empower communities of every size, shade, and shape.</p> <p>Finally, Archipelago tries to stay humble, slim, and nimble in nature with a small codebase full of inline comments and <code>@todos</code>. All of our work is driven by a clear and concise but thoughtful planned technical roadmap --updated in tandem with new releases.</p> <p>We recommend you start with the Core Documentation Guides listed here as you begin your Archipelago explorations. </p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p>"},{"location":"101_guides_list/","title":"Archipelago 101: Core Documentation Guides","text":"<p>Top 10 guides we recommend you review as you get started working with Archipelago:</p> <ol> <li> <p>Metadata in Archipelago: a long and worthwhile read that covers the fundamentals of Archipelago's architecture and approach to metadata and data</p> </li> <li> <p>Strawberryfield Formatters: overview of the general setup of an Archipelago Digital Object (ADO) page and the way your ADO JSON metadata and data are output</p> </li> <li> <p>Primer on Display Modes &amp; How to Create a Webform as an Input Method: deeper look at Display Modes and Form Modes, two ways you'll be interacting with your ADOs most frequently</p> </li> <li> <p>Twig Templates and Archipelago: a great place to dive into one of Archipelago's best loved feature areas</p> </li> <li> <p>Archipelago Multi Importer: all about Archipelago's batch ingest and update functionality</p> </li> <li> <p>Search and Solr Overview: for repositories, it's all about the search</p> <ul> <li>In-a-nutshell : JSON data to Strawberry Keyname Providers to Solr: essential overview of the pipeline from JSON data into and out of Solr</li> <li>Strawberry Key Name Providers, Solr Field, and Facet Configuration: fundamental information for site adminisrators</li> </ul> </li> <li> <p>Advanced Batch Find and Replace: targetted batch updates for your ADO metadata</p> </li> <li> <p>Strawberry Runners Post-Processing Configuration: background post-processing defaults and options for all your file transformation and data indexing needs</p> </li> <li> <p>Archipelago Local Deployment Guide: get your own local Archipelago up and running in about 15 minutes</p> </li> <li> <p>Archipelago Presentations, Events, and Additional Resources: features recordings and links to different Archipelago workshops, conference presentations, and other helpful references</p> </li> </ol> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p>","tags":["Archipelago 101","Documentation"]},{"location":"AMIviaSpreadsheets/","title":"Ingesting New Digital Objects and Collections using Spreadsheets or Google Sheets","text":"<p>Ingesting Only Digital Objects or Both Digital Objects and Collections uses similar processes, with a few key differences. Click here to jump to the Ingesting both Digital Objects and Collections and/or Creative Work Series (Compound) Objects section of this guide page.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#ingesting-only-new-digital-objects","title":"Ingesting Only New Digital Objects","text":"<p>From either the main Content page or the AMI Sets List page, select the 'Start an AMI set' button to begin.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-1-plugin-selection","title":"Step 1: Plugin Selection","text":"<p>Select the Plugin type you will be using from the dropdown menu.</p> <ul> <li>Google Sheets Importer</li> <li> <p>Spreadsheet Importer  (if using local CSV file)</p> <p></p> </li> </ul> <p>*The <code>Remote JSON API Importer</code> and additional remote import source options (for other repository systems) will be covered in separate tutorials following future releases.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-2-operation-and-spreadsheet-source-selection","title":"Step 2: Operation and Spreadsheet Source Selection","text":"<p>Select 'Create New ADOs' as the Operation you would like to perform.</p> <ul> <li> <p>If using Google Sheets Importer:</p> <ul> <li>Enter the ID of your Google Sheet</li> <li>Enter the Cell Range for your Google Sheet</li> </ul> <p></p> </li> <li> <p>If using Spreadsheet Importer:</p> <ul> <li>Under the 'Upload your file' section, select 'Choose File' to upload the CSV you will be using. After the file is finished uploading, you will have the option to 'Remove' and select a different file if needed.</li> </ul> <p> </p> </li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-3-data-transformation-selections","title":"Step 3: Data Transformation Selections","text":"<p>Select the data transformation approach--how your source data will be transformed into ADO (Archipelago Digital Object) Metadata.</p> <ul> <li> <p>You will have 3 options for your data transformation approach:</p> <ol> <li>Direct<ul> <li>Columns from your spreadsheet source will be cast directly to ADO metadata (JSON), without transformation/further processing (only intended for use with simple data strings or already JSON-encoded snippets/values).</li> </ul> </li> <li>Custom (Expert Mode)<ul> <li>Provides very granular custom data transformation and mapping options</li> <li>Needs to be used if importing Digital Objects and Digital Object Collections at the same time/from same spreadsheet source (see separate instructions below).</li> </ul> </li> <li>Template<ul> <li>Columns from your spreadsheet source will be cast to ADO metadata (JSON) using a Twig template setup for JSON output.</li> </ul> </li> </ol> </li> <li> <p>You will also need to Select which columns contain filenames, entities or URLS where files can be fetched from. Select what columns correspond to the Digital Object types found in your spreadsheet source.</p> </li> <li> <p>Lastly, for this step, you will need to select the destination Fields and Bundles for your New ADOs. If your spreadsheet source only contains Digital Objects, select <code>Strawberry (Descriptive Metadata source) for Digital Object</code></p> <p></p> </li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-4-global-ado-mappings","title":"Step 4: Global ADO Mappings","text":"<p>Select your global ADO mappings.</p> <ul> <li>Make sure to select the <code>ismemberof</code> collection membership relationship predicate column if applicable. For AMI source spreadsheets containing only non-Creative Work Series (Compound) Objects, only <code>ismemberof</code> can be mapped properly. To use <code>ispartof</code> relationship setup, please refer to the steps outlined in the separate section below.</li> </ul> Click to read more about Archipelago's default relationship mappings <pre><code>- `ismemberof` and/or `ispartof` (and/or whatever predicate corresponds with the relationship you are mapping)\n- these columns can be used to connect related objects using the object-to-object relationship that matches your needs\n- in default Archipelago configurations, `ismemberof` is used for Collection Membership and `ispartof` is used for Parent-Child Object Relationships (so a Child ADO would reference the Parent ADO in `ispartof`)\n- these columns can hold 3 types of values\n    - empty (no value)\n    - an integer to connect an object to another object's corresponding row in the same spreadsheet/CSV\n      * Ex: Row 2 corresponds to a Digital Object Collection; for a Digital Object corresponding to Row 3, the 'ismemberof' column contains a value of '2'. The Digital Object in Row 3 would be ingested as a member of the Digital Object Collection in Row 2.\n    - a UUID to connect with an already ingested object\n</code></pre> <ul> <li>By default, the option to automatically assigns UUIDs is selected. Keep 'Automatically assign UUID' checked unless your source data already contains UUIDs in a <code>node_uuid</code> column.</li> <li> <p>Under the 'Base ADO mappings', select the <code>label</code> column for ADO Label. This selection is only used as a fail-safe (in case your AMI JSON Ingest Template does not have any mapping for a column to be mapped to the JSON <code>label</code> key, or your source data csv does not contain a <code>label</code> if going Direct for data transformation).</p> <p></p> </li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-5-zip-upload","title":"Step 5: ZIP upload","text":"<p>Provide an optional ZIP file containing your assets.</p> <ul> <li>You may choose to upload a ZIP file containing all or some of the corresponding files specified in your csv/spreadsheet.</li> <li> <p>The file upload size restrictions specified in your Archipelago instance will apply here (512MB maximum by default). </p> <p> </p> <ul> <li>Please note, when creating your ZIP file (in particular, within an OSX environment): only select the folders and files needed, not the top/enclosing folder they are in. </li> </ul> </li> </ul> Click to view screenshot of example ZIP file creation in OSX <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-6-ami-set-confirmation","title":"Step 6: AMI Set Confirmation","text":"<p>You will now see a message letting you know that 'Your source data was saved and is available as a CSV at <code>linktotheAMIgenerated.csv</code></p> <p>The message will also let you know that your New AMI Set was created and provide a link to the AMI Set page.</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-7-ami-set-processing","title":"Step 7: AMI Set Processing","text":"<p>Your newly created AMI Set will now need to be Processed.</p> <p>If you clicked on the 'see it here' link in Step 6, you will be brought to the AMI Set page for review. You may also select <code>Process</code> from the <code>Operations</code> menu for the AMI set from the main <code>AMI sets</code> page. From the <code>Process</code> page you can review the JSON configuration for your set (determined by your selections in the preceding steps).</p> Optional step to review the settings configured in your AMI Set <p>You may wish to double check the settings configured in your AMI Set in the Raw Metadata (JSON) on the AMI Set <code>View</code> tab before Processing.</p> <p></p> <p></p> <p>To Process this set, navigate to the <code>Process</code> tab. You will have multiple options related to the Processing outcome for your AMI Set.</p> <ul> <li>Skip ADO processing on missing File : If enabled a referenced missed file or one that can not be processed from the source, remote or local will make AMI skip the affected ROW. Enabled by default for better QA during processing. </li> <li>Desired ADOS Statuses After Process<ul> <li>The Statuses you have available will reflect the publication workflow/moderation states (such as Draft, Published, Archived/Unpublished) setup in your Archipelago instance, and the permissions associated your user account.</li> </ul> </li> <li>Please review the note about the 'remaining free space on your Drupal temporary filesystem. Please be aware of that before running a batch with large files'. If the amount of remaining free space you see does not seem sufficient for your AMI set processing needs, we recommend contacting your system administrator.  </li> <li> <p>Enqueuing and File Processing Options</p> <ul> <li>Enqueue but do not process Batch in realtime : Check this to enqueue but not trigger the interactive Batch processing. Cron or any other mechanism you have enabled will do the actual operation. This queue is shared by all AMI Sets in this repository and will be processed on a First-In First-Out basis.</li> <li>Force every File attached to an ADO to be processed in its own Queue item : Warning: This may make your ingest slower. Check this to force every file attached to an ADO to be downloaded and characterized as an independent process. This bypasses the Number of files Global setting that would otherwise trigger this behavior.</li> <li>Re download and reprocess every file : Check this to force every file attached to an ADO to be downloaded and characterized again, even if on a previous Batch run that data was already generated for reuse. IMPORTANT: Needed if e.g the URL of a file is the same but the remote source changed, if you have custom code that modifies the backend naming strategy of files.    </li> </ul> </li> <li> <p>Select <code>Confirm</code> to continue. </p> </li> </ul> <p>You will be returned to <code>AMI sets</code> page and see a brief confirmation message regarding the Enqueuing and Processing options you selected.</p> <p>If you chose to 'Confirm\" and Process your AMI Set immediately, proceed to Step 9: Processing and ADO Creation.</p> <p>If the chose to 'Enqueue' your AMI Set and the Queue operations for your Archipelago instance have been configured, you can simply leave your AMI Set in the Queue for Processing on the preconfigured schedule. Common timing for AMI Set Processes schedules are typically setup to run every three to six hours. Contact your Archipelago Administrators for details about your particular Archipelago's Processing schedule.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-8-queue-manager-push-may-be-restricted-to-administrator-users-only","title":"Step 8: Queue Manager Push (may be restricted to Administrator Users only)","text":"<p>If you chose to place your AMI set in the Queue to Process in step 7 and you wish to manually kickstart the Queue Processes, navigate to the Queue Manager found at <code>/admin/config/system/queue-ui</code>. (Be sure to select the <code>Queue Manager</code> under the System section, not the <code>Queue Manager for Hydroponic Service</code> under the Archipelago section).</p> <p></p> <p>To Process your AMI Set immediately from the Queue Manager page, select the checkbox next to the 'AMI Digital Object Ingester Queue Worker'. Keep the <code>Action</code> menu set to <code>Batch Process</code> and click the <code>Apply to selected items</code> button.</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-9-processing-and-ado-creation","title":"Step 9: Processing and ADO Creation","text":"<p>Your AMI set will now be Processed. You can follow the set's progress through the <code>Processing queues</code> loading screen.</p> <p></p> <p>After your AMI set is Processed, you will receive confirmation messages letting you know your Digital Objects were successfully created. </p> <p></p> <p>From this message, you can click on each ADO title to review the new created Digital Object (or Collection) if you wish. Or, you may proceed to step 10.  </p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-10-review-your-newly-created-digital-objects-directly-or-via-ami-set-report","title":"Step 10: Review your newly created Digital Objects directly or via AMI Set Report","text":"<ul> <li>Option 1: Return to the main Content page found at <code>/admin/content</code> and review your newly created Digital Objects. After ensuring that files and metadata elements were mapped correctly, you may choose to change the Status for your Digital Objects to 'Published'.</li> <li> <p>Option 2: Use the AMI Set Report</p> <ul> <li> <p>From the main <code>AMI sets</code> page, select <code>Report</code> from the <code>Operations</code> menu for the AMI set you wish to review. </p> </li> <li> <p>This Report will contain information related to the last Processing operation run against your AMI Set.</p> </li> <li>For each Digital Object or Collection row that was Processed, you will see:<ul> <li>a <code>datetime</code> stamp</li> <li>one of three <code>level</code> (INFO, WARNING, or ERRORS) applicability</li> <li>a <code>message</code> summarizing the Processing outcome--including a title/label link to the created ADO if successful</li> <li>a <code>details</code> summary containing system information related to the operations.</li> </ul> </li> </ul> <p></p> <ul> <li>You can use information found in the Reports tab to identify review your created ADOs one-by-one and identify any errors or issues that may have come up during the Process if needed.  </li> </ul> </li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#ingesting-both-new-digital-objects-and-collections-andor-creative-work-series-compound-objects-in-the-same-spreadsheet","title":"Ingesting Both New Digital Objects and Collections and/or Creative Work Series (Compound) Objects in the same spreadsheet","text":"<p>From either the main Content page or the AMI Sets List page, select the 'Start an AMI set' button to begin.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#steps-1-plugin-selection-step-2-operation-and-spreadsheet-source-selection","title":"Steps 1: Plugin Selection &amp; Step 2: Operation and Spreadsheet Source Selection","text":"<p>Follow the same instructions found above for Ingesting New Digital Objects.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-3-data-transformation-selections_1","title":"Step 3: Data Transformation Selections","text":"<p>To import Digital Objects and Digital Object Collections and/or Creative Work Series (Compound) Objects at the same time/from same spreadsheet source, you will need to select the <code>Custom (Expert Mode)</code> option for your data transformation approach.</p> <ul> <li>Custom (Expert Mode)<ul> <li>Provides very granular custom data transformation and mapping options</li> </ul> </li> </ul> <p>You will then need to 'Select your Custom Data Transformation and Mapping Options' for each of your Digital Object, Collection, and Creative Work Series (Compound) types.</p> <ul> <li> <p>For Collection and Creative Work Series (Compound) objects:</p> <ul> <li>Select either the Direct or Template (and corresponding JSON template) option for your data transformation approach.</li> <li>Select the destination Fields and Bundles for <code>Strawberry (Descriptive Metadata source) for Digital Object Collection</code><ul> <li>Use this same destination for your Creative Work Series (Compound) objects</li> </ul> </li> <li>You may also wish to Select which columns contain filenames, entities or URLS where files can be fetched from. For most Collection objects, you will likely leave unselected or choose <code>images</code> if you are uploading a thumbnail image for your Collection.</li> </ul> <p></p> </li> <li> <p>For each non-Creative Work Series (Compound) Digital Object type in your spreadsheet source:</p> <ul> <li>You will also need to select either the Direct or Template (and corresponding JSON template) option for your data transformation approach.</li> <li>Then Select the destination Fields and Bundles for <code>Strawberry (Descriptive Metadata source) for Digital Object</code></li> <li>Then select which columns contain filenames, entities or URLS where files can be fetched from. Select what columns correspond to the Digital Object types found in your spreadsheet source.</li> <li> <p>For example, for 'Map' type Digital Objects, you would select the following options (as depicted in this screenshot):</p> <p> </p> </li> </ul> </li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-4-global-ado-mappings_1","title":"Step 4: Global ADO Mappings","text":"<p>Select your global ADO mappings.</p> <ul> <li>Make sure to select the applicable relationship predicate columns (such as <code>ismemberof</code> and <code>ispartof</code>).</li> </ul> Click to read more about Archipelago's default relationship mappings <pre><code>- `ismemberof` and/or `ispartof` (and/or whatever predicate corresponds with the relationship you are mapping)\n- these columns can be used to connect related objects using the object-to-object relationship that matches your needs\n- in default Archipelago configurations, `ismemberof` is used for Collection Membership and `ispartof` is used for Parent-Child Object Relationships (so a Child ADO would reference the Parent ADO in `ispartof`)\n- these columns can hold 3 types of values\n    - empty (no value)\n    - an integer to connect an object to another object's corresponding row in the same spreadsheet/CSV\n      * Ex: Row 2 corresponds to a Digital Object Collection; for a Digital Object corresponding to Row 3, the 'ismemberof' column contains a value of '2'. The Digital Object in Row 3 would be ingested as a member of the Digital Object Collection in Row 2.\n    - a UUID to connect with an already ingested object\n</code></pre> <ul> <li>By default, the option to automatically assigns UUIDs is selected. Keep 'Automatically assign UUID' checked unless your source data already contains UUIDs in a <code>node_uuid</code> column.</li> <li>Under the 'Base ADO mappings', select the <code>label</code> column for ADO Label. This selection is only used as a fail-safe (in case your AMI JSON Ingest Template does not have any mapping for a column to be mapped to the JSON <code>label</code> key, or your source data csv does not contain a <code>label</code> if going Direct for data transformation).</li> <li>In order to make sure that Digital Objects containing the corresponding UUID or spreadsheet row number for any corresponding Collections, make sure <code>ismemberof</code> is also selected in the ADO Parent Columns. In order to make sure that Digital Objects containing the corresponding UUID or spreadsheet row number for any corresponding Parent ADOs (Creative Work Series/Compounds), make sure <code>ispartof</code> is also selected in the ADO Parent Columns.</li> <li> <p>Select the corresponding Columns for the Required ADO mappings.</p> <p></p> </li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-5-10","title":"Step 5-10:","text":"<p>Follow the same instructions found in Steps 5-10 above. As part of step 10, make sure your Digital Objects were ingested into the corresponding Collections you mapped them to in your spreadsheet source. Please note, you will need to Publish the Digital Objects before the Objects will appear in the Collection's View page (whether accessed as a logged-in Admin user or Anonymous/Public user). Celebrate your next AMI success with another fresh coffee, tea, or cookie!</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"CODE_OF_CONDUCT/","title":"Archipelago - code of conduct / anti-harassment policy","text":"<p>The Archipelago Commons community and the Metropolitan New York Library Council (METRO) are dedicated to providing a welcoming and positive experience for all participants, whether they are at a formal gathering, in a social setting, or taking part in activities online. This includes any forum, mailing list, wiki, web site, IRC channel, public meeting, conference, workshop/training or private correspondence. The Archipelago community welcomes participation from people all over the world, and these community members bring with them a wide variety of professional, personal and social backgrounds; whatever these may be, we treat colleagues with dignity and respect.</p> <p>This Code of Conduct governs how we behave in public or in private. We expect it to be honored by everyone who represents the project officially or informally, claims affiliation with the project, or participates directly.</p> <p>We ask that all community members adhere to the following expectations:</p> <ul> <li> <p>METRO and Archipelago have a zero-tolerance policy for verbal, physical, and sexual harassment. Anyone who is asked to stop a hostile or harassing behavior is expected to do so immediately. Here, for reference, are New York State\u2019s requirements.</p> <p>Harassment includes: Offensive verbal comments related to sex, gender, ethnicity, nationality, socioeconomic status, sexual orientation, disability, physical appearance, body size, age, race, religion; sexual or discriminatory images in public spaces; deliberate intimidation; stalking; harassing photography or recording; sustained disruption of talks or other events; inappropriate physical contact; and unwelcome sexual attention.</p> </li> <li> <p>Participation in discussions and activities should be respectful at all times. Please refrain from making inappropriate comments. Create opportunities for all people to speak, exercising tolerance of the perspectives and opinions of others. When we disagree, we do this in a polite and professional manner. We may not always agree. When frustrated, we back away and look for good intentions, not reasons to be more frustrated. When we see a flaw in a contribution, we offer guidance on how to fix it.</p> </li> <li>METRO and Archipelago honor the ability to be anonymous. If a person is going by their handle, a pseudonym, or doesn\u2019t wish to use their name, please respect their wishes and privacy. This also includes \u2018outing\u2019 someone\u2019s workplace, their age, their gender, their real name, etc, without their consent. We take people\u2019s privacy and their comfort very seriously.</li> </ul>"},{"location":"CODE_OF_CONDUCT/#protocol-for-conflict-resolution","title":"Protocol for Conflict Resolution","text":"<p>Participants in METRO and Archipelago communication channels violating this code of conduct may be sanctioned or expelled at the discretion of the organizers of the meeting (if the channel is an in-person event) or the Archipelago Advisory Board (if the channel is online).</p>"},{"location":"CODE_OF_CONDUCT/#initial-incident","title":"Initial Incident","text":"<p>If you are being harassed, notice that someone else is being harassed, or have any other concerns, and you feel comfortable speaking with the offender, please inform the offender that he/she/they has affected you negatively. Oftentimes, the offending behavior is unintentional, and the accidental offender and offended will resolve the incident by having that initial discussion. Participants asked to stop any harassing behavior are expected to comply immediately.</p>"},{"location":"CODE_OF_CONDUCT/#escalation","title":"Escalation","text":"<p>If the offender insists that they did not offend, if offender is actively harassing you, or if direct engagement is not a good option for you at this time, then you will need a third party to step in. To report any violation of the following code of conduct or if you have any questions or suggestions about this code of conduct, please contact archipelago-community@metro.org or fill out this form anonymously. This will be sent to leadership at METRO and the advisory board member currently acting as the Code of Conduct liaison. Our enforcement guidelines work in accordance with those published at the Contributor Covenant. </p> <p>Upon review, if METRO leadership and the Code of Conduct Liaison determine that the incident constitutes harassment they may take any action they deem appropriate, including warning the offender, expulsion from the meeting or other community channels, or contacting a higher authority such as a representative from the offender's institution.</p> <p>These policies draw from many other code of conduct documents, including but not limited to: code4lib, DLF, Islandora, ICG, Samvera, WikimediaNYC, and IDOCE</p>"},{"location":"I7solrImporter/","title":"Using the Islandora 7 Solr Importer","text":"<p>From either the main Content page or the AMI Sets List page, select the 'Start an AMI set' button to begin.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"I7solrImporter/#step-1-plugin-selection","title":"Step 1: Plugin Selection","text":"<p>Select the Islandora 7 Solr Importer from the dropdown menu.</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"I7solrImporter/#step-2-section-1-solr-server-configuration","title":"Step 2, section 1: Solr Server Configuration","text":"<p>You will only have the option to select 'Create New ADOs' as the Operation you would like to perform.</p> <p>For the Solr Server Configuration section, you will need to provide all of the following information:</p> <ul> <li>PID of the Islandora Collection Members you want to fetch (example: islandora:root)</li> <li>Host of your Solr Server (example: repositorydomain.org)</li> <li>Port (example: 8080)</li> <li>Path (example: /) <ul> <li>Note: if your Solr can be found at http://myrepo.com:8080/solr, then the path is always a single \"/\" (as in screenshot depiction below)</li> </ul> </li> <li>Type of Solr Deployment<ul> <li>Either Single Solr Server (most common) or Solr Cloud Ensemble</li> </ul> </li> <li>Core (example: islandora)</li> </ul> <p>You will also need to select the Starting Row you would like to begin fetching results from, and the Number of Rows to fetch. </p> <p>The Starting Row is an offset and defaults to 0, which is the most common (and recommended) approach. For the Total Number of Rows to Fetch, setting this to empty or null will automatically (refresh when selecting 'Next' button at bottom of page) prefill with the real Number Rows found by the Solr Query invoked. If you set this number higher than the actual results we will only fetch what can be fetched.</p> <p>For larger collections, you may wish to create multiple/split AMI ingest sets by selecting a specified number of rows. </p> <ul> <li>As an example, for a collection of 1500 objects that you wanted to split into three AMI ingests of 500 objects, you would specify the Starting Row as 0 for the first set and Number of Rows as 500. For the second set, your Starting Row would be Row 501; for the third set, 1001). In this example, the Number of Rows would always be 500.</li> </ul> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"I7solrImporter/#step-2-section-2-islandora-mappings","title":"Step 2, section 2: Islandora Mappings","text":"<p>In this step you will need to make determinations on how you would like to map your Islandora 7 digital objects to your Archipelago repository and whether or not you would like to fetch additional file datastreams, such as those for thumbnail images, transcripts, OCRs/HOCRs, etc.</p> <ul> <li> <p>Selecting \"Collapse Multi Children Objects\" will collapse Children Datastreams into a single ADO with many attached files (single row in the generated AMI set .csv file). Book Pages will be fetched but also the Top Level PDF (if one exists in your Islandora instance).</p> </li> <li> <p>In the Required ADO mappings, you will need to specify which Archipelago type you want to map each Islandora Content Model found in your source collection. </p> <ul> <li>For example, for info:fedora/islandora:sp_large_image_cmodel you may want to use Photograph.</li> </ul> </li> <li> <p>If you had left \"Collapse Multi Children Objects\" unselected, you will also need to specify the Islandora Content Model to ADO types mapping for possible Children.</p> </li> </ul> <p></p> <ul> <li> <p>You can also specify an ADO (Object or Collection) to be used as the Parent of Imported Objects. By selecting an existing ADO (Object or Collection) here using the autocomplete/search, the generated AMI set .csv file will contain an 'ismemberof' column containing the UUID of the selected ADO for every row.</p> </li> <li> <p>Under \"Additional Datastreams to Fetch\", you can select any number and/or combination of extra file datastreams to retrieve from your harvest. Please note that the I7 Importer will fetch every possible datastream that is present in your source I7 repository, but the additional file datastreams referenced may not be associated with actual files for every digital object.</p> </li> </ul> <p></p> <p>Language from form itself:</p> <p>Additional datastreams to fetch. OBJ datastream will always be fetched. Not all datastreams listed here might be present once your data is fetched.</p> <ul> <li>Selection of any additonal datastreams will generate a dropdown menu under \"Where extra datastreams should go.\" Here you will decide the organization and location of those datastreams by selecting one of the two options<ul> <li>Organize by mime type e.g TRANSCRIPT will go into the \"texts\" column</li> <li>Each in a separate column based on the datastream neame, TRANSCRIPT will go into the \"transcripts\" column</li> </ul> </li> </ul> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"I7solrImporter/#step-3-data-transformation-selections","title":"Step 3: Data Transformation Selections","text":"<p>Select the data transformation approach--how your source data will be transformed into ADO (Archipelago Digital Object) Metadata. As noted in the list below, 'Custom (Expert Mode)' is the recommended choice for AMI sets generated using the Islandora 7 Solr Importer plugin.</p> <ul> <li> <p>You will have 3 options for your data transformation approach:</p> <ol> <li>Direct<ul> <li>Columns from your spreadsheet source will be cast directly to ADO metadata (JSON), without transformation/further processing (only intended for use with simple data strings).</li> </ul> </li> <li>Custom (Expert Mode) Recommended choice for AMI sets generated using the Islandora 7 Solr Importer plugin<ul> <li>Provides very granular custom data transformation and mapping options</li> <li>Needs to be used if importing Digital Objects and Digital Object Collections at the same time/from same spreadsheet source (see separate instructions below).</li> </ul> </li> <li>Template<ul> <li>Columns from your spreadsheet source will be cast to ADO metadata (JSON) using a Twig template setup for JSON output.</li> </ul> </li> </ol> </li> <li> <p>You will also need to Select which columns contain filenames, entities or URLS where files can be fetched from. Select what columns correspond to the Digital Object types found in your spreadsheet source. If you fetched additional file datastreams during Step 2, you will see those columns listed here as well (see screenshot below for examples).</p> </li> <li> <p>Lastly, for this step, you will need to select the destination Fields and Bundles for your New ADOs. If your spreadsheet source only contains Digital Objects, select <code>Strawberry (Descriptive Metadata source) for Digital Object</code></p> <ul> <li>Select <code>Template</code> and use the AMI Ingest JSON template that corresponds with your metadata elements.</li> <li>Examples of potential file source columns include: <code>images</code>, <code>documents</code>, <code>audios</code>, <code>videos</code>, etc.</li> </ul> <p></p> </li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"I7solrImporter/#step-4-global-ado-mappings","title":"Step 4: Global ADO Mappings","text":"<p>Select your global ADO mappings.</p> <ul> <li>Even if empty (no values), select any relationship predicate columns (such as <code>ismemberof</code> and <code>ispartof</code>, if present) for ADO Parent columns.</li> <li>By default, the option to automatically assigns UUIDs is selected. Unchecking this option is not permitted when using the I7 Solr Importer.</li> <li> <p>Select the corresponding Columns for the Required ADO mappings.</p> <ul> <li>Select the <code>mods_titleinfo_title</code> column for ADO Label</li> </ul> <p></p> </li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"I7solrImporter/#step-5-zip-upload-and-ami-set-naming","title":"Step 5: ZIP upload and AMI Set naming","text":"<p>For standard Spreadsheet or Google Sheets AMI ingests, you would use this step to provide an optional ZIP file containing your assets. </p> <p>For your Islandora 7 Solr Importer process, the generated AMI set.csv file will contain the necessary URLs to the corresponding Islandora 7 file datastreams for each object as needed. Select next to skip this ZIP upload step and proceed. </p> <p>After you provide a title for your AMI set under \"Please Name your AMI set\", select \"Press to Create Set\"</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"I7solrImporter/#step-6-ami-set-confirmation","title":"Step 6: AMI Set Confirmation","text":"<p>You will now see a message letting your know your \"New AMI Set was created\". You will be able to review the generated .csv file directly from this page under Source Data File.</p> <p></p> <p>While you may immediately select \"Process\" from this AMI Set Confirmation page to use the Islandora 7 Importer generated .csv file as-is to ingest the ADOs in your AMI set, it is strongly recommended that you review the .csv file first. AMI is configured to trim unecessary (for Archipelago) and de-duplicate redundant Solr source data, but you may wish to pare down the sourced data even further and/or conduct general metadata review and cleanup before migrating your content. You will also likely want to make adjustments to your AMI Ingest JSON Template based on your review, depending on the variation of metadata columns/keys found in your source repostiory. </p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"I7solrImporter/#next-steps","title":"Next Steps","text":"<p>To proceed with Processing your AMI Set, click here to be directed to the main Ingesting Digital Objects via Spreadsheets.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"about/","title":"About this Documentation","text":"<p>This documentation was generated with Material for MkDocs. The repo/branch is at https://github.com/esmero/archipelago-documentation/tree/1.3.0, and the site is built using the following Github workflow: https://github.com/esmero/archipelago-documentation/blob/1.3.0/.github/workflows/ci.yml.</p>","tags":["Documentation","Contributing"]},{"location":"about/#contributing","title":"Contributing","text":"<ol> <li>First, please see this guide to set up the repo and branch locally and to create an issue and corresponding pull request.</li> <li>Install mkdocs-material and mike: <code>pip install mkdocs-material mike</code>.</li> <li>Make changes to local</li> <li>Run <code>mike delete --all &amp;&amp; mike deploy 1.0.0 default &amp;&amp; mike set-default 1.0.0 &amp;&amp; mike serve</code> to see and test changes.</li> <li>Follow the above guide to contribute the changes back.</li> </ol>","tags":["Documentation","Contributing"]},{"location":"acknowledgments/","title":"Caring &amp; Coding + Fixing","text":"<ul> <li>Diego Pino</li> <li>Giancarlo Birello</li> <li>Allison Sherrick</li> </ul>"},{"location":"acknowledgments/#acknowledgments","title":"Acknowledgments","text":"<p>Archipelago Commons is an Open Source repository software that is developed and supported at the Metropolitan New York Library Council (METRO).</p>"},{"location":"acknowledgments/#license","title":"License","text":"<p>GPLv3</p>"},{"location":"ami_index/","title":"Archipelago Multi-Importer (AMI)","text":"<p>Archipelago Multi-Importer (AMI) is a module for batch/bulk/mass ingests of Archipelago digital objects (ADOs) and collections. AMI also enables you to perform batch administrative actions, such as updating, patching/revising, or deleting digital objects and collections. AMI's Solr Importer plugin can be used to create AMI ingests and migrating content from existing Solr-sourcable digital repositories (such as Islandora 7).</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#ami-overview-and-under-the-hood-explanations","title":"AMI Overview and Under-the-Hood Explanations","text":"<p>From the desk of Diego Pino</p> <p>AMI provides Tabulated data ingest for ADOs with customizable input plugins. Each Spreadsheet (or Google Spreadsheet) goes through a Configuration Multi-step setup and generates at the end an AMI Set. AMI Sets then can be enqueued or directly ingested, its generated Objects purged and reingested again, its source data (generated and enriched with UUIDS) CSV replaced, improved and uploaded again and ingested.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#learn-more-about-metadata-in-archipelago-and-ami","title":"Learn More about Metadata in Archipelago and AMI","text":"<p>Please review the Metadata in Archipelago overview to learn about Archipelago's unique approach to metadata and how this applies in the context of AMI set adminstration.</p> Click to read the full AMI 0.4.0 (Archipelago - 1.0.0) Pre-Release Notes.","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#setup-steps","title":"Setup Steps","text":"<p>AMI has Ingest, Update and Patch capabilities. AMI has a plugin system to fetch data. The data can come from multiple sources and right now CSV/EXCEL or Google Spreadsheets are the ones enabled. It does parent/children validation, makes sure that parents are ingested first, cleans broken relationships, allows arbitrary multi relations to be generated in a single ROW (ismemberof, partOf, etc)  pointing to other rows or existing ADOs (via UUIDs) and can process rows directly as JSON or preprocessed via a Metadata Display entity (twig template) capable of producing JSON output. These templates can be configured by \u201ctype\u201d, Articles v/s 3DModel can have different ones. Even which columns contain Files can be configured at that level.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#ami-set-entity","title":"AMI Set Entity","text":"<p>Ami Sets are special custom entities that hold an Ingest Strategy generated via the previous Setup steps (as JSON with all it's settings), a CSV with data imported from the original source (with UUIDs prepopulated if they were not provided by the user). These AMI sets are simpler and faster than \u201cbatch sets\u201d because they do not have a single entry per Object to be ingested. All data lives in a CSV. This means the CSV of an AMI set can be corrected and reuploaded. Users can then Process a Set either putting the to be ingested ADOs in the queue and let Hydroponics Service do the rest or directly via Batch on the UI. ADOs generated by a set can also be purged from there. These sets can also be created manually if needed of any of the chosen settings modified anytime. Which AMI set generated the Ingest is also tracked in a newly created ADO\u2019s JSON and any other extra data (or fixed data e.g common Rights statements, or LoD) can be provided by a Twig Template. Ingest is amazingly fast. We monitored Ingest with Remote URL(islandora Datastreams) files of 15Mbytes average at a speed of 2 seconds per Object (including all post processing) continuously for a set of 100+.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#search-and-replace","title":"Search and Replace","text":"<p>This module also provides a simple search/replace text VBO action (handles JSON as text) and a full blown JSONPATCH VBO action to batch modify ADOs. The last one is extremely powerful permitting multiple operations at the same time with tests. E.g replace a certain value, add another value, remove another value only if a certain test (e.g \u201ctype\u201d:\u201dArticle\u201d and \u201cdate_of_digital\u201d: \u201c2020-09-09\u201d) matches. If any tests fail the whole operation will be canceled for that ADO. An incomplete \u201cWebform\u201d VBO action is present but not fully functional yet. This one allows you to choose a Webform, a certain element inside that Webform and then find and replace using the same Interface you would see while editing/adding a new ADO via the web form workflow.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#getting-started-with-ami","title":"Getting started with AMI","text":"<p>You can access AMI through the <code>AMI Sets</code> tab on the main Content page found at <code>/admin/content</code> or directly at <code>/amiset/list</code>.</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#google-sheets-api-configuration","title":"Google Sheets API Configuration","text":"<p>If you plan on using the Google Sheets Importer option, you will need to Configure the Google Sheets API.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#example-spreadsheetcsv","title":"Example Spreadsheet/CSV","text":"<p>Please refer to or use a fresh/new copy of the Demo Archipelago Digital Objects (ADOs) spreadsheet to import a small set of Digital Objects, using the same assets part of the One-Step Demo content ingest guide.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#example-json-template","title":"Example JSON template","text":"<p>This JSON template can be used during the Data Transformation (step 3) of your AMI Import. This particular template corresponds with the metadata elements found in the Default Descriptive Metadata and Default Digital Object Collection webforms shipped with Archipelago 1.0.0.</p> Click to view the example 1.0.0 AMI JSON template <p>To use this template, copy and paste the JSON below directly into a new Metadata Display, found here for a local <code>http://localhost:8001/metadatadisplay/list</code> or <code>http://yoursite.org/metadatadisplay/list</code>. Select <code>JSON</code> as the 'Primary mime type this Twig Template entity will generate as output' for this new Metadata Display.</p> <pre><code>  {\n      \"type\": {{ data.type|json_encode|raw }},\n      \"label\": {{ data.label|json_encode|raw }},\n      \"issue_number\": {{ data.issue_number|json_encode|raw }},\n      \"interviewee\": {{ data.interviewee|json_encode|raw }},\n      \"interviewer\": {{ data.interviewer|json_encode|raw }},\n      \"duration\": {{ data.duration|json_encode|raw }},\n      \"website_url\": {{ data.website_url|json_encode|raw }},\n      \"description\": {{ data.description|json_encode|raw }},\n      \"date_created\": {{ data.date_created|json_encode|raw }},\n      \"date_created_edtf\": {{ data.date_created_edtf|json_encode|raw }},\n      \"date_created_free\": {{ data.date_created_free|json_encode|raw }},\n      \"creator\": {{ data.creator|json_encode|raw }},\n      \"creator_lod\": {{ data.creator_lod|json_encode|raw }},\n      \"publisher\": {{ data.publisher|json_encode|raw }},\n      \"language\": {{ data.language|json_encode|raw }},\n      \"ismemberof\": [],\n      \"ispartof\": [],\n      \"sequence_id\": {{ data.sequence_id|json_encode|raw }},  \n      \"owner\": {{ data.owner|json_encode|raw }},\n      \"local_identifier\": {{ data.local_identifier|json_encode|raw }},\n      \"related_item_host_title_info_title\": {{ data.related_item_host_title_info_title|json_encode|raw }},\n      \"related_item_host_display_label\": {{ data.related_item_host_display_label|json_encode|raw }},\n      \"related_item_host_type_of_resource\": {{ data.related_item_host_type_of_resource|json_encode|raw }},\n      \"related_item_host_local_identifier\": {{ data.related_item_host_local_identifier|json_encode|raw }},\n      \"related_item_note\": {{ data.related_item_note|json_encode|raw }},\n      \"related_item_host_location_url\": {{ data.related_item_host_location_url|json_encode|raw }},\n      \"note\": {{ data.note|json_encode|raw }},\n      \"physical_description_note_condition\": {{ data.physical_description_note_condition|json_encode|raw }},\n      \"note_publishinginfo\": {{ data.note_publishinginfo|json_encode|raw }},\n      \"physical_location\": {{ data.physical_location|json_encode|raw }},\n      \"physical_description_extent\": {{ data.physical_description_extent|json_encode|raw }},\n      \"date_published\": {{ data.date_published|json_encode|raw }},\n      \"date_embargo_lift\": {{ data.date_embargo_lift|json_encode|raw }},\n      \"rights_statements\": {{ data.rights_statements|json_encode|raw }},\n      \"rights\": {{ data.rights|json_encode|raw }},\n      \"subject_loc\": {{ data.subject_loc|json_encode|raw }},\n      \"subject_lcnaf_personal_names\": {{ data.subject_lcnaf_personal_names|json_encode|raw }},\n      \"subject_lcnaf_corporate_names\": {{ data.subject_lcnaf_corporate_names|json_encode|raw }},\n      \"subject_lcnaf_geographic_names\": {{ data.subject_lcnaf_geographic_names|json_encode|raw }},\n      \"subject_lcgft_terms\": {{ data.subject_lcgft_terms|json_encode|raw }},\n      \"subject_wikidata\": {{ data.subject_wikidata|json_encode|raw }},\n      \"edm_agent\": {{ data.edm_agent|json_encode|raw }},\n      \"term_aat_getty\": {{ data.term_aat_getty|json_encode|raw }},\n      \"viaf\": {{ data.viaf|json_encode|raw }},\n      \"pubmed_mesh\": {{ data.pubmed_mesh|json_encode|raw }},\n      \"europeana_concepts\": {{ data.europeana_concepts|json_encode|raw }},\n      \"europeana_agents\": {{ data.europeana_agents|json_encode|raw }},\n          \"europeana_places\": {{ data.europeana_places|json_encode|raw }},\n      \"geographic_location\": {{ data.geographic_location|json_encode|raw }},\n      \"subjects_local_personal_names\": {{ data.subjects_local_personal_names|json_encode|raw }},\n      \"subjects_local\": {{ data.subjects_locals|json_encode|raw }},\n      \"audios\": [],\n      \"images\": [],\n      \"models\": [],\n      \"videos\": [],\n      \"documents\": [],\n      \"as:generator\": {\n          \"type\": \"Create\",\n          \"actor\": {\n              \"url\": {{ setURL|json_encode|raw }},\n              \"name\": \"ami\",\n              \"type\": \"Service\"\n          },\n          \"endTime\": \"{{\"now\"|date(\"c\")}}\",\n          \"summary\": \"Generator\",\n          \"@context\": \"https:\\/\\/www.w3.org\\/ns\\/activitystreams\"\n      },\n      \"upload_associated_warcs\": []\n  }\n</code></pre> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/","title":"Using AMI's Linked Data Reconciliation","text":"<p>Archipelago Multi Importer (AMI)'s Linked Data Reconciliation tool can be used to enrich your metadata with Linked Data (LoD). Using this tool, you can map values from your topical/subject metadata elements to your preferred LoD vocabulary source. These mappings can then be transformed via a corresponding Metadata Display (Twig) template to process the values into JSON-formatted metadata for your specified AMI set.</p> <p>The aim of this tool is to automize as much of the reconciliation process as feasible within Archipelago. Please be aware that data reconciliation will still be in part a manual and potentially time intensive process.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/#important-note-preliminary-pre-requisite-ami-set-configuration","title":"Important Note: Preliminary / Pre-requisite AMI Set Configuration","text":"<p>In order to Reconciliate an AMI Set, you will need to have selected the 'Template' or 'Custom' data transformation approach (then also, via 'Template' for your Digital Object or Collection types) during Step 3 : Data Transformation of your AMI Set configuration.</p> <p>Your source spreadsheet will also need to contain at least one column containing terms/names (values) you want to reconcile against an LoD Authority Source. Multiple values should be separated by '|@|'.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/#step-1-select-the-ami-set-you-will-be-working-with","title":"Step 1: Select the AMI Set you will be working with.","text":"<p>From the main AMI Sets List page, click on your AMI Set's Name, or select the 'Edit' option from the Operations menu on the right-hand side of the Sets list.</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/#step-2-reconcile-lod-tab","title":"Step 2: Reconcile LoD Tab","text":"<p>Navigate to the Reconcile LoD tab.</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/#step-3-lod-reconciling-selections","title":"Step 3: LoD Reconciling Selections","text":"<p>From the list of columns from your spreadsheet source, select which columns you want to reconcile against LoD providers. </p> <p>Under the LoD Sources section, select how your chosen Columns will be LoD reconciled. - LoD reconcile options will be on the left, LoD Authority Sources will be on the right. - Example: 'local_subjects' will be mapped to 'LoC subjects (LCSH)'</p> <p></p> Full list of potential LoD Authority Sources <ul> <li>LoC subjects(LCSH)</li> <li>LoC Name Authority File (LCNAF)</li> <li>LoC Genre/Form Terms (LCGFT)</li> <li>LoC Thesaurus of Graphic Materials (TGN)</li> <li>LoC MARC List for Geographic Areas</li> <li>LoC Relators Vocabulary (Roles)</li> <li>LoC MADS RDF by type:</li> <li>Corporate Name</li> <li>Personal Name</li> <li>Family Name</li> <li>Topic</li> <li>Genre Form</li> <li>Geographic</li> <li>Temporal</li> <li>Extraterrestrial Area</li> <li>VIAF</li> <li>Getty aat Fuzzy / Terms / Exact Label Match</li> <li>Wikidata Q Items</li> </ul> <p>To preview the values contained in the column(s) you selected, click the 'Inspect cleaned/split up column values' button. </p> <p></p> <p>Tip: This preview step provides you with the opportunity to return to your AMI Set source CSV and make any necessary label/term corrections such as outliers and formatting errors before processing. This can be done multiple times until your source set is fully prepared. If using this workflow, you will tick the 'Re-process only adding new terms/LoD Authority Sources' processing option after replacing your updated source CSV (see screenshot below)</p> <p>When ready, there are multiple processing options to select from depending on your current need/workflow.  - To process immediately, select 'Process LoD from Source'  - To enqueue the batch process, select 'Enqueue but do not process Batch in real time.  - To add new data (i.e. terms, LoD Authority Sources) to existing reconciliation (e.g after replacing     source CSV data), select 'Re-process only adding new terms/LoD Authority Sources</p> <p></p> <p>Important note: if you have previously run LoD Reconciliation for your AMI set, this action will overwrite any manually corrected LoD on your Processed CSV. Please make sure you have a backup if unsure.</p> <p>Depending on the size of your AMI Set, the Reconciliation processing may take a few minutes. </p> <p></p> <p>When the process is finished, you will see a brief confirmation message.</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/#step-4-edit-reconciled-lod","title":"Step 4: Edit Reconciled LoD","text":"<p>Open the 'Edit Reconciled LoD' tab.</p> <p>You will see a table (form) containing: - Your Original term values (labels) - The CSV Column Header/Key from the source spreadsheet where the value is found - A Checked option you can use to denote that an LoD mapping has been reviewed/revisioned - The Linked Data Label and URL pairing selected during the LoD reconciliation process</p> <p></p> <p>The results table will show 10 original terms and mappings per page. You can advance through the pages using the page numbers and navigational arrows above and below the table.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/#step-5-review-and-edit-your-reconciled-lod-mappings","title":"Step 5: Review and Edit your Reconciled LoD Mappings","text":"<p>Review the LoD reconciliation mappings, to make sure the best terms were selected for your metadata.</p> <ul> <li>To revise and select a different term mapping, begin by typing in the 'Label' box in the corresponding LoD lookup element. (You can type directly over an incorrect term or within an empty cell if no value was mapped/identified.)</li> <li>Select your preferred term and URL pairing from the list.</li> <li>You can also add or remove multiple mappings using the +/- buttons beside the LoD lookup element.</li> <li>If desired, click the Checked option to mark that the term was reviewed/revisioned.</li> </ul> <p>As you advance through your review process, it is recommended that you use the 'Save Current LoD Page' at the bottom of each results page as you work. This will preserve the corrections you may have made and update the LoD Reconciled data for your AMI Set within the editing form.</p> <p></p> <p>When you have finished editing/reviewing your data, you must select 'Save all LoD back to CSV File' or else your LoD selections will not be preserved.</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/#step-6-ami-set-review-and-twig-metadata-display-preparation","title":"Step 6: AMI Set Review and Twig (Metadata Display) Preparation","text":"<p>You will now need to make sure that the Metadata Display (Twig) Template you selected to use during your initial AMI Set configuration is setup to Process your LoD mapped Label and URL selections into your Digital Objects and Collections JSON metadata.</p> <p>For every JSON key/element in your metadata that you need to process the LoD Reconciled data into, you need to specify in your Template that data for this element will be read from the 'Processed Data' LoD information. </p> <p>In the following example Twig snippet, the \"subject_loc\" JSON key will map corresponding values from the 'Processed Data' (data.lod) LoD information into a newly created Digital Object/Collection during the AMI Set Processing.</p> <pre><code>\"subject_loc\": {{ data_lod.mods_subject_topic.loc_subjects_thing|json_encode|raw }},\n</code></pre> <ul> <li>\"subject_loc\" = destination JSON Key or Property for your LoD values</li> <li>data.lod = directs the Twig template to source from the Processed Data LoD information (instead of the original AMI Source CSV accessed via 'data.xx_property_name')</li> <li>mods.subject.topic = the Column header in the original AMI Source CSV</li> <li>loc_subjects_thing = the Column containing the Label and URI/L pairs in the Processed Data LoD editable Table/Form (and reference CSV)</li> </ul> <p>The same general pattern can be adapted to apply to different mapping scenarios (original CSV source columns to Reconciled LoD Sources) as needed.</p> Full list of Column Options =&gt; Corresponding LoD Sources <ul> <li>'loc_subjects_thing' =&gt; LoC subjects(LCSH)</li> <li>'loc_names_thing' =&gt; LoC Name Authority File (LCNAF)</li> <li>'loc_genreForms_thing' =&gt; LoC Genre/Form Terms (LCGFT)</li> <li>'loc_graphicMaterials_thing' =&gt; LoC Thesaurus of Graphic Materials (TGN)</li> <li>'loc_geographicAreas_thing' =&gt; LoC MARC List for Geographic Areas</li> <li>'loc_relators_thing' =&gt; LoC Relators Vocabulary (Roles)</li> <li>'loc_rdftype_CorporateName' =&gt; LoC MADS RDF by type: Corporate Name</li> <li>'loc_rdftype_PersonalName' =&gt; LoC MADS RDF by type: Personal Name</li> <li>'loc_rdftype_FamilyName' =&gt; LoC MADS RDF by type: Family Name</li> <li>'loc_rdftype_Topic' =&gt; LoC MADS RDF by type: Topic</li> <li>'loc_rdftype_GenreForm' =&gt;  LoC MADS RDF by type: Genre Form</li> <li>'loc_rdftype_Geographic' =&gt; LoC MADS RDF by type: Geographic</li> <li>'loc_rdftype_Temporal' =&gt;  LoC MADS RDF by type: Temporal</li> <li>'loc_rdftype_ExtraterrestrialArea' =&gt; LoC MADS RDF by type: Extraterrestrial Area </li> <li>'viaf_subjects_thing' =&gt; VIAF</li> <li>'getty_aat_fuzzy' =&gt; Getty aat Fuzzy</li> <li>'getty_aat_terms' =&gt; Getty aat Terms</li> <li>'getty_aat_exact' =&gt; Getty aat Exact Label Match</li> <li>'wikidata_subjects_thing' =&gt; Wikidata Q Items</li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/#next-steps","title":"Next Steps","text":"<p>To proceed with Processing your AMI Set, click here to be directed to the main Ingesting Digital Objects via Spreadsheets.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_spreadsheet_overview/","title":"Spreadsheet Formatting Overview","text":"","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_spreadsheet_overview/#spreadsheet-formatting-overview","title":"Spreadsheet Formatting Overview","text":"<p>There are multiple ways a spreadsheet/CSV file can be structured to work with AMI, depending on the data transformation and mapping you will be using.</p> <ul> <li>For most standard AMI ingests, each Row of your spreadsheet/CSV will correspond to a single Digital Object, Creative Work Series (Compound Object Parent) or Collection.</li> <li> <p>Columns in your spreadsheet/CSV can be mapped to different data (files) and metadata elements (label, description, subjects, etc.).</p> </li> <li> <p>It is recommended that different types of files are placed into separate columns--\"images\", \"documents\", \"models\", \"videos\", \"audios\", \"texts\".</p> <ul> <li>Filepaths can point to remote files, to existing files within your docker container, s3 (or other storage type/location that is accessible to Archipelago), and to paths within zip files.<ul> <li>Example path for existing file within docker container:     <code>/var/www/html/d8content/myAMIimage.jpg</code></li> <li>Example s3 path:     <code>s3://myAMIuploads/myAMIdocument.pdf</code></li> <li>Example remote filepath:     <code>https://dogsaregreat.edu/dogs.tiff</code></li> </ul> </li> <li>Multiple files (of the same type) can be placed in a single cell, separated by a semicolon ( ; ).</li> <li>For Digital Objects comprised of multiple types of files, such as an Oral History Interview with an audio file and a PDF transcript file, you can place different file types within different corresponding columns for the same Row.</li> <li>It is recommended that filepaths are copied/stored as plain (non-hyperlinked) formatted text.</li> </ul> </li> </ul> <p>Caution with using Quotation Marks</p> <p>Archipelago is very JSON-forward (friendly!) and makes assumptions about JSON encoding coming from CSV documents. If you are planning to start and end values with quotation marks in your CSV file, such as for <code>label</code> values, you need to enter the values encapsulated between the UTF-8 (unicode) character for straight quotations--in the CSV cell (corresponding row + column).  For example, in order for the ingested object to have the label (title) value processed as \"Strawberry-Bed\" for the ingested Archipelago Digital Object you need to enter the data as \"\\u0022Strawberry-Bed\\u0022\" in the corresponding <code>label</code> cell in your AMI Set CSV.</p> <ul> <li> <p>Every spreadsheet/CSV file should contain the following Columns:</p> <ul> <li><code>type</code><ul> <li>the Digital Object or Digital Object Collection Type, such as 'Photograph' or 'Collection'</li> </ul> </li> <li><code>label</code><ul> <li>the title of the Digital Object or Collection</li> </ul> </li> <li>Soft-requirement <code>node_uuid</code><ul> <li>this can be empty</li> <li>if empty, Archipelago will automatically generate UUIDs</li> <li>can be used with existing UUIDs during migrations</li> </ul> </li> <li>Soft-requirement <code>sequence_id</code> for Creative Work Series (compound) children objects<ul> <li>this is used to determine the sequence order for children objects within a Creative Work Series (compound) object</li> <li>should be an integer only (ie, '1' and not 'Page 1')</li> <li>if not present, the objects will present in the original ingest order</li> <li>we strongly recommend mapping the correct sequence using 'sequence_id'</li> </ul> </li> </ul> </li> <li> <p>Recommended Columns:</p> <ul> <li>Files as defined above<ul> <li>\"images\", \"documents\", \"models\", \"videos\", \"audios\", \"text\"</li> <li>.warc/.wacz files should be placed in a column \"upload_associated_warcs\"</li> </ul> </li> <li><code>ismemberof</code> and/or <code>ispartof</code> (and/or whatever predicate corresponds with the relationship you are mapping)<ul> <li>these columns can be used to connect related objects using the object-to-object relationship that matches your needs</li> <li>in default Archipelago configurations, <code>ismemberof</code> is used for Collection Membership and <code>ispartof</code> is used for Parent-Child Object Relationships (so a Child ADO would reference the Parent ADO in <code>ispartof</code>)</li> <li>these columns can hold 3 types of values<ul> <li>empty (no value)</li> <li>an integer to connect an object to another object's corresponding row in the same spreadsheet/CSV</li> <li>Ex: Row 2 corresponds to a Digital Object Collection; for a Digital Object corresponding to Row 3, the 'ismemberof' column contains a value of '2'. The Digital Object in Row 3 would be ingested as a member of the Digital Object Collection in Row 2.</li> <li>a UUID to connect with an already ingested object</li> </ul> </li> </ul> </li> <li>Metadata - for all the rich, detailed information associated with your Digital Objects and Collections<ul> <li>Every Column header will become a JSON Key and each cell a JSON value for that Key</li> <li> <p>You can use direct JSON snippets such as:</p> <p><pre><code>[{\"uri\": \"http://id.loc.gov/authorities/subjects/sh95008857\",\"label\": \"Digital libraries\"}]\n</code></pre>         - If you have an advanced twig template with the necessary logic, you can place data in cells that can be parsed and structured in various ways (such as multiple values separated by semicolons split accordingly, capitalization of values based on defined patterns, etc.)</p> </li> </ul> </li> </ul> </li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_update/","title":"Using AMI's Update Operations","text":"<p>Archipelago Multi Importer (AMI)'s Update Operations can be used to Update, Replace, or Append metadata values or files for existing Digital Objects and Collections found in your Archipelago. You can prepare and use AMI Update Sets in different ways using one of three functional operations, depending on your update needs. This guide will provide a general overview of the three main functions and how each operation may be useful.</p>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"ami_update/#important-notes-preliminary-pre-requisites","title":"Important Notes: Preliminary / Pre-requisites","text":"<p>You need to have existing Digital Objects or Collections (ADOs) in your Archipelago to work with. You should have a prepared AMI Update Set CSV that contains at least the following columns/headers:</p> <ul> <li>node_uuid<ul> <li>required</li> <li>should contain the values for any existing ADOs you wish to update</li> <li>needs to be a 1:1 match; AMI Update functionality cannot be used to update/change node_uuid values</li> </ul> </li> <li>label<ul> <li>required</li> <li>contains the corresponding label (title) for your existing ADOs</li> <li>can contain modifications for the label (title) values</li> </ul> </li> <li>type<ul> <li>optional, only required if using the Custom (Expert Mode) approach for your AMI set configuration or targeting changes for 'type' mappings for your ADOs</li> <li>contains the corresponding type values for your existing ADOs</li> <li>can contain modifications for the type values  </li> <li>related to this, AMI Update functionality cannot be used to update/change the underlying Drupal bundle mappings for already-ingested objects. In other words, you cannot use an AMI Update Set to change a Child Object with a 'Page' type originally ingested as a Digital Object -&gt; to a Parent Object with a 'Book' type mapped to the Digital Object Collection/Compound bundle. For these changes, you will need to delete and re-ingest your ADOs, or use an external (not provided in default Archipelagos) Drupal module for bundle/Drupal Content Type changes.</li> </ul> </li> <li>additional metadata elements you wish to Update, Replace, or Append values for, such as \"subjects\"<ul> <li>optional (but likely pertains to the reason you are Updating your ADOs)</li> <li>follow recommendations and practices described in more below detail</li> </ul> </li> <li>files<ul> <li>optional</li> <li>on the final 'Process' tab of your AMI Update Set Configuration, if the \"Do not touch existing files\" option is checked, then existing files will be left untouched. It is recommended to always keep this option checked if you are targeting only Metadata updates.</li> </ul> </li> </ul> <p>You should be familiar with the basic mechanics of AMI Set Configuration noted in Steps 1-6.</p> <p>Best Practices</p> <p>For all AMI Update operations, it is strongly recommended to both:</p> <ol> <li> <p>Before Updating, use the 'Export Archipelago Digital Objects to CSV content item' Action available on the main <code>Content</code> page and the <code>Find and Replace</code> page menus to generate a CSV of your non-modified objects. If something unintended occurs with your Update execution, you could use this CSV of your non-modified objects to restore your objects (or just a field or two) as needed.</p> </li> <li> <p>Create a small test batch CSV referencing one to two/three ADOs to test the execution of your desired Update actions on before running your larger Update Sets. There is no 'Undo' or 'Revert Changes' button that can be used for an AMI Update Set. You do have the option to 'Revert Revisions' on an object-by-object basis, but that is not ideal for reverting changes that were executing across large batches of ADOs. See the 'Checking Your Changes' documentation section for more information about reviewing and potentially reverting Revisions.</p> </li> </ol>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"ami_update/#data-transformation-options-for-ami-update-sets","title":"Data Transformation Options for AMI Update Sets","text":"<p>As with regular/Create New AMI Sets, you will have to select your preferred Data Tranformation configuration during Step 3 : of your AMI Update Set Configuration.</p> <ul> <li>Direct<ul> <li>Columns from your spreadsheet source will be cast directly to ADO metadata (JSON), without transformation/further processing (only intended for use with simple data strings or already JSON-encoded snippets/values).</li> <li>This is likely the most common Data Transformation configuration you will use for simple AMI Update Replace and Append Sets.</li> </ul> </li> <li>Custom (Expert Mode)<ul> <li>Provides very granular custom data transformation and mapping options</li> <li>You will likely only use this Data Transformation configuration for more complex AMI Update Sets when you want to differentiate between Data Transformation setups for Digital Objects and Digital Object Collections/Compound Objects/Creative Work Series (such as passing Digital Objects through 'Template' tranformation, and Collections/Compounds through 'Direct' transformation).</li> </ul> </li> <li>Template<ul> <li>Columns from your spreadsheet source will be cast to ADO metadata (JSON) using a Twig template setup for JSON output.</li> <li>This is likely the most common Data Transformation configuration you will use for more complex AMI Update Sets. This is the setup you would need to use if you want to use an AMI Update Set with AMI's LoD Reconciliation to update existing ADOs subject metadata with enriched LoD.</li> </ul> </li> </ul> <p>Caution with using Templates for Data Transformation</p> <p>If you are planning to use the 'Template' or 'Custom (Export Mode)' data transformation approach for your AMI Update Set configuration, you will need to have prepared your corresponding AMI Ingest Template to account for the specific Update actions you have planned. </p> <p>It is important to keep in mind that all of the metadata elements for your existing ADOs metadata may not necessarily be present in your AMI Update Set Source CSV. For example, you may have only prepared your AMI Update Set Source CSV to contain a limited number of headers/columns, such as only those required (node_uuid, label) and one or two metadata elements you wish to update (such as \"subjects\"). If you choose to pass your AMI Update Set through a Twig template, the output after Processing your AMI Update Set may overwrite your existing data if you do not have all of the necessary logic/checks in place to preserve the existing metadata if desired.</p> <p>In other words, imagine your twig template contains this statement: <pre><code>  \"subjects\": {{ data.subjects|json_encode|raw }}, \n</code></pre></p> <p>Independently of IF your CSV contains \"subjects\" as a header/column, the Twig template will still output an empty \"subjects\", which, when using the \"Replace\" mode will wipe out any existing \"subjects\" in your ADO.</p> <p>During any update operation (independently of the functional operation chosen) and IF you are using/passing your CSV through a template, AMI will provide an extra Context key for you to reference in your Twig Template. You can always reference 'dataOriginal.subjects' for example -- all dataOriginal.xx keys will contain the values of the existing metadata for your ADOs. This allows you to make \"smart\" templates that check IF a certain key/values exists, compare the unmodified (and to be modified) ADO(s) with the new data passed, then generate the desired output. </p>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"ami_update/#update-set-processing-options","title":"Update Set Processing Options","text":"<p>Beginning from Step 7, Processing of your AMI Set Configuration, select the Update operation that best corresponds to your targeted Update scenario.</p> <p></p> <p></p>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"ami_update/#1-replace-update-operation","title":"1. Replace Update Operation","text":"<p>The Replace Update Operation Replace 'will replace JSON keys found in an ADO's configured target field(s) with new JSON values. Not provided JSON keys will be kept.'</p> <ul> <li>If the processed data contains a JSON key that is already in the ADO's metadata to be updated, the values in the AMI Update Set CSV will be used, replacing completely the values found in that key in the existing ADO.</li> <li>The Replace update operation paired with the 'Direct' data transformation is likely the update operation you will use.</li> </ul>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"ami_update/#common-use-case-scenario-for-replace-updates","title":"Common use case scenario for Replace updates:","text":"<ul> <li>You notice a missing or incorrectly processed field in your original AMI Set/ADOs.</li> <li>You create an AMI set that contains only the 'node_uuid', 'label', and one column for the missing field and values.<ul> <li>If the values are singular, you do not need to JSON-encode the values in the CSV</li> <li>If the values are multiple, you need to JSON-encode them, formatted as: [\"value1\",\"value2\"]</li> </ul> </li> <li>You select the Direct data transformation approach on the AMI configuration.</li> <li>You select the Replace update operation and keep 'Do not touch existing files' checked.</li> <li>With this setup, the new field and values are added to the existing JSON for the impacted ADOs.</li> </ul>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"ami_update/#2-complete-all-json-keys-update-operation","title":"2. Complete (All JSON keys) Update Operation","text":"<p>The Complete (All JSON keys) Update Operation (formerly labeled 'Normal' in previous Archipelago releases) 'will update a complete existing ADO's JSON data with all new JSON data.' This will replace all the existing JSON, everything in an ADO with new processed data. </p> <ul> <li>The Complete (All JSON keys) update operation is powerful and can overwrite your whole JSON object record if not paired with a template that has all the extra checks/logic needed to preserve existing data if desired (see note of 'Caution with Templates for Data Transformation' above). </li> <li>It is also recommended to only use the Complete (All JSON keys) approach if you need to re-process the majority of the metadata fields for ADOs.</li> </ul>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"ami_update/#3-append-update-operation","title":"3. Append Update Operation","text":"<p>The Append update operation 'will append values to existing JSON key(s) in an ADO's configured target field(s). New JSON keys will be added too.' </p> <ul> <li>If the processed data contains a key that is already in the ADO\u2019s metadata to be updated, attempts will be made to match the \"source\" type (array, complex object) to add to it. If you have 2 values in a key, and your original/existing data contains a single value, the result will have 3 values (and then it will try to deduplicate too). If the Source data did not contain a key present in the processed data, then it will be added.</li> <li>The Append operation can be very useful, but it should be used with caution if targeting single values versus arrays. AMI will not permit malformed JSON data to be generated. But you need to consider if your Append update tranforms a previously single-value key into a multiple-value array, how this change may impact any references made in you display or other templates, Views throughout your Archipelago. <ul> <li>For example, if your Object Description Display template is not setup to check for iterable (multiple value/array) values for a given element, then the multiple values for an updated ADO may not output as expected.</li> </ul> </li> </ul>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"ami_update/#other-process-setup-options","title":"Other Process Setup Options","text":"<p>For the other AMI Set Process options and steps, please refer to the information found from Steps 7-10 in this complementary documentation for Create New ADOs AMI Sets. See the 'Checking Your Changes' documentation section for more information about reviewing and potentially reverting Revisions.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"annotations/","title":"Annotations in Archipelago","text":"<p>Archipelago extends Annotorius to provide W3C-compliant Web Annotations for Digital Objects. These annotations can be added per image (when multiple), edited for text and shape adjustments, and saved/discarded using the regular Edit mode (bonus track 1: temp storage that persists when you log out and come back in to your session). Archipelago also exposes a full API for WebAnnotations, that keeps track of which Images (referenced in the Strawberryfield @ <code>as:image</code>) were annotated and creates the W3C valid entries inside your Digital Object's JSON @ <code>ap:annotationCollection</code> (bonus track 2: multiple users can annotate the same resource, enabling digital scholarship collaboration opportunities). </p> <p>Important Note: For any image-based Digital Objects you would like to apply annotations to, the Digital Object <code>type</code> must be setup to display the image file(s) using the Open SeaDragon viewer. More information about about Managing Display Modes in Archipelago can be found here. Please stay tuned for updates announcing web annotation integration for Mirador 3.</p>"},{"location":"annotations/#enabling-annotations","title":"Enabling Annotations","text":"<ol> <li>Navigate to Admin --&gt; Structure --&gt; Content types --&gt; Digital Object --&gt; Manage Display and select the \"Digital Object Full view\" mode. <code>https://yoursite.org/admin/structure/types/manage/digital_object/display/digital_object_viewmode_fullitem</code> </li> <li>On the \u201cFragola\u201d row, click on the small gear icon on the far right, which will be open the configurations for this display type. </li> <li>Select the \u201cEnable loading/editing of W3C webAnnotations\u201d option.   <ul> <li>Learn more about the JSON format of WebAnnotations here: https://www.w3.org/TR/annotation-model/#index-of-json-keys.</li> </ul> </li> <li>Under \"What tool to enable\", select either the Rectangular or Polygon (freehand drawing) tool for your annotation style.</li> <li>Select the 'Update' button.</li> <li>Also Save your settings using the button at the bottom of the page.</li> </ol> <p> You are now ready to get started adding annotations! </p>"},{"location":"annotations/#adding-and-saving-annotations","title":"Adding and Saving Annotations","text":"<ol> <li>Navigate to the image-based Digital Object you would like to apply annotations to.</li> <li>To add a new annotation, select and hold the <code>Shift</code> key. Click and then drag to apply either a Rectangular box or multi-point Polygon shape.</li> <li>Double click to exit the annotation drawing mode.</li> <li>Enter the text for your annotation in the pop-up window.     </li> <li>Click the \"Ok\" button when you are ready.</li> <li>To save your annotation (or annotations if you created multiple), navigate to the main Digital Object \"Edit\" tab, where you will see a message about Unsaved Web Annotation Changes.     </li> <li>Select \"Save\" to preserve your Annotation(s). They will now become part of your Digital Object's JSON, found under the <code>ap:annotationCollection</code> key.<ul> <li>Pressing the \"Discard\" button will discard only the unsaved Annotations, and will reload the page.</li> </ul> </li> </ol>"},{"location":"annotations/#editing-and-deleting-annotations","title":"Editing and Deleting Annotations","text":"<ol> <li>Navigate to the image-based Digital Object you would whose annotation(s) you want to edit or delete.</li> <li>Click within the Annotation and select the downwards arrow in the upper right-hand corner of the pop-up window.</li> <li>Select either the \"Edit\" option and Edit the Annotation as desired; Or select the \"Delete\" option.     </li> <li>To preserve your editing or deleting actions, navigate to the main Digital Object \"Edit\" tab, where you will see a message about Unsaved Web Annotation Changes. (See screenshot in Step 6 of Adding and saving Annotations above.)</li> <li>Select \"Save\" to preserve your Annotation(s) edits or deletions. Pressing the \"Discard\" button will discard only the unsaved Annotations changes, and will reload the page.</li> </ol> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"archifilepersistencestrategy/","title":"Archipelago's File Persistence Strategy","text":""},{"location":"archifilepersistencestrategy/#how-are-files-for-archipelago-digital-objects-ados-persisted-what-happens-with-those-fishtanks","title":"How are files for Archipelago Digital Objects (ADOs) persisted? (What happens with those fishtanks?)","text":"<p>A few Event Subscribers/Data describing logics happen in a certain order:</p> <ol> <li> <p>User Uploads via a webform Element a new File or via Drush/Batch ingest that attaches (via JSONAPI) a file.</p> </li> <li> <p>If the webform is involved, Archipelago acts quickly and calls directly (before the Node even exists) the file classifier, that will:</p> <ol> <li>Add/complement a <code>as:somefiletype</code> JSON structure into the main <code>ADO</code> SBF <code>JSON</code>, with info about the file, checksums, size, Drupal fids, uuid, etc. This is a heavy function part of the <code>StrawberryfieldFilePersisterService</code>. It does a lot, making use of optimized logic, but may do more in the future to handle too-many/too-big files needs (FYI: solution will be simple, add to a queue and process later).</li> <li>The most (yes) important info added here is the desired future storage location of the file.</li> </ol> </li> <li> <p>The user finishes the form, saves and and confirms the ADO creation, and finally all the Node events fire.</p> </li> <li> <p>On presave <code>StrawberryfieldEventPresaveSubscriberAsFileStructureGenerator</code> runs and checks if 2.1 already was processed. This is needed since the user could have triggered an ingest via drush/JSONAPI/Webhooks etc. If all is well (this is a less expensive check) Archipelago continues.</p> </li> <li> <p>On presave (next) <code>StrawberryfieldEventPresaveSubscriberFilePersister</code> runs, checking all TEMPORARY files described in <code>as:somefiletype</code> and actually copying them to the right \"desired\" location.</p> </li> <li> <p>And on Save <code>StrawberryfieldEventInsertFileUsageUpdater</code> also marking the file as \"being\" used by a Strawberry driven Node (different Event).</p> </li> </ol> <p>Note: Anytime we remove directly from the raw JSON a full <code>as:somefiletype</code> structure of a sub-element from an <code>as:structure</code> we force Archipelago to do all the above again, and Archipelago can regenerate technical metadata. This has been used when updating EXIF binaries or even when something went wrong (while testing, but this stuff is safe no worries). Eventually, there will be a BIG red button that does that if you do not like JSON editing.</p> <p>Discussions related to Archipelago's file persistence strategy and planned potential strategies can be be found here: Strawberryfield Issues: 107, and here: Strawberryfield Issues: 76. This page will be updated with additional information following future developments.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"archipelag-logo-usage/","title":"Archipelago Commons Logo Usage Guidelines","text":"","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#our-logo","title":"Our Logo","text":"<p>The Archipelago Commons Logo features an illustrated dumbo octopus, named Dumpling, swimming beside the Archipelago Commons name. This logo is used for branding identity and visual representation for Archipelago Commons, an open-source repository software that is developed and supported at the Metropolitan New York Library Council (METRO).</p> <p>This logo was developed with care by METRO's Digital Services Team, and was drawn by Diego Pino Navarro.</p>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#logo-license","title":"Logo License","text":"<p>The Archipelago Commons Logo is protected by a Mozilla Public License 2.0. Please refer to the full License Description and Terms found at https://opensource.org/license/mpl-2-0.</p>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#logo-versions","title":"Logo Versions","text":"<p>The Archipelago Commons Logo is available in a few different iterations. You should use our Primary version for the vast majority of use cases.</p>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#primary","title":"Primary","text":"<ul> <li>Link to Zip File Containing 720px, 480px, 320px, and 240px versions</li> </ul>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#alternate-colors","title":"Alternate Colors","text":"<ul> <li>Link to Zip File Containing 480px, 320px, and 240px versions</li> </ul> <ul> <li>Link to Zip File Containing 480px, 320px, and 240px versions</li> </ul>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#solid-black-outline","title":"Solid Black Outline","text":"<ul> <li>Link to Zip File Containing 480px, 320px, and 240px versions</li> </ul>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#solid-white-outline","title":"Solid White Outline","text":"<ul> <li>Link to Zip File Containing 480px, 320px, and 240px versions</li> </ul>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#stamp-versions-black-or-white-outline","title":"Stamp Versions: Black or White Outline","text":"<p>Please only use the Stamp Versions of the Archipelago Commons logo if you do not have space to use the Primary or Alternate Color versions of the Logo.</p> <p></p> <ul> <li>Link to Zip File Containing Multiple Versions</li> </ul>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#favicon","title":"Favicon","text":"<ul> <li>Link to Zip File Containing Multiple Versions</li> </ul>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#placement-clear-space","title":"Placement &amp; Clear Space","text":"<p>To ensure our Archipelago Commons Logo maintains legibility and integrity when placing within your own website or documents, always preserve a minimum clear space between the logo and other elements. Please see further notes relating to proper Logo placement in the Logo Improper Uses section below.</p>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#scaling-proportionally","title":"Scaling proportionally","text":"<p>The proportions of the Archipelago Commons Logo should never be altered. When resizing any version of the Archipelago Commons logo, use the original scaling to maintain the correct proportions.</p>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#background-colors","title":"Background colors","text":"<p>Our primary Archipelago Commons Logo (black type with a pink dumbo octopus) should be used over most backgrounds. </p> <p>Do not use the Archipelago Commons Logo over patterned backgrounds.</p>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#logo-improper-uses","title":"Logo Improper Uses","text":"<p>We are very excited to see the Archipelago Commons Logo presented on your institutional websites. We are proud to have you and your colleagues as part of the wider Archipelago Community. We kindly ask that you abide by the following notes to help ensure that the Archipelago Commons Logo is well presented, and thank you in advance for your respectful usage of the Archipelago Commons Logo.</p> <ol> <li>Do not use the Archipelago Commons Logo in a way that incorrectly implies affiliation with, or sponsorship, endorsement, or approval by Archipelago Commons or METRO of your products or services.</li> <li>Do not display the Archipelago Commons Logo more prominently than your own product, service, or company name.</li> <li>Do not use the Archipelago Commons Logo on merchandise for sale (e.g., selling t-shirts, tote bags, mugs, stickers, etc.).</li> <li>Do not use the Archipelago Commons Logo on merchandise produced for free distribution (e.g., t-shirts, tote bags, mugs, stickers, etc.) without prior written consent of the METRO Digital Services Team.</li> <li>Do not use the Archipelago Commons Logo for any other form of commercial use (e.g. offering technical support services), unless such use is limited to a truthful and descriptive reference (e.g. \u201cIndependent technical support for Archipelago Commons\u201d or \"This repository runs on Archipelago Commons\").</li> <li>Do not modify any Archipelago Commons Logo versions, abbreviate them, or combine them with any other symbols, words, or images, or incorporate them into a tagline or slogan.</li> <li>Do not separate the illustration of Dumpling (the dumbo octopus) and the Archipelago Commons Wordmark.</li> <li>Do not rearrange the elements, change the scale of elements, or flip or rotate the elements in the Archipelago Commons Logo.</li> <li>Do not stretch, distort, or add decorative effects such as emboss or drop shadow, or otherwise modify the Archipelago Commons Logo.</li> <li>Do not alter the colors of the Archipelago Commons Logo or make it transparent.</li> <li>Do not place or embed the Archipelago Commons Logo within a box or carrier.</li> <li>Do not use the Archipelago Commons Favicon images outside actual Favicon application.</li> </ol>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#questions-about-proper-usage","title":"Questions About Proper Usage","text":"<p>If you have any questions or concerns about proper usage of the Archipelago Commons Logo not specified in this guide, please reach out to METRO's Digital Services Team, Diego Pino Navarro and Allison Sherrick.</p>","tags":["Archipelago Commons","Logo"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/","title":"Archipelago-deployment: upgrading Drupal 9 to Drupal 10 (1.1.0 to 1.3.0)","text":"","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#what-is-this-documentation-for","title":"What is this documentation for?","text":"<p>If you already have a well-set-up and well-loved Archipelago (1.1.0) running Drupal 9 (D9), this documentation will allow you to update to 1.3.0 on Drupal 10 (D10) without any major issues.</p> <p>D9 is no longer supported as of the end of November 2023. D10 has been around for a little while, and even if every module is not supported yet, what you need and want for Archipelago has long been ready for D10. However, Archipelago is still D9 compatible if it's necessary for you to stay back a little longer.</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#requirements","title":"Requirements","text":"<ul> <li>An archipelago-deployment local instance 1.1.0 (working, tested) deployed using provided instructions via Docker and running Drupal 9.</li> <li>Basic knowledge and instincts (+ courage) on how to run Terminal Commands, <code>composer</code> and <code>drush</code>.</li> <li>Patience. You can't skip steps here.</li> <li>For shell Commands documented here please copy line by line--not the whole block.</li> </ul>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#backing-up-and-preparing-for-the-upgrade","title":"Backing up and preparing for the upgrade","text":"<p>Backups are always going to be your best friends. Archipelago's code, database, and settings are mostly self-contained in your current <code>archipelago-deployment</code> repo folder, and backing up is simple because of that.</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-1","title":"Step 1:","text":"<p>On a terminal, <code>cd</code> into your running <code>archipelago-deployment</code> folder and shut down your <code>docker-compose</code> ensemble by running the following:</p> <pre><code>docker-compose down\n</code></pre>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-2","title":"Step 2:","text":"<p>Verify that all containers are actually down. The following command should return an empty listing:</p> <pre><code>docker ps\n</code></pre> <p>If anything is still running, wait a little longer and run the command again.</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-3","title":"Step 3:","text":"<p>Now let's <code>tar.gz</code> the whole ensemble with data and configs. As an example we will save this into your <code>$HOME</code> folder. As a good practice we append the current date (YEAR-MONTH-DAY) to the filename. Here we assume today is October 31st of 2023.</p> <pre><code>cd ..\nsudo tar -czvpf $HOME/archipelago-deployment-D9-20231031.tar.gz archipelago-deployment\ncd archipelago-deployment\n</code></pre> <p>The process may take a few minutes. Now let's verify that all is there and that the <code>tar.gz</code> is not corrupt.</p> <pre><code>tar -tvvf $HOME/archipelago-deployment-D9-20231031.tar.gz\n</code></pre> <p>You will see a listing of files, and at the end you will see something like this: <code>Archive Format: POSIX pax interchange format, Compression: gzip</code>. If corrupt (Do you have enough space? Did your ssh connection drop?) you will see the following:</p> <pre><code>tar: Unrecognized archive format\n</code></pre>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-4","title":"Step 4:","text":"<p>Restart your <code>docker-compose</code> ensemble, and wait a little while for all to start.</p> <pre><code>docker-compose up -d\n</code></pre>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-5","title":"Step 5:","text":"<p>Export/backup all of your live Archipelago configurations (this allows you to compare/come back in case you lose something custom during the upgrade).</p> <pre><code>docker exec esmero-php mkdir config/backup\ndocker exec esmero-php drush cex --destination=/var/www/html/config/backup\n</code></pre> <p>Good. Now it's safe to begin the upgrade process.</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#upgrading-to-130","title":"Upgrading to 1.3.0","text":"","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-1-edit-docker-composeryml","title":"Step 1: Edit docker-composer.yml","text":"<p>First we are going to edit your docker-compose.yml file to reference the latest PHP container as needed. Starting in your Archipelago deployment directory location, run the following commands:</p> <p>If you have not already, run:</p> <pre><code>docker-compose down\n</code></pre> <p>Then open your docker-compose.yml file:</p> <pre><code>nano docker-compose.yml\n</code></pre> <p>Inside your <code>docker-compose.yml</code> file, page down to the <code>php</code> section and change the <code>image</code> section to match exactly as follows:</p> <pre><code>image: \"esmero/php-8.1-fpm:1.2.0-multiarch\"\n</code></pre> <p>Next page down to the <code>iiif</code> section and change the <code>image</code> section to match exactly as follows:</p> <pre><code>image: \"esmero/cantaloupe-s3:6.0.1-multiarch\"\n</code></pre> <p>Save your changes.</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-2-docker-pull-and-check","title":"Step 2: docker pull and check","text":"<p>Time to fetch the latest branch and update our <code>docker compose</code> and <code>composer</code> dependencies. To pull the images and bring up the ensemble, run:</p> <pre><code>docker compose pull\ndocker compose up -d\n</code></pre> <p>Give all a little time to start. Please be patient. To ensure all is well, run (more than once if necessary) the following:</p> <pre><code>docker ps\n</code></pre> <p>You should see something like this:</p> <pre><code>CONTAINER ID   IMAGE                                      COMMAND                  CREATED          STATUS          PORTS                              NAMES\n355e13878b7e   nginx                                      \"/docker-entrypoint.\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8001-&gt;80/tcp               esmero-web\n86b685008158  solr:8.11.2                                 \"docker-entrypoint.s\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8983-&gt;8983/tcp             esmero-solr\na8f0d9c6d4a9   esmero/cantaloupe-s3:6.0.1-multiarch       \"sh -c 'java -Dcanta\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8183-&gt;8182/tcp             esmero-cantaloupe\n6642340b2496   mariadb:10.6.12-focal                      \"docker-entrypoint.s\u2026\"   10 minutes ago   Up 10 minutes   3306/tcp                           esmero-db\n0aef7df34037   minio/minio:RELEASE.2022-06-11T19-55-32Z   \"/usr/bin/docker-ent\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:9000-9001-&gt;9000-9001/tcp   esmero-minio\n28ee3fb4e7a7   esmero/php-8.1-fpm:1.2.0-multiarch         \"docker-php-entrypoi\u2026\"   10 minutes ago   Up 10 minutes   9000/tcp                           esmero-php\na81c36d51a81   esmero/esmero-nlp:fasttext-multiarch       \"/usr/local/bin/entr\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:6400-&gt;6400/tcp             esmero-nlp\n</code></pre> <p>Important here is the <code>STATUS</code> column. It needs to be a number that goes up in time every time you run <code>docker ps</code> again (and again).</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-3-updates-for-key-drupal-and-archipelago-modules","title":"Step 3. Updates for key Drupal and Archipelago modules","text":"<p>Now we are going to tell <code>composer</code> to update the key Drupal and Archipelago modules.</p> <p>First we are going to disable and remove a few minor Drupal modules. Run the following commands (in order):</p> <pre><code>docker exec -ti esmero-php bash -c \"drush pm-uninstall fancy_file_delete\"\ndocker exec -ti esmero-php bash -c \"drush pm:uninstall quickedit\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/fancy_file_delete\"\n</code></pre> <p>Now update the versions used for these Drupal modules:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require drupal/jquery_ui_slider:^2 drupal/jquery_ui_effects:^2 drupal/jquery_ui:1.6 drupal/jquery_ui_datepicker:^2 drupal/jquery_ui_touch_punch:^1 drupal/better_exposed_filters:6.0.3 --no-update --with-all-dependencies\"\n</code></pre> <p>And now update one other Drupal module and the main Archipelago modules:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/views_bulk_operations:^4.2' 'strawberryfield/strawberryfield:1.3.0.x-dev' 'strawberryfield/webform_strawberryfield:1.3.0.x-dev' 'strawberryfield/format_strawberryfield:1.3.0.x-dev' 'strawberryfield/strawberry_runners:0.7.0.x-dev' 'archipelago/ami:0.7.0.x-dev' --no-update --with-all-dependencies\"\n</code></pre> <p>From inside your <code>archipelago-deployment</code> repo folder we are now going to open up file <code>permissions</code> for some of your most protected Drupal files.</p> <pre><code>sudo chmod 777 web/sites/default\nsudo chmod 666 web/sites/default/*settings.php\nsudo chmod 666 web/sites/default/*services.yml\n</code></pre>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-4-disableremove-additional-drupal-modules-and-loosen-up-dependencies","title":"Step 4: Disable/Remove additional Drupal modules and loosen up dependencies","text":"<p>We are going to remove additional select Drupal modules that are not 1.3.0 or D10 compliant.</p> <p>Please run each of the following commands separately, in order, and do not skip any commands.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer remove symfony/http-kernel symfony/yaml --no-update\"\ndocker exec -ti esmero-php bash -c 'composer remove egulias/email-validator --no-update'\ndocker exec -ti esmero-php bash -c \"composer require drupal/config_inspector:^2 --no-update\"\ndocker exec -ti esmero-php bash -c \"drush pm:uninstall jsonapi_earlyrendering_workaround\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/jsonapi_earlyrendering_workaround --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/core:^10' 'drupal/core-recommended:^10' 'drupal/core-composer-scaffold:^10' 'drupal/core-project-message:^10' --update-with-dependencies --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/core-dev:^10' --dev --update-with-dependencies --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drush/drush:^12' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/twig_tweak:^2 --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/config_inspector:^2 --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/config_update:2.0.x-dev --no-update\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/config_update_ui --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/context:^5.0@RC' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/devel:^5.1' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/sophron --no-update\"\ndocker exec -ti esmero-php bash -c \"composer remove fileeye/mimemap --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/imagemagick:^3 --no-update\"  \ndocker exec -ti esmero-php bash -c \"composer remove fileeye/mimemap --no-update\"    \ndocker exec -ti esmero-php bash -c \"composer require 'drupal/imce:^3.0' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/search_api_attachments:^9.0' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/search_api_solr:^4.3 solarium/solarium ^6.3 --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/twig_tweak:^3.2' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/webform_entity_handler:^2.0' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/webformnavigation:^2.0' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/form_mode_manager --no-update\"\n</code></pre> <p>Well done! If you see no issues and all ends in Green colored messages, all is good!  Jump to Step 5</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#what-if-all-is-not-ok-and-i-see-red-and-a-lot-of-dependency-explanations","title":"What if all is not OK, and I see red and a lot of dependency explanations?","text":"<p>If you have manually installed packages via composer in the past that are NO longer Drupal 10 compatible you may see errors. In that case you need to check each package website's (normally https://www.drupal.org/project/the_module_name) and check if there is a Drupal 10 compatible version.</p> <p>If so run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require drupal/the_module_name:^VERSION_NUMBER_THAT_WORKS_ON_DRUPAL10_ --update-with-dependencies --no-update\" and run **Step 3** again (and again until all is cleared)\n</code></pre> <p>If not, try to find a replacement module that does something similar, but in any case you may end up having to remove before proceeding. Give us a ping/slack/google group/open a github ISSUE if you find yourself uncertain about this.</p> <pre><code>docker exec -ti esmero-php bash -c \"drush pm-uninstall the_module_name\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/the_module_name --no-update\"\n</code></pre>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-5-update-composerjson","title":"Step 5: Update composer.json","text":"<p>Now you need to update your <code>composer.json</code> file to include an important patch. Starting in your Archipelago deployment directory location, run the following commands:</p> <pre><code>nano composer.json\n</code></pre> <p>Inside your <code>composer.json</code> file, page down to the <code>\"patches\"</code> section and change the section to match exactly as follows:</p> <pre><code>\"patches\": {\n      \"drupal/form_mode_manager\": {\n       \"D10 compatibility\": \"https://www.drupal.org/files/issues/2023-10-11/3297262-20.patch\"\n      },\n      \"drupal/ds\": {\n       \"https://www.drupal.org/project/ds/issues/3338860\": \"https://www.drupal.org/files/issues/2023-04-04/3338860-5-d10-compatible_0.patch\"\n      }\n    }\n</code></pre> <p>Save your changes and then run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer update -W\"\n</code></pre>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-6-one-final-round-of-drupal-module-updates","title":"Step 6: One final round of Drupal module updates","text":"<p>We will now run a few more updates for additional Drupal modules.</p> <p>Please run each of the following commands separately, in order, and do not skip any commands.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require mglaman/composer-drupal-lenient\"\ndocker exec -ti esmero-php bash -c \"composer config --merge --json extra.drupal-lenient.allowed-list '[\\\"drupal/form_mode_manager\\\"]'\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/form_mode_manager:2.x-dev@dev'\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/color:^1.0'\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/hal\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/aggregator\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/ckeditor\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/seven\"\ndocker exec -ti esmero-php bash -c \"composer require archipelago/archipelago_subtheme:1.3.0.x-dev\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/quickedit drupal/classy drupal/stable\"\ndocker exec -ti esmero-php bash -c \"drush pm:uninstall quickedit\"\ndocker exec -ti esmero-php bash -c \"drush pm:uninstall hal\"\ndocker exec -ti esmero-php bash -c \"drush pm:install jquery_ui\"\n</code></pre> <p>Whew, that's a lot of module updates! Now run one final database update command:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-7-optional-syncs","title":"Step 7: Optional Syncs","text":"<p>Optionally, you can sync your new Archipelago 1.3.0 and bring in all the latest configs and settings. For this you have two options (no worries, remember you made a backup!):</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#a-partial-sync-which-will-bring-new-configs-and-update-existing-ones-but-will-not-remove-ones-that-only-exist-in-your-custom-setup-eg-new-webforms-or-view-modes","title":"A Partial Sync, which will bring new configs and update existing ones but will not remove ones that only exist in your custom setup, e.g. new Webforms or View Modes.","text":"<pre><code>docker exec esmero-php drush cim -y --partial   \n</code></pre>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#a-complete-sync-which-will-bring-new-things-and-update-existing-but-will-also-remove-all-the-ones-that-are-not-part-of-130-its-a-like-clean-factory-reset","title":"A Complete Sync, which will bring new things and update existing but will also remove all the ones that are not part of 1.3.0. It's a like clean factory reset.","text":"<pre><code>docker exec esmero-php drush cim -y\n</code></pre> <p>If all goes well here and you see no errors it's time to reindex <code>Solr</code> because there are new Fields. Run the following:</p> <pre><code>docker exec esmero-php drush search-api-reindex\ndocker exec esmero-php drush search-api-index\n</code></pre> <p>You might see some warnings related to modules dealing with previously non-existent data\u2014no worries, just ignore those.</p> <p>If you made it this far you are done with code/devops (are we ever ready?), and that means you should be able to (hopefully) stay in the Drupal 10 realm for a few years!</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-8-update-or-not-your-metadata-display-entities-and-menu-items","title":"Step 8: Update (or not) your Metadata Display Entities and menu items","text":"<p>Recommended: If you want to add new templates and menu items 1.3.0 provides, run this:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/deploy.sh'\n</code></pre> <p>Once that is done, you can choose to update all Metadata Displays (twig templates) we ship with new 1.3.0 versions (heavily adjusted IIIF manifests, better Object description template using Macros). Before you do this, we strongly recommend that you first make sure to manually (copy/paste) backup any Twig templates you have modified. If unsure, do not run the command that comes after this warning! You can always manually copy the new templates from the <code>d8content/metadatadisplays</code> folder which contains text versions (again, copy/paste) of each shipped template you can pick and use when you feel ready.</p> <p>If you are sure (like really?) you want to overwrite the ones you modified (sure, just checking?), then you can run this:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/update_deployed.sh'\n</code></pre> <p>Done! (For realz now)</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-9-or-should-we-say-10","title":"Step 9 (or should we say 10)","text":"<p>Please login to your Archipelago and test/check all is working! Enjoy 1.3.0 and Drupal 10. Thanks!</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#need-help-blue-screen-missed-a-step-need-a-hug-and-such","title":"Need help? Blue Screen? Missed a step? Need a hug and such?","text":"<p>If you see any issues or errors or need help with a step, please let us know (ASAP!). You can either open an <code>issue</code> in this repository or use the Google Group. We are here to help.</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Sherrick</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-democontent/","title":"Adding Demo Archipelago Digital Objects (ADOs) to your Repository","text":"<p>We make this optional since we feel  not everyone wants to have Digital Objects from other people using space in their system.  Still, if you are new to Archipelago we encourage you to do this. Its a simply way to get started without thinking too much.  You can learn and test. Then delete and move over. </p>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#prerequisites","title":"Prerequisites","text":"","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#the-new-way-archipelago-100-rc2-or-higher","title":"The new way Archipelago 1.0.0-RC2 or higher.","text":"<ul> <li>You installed it either via the Step by Step deployment on OSX, the one for Ubuntu or using your secret powers directly on a VM/Metal/Cloud/EC2 or even a raspberryPI.</li> <li>You followed the guides without being too creative which means you have an <code>jsonapi</code> drupal user and an <code>admin</code> one and you can login and out of your server.</li> <li>You remember your <code>admin</code> user. (If you followed one of the deployment guides, password will be <code>archipelago</code>)</li> </ul>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#step-1-only-step","title":"Step 1: (only step)","text":"<ul> <li>Log into your Archipelago using the <code>admin</code> user. </li> <li>Navigate to <code>Content</code> -&gt; <code>Ami Sets</code>. You will see a single <code>AMI Set</code> already in place. </li> <li>On the Drop Down Menu to the right (The <code>edit</code> Button), press on the little <code>down arrow</code> and choose <code>Process</code>. </li> <li>A new Form will appear. Under <code>DESIRED ADOS STATUSES AFTER PROCESS</code>, change all from Draft to Published, leave <code>Enqueue but do not process Batch in realtime</code> unchecked and press \"Confirm\". The Ingest will start and a progress bar will advance. Once ready a list of Ingest Objects should appear.</li> <li>You are done!</li> </ul>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#old-way-a-running-archipelago-10-beta3-or-higher","title":"Old way, A running Archipelago 1.0-Beta3 or higher.","text":"<ul> <li>You installed it either via the Step by Step deployment on OSX, the one for Ubuntu or using your secret powers directly on a VM/Metal/Cloud/EC2 or even a raspberryPI.</li> <li>You followed the guides without being too creative which means you have a <code>jsonapi</code> drupal user and you can login and out of your server.</li> </ul>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#step-1-get-the-content","title":"Step 1: Get the content","text":"<p>Go into your <code>archipelago-deployment</code> folder and into the <code>d8content</code> folder that is inside it, e.g.</p> <pre><code>cd archipelago-deployment/d8content\ngit clone https://github.com/esmero/archipelago-recyclables\n</code></pre>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#step-2-ingest-the-objects","title":"Step 2: Ingest the Objects","text":"<ul> <li>If running Docker execute:</li> </ul> <pre><code>docker exec -ti esmero-php bash -c 'd8content/archipelago-recyclables/deploy_ados.sh'\n</code></pre> <p>You will see multiple outputs similar to this:</p> <pre><code>Files in provided location:\n - anne_001.jpg\n - anne_002.jpg\n - anne_003.jpg\n - anne_004.jpg\n - anne_005.jpg\n - anne_006.jpg\n - anne_007.jpg\n - anne_008.jpg\n - anne_009.jpg\n - anne_010.jpg\nFile anne_001.jpg sucessfully uploaded with Internal Drupal file ID 5\nFile anne_002.jpg sucessfully uploaded with Internal Drupal file ID 6 \nFile anne_003.jpg sucessfully uploaded with Internal Drupal file ID 7\nFile anne_004.jpg sucessfully uploaded with Internal Drupal file ID 8\nFile anne_005.jpg sucessfully uploaded with Internal Drupal file ID 9 \nFile anne_006.jpg sucessfully uploaded with Internal Drupal file ID 10 \nFile anne_007.jpg sucessfully uploaded with Internal Drupal file ID 11 \nFile anne_008.jpg sucessfully uploaded with Internal Drupal file ID 12\nFile anne_009.jpg sucessfully uploaded with Internal Drupal file ID 13 \nFile anne_010.jpg sucessfully uploaded with Internal Drupal file ID 14\nNew Object 'Anne of Green Gables : Chapters 1 and 2' with UUID 9eb28775-d73a-4904-bc79-f0e925075bc5 successfully ingested. Thanks!\n</code></pre> <p>The gist here is that if the script says <code>Thanks</code> you are good.</p> <ul> <li>If you are not running Docker (You are a unicorn or at least a hacker) you will need to tune/copy/modify the following script: <code>archipelago-deployment/d8content/archipelago-recyclables/deploy_ados.sh</code></li> </ul> <p>Inside you will find lines like this one: </p> <pre><code>drush archipelago:jsonapi-ingest /var/www/html/d8content/archipelago-recyclables/ado/0c2dc01a-7dc2-48a9-b4fd-3f82331ec803.json --uuid=0c2dc01a-7dc2-48a9-b4fd-3f82331ec803 --bundle=digital_object --uri=http://esmero-web --files=/var/www/html/d8content/archipelago-recyclables/ado/0c2dc01a-7dc2-48a9-b4fd-3f82331ec803 --user=jsonapi --password=jsonapi --moderation_state=published;\n</code></pre> <p>What you want here is to modify/replace the absolute paths that point your demo objects (.json) and their assets (folders with the same name). Basically replace every entry of <code>/var/www/html/d8content/archipelago-recyclables/</code> with the path to <code>archipelago-recyclables</code>.</p>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#need-help-blue-screen-missed-a-step-need-a-hug-another-hug","title":"Need help? Blue Screen? Missed a step? Need a hug? Another Hug?","text":"<p>If you have trouble running this or see errors or need help with a step (its only two steps), please let us know (ASAP!). You can either open an <code>issue</code> in this repository or use the Google Group. We are here to help.</p>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#caring-coding-fixing","title":"Caring &amp; Coding + Fixing","text":"<ul> <li>Diego Pino</li> <li>Giancarlo Birello</li> <li>Allison Lund</li> </ul>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/","title":"Archipelago-deployment-live: upgrading Drupal 9 to Drupal 10 (1.1.0 to 1.3.0)","text":"","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#what-is-this-documentation-for","title":"What is this documentation for?","text":"<p>If you already have a well-set-up and well-loved Archipelago (1.1.0) running Drupal 9 (D9), this documentation will allow you to update to 1.3.0 on Drupal 10 (D10) without any major issues.</p> <p>D9 is no longer supported as of the end of November 2023. D10 has been around for a little while, and even if every module is not supported yet, what you need and want for Archipelago has long been ready for D10. However, Archipelago is still D9 (1.2.0) compatible if it's necessary for you to stay back a little longer.</p> <p>This documentation is meant to be a guide/helper, not a single-click-magic solution. Why? Because your Drupal environment and your local github branches for this whole project might have diverged really a lot from a vanilla Archipelago base deployment. You might have new templates, custom composer.json, new modules, changed webforms and Views. That is not an Archipelago issue, more like a <code>How do i keep any Drupal instance, YAML files, custom configurations and modules</code> in version control and in sync as i upgrade other parts, issue.</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#requirements","title":"Requirements","text":"<ul> <li>An archipelago-deployment-live instance 1.1.0 (working, tested) deployed using provided instructions via Docker and running Drupal 9.</li> <li>Basic knowledge, patience and instincts (+ courage) on how to run Terminal Commands, <code>composer</code> and <code>drush</code>.</li> <li>Patience(again). You can't skip steps here.</li> <li>For shell Commands documented here please copy line by line--not the whole block.</li> <li>You are running already version control and know how to git pull/push/merge.</li> </ul>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#backing-up-and-preparing-for-the-upgrade","title":"Backing up and preparing for the upgrade","text":"<p>Backups are always going to be your best friends. Archipelago's code, database, and settings are mostly self-contained in your current <code>archipelago-deployment-live</code> repo folder, and backing up is simple because of that.</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-1","title":"Step 1:","text":"<p>On a terminal, <code>cd</code> into your running <code>archipelago-deployment-live</code> folder, then <code>cd</code> inside the <code>deploy/ec2-docker</code> subfolders and shut down your <code>docker-compose</code> ensemble by running the following:</p> <pre><code>docker-compose down\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-2","title":"Step 2:","text":"<p>Verify that all containers are actually down. The following command should return an empty listing:</p> <pre><code>docker ps\n</code></pre> <p>If anything is still running, wait a little longer and run the command again.</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-3","title":"Step 3:","text":"<p>Now let's <code>tar.gz</code> the whole ensemble with data and configs. We will exclude here the local source caches generated by Cantaloupe. If these or not exist will depend on how custom your deployment is. As an example we will save this into your <code>$HOME</code> folder. As a good practice we append the current date (YEAR-MONTH-DAY) to the filename. Here we assume today is December 12th of 2023.</p> <p>We will cd back to the parent folder of your running <code>archipelago-deployment-live</code> folder, so three levels down, assuming you are right now inside <code>archipelago-deployment-live/deploy/ec2-docker</code></p> <pre><code>cd ../../..\nsudo tar --exclude=archipelago-deployment-live/data_storage/iiifcache --exclude=archipelago-deployment-live/data_storage/iiiftmp -czvpf $HOME/archipelago-deployment-D9-20231212.tar.gz archipelago-deployment-live\n</code></pre> <p>The process may take a few minutes. Now let's verify that all is there and that the <code>tar.gz</code> is not corrupt.</p> <pre><code>tar -tvvf $HOME/archipelago-deployment-D9-20231212.tar.gz\n</code></pre> <p>You will see a listing of files, and at the end you will see something like this: <code>Archive Format: POSIX pax interchange format, Compression: gzip</code>. If corrupt (Do you have enough space? Did your ssh connection drop?) you will see the following:</p> <pre><code>tar: Unrecognized archive format\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-4","title":"Step 4:","text":"<p><code>cd</code> again into your running <code>archipelago-deployment-live</code> folder, then <code>cd</code> inside the <code>deploy/ec2-docker</code> Restart your <code>docker-compose</code> ensemble, and wait a little while for all to start.</p> <pre><code>docker-compose up -d\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-5","title":"Step 5:","text":"<p>Export/backup all of your live Archipelago 1.1.0, Drupal 9 configurations (this allows you to compare/come back in case you lose something custom during the upgrade).</p> <pre><code>docker exec esmero-php mkdir config/backup\ndocker exec esmero-php drush cex --destination=/var/www/html/config/backup\n</code></pre> <p>Good. Now it's safe to begin the upgrade process.</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#upgrading-to-130","title":"Upgrading to 1.3.0","text":"","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-1-edit-docker-composeryml","title":"Step 1: Edit docker-composer.yml","text":"<p>First we are going to edit your docker-compose.yml file to reference the latest PHP container as needed. Starting in your Archipelago deployment directory location, run the following commands:</p> <p>If you have not already, run:</p> <pre><code>docker-compose down\n</code></pre> <p>Then open your docker-compose.yml file:</p> <pre><code>nano docker-compose.yml\n</code></pre> <p>Inside your <code>docker-compose.yml</code> file, page down to the <code>php</code> section and change the <code>image</code> section to match exactly as follows:</p> <pre><code>image: \"esmero/php-8.1-fpm:1.2.0-multiarch\"\n</code></pre> <p>Next page down to the <code>iiif</code> section and change the <code>image</code> section to match exactly as follows:</p> <pre><code>image: \"esmero/cantaloupe-s3:6.0.1-multiarch\"\n</code></pre> <p>Save your changes.</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-2-docker-pull-and-check","title":"Step 2: docker pull and check","text":"<p>Time to fetch the latest branch and update our <code>docker compose</code> and <code>composer</code> dependencies. To pull the images and bring up the ensemble, run:</p> <pre><code>docker compose pull\ndocker compose up -d\n</code></pre> <p>Give all a little time to start. Please be patient. To ensure all is well, run (more than once if necessary) the following:</p> <pre><code>docker ps\n</code></pre> <p>You should see something like this (your versions might vary):</p> <pre><code>CONTAINER ID   IMAGE                                      COMMAND                  CREATED          STATUS          PORTS                              NAMES\n355e13878b7e   nginx                                      \"/docker-entrypoint.\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8001-&gt;80/tcp               esmero-web\n86b685008158   solr:8.11.2                                \"docker-entrypoint.s\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8983-&gt;8983/tcp             esmero-solr\na8f0d9c6d4a9   esmero/cantaloupe-s3:6.0.1-multiarch       \"sh -c 'java -Dcanta\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8183-&gt;8182/tcp             esmero-cantaloupe\n6642340b2496   mariadb:10.6.12-focal                      \"docker-entrypoint.s\u2026\"   10 minutes ago   Up 10 minutes   3306/tcp                           esmero-db\n0aef7df34037   minio/minio:RELEASE.2022-06-11T19-55-32Z   \"/usr/bin/docker-ent\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:9000-9001-&gt;9000-9001/tcp   esmero-minio\n28ee3fb4e7a7   esmero/php-8.1-fpm:1.2.0-multiarch         \"docker-php-entrypoi\u2026\"   10 minutes ago   Up 10 minutes   9000/tcp                           esmero-php\na81c36d51a81   esmero/esmero-nlp:fasttext-multiarch       \"/usr/local/bin/entr\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:6400-&gt;6400/tcp             esmero-nlp\n</code></pre> <p>Important here is the <code>STATUS</code> column. It needs to be a number that goes up in time every time you run <code>docker ps</code> again (and again). <code>Note</code>: We will update from Solr 8.11.2 to Solr 9.1.1 only after we are sure all is running correctly.</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-3-updates-for-key-drupal-and-archipelago-modules","title":"Step 3. Updates for key Drupal and Archipelago modules","text":"<p>Now we are going to tell <code>composer</code> to update the key Drupal and Archipelago modules.</p> <p>First we are going to disable and remove a few minor Drupal modules. Run the following commands (in order):</p> <pre><code>docker exec -ti esmero-php bash -c \"drush pm-uninstall fancy_file_delete\"\ndocker exec -ti esmero-php bash -c \"drush pm:uninstall quickedit\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/fancy_file_delete\"\n</code></pre> <p>Now update the versions used for these Drupal modules:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require drupal/jquery_ui_slider:^2 drupal/jquery_ui_effects:^2 drupal/jquery_ui:1.6 drupal/jquery_ui_datepicker:^2 drupal/jquery_ui_touch_punch:^1 drupal/better_exposed_filters:6.0.3 --no-update --with-all-dependencies\"\n</code></pre> <p>And now update one other Drupal module and the main Archipelago modules:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/views_bulk_operations:^4.2' 'strawberryfield/strawberryfield:1.3.0.x-dev' 'strawberryfield/webform_strawberryfield:1.3.0.x-dev' 'strawberryfield/format_strawberryfield:1.3.0.x-dev' 'strawberryfield/strawberry_runners:0.7.0.x-dev' 'archipelago/ami:0.7.0.x-dev' --no-update --with-all-dependencies\"\n</code></pre> <p>From inside your <code>archipelago-deployment-live</code> repo folder, and inside the <code>drupal</code> subfolder, we are now going to open up file <code>permissions</code> for some of your most protected Drupal files.</p> <pre><code>sudo chmod 777 web/sites/default\nsudo chmod 666 web/sites/default/*settings.php\nsudo chmod 666 web/sites/default/*services.yml\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-4-disableremove-for-additional-select-drupal-modules","title":"Step 4: Disable/Remove for additional select Drupal modules","text":"<p>We are going to remove additional select Drupal modules that are not 1.3.0 or D10 compliant.</p> <p>Please run each of the following commands separately, in order, and do not skip any commands.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer remove symfony/http-kernel symfony/yaml --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/config_inspector:^2 --no-update\"\ndocker exec -ti esmero-php bash -c \"drush pm:uninstall jsonapi_earlyrendering_workaround\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/jsonapi_earlyrendering_workaround --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/core:^10' 'drupal/core-recommended:^10' 'drupal/core-composer-scaffold:^10' 'drupal/core-project-message:^10' --update-with-dependencies --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/core-dev:^10' --dev --update-with-dependencies --no-update\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/google_api_client --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drush/drush:^12' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/twig_tweak:^2 --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/config_inspector:^2 --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/config_update:2.0.x-dev --no-update\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/config_update_ui --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/context:^5.0@RC' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/devel:^5.1' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/sophron --no-update\"\ndocker exec -ti esmero-php bash -c \"composer remove fileeye/mimemap --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/imagemagick:^3 --no-update\"  \ndocker exec -ti esmero-php bash -c \"composer remove fileeye/mimemap --no-update\"    \ndocker exec -ti esmero-php bash -c \"composer remove drupal/jsonapi_earlyrendering_workaround --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/imce:^3.0' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/search_api_attachments:^9.0' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/twig_tweak:^3.2' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/webform_entity_handler:^2.0' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/webformnavigation:^2.0' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/webform_jqueryui_datepicker:^6 --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/rdf --no-update \"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/form_mode_manager --no-update\"\n</code></pre> <p>Well done! If you see no issues and all ends in Green colored messages, all is good!  Jump to Step 5</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#what-if-all-is-not-ok-and-i-see-red-and-a-lot-of-dependency-explanations","title":"What if all is not OK, and I see red and a lot of dependency explanations?","text":"<p>If you have manually installed packages via composer in the past that are NO longer Drupal 10 compatible you may see errors. In that case you need to check each package website's (normally https://www.drupal.org/project/the_module_name) and check if there is a Drupal 10 compatible version.</p> <p>If so run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/the_module_name:^VERSION_NUMBER_THAT_WORKS_ON_DRUPAL10_' --update-with-dependencies --no-update\" and run **Step 3 ** again (and again until all is cleared)\n</code></pre> <p>If not, try to find a replacement module that does something similar, but in any case you may end up having to remove before proceeding. Give us a ping/slack/google group/open a github ISSUE if you find yourself uncertain about this.</p> <pre><code>docker exec -ti esmero-php bash -c \"drush pm-uninstall the_module_name\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/the_module_name --no-update\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-5-update-composerjson","title":"Step 5: Update composer.json","text":"<p>Now you need to update your <code>composer.json</code> file to include an important patch. Starting in your Archipelago deployment directory location, run the following commands:</p> <pre><code>nano composer.json\n</code></pre> <p>Inside your <code>composer.json</code> file, page down to the <code>\"patches\"</code> section and change the section to match exactly as follows:</p> <pre><code>\"patches\": {\n      \"drupal/form_mode_manager\": {\n       \"D10 compatibility\": \"https://www.drupal.org/files/issues/2023-11-07/3297262-62-form-mode-manager-D10.patch\"\n      },\n      \"drupal/ds\": {\n       \"https://www.drupal.org/project/ds/issues/3338860\": \"https://www.drupal.org/files/issues/2023-04-04/3338860-5-d10-compatible_0.patch\"\n      }\n    }\n</code></pre> <p>Save your changes and then run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer update -W\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-6-one-final-round-of-drupal-module-updates","title":"Step 6: One final round of Drupal module updates","text":"<p>We will now run a few more updates for additional Drupal modules.</p> <p>Please run each of the following commands separately, in order, and do not skip any commands.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require mglaman/composer-drupal-lenient\"\ndocker exec -ti esmero-php bash -c \"composer config --merge --json extra.drupal-lenient.allowed-list '[\\\"drupal/form_mode_manager\\\"]'\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/form_mode_manager:2.x-dev@dev'\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/color:^1.0'\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/hal\"  \ndocker exec -ti esmero-php bash -c \"composer require drupal/aggregator\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/ckeditor\"  \ndocker exec -ti esmero-php bash -c \"composer require drupal/seven\"\ndocker exec -ti esmero-php bash -c \"composer require archipelago/archipelago_subtheme:1.3.0.x-dev\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/classy drupal/stable\"\ndocker exec -ti esmero-php bash -c \"drush pm:uninstall hal\"\ndocker exec -ti esmero-php bash -c \"drush pm:uninstall rdf\"\ndocker exec -ti esmero-php bash -c \"drush pm:install jquery_ui\"\ndocker exec -ti esmero-php bash -c \"drush pm:install jquery_ui_effects\"\n</code></pre> <p>Whew, that's a lot of module updates! Now run one final database update command:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-7-optional-syncs","title":"Step 7: Optional Syncs","text":"<p>Optionally, you can sync your new Archipelago 1.3.0 and bring in all the latest configs and settings. This requires you do fetch, either via <code>git</code> or manually via <code>wget</code> or <code>curl</code> newer configs from https://github.com/esmero/archipelago-deployment-live/tree/1.3.0/drupal/config/sync into the same folder structure/location of your deployment.</p> <p>For this you have two options (no worries, remember you made a backup!):</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#a-partial-sync-which-will-bring-new-configs-and-update-existing-ones-but-will-not-remove-ones-that-only-exist-in-your-custom-setup-eg-new-webforms-or-view-modes","title":"A Partial Sync, which will bring new configs and update existing ones but will not remove ones that only exist in your custom setup, e.g. new Webforms or View Modes.","text":"<pre><code>docker exec esmero-php drush cim -y --partial\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#a-complete-sync-which-will-bring-new-things-and-update-existing-but-will-also-remove-all-the-ones-that-are-not-part-of-130-its-a-like-clean-factory-reset","title":"A Complete Sync, which will bring new things and update existing but will also remove all the ones that are not part of 1.3.0. It's a like clean factory reset.","text":"<pre><code>docker exec esmero-php drush cim -y\n</code></pre> <p>If all goes well here and you see no errors it's time to reindex <code>Solr</code> because there are new Fields. Run the following:</p> <pre><code>docker exec esmero-php drush search-api-reindex\ndocker exec esmero-php drush search-api-index\n</code></pre> <p>You might see some warnings related to modules dealing with previously non-existent data\u2014no worries, just ignore those.</p> <p>If you made it this far you are done with code/devops (are we ever ready?), and that means you should be able to (hopefully) stay in the Drupal 10 realm for a few years!</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-8-update-or-not-your-metadata-display-entities-and-menu-items","title":"Step 8: Update (or not) your Metadata Display Entities and menu items","text":"<p>Recommended: If you want to add new templates and menu items 1.3.0 provides, you need to fetch everything from this remote https://github.com/esmero/archipelago-deployment-live/tree/1.3.0/drupal/d8content to your local installation into the same folder structure/location and then run:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/deploy.sh'\n</code></pre> <p><code>Important</code>: If you don't download/sync/git merge (or your prefered method) then the command will add nothing, since you will be running this command against 1.1.0 content.</p> <p>Once that is done, you can choose to update all Metadata Displays (twig templates) we ship with new 1.3.0 versions (heavily adjusted IIIF manifests, better Object description template using Macros). Before you do this, we strongly recommend that you first make sure to manually (copy/paste) backup any Twig templates you have modified. If unsure, do not run the command that comes after this warning! You can always manually copy the new templates from the <code>d8content/metadatadisplays</code> folder which contains text versions (again, copy/paste) of each shipped template you can pick and use when you feel ready.</p> <p>If you are sure (like really?) you really want to overwrite the ones you modified (sure, just checking?), then you can run this (make sure you edit that file):</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/update_deployed.sh'\n</code></pre> <p>Done! (For realz now)</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-9","title":"Step 9","text":"<p>Please login to your Archipelago and test/check all is working! Enjoy 1.3.0 and Drupal 10. Thanks! Also please keep all your new changes under version control. Check what has changed. <code>git add</code> and <code>git commit -m \"What i changed\"</code> as needed.</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-10","title":"Step 10","text":"<p>If all looks good and you are not missing any functionality (please be thorough about testing), it is time to Jump to Upgrading to Solr 9.x</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#need-help-blue-screen-missed-a-step-need-a-hug-or-someone-that-listens-to-you-in-silence","title":"Need help? Blue Screen? Missed a step? Need a hug or someone that listens to you in silence?","text":"<p>If you see any issues or errors or need help with a step, please let us know (ASAP!). You can either open an <code>issue</code> in this repository or use the Google Group. We are here to help.</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Sherrick</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/","title":"Archipelago-deployment-live: upgrading Archipelago 1.3.0 to 1.4.0 (Drupal 10.1 to 10.2.x)","text":"","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#what-is-this-documentation-for","title":"What is this documentation for?","text":"<p>If you already have a well-set-up and well-loved Archipelago (1.3.0) running Drupal 10.1 or 10.2 (D10), this documentation will allow you to update to 1.4.0 on latest Drupal 10.2 without any major issues. Even if Archipelago is 10.3.1 (as the time of writing this guide) compatible we do not recommend upgrading still until some of the Drupal core and contributed module issues are solved and that minor release is more stable.</p> <p>This documentation is meant to be a guide/helper, not a single-click-magic solution. Why? Because your Drupal environment and your local github branches for this whole project might have diverged really a lot from a vanilla Archipelago base deployment. You might have new templates, custom composer.json, new modules, changed webforms and Views. That is not an Archipelago issue, more like a 'How do I keep any Drupal instance, YAML files, custom configurations and modules in version control and in sync as I upgrade other parts' issue.</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#requirements","title":"Requirements","text":"<ul> <li>An archipelago-deployment-live instance 1.3.0 (working, tested) deployed using provided instructions via Docker and running Drupal 10.</li> <li>Basic knowledge, patience and instincts (+ courage) on how to run Terminal Commands, <code>composer</code> and <code>drush</code>.</li> <li>Patience(again). You can't skip steps here.</li> <li>For shell Commands documented here please copy line by line--not the whole block.</li> <li>You are running already version control and know how to git pull/push/merge.</li> </ul>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#backing-up-and-preparing-for-the-upgrade","title":"Backing up and preparing for the upgrade","text":"<p>Backups are always going to be your best friends. Archipelago's code, database, and settings are mostly self-contained in your current <code>archipelago-deployment-live</code> repo folder, and backing up is simple because of that.</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-1","title":"Step 1:","text":"<p>On a terminal, <code>cd</code> into your running <code>archipelago-deployment-live</code> folder, then <code>cd</code> inside the <code>deploy/ec2-docker</code> subfolders and shut down your <code>docker-compose</code> ensemble by running the following:</p> <pre><code>docker-compose down\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-2","title":"Step 2:","text":"<p>Verify that all containers are actually down. The following command should return an empty listing:</p> <pre><code>docker ps\n</code></pre> <p>If anything is still running, wait a little longer and run the command again.</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-3","title":"Step 3:","text":"<p>Now let's <code>tar.gz</code> the whole ensemble with data and configs. We will exclude here the local source caches generated by Cantaloupe. If these or not exist will depend on how custom your deployment is. As an example we will save this into your <code>$HOME</code> folder. As a good practice we append the current date (YEAR-MONTH-DAY) to the filename. Here we assume today is July 7th of 2024.</p> <p>We will <code>cd</code> back to the parent folder of your running <code>archipelago-deployment-live</code> folder, so three levels down, assuming you are right now inside <code>archipelago-deployment-live/deploy/ec2-docker</code></p> <pre><code>cd ../../..\nsudo tar --exclude=archipelago-deployment-live/data_storage/iiifcache --exclude=archipelago-deployment-live/data_storage/iiiftmp -czvpf $HOME/archipelago-deployment-D10-20240707.tar.gz archipelago-deployment-live\n</code></pre> <p>The process may take a few minutes. Now let's verify that all is there and that the <code>tar.gz</code> is not corrupt.</p> <pre><code>tar -tvvf $HOME/archipelago-deployment-D10-20240707.tar.gz\n</code></pre> <p>You will see a listing of files, and at the end you will see something like this: <code>Archive Format: POSIX pax interchange format, Compression: gzip</code>. If corrupt (Do you have enough space? Did your ssh connection drop?) you will see the following:</p> <pre><code>tar: Unrecognized archive format\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-4","title":"Step 4:","text":"<p><code>cd</code> again into your running <code>archipelago-deployment-live</code> folder, then <code>cd</code> inside the <code>deploy/ec2-docker</code> Restart your <code>docker-compose</code> ensemble, and wait a little while for all to start.</p> <pre><code>docker-compose up -d\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-5","title":"Step 5:","text":"<p>Export/backup all of your live Archipelago 1.3.0, Drupal 10 configurations (this allows you to compare/come back in case you lose something custom during the upgrade).</p> <pre><code>docker exec esmero-php mkdir config/backup\ndocker exec esmero-php drush cex --destination=/var/www/html/config/backup\n</code></pre> <p>Good. Now it's safe to begin the upgrade process.</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#upgrading-to-140","title":"Upgrading to 1.4.0","text":"","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-0-get-familiar-with-what-changed","title":"Step 0: Get familiar with what changed.","text":"<p>We mean this. This is a new step. Running a Production Server requires some informed decision making and thus, we believe, a good pre-step is reviewing what changed between releases. </p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-1-edit-docker-composeryml-optional","title":"Step 1: Edit docker-composer.yml (optional)","text":"<p>For this release there are no larger updates on the Docker backend level (good!). Solr 9 was updated from 9.1 to 9.2 and Database (MYSQL and MariaDB) also got a version bump. Esmero NLP with Machine Learning Model Endpoints is not meant for production/public access and right now is for community evaluation only. Reasons are many: some technical (CPU/GPU and memory consumption not suited for a medium server) but mostly about usefulness, use cases, real needs driving the use of the tools we developed and the very much needed Community Discussion and Consensus on our Field's (GLAM) ethical concerns about this.</p> <p>If you decide to upgrade your services please review one of these (based on your running Platform): - https://github.com/esmero/archipelago-deployment-live/blob/1.4.0/deploy/ec2-docker/docker-compose-aws-s3-arm64.yml - https://github.com/esmero/archipelago-deployment-live/blob/1.4.0/deploy/ec2-docker/docker-compose-aws-s3.yml</p> <p>And selectively copy images/enviromentals into your production <code>docker-compose.yml</code> file.</p> <p>For this, if you have not already, run:</p> <pre><code>docker-compose down\n</code></pre> <p>Then open your docker-compose.yml file and with one of the previous links open, edit the corresponding service definitions:</p> <pre><code>nano docker-compose.yml\n</code></pre> <p>Save your changes.</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-2-docker-pull-and-check","title":"Step 2: docker pull and check","text":"<p>Time to fetch the latest branch and update our <code>docker compose</code> and <code>composer</code> dependencies. To pull the images and bring up the ensemble, run:</p> <pre><code>docker compose pull\ndocker compose up -d\n</code></pre> <p>Give all a little time to start. Please be patient. To ensure all is well, run (more than once if necessary) the following:</p> <pre><code>docker ps\n</code></pre> <p>You should see something like this if you synced all containers to the latest (your versions and databse might vary depending on your server's platform):</p> <pre><code>CONTAINER ID   IMAGE                                      COMMAND                  CREATED          STATUS          PORTS                              NAMES\n5b06ee366f58   jonasal/nginx-certbot                      \"/docker-entrypoint.\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8001-&gt;80/tcp               esmero-web\n86b685008158   solr:9.2.1                                 \"docker-entrypoint.s\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8983-&gt;8983/tcp             esmero-solr\na4872b237e17   esmero/cantaloupe-s3:6.0.1-multiarch       \"sh -c 'java -Dcanta\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8183-&gt;8182/tcp             esmero-cantaloupe\nbec0b31f3421   mariadb:10.6.18-focal                      \"docker-entrypoint.s\u2026\"   10 minutes ago   Up 10 minutes   3306/tcp                           esmero-db\n85bedadf9732   redis:6.2-alpine                           \"docker-entrypoint.s\u2026\"   10 minutes ago   10 minutes ago                                     esmero-redis\n6a9e9d8647a9   minio/minio:RELEASE.2022-06-11T19-55-32Z   \"/usr/bin/docker-ent\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:9000-9001-&gt;9000-9001/tcp   esmero-minio\nbc5327680ca7   esmero/php-8.1-fpm:1.2.0-multiarch         \"docker-php-entrypoi\u2026\"   10 minutes ago   Up 10 minutes   9000/tcp                           esmero-php\nd53729be1211   esmero/esmero-nlp:fasttext-multiarch       \"/usr/local/bin/entr\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:6400-&gt;6400/tcp             esmero-nlp\n</code></pre> <p>Important here is the <code>STATUS</code> column. It needs to be a number that goes up in time every time you run <code>docker ps</code> again (and again).</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-3-updates-for-key-drupal-and-archipelago-modules","title":"Step 3. Updates for key Drupal and Archipelago modules","text":"<p>Now we are going to tell <code>composer</code> to update the key Drupal and Archipelago modules.</p> <p>We don't need to disable any modules this time (great) but we will need to lose up some dependencies to target a stable Drupal 10.2.7.</p> <p>Notice how we will run composer commands using the <code>--no-update</code> argument which means composer won't have to resolve (nor fetch) library dependencies on every execution.</p> <p>First, update the versions used for these Drupal modules:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/color:1.x-dev@dev' 'drupal/config_inspector:dev-2.1.x' 'drupal/search_api_solr:4.3.4' 'drupal/search_api:^1.3' 'solarium/solarium:^6' 'egulias/email-validator:^4' 'drupal/upgrade_status:^4.3'  --no-update --with-all-dependencies\"\n</code></pre> <p>And now update the main Archipelago modules:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require cebe/php-openapi:^1 'league/openapi-psr7-validator:0.22' 'devizzent/cebe-php-openapi:1.0.3' 'strawberryfield/strawberryfield:1.4.0.x-dev' 'strawberryfield/webform_strawberryfield:1.4.0.x-dev' 'strawberryfield/format_strawberryfield:1.4.0.x-dev' 'strawberryfield/strawberry_runners:0.8.0.x-dev' 'archipelago/ami:0.8.0.x-dev'  'archipelago/archipelago_subtheme:1.4.0.x-dev' --no-update --with-all-dependencies\"\n</code></pre> <p>From inside your <code>archipelago-deployment-live</code> repo folder, and inside the <code>drupal</code> subfolder, we are now going to open up file <code>permissions</code> for some of your most protected Drupal files.</p> <pre><code>sudo chmod 777 web/sites/default\nsudo chmod 666 web/sites/default/*settings.php\nsudo chmod 666 web/sites/default/*services.yml\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-4-update-drupal-core-to-stable-1027","title":"Step 4: Update Drupal Core to stable 10.2.7","text":"<p>We are going to remove additional select Drupal modules that are not 1.3.0 or D10 compliant.</p> <p>Please run each of the following commands separately, in order, and do not skip any commands.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/core:10.2.7' 'drupal/core-recommended:10.2.7' 'drupal/core-composer-scaffold:10.2.7' 'drupal/core-project-message:10.2.7' --update-with-dependencies --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/core-dev:^10' --dev --update-with-dependencies --no-update\"\n</code></pre> <p>Well done! If you see no issues and all ends in Green colored messages, all is good!  Jump to Step 5</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#what-if-all-is-not-ok-and-i-see-red-and-a-lot-of-dependency-explanations","title":"What if all is not OK, and I see red and a lot of dependency explanations?","text":"<p>If you have manually installed packages via composer in the past that are NO longer Drupal 10.2 compatible you may see errors. In that case you need to check each package website's (normally https://www.drupal.org/project/the_module_name) and check if there is a Drupal 10 compatible version.</p> <p>If so run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/the_module_name:^VERSION_NUMBER_THAT_WORKS_ON_DRUPAL10_' --update-with-dependencies --no-update\" and run **Step 3 ** again (and again until all is cleared)\n</code></pre> <p>If not, try to find a replacement module that does something similar, but in any case you may end up having to remove before proceeding. Give us a ping/slack/google group/open a github ISSUE if you find yourself uncertain about this.</p> <pre><code>docker exec -ti esmero-php bash -c \"drush pm-uninstall the_module_name\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/the_module_name --no-update\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-5-update-composerjson","title":"Step 5: Update composer.json","text":"<p>Now you need to update your <code>composer.json</code> file to include an important patch. Starting in your Archipelago deployment directory location, run the following commands:</p> <pre><code>nano composer.json\n</code></pre> <p>Inside your <code>composer.json</code> file, page down to the <code>\"patches\"</code> section and change the section to match exactly as follows:</p> <pre><code>\"patches\": {\n           \"drupal/core\": {\n              \"https://www.drupal.org/project/drupal/issues/3364109\": \"https://git.drupalcode.org/project/drupal/-/merge_requests/4092.diff\"\n           }\n    }\n</code></pre> <p>Save your changes and then run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer update -W\"\n</code></pre> <p>Note: again. If you see error messages of missing dependencies, invalid (stuck/old versions) libraries or conflicts, review each message and check what modules/libraries are not allowing you to upgrade. One way of understanding the chained dependency errors is to run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer why some_namespace/some_library\" \n</code></pre> <p>Example given:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer why solarium/solarium\"\n</code></pre> <p>Will output:</p> <pre><code>drupal-composer/drupal-project 1.4.0.x-dev requires solarium/solarium (^6)\ndrupal/search_api_solr         4.3.4       requires solarium/solarium (^6.3.5)\n</code></pre> <p>If a given library/module/dependency is only required by <code>drupal-composer/drupal-project</code> you have space to loosen it up to a higher/more permissive version. If it dependends on other library/modules, those might be the culprits of not being able to upgrade it and normally the solution is to upgrade that library first (always using the <code>--no-update</code> argument) and then run <code>docker exec -ti esmero-php bash -c \"composer update -W\"</code> again.</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-6-update-your-database-and-run-hook-updates","title":"Step 6: Update your Database (and run hook updates)","text":"<p>Now run one final database update command:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-7-optional-syncs","title":"Step 7: Optional Syncs","text":"<p>Optionally, you can sync your new Archipelago 1.4.0 and bring in all the latest configs and settings. This requires you do fetch, either via <code>git</code> or manually via <code>wget</code> or <code>curl</code> newer configs from https://github.com/esmero/archipelago-deployment-live/tree/1.4.0/drupal/config/sync into the same folder structure/location of your deployment.</p> <p>For this you have two options (no worries, actually do worry, just remember you made a backup! Did you? Double check!):</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#option-1-a-partial-sync-which-will-bring-new-configs-and-update-existing-ones-but-will-not-remove-ones-that-only-exist-in-your-custom-setup-eg-new-webforms-or-view-modes","title":"Option 1. A Partial Sync, which will bring new configs and update existing ones but will not remove ones that only exist in your custom setup, e.g. new Webforms or View Modes.","text":"<pre><code>docker exec esmero-php drush cim -y --partial\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#option-2-a-complete-sync-which-will-bring-new-things-and-update-existing-but-will-also-remove-all-the-ones-that-are-not-part-of-130-its-a-like-clean-factory-reset","title":"Option 2. A Complete Sync, which will bring new things and update existing but will also remove all the ones that are not part of 1.3.0. It's a like clean factory reset.","text":"<pre><code>docker exec esmero-php drush cim -y\n</code></pre> <p>If all goes well here and you see no errors it's time to reindex <code>Solr</code> because there are new Fields. Run the following:</p> <pre><code>docker exec esmero-php drush search-api-reindex\ndocker exec esmero-php drush search-api-index\n</code></pre> <p>You might see some warnings related to modules dealing with previously non-existent data\u2014-no worries, just ignore those.</p> <p>If you made it this far you are done with code/devops (are we ever really?), and that means you should be able to (hopefully) stay in the Drupal 10 realm for a few years!</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-8-update-or-not-your-metadata-display-entities-and-menu-items","title":"Step 8: Update (or not) your Metadata Display Entities and menu items","text":"<p>Recommended: If you want to add new templates and menu items 1.4.0 provides, you need to fetch everything from this remote https://github.com/esmero/archipelago-deployment-live/tree/1.4.0/drupal/d8content to your local installation into the same folder structure/location and then run:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/deploy.sh'\n</code></pre> <p><code>Important</code>: If you don't download/sync/git merge (or your prefered method) then the command will add nothing, since you will be running this command against 1.3.0 content.</p> <p>Once that is done, you can choose to update all Metadata Displays (twig templates) we ship with new 1.4.0 versions (e.g heavily adjusted IIIF manifests with Content Search API service definitions). Before you do this, we strongly recommend that you first make sure to manually (copy/paste) backup any Twig templates you have modified. If unsure, do not run the command that comes after this warning! You can always manually copy the new templates from the <code>d8content/metadatadisplays</code> folder which contains text versions (again, copy/paste) of each shipped template you can pick and use when you feel ready.</p> <p>If you are sure (like really?) you really want to overwrite the ones you modified (sure, just checking?), then you can run this (make sure you edit that file):</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/update_deployed.sh'\n</code></pre> <p>Done! (For realz now)</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-9","title":"Step 9","text":"<p>Please login to your Archipelago and test/check all is working! Enjoy 1.4.0 under Drupal 10. Thanks! Also please keep all your new changes under version control. Check what has changed. <code>git add</code> and <code>git commit -m \"What i changed\"</code> as needed.</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-10","title":"Step 10","text":"<p>If all looks good and you are not missing any functionality (please be thorough about testing), it is time to Jump to Upgrading to Solr 9.x</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#need-help-blue-screen-missed-a-step-need-a-hug-or-someone-that-listens-to-you-in-silence","title":"Need help? Blue Screen? Missed a step? Need a hug or someone that listens to you in silence?","text":"<p>If you see any issues or errors or need help with a step, please let us know (ASAP!). You can either open an <code>issue</code> in this repository or use the Google Group. We are here to help.</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Sherrick</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-gitworkflow/","title":"Managing, sheltering, pruning and nurturing your own custom Archipelago","text":"<p>Now that you have your base Archipelago Live Deployment running (Do you? If not, go back!) you may be wondering about things like:</p> <ol> <li>What happens when I need to update to the next release?</li> <li>How do I keep my Drupal and Modules updated in between releases?</li> <li>Can I add Drupal Modules?</li> <li>Will a new release overwrite all my customizations?</li> <li>What things are safe to customize?</li> <li>How do I keep my very own things in Version Control and safe from others?</li> <li>And many (many) other similar questions.</li> </ol>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#1-keep-your-archipelago-under-version-control-via-github","title":"1. Keep your Archipelago under Version Control via Github","text":"<p><code>Archipelagos</code> are living beings. They evolve and become beautiful, closer and closer to your needs. Because of that <code>resetting</code> your particularities on every <code>Archipelago</code> code release is not a good idea, nor even recommended. What you want is to keep your own <code>Drupal Settings</code>\u2014your facets, your themes, your Solr fields, your own modules, and all their configurations\u2014safe and be able to restore all in case something goes wrong.</p> <p>The ones we ship with every Release will <code>reset</code> your Archipelago's settings to Factory defaults if applied <code>wildly</code>.</p> <p>This is where <code>Github</code> comes in place.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#basic-steps","title":"Basic steps","text":"<p>Prerequisites:</p> <ul> <li>Have an Archipelago Deployment Live instance running</li> <li>Have Terminal access to your Live Instance</li> <li>Have a Github account</li> <li>Have a personal Github Access Token Created</li> <li>Run <code>git config --global --edit</code> on your Live Instance and Set your user name/email/etc.</li> <li>Note: Opens in <code>Vi</code>! In case of emergency/panic press <code>ESC</code> and type <code>:x</code> to escape and/or run away in terror. To edit Press <code>i</code> and uncomment the lines. Once Done press <code>ESC</code> and type <code>:x</code> to save.</li> </ul>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#11-start-by-creating-a-git-fork","title":"1.1 Start by creating a Git Fork","text":"<p>Let's fork https://github.com/esmero/archipelago-deployment-live under your own Account via the web. Happy Note: Since 2021 also keeping forked branches in sync with the origin can be done via the UI directly.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#12-connect-your-live-instance-terminal","title":"1.2 Connect your Live instance terminal.","text":"<p>Move to your repository's base folder, and let's start by adding your New Fork as a secondary Git <code>Origin</code>. Replace in this command <code>yourOwnAccount</code> with (guess what?) your own account:</p> <pre><code>git remote add upstream https://github.com/yourOwnAccount/archipelago-deployment-live\n</code></pre> <p>Now check if you have two remotes (<code>origin</code> =&gt; This repository, <code>upstream</code> =&gt; your own fork):</p> <pre><code>git remote -v\n</code></pre> <p>You will see this:</p> <pre><code>origin  https://github.com/esmero/archipelago-deployment-live (fetch)\norigin  https://github.com/esmero/archipelago-deployment-live (push)\nupstream    https://github.com/yourOwnAccount/archipelago-deployment-live (fetch)\nupstream    https://github.com/yourOwnAccount/archipelago-deployment-live (push)\n</code></pre> <p>Good!</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#13-now-lets-create-from-your-current-live-instance-a-new-branch","title":"1.3 Now let's create from your current Live Instance a new Branch.","text":"<p>We will push this branch into your Fork and it will be all yours to maintain. Please replace <code>yourOwnOrg</code> with any Name you want for this. We like to keep the current Branch name in place after your personal prefix:</p> <pre><code>git checkout -b yourOwnOrg-1.0.0-RC3\n</code></pre> <p>Good, you now have a new <code>local</code> branch named <code>yourOwnOrg-1.0.0-RC3</code>, and it's time to decide what we are going to push into Github.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#14-push-the-basics","title":"1.4 Push the Basics.","text":"<p>By default our deployment strategy (this repository) ignores a few files you want to have in Github.  Also, there are things like the Installed Drupal Modules and PHP Libraries (the Source Code), the Database, Caches, your Secrets (<code>.env</code> file), and your Drupal <code>settings.php</code> file. You FOR SURE do not want to have these in Github and are better suited for a private Backup Storage.</p> <p>Let's start by <code>push</code>ing what you have (no commits, your new <code>yourOwnOrg-1.0.0-RC3</code> as it is) to your new Fork. From there on we can add new Commits and files:</p> <pre><code>git push upstream yourOwnOrg-1.0.0-RC3\n</code></pre> <p>And Git will respond with the following (use your <code>yourOwnAccount</code> personal Github Access Token as password):</p> <pre><code>Username for 'https://github.com': yourOwnAccount\nPassword for 'https://yourOwnAccount@github.com': \nTotal 0 (delta 0), reused 0 (delta 0)\nremote: \nremote: Create a pull request for 'yourOwnOrg-1.0.0-RC3' on GitHub by visiting:\nremote:      https://github.com/yourOwnAccount/archipelago-deployment-live/pull/new/yourOwnOrg-1.0.0-RC3\nremote: \nTo https://github.com/yourOwnAccount/archipelago-deployment-live\n * [new branch]      yourOwnOrg-1.0.0-RC3 -&gt; yourOwnOrg-1.0.0-RC3\n</code></pre>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#15-first-commit","title":"1.5 First Commit","text":"<p>Right now this new Branch (go and check it out at https://github.com/yourOwnAccount/archipelago-deployment-live/tree/yourOwnOrg-1.0.0-RC3) will not differ at all from 1.0.0-RC3. That is OK. To make your Branch unique, what we want is to \"commit\" our changes. How do we do this?</p> <p>Let's add our <code>composer.json</code> and <code>composer.lock</code> to our change list. Both of these files are quite personal, and as you add more Drupal Modules, dependencies, or Upgrade your Archipelgo and/or Drupal Core and Modules, all of these corresponding files will change. See the <code>-f</code>? Because our base deployment ignores that file and you want it, we \"Force\" add it. Note: At this stage <code>composer.lock</code> won't be added at all because it's still the same as before. So you can only \"add\" files that have changes.</p> <pre><code>git add drupal/composer.json \ngit add -f drupal/composer.lock\n</code></pre> <p>Now we can see what is new and will be committed by executing:</p> <pre><code>git status\n</code></pre> <p>You may see something like this:</p> <pre><code>On branch yourOwnOrg-1.0.0-RC3 \nChanges to be committed:\n  (use \"git restore --staged &lt;file&gt;...\" to unstage)\n    new file:   drupal/composer.json\n\nChanges not staged for commit:\n  (use \"git add &lt;file&gt;...\" to update what will be committed)\n  (use \"git restore &lt;file&gt;...\" to discard changes in working directory)\n    modified:   drupal/scripts/archipelago/deploy.sh\n    modified:   drupal/scripts/archipelago/update_deployed.sh\n\nUntracked files:\n  (use \"git add &lt;file&gt;...\" to include in what will be committed)\n    deploy/ec2-docker/docker-compose.yml\n    drupal/.editorconfig\n    drupal/.gitattributes\n</code></pre> <p>If you do not want to add each <code>Changes not staged for commit</code> individually (WE recommend you only commit what you need. Be warned and take caution.), you can also issue a <code>git add .</code>, which means add all.</p> <pre><code>git add drupal/scripts/archipelago/deploy.sh\ngit add drupal/scripts/archipelago/update_deployed.sh\ngit add deploy/ec2-docker/docker-compose.yml\n</code></pre> <p>In this case we are also committing <code>docker-compose.yml</code>, which you may have customized and modified to your domain (See Install Guide Step 3), <code>deploy.sh</code> and <code>update_deployed.sh</code> scripts. If you ever need to avoid tracking certain files at all, you can edit the <code>.gitignore</code> file and add more patterns to it (look at it, it's fun!).</p> <pre><code>git commit -m \"Fresh Install of Archipelago for yourOwnOrg\"\n</code></pre> <p>If you had your email/user account setup correctly (see Prerequisites) you will see:</p> <pre><code>Fresh Install of Archipelago yourOwnOrg\n 4 files changed, 360 insertions(+), 46 deletions(-)\n create mode 100644 deploy/ec2-docker/docker-compose.yml\n create mode 100644 drupal/composer.json\n</code></pre> <p>And now finally you can push this back to your Fork:</p> <pre><code>git push upstream yourOwnOrg-1.0.0-RC3\n</code></pre> <p>And Git will respond with the following (use your <code>yourOwnAccount</code> personal Github Access Token as password):</p> <pre><code>Username for 'https://github.com': yourOwnAccount\nPassword for 'https://yourOwnAccount@github.com': \nEnumerating objects: 18, done.\nCounting objects: 100% (18/18), done.\nCompressing objects: 100% (10/10), done.\nWriting objects: 100% (10/10), 2.26 KiB | 2.26 MiB/s, done.\nTotal 10 (delta 5), reused 0 (delta 0)\nremote: Resolving deltas: 100% (5/5), completed with 5 local objects.\nTo https://github.com/yourOwnAccount/archipelago-deployment-live\n   d9fa835..3427ce5  yourOwnOrg-1.0.0-RC3 -&gt; yourOwnOrg-1.0.0-RC3\n</code></pre> <p>And done.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#2-keeping-your-archipelago-modules-updated-during-releases","title":"2. Keeping your Archipelago Modules Updated during releases","text":"<p>Releases in Archipelago are a bit different to other OSS projects. When a Release is done (let's say 1.0.0-RC2) we freeze the current release branches in every module we provide, package the release, and inmediatelly start with a new Release Cycle (6 months long normally) by creating in each repository a new Set of Branches (for this example 1.0.0-RC3). All new commits, fixes, improvements, features now will ALWAYS go into the Open/on-going new cycle branches (for this example 1.0.0-RC3), and once we are done we do it all over again. We freeze (for this example 1.0.0-RC3), and a new release cycle starts with fresh new \"WIP\" branches (for this example 1.1.0).</p> <p>Some Modules like AMI or Strawberry Runners have their independent Version but are released together anyway, e.g. for 1.0.0-RC3 both AMI and Strawberry Runners are 0.2.0. Why? Because work started later than the core Archipelago and also because they are not really CORE. So what happens with <code>main</code> branches? In our project <code>main</code> branches are never experimental. They are always a 1:1 with the latest stable release. So <code>main</code> will contain a full commit of 1.0.0-RC2 until we freeze 1.0.0-RC3 when <code>main</code> gets that code. Over and over. Nice, right?</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#which-modules-belong-to-archipelago-and-follow-a-release-cycle","title":"Which modules belong to Archipelago and follow a release cycle?","text":"<p>The following modules are the ones we update on every release:</p> <ul> <li>https://github.com/esmero/strawberryfield -&gt; Composer name: <code>strawberryfield/strawberryfield</code></li> <li>https://github.com/esmero/format_strawberryfield -&gt; Composer name: <code>strawberryfield/format_strawberryfield</code></li> <li>https://github.com/esmero/webform_strawberryfield -&gt; Composer name: <code>strawberryfield/webform_strawberryfield</code></li> <li>https://github.com/esmero/ami -&gt; Composer name: <code>archipelago/ami</code></li> <li>https://github.com/esmero/strawberry_runners    -&gt; Composer name: <code>strawberryfield/strawberry_runners</code></li> </ul> <p>We also update macro modules that are meant for deployment like this Repository and https://github.com/esmero/archipelago-deployment.</p> <p>To keep your Archipelago up to date, especially once you \"go custom\" as described in this Documentation, the process is quite simple, e.g. to fetch latest <code>1.0.0-RC3</code> updates during the <code>1.0.0-RC3</code> release cycle run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require strawberryfield/strawberryfield:dev-1.0.0-RC3 strawberryfield/format_strawberryfield:dev-1.0.0-RC3 strawberryfield/webform_strawberryfield:dev-1.0.0-RC3 archipelago/ami:0.2.0.x-dev strawberryfield/strawberry_runners:0.2.0.x-dev strawberryfield/strawberry_runners:0.2.0.x-dev archipelago/archipelago_subtheme:dev-1.0.0-RC3 -W\"\n</code></pre> <p>And then run any Database updates that may be needed:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre> <p>This will bring all the new code and all (except if there are BUGS!) should work as expected.</p> <p>Note: Archipelago really tries hard to be as backwards compatible as possible and rarely will you see a non-documented or non-dealt-with deprecation. </p> <p>Note 2: We of course recommend always running the Stable (frozen) release, but since code is plastic and fixes will go into a WIP open branch, you should be safe enough to move all modules together.</p> <p>You can run these commands any time you need, and while the release is open you will always get the latest code (even if it's always the same branch). Please follow/subscribe to each Module's Github to be aware of changes/issues and improvements.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#3-keeping-your-archipelagos-drupal-contributed-modules-and-core-updated","title":"3. Keeping your Archipelago's Drupal Contributed Modules and Core updated","text":"","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#31-contributed-modules","title":"3.1 Contributed Modules.","text":"<p>To keep your Archipelago's Drupal up to date check your Drupal at https://yoursite.org/admin/modules/update. Make sure you check mostly (yes mostly, no need to overreact) for Security Updates. Not every Drupal contributed module (project) keeps backwards compatibility, and we try to test every version we ship (as in this repository's <code>composer.lock</code> files) before releasing. Once you detect a major change/requirement, visit the Project's Changelog Website, and take some time reading it. If you feel confident it's not going to break all, copy the suggested Composer command, e.g. if you visit https://www.drupal.org/project/google_api_client/releases/8.x-3.2 you will see that the update is suggested as:</p> <pre><code>Install with Composer: $ composer require 'drupal/google_api_client:^3.2'\n</code></pre> <p>Using the same module as an example, before doing any final updates, check your current running version (take note in case you need to revert):</p> <pre><code>docker exec -ti esmero-php bash -c \"composer info 'drupal/google_api_client\"\n</code></pre> <p>Keep the version around.</p> <p>Now let's update, which means using the suggested command translated to our own Docker world like this (notice the <code>-W</code>):</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/google_api_client:^3.2 -W\"\n</code></pre> <p>And then run any Database updates that may be needed:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre> <p>This will update that module. Test your website. Depending on what you update, you want to focus first on the functionality it provides, and then create/edit/delete a fictitious Digital Object to ensure it did not add any errors to your most beloved Digital Objects workflows.</p> <p>If you see errors or you feel it's not acting as it should, you can revert by doing:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/google_api_client:^VERSION_YOU_KEPT_AROUND -W\"\n</code></pre> <p>And then run any Database updates that may be needed:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre> <p>If this happens we encourage you to please \ud83d\udc4f share your findings with our community/slack/Github ISSUE here.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#31-drupal-core-inside-the-same-major-version","title":"3.1 Drupal Core inside the same major version:","text":"<p>This is quite similar to a contributed module but normally involves at least 3 dependencies and of course larger changes.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#exact-version","title":"Exact Version","text":"<p>Inside the same major version, e.g. inside Drupal 9, if you are currently running Drupal <code>9.0.1</code> and you want to update to an exact latest (as I write <code>9.2.4</code>):</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require drupal/core:9.2.4 drupal/core-dev:9.2.4 drupal/core-composer-scaffold:9.2.4 drupal/core-project-message:9.2.4 --update-with-dependencies\"\n</code></pre> <p>Or under Drupal 8, if you are currently running Drupal <code>8.9.14</code> and you want to update to an exact latest (as I write <code>8.9.18</code>):</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require drupal/core-dev:8.9.18 drupal/core:8.9.18 drupal/core-composer-scaffold:8.9.18 --update-with-dependencies\"\n</code></pre> <p>And then for both cases run any Database updates that may be needed:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#alternative-major-version","title":"Alternative Major Version","text":"<p>If you want to only remember a <code>single command</code> and want to be sure to also get all extra packages for Drupal 9, run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require drupal/core-dev:^9 drupal/core:^9 drupal/core-composer-scaffold:^9 drupal/core-project-message:^9 -W\"\n</code></pre> <p>Or for Drupal 8:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require drupal/core-dev:^8 drupal/core:^8 drupal/core-composer-scaffold:^8 drupal/core-project-message:^8 -W\"\n</code></pre> <p>And then for both cases run any Database updates that may be needed:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre> <p>This will always get you the latest <code>Drupal</code> and <code>dependencies</code> allowed by your <code>composer.json</code>.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#32-drupal-core-between-major-versions","title":"3.2 Drupal Core between major versions:","text":"<p>Since major versions may bring larger deprecations, contributed modules will stay behind, and the world (and your database may collapse), we really recommend that you do some tests first (locally) or follow one of our guides. We at Archipelago will always document a larger version update. Currently, the Drupal 8 to Drupal 9 Update is documented in detail here.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback, or open an ISSUE in this Archipelago Deployment Live Repository.</p> <p>Return to Archipelago Live Deployment.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-moveToLive/","title":"Moving from <code>archipelago-deployment</code> to <code>archipelago-deployment-live</code>","text":"","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#what-is-this-documentation-for","title":"What is this documentation for?","text":"<p>If you have been using/running/populating an instance with Archipelago Digital Objects that was set up using our simpler-to-deploy but harder-to-customize archipelago-deployment strategy and can't wait to move to this one\u2014meant for a larger (and somehow easier to maintain and upgrade on the long run) instance\u2014but (wait!) you do not want to ingest again, set up again, configure users, etc. (You already did that!), this is your documentation.</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#what-is-this-documentation-not-for","title":"What is this documentation not for?","text":"<p>To install an <code>archipelago-deployment-live</code> from scratch or to keep (forever) syncing between the two deployment options in a quantum phase shifting eternum like a time crystal.</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#requirements","title":"Requirements","text":"<ul> <li>An instance of Archipelago (working, tested) installed using <code>archipelago-deployment</code> as a basis.</li> <li>Basic knowledge and instincts on how to run Terminal Commands, copy files, run <code>composer</code>, <code>drush</code>, Linux Permissions, and <code>git</code> of course.</li> <li>Patience. You can't skip steps here. Also, Patience again since you may have stretched (good) your current instance to do way more than we thought it could.</li> <li>Time to read the main Documentation of this Repo to have a basic knowledge of how this is deployed. Recommended even if you are not going to deploy one from scratch here.</li> <li>For Shell Commands documented here please copy line by line\u2014not the whole block.</li> </ul>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#differences-between-both-deployments-strategies","title":"Differences between both deployments strategies.","text":"<p>In a nutshell: <code>archipelago-deployment-live</code> uses a different folder structure moving configuration storage, data storage outside of your webroot, and allows a much finer control of your settings (safer) and Docker containers. In a nutshell inside the first nutshell: <code>archipelago-deployment-live</code> also ignores more files so keeping customized versions, your own packages, your own settings around, and version controlled is much easier. Lastly: <code>archipelago-deployment-live</code> makes more use of Cloud Services, e.g. so if you have been running <code>min.io</code> as local mounted storage you may now consider moving storage (files) to a cloud service like AWS S3.</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#commonalities-between-both-deployment-strategies","title":"Commonalities between both deployment strategies.","text":"<p>In a nutshell: Since both run the same code and use the same Docker Containers, the data is actually the same. Everything is just persisted in different places.</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#getting-the-new-repo-in-place","title":"Getting the new repo in place","text":"<p>First you need to clone this repository and (hopefully) store in the same parent folder to your current <code>archipelago-deployment</code> one. For the purpose of this tutorial we will assume you have <code>archipelago-deployment</code> cloned in this location: <code>$HOME/archipelago-deployment</code>.</p> <p>Locate your <code>archipelago-deployment</code> folder in your terminal. Do an <code>ls</code> to make sure you can see the folder (not the content) and run:</p> <pre><code>git clone https://github.com/esmero/archipelago-deployment-live\ncd archipelago-deployment-live\ngit checkout 1.0.0-RC3\ncd ..\ncd archipelago-deployment\n</code></pre> <p>Now you have side by side <code>$HOME/archipelago_deployment</code> and <code>$HOME/archipelago-deployment-live</code>.</p> <p>This will give you the base structure.</p> <p>Before touching anything let's start by generating a backup of your current deployment (safety first).</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#backing-up","title":"Backing up","text":"","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#step-1","title":"Step 1:","text":"<p>Shut down your <code>docker-compose</code> ensemble. Inside your original <code>archipelago-deployment</code> folder run this:</p> <pre><code>docker-compose down\n</code></pre>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#step-2","title":"Step 2:","text":"<p>Verify all containers are actually down:</p> <pre><code>docker ps\n</code></pre> <p>The following command should return an empty listing. If anything is still running, wait a little longer and run the previous command again.</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#step-3","title":"Step 3:","text":"<p>Now let's tar.gz the whole ensemble with data and configs. As an example we will save this into your <code>$HOME</code> folder. As a good practice we append the current date (YEAR-MONTH-DAY) to the filename. Here we assume today is December 1st of 2021:</p> <pre><code>sudo tar -czvpf $HOME/archipelago-deployment-backup-20211201.tar.gz ../archipelago-deployment\n</code></pre> <p>The process may take a few minutes. Now let's verify that all is there and that the <code>tar.gz</code> is not corrupt:</p> <pre><code>tar -tvvf $HOME/archipelago-deployment-backup-20211201.tar.gz \n</code></pre> <p>You will see a listing of files. If corrupt (do you have enough space? did your ssh connection drop?) you will see:</p> <pre><code>tar: Unrecognized archive format\n</code></pre> <p>Done! If you are running a public instance we can allow ourselves to start Docker again to avoid downtime:</p> <pre><code>docker-compose up -d\n</code></pre>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#the-directory-structures","title":"The directory structures","text":"<p>Now that you backed all up we can spend some minutes looking at both directory structures.</p> <p>If you observe both deployment strategies side by side you will inmediately notice the most important similarities and also differences:</p> archipelago-deployment Live archipelago-deployment <pre>.\n\u251c\u2500\u2500 config_storage\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiifconfig\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 nginxconfig\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 nginxconfig_selfcert\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 php-fpm\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 solrconfig\n\u251c\u2500\u2500 data_storage\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 db\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiifcache\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiiftmp\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 letsencrypt\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 minio-data\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 ngnixcache\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 selfcert\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 solrcore\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 solrlib\n\u251c\u2500\u2500 deploy\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 azure-kubernetes\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 ec2-docker\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 kubernetes\n\u251c\u2500\u2500 docs\n\u2514\u2500\u2500 drupal\n\u2502   \u251c\u2500\u2500 config\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 d8content\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 docs\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 drush\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 patches\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 persistent\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 private\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 scripts\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 vendor\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 web\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 xdebug\n</pre> <pre>.\n\u251c\u2500\u2500 config\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 sync\n\u251c\u2500\u2500 d8content\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 metadatadisplays\n\u251c\u2500\u2500 docs\n\u251c\u2500\u2500 drush\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 Commands\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 sites\n\u251c\u2500\u2500 nginxconfigford8\n\u251c\u2500\u2500 patches\n\u251c\u2500\u2500 persistent\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 db\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiifcache\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiifconfig\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 miniodata\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 solrconfig\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 solrcore\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 solrlib\n\u251c\u2500\u2500 private\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 webform\n\u251c\u2500\u2500 scripts\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 archipelago\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 composer\n\u251c\u2500\u2500 vendor\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 archipelago\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 asm89\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 aws\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 behat\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 bin\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 brick\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 chi-teck\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 composer\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 consolidation\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 container-interop\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 cweagans\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 data-values\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 dflydev\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 doctrine\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 drupal\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 drush\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 easyrdf\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 egulias\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 enlightn\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 erusev\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 evenement\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 ezyang\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 fabpot\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 fileeye\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 firebase\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 frictionlessdata\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 google\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 graham-campbell\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 grasmash\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 guzzlehttp\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 instaclick\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 jcalderonzumba\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 jean85\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 jmikola\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 justinrainbow\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 laminas\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 league\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 lsolesen\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 maennchen\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 markbaker\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 masterminds\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 mglaman\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 mhor\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 mikey179\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 mixnode\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 ml\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 monolog\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 mtdowling\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 myclabs\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 nesbot\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 nette\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 nikic\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 paragonie\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 pear\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phar-io\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phenx\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phpdocumentor\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phplang\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phpmailer\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phpoffice\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phpoption\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phpseclib\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phpspec\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phpstan\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phpunit\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 professional-wiki\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 psr\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 psy\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 ralouphie\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 ramsey\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 react\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 sebastian\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 seld\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 sirbrillig\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 solarium\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 squizlabs\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 stack\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 strawberryfield\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 swaggest\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 symfony\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 symfony-cmf\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 theseer\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 twbs\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 twig\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 typo3\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 vlucas\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 web64\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 webflo\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 webmozart\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 wikibase\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 wikimedia\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 zaporylie\n\u251c\u2500\u2500 web\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 core\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 libraries\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 modules\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 profiles\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 sites\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 themes</pre>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#the-data","title":"The Data","text":"<p>Let's start by focusing on the <code>data</code>, in our case the Database, Solr, and File (S3 + Private) storage. Collapsing here a few folders will make this easier to read. Marked with a <code>*</code> are matching folders that contain DB, Solr Core, the S3 min.io data (if you are using local storage) and also Drupal's very own <code>private</code> folder:</p> archipelago-deployment Live archipelago-deployment <pre>.\n\u251c\u2500\u2500 config_storage\n\u251c\u2500\u2500 data_storage\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 * db *\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiifcache\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiiftmp\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 letsencrypt\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 * minio-data *\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 ngnixcache\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 selfcert\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 solrcore\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 solrlib\n\u251c\u2500\u2500 deploy\n\u251c\u2500\u2500 docs\n\u2514\u2500\u2500 drupal\n\u2502   \u251c\u2500\u2500 config\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 d8content\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 docs\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 drush\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 patches\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 persistent\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 * private *\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 scripts\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 vendor\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 web\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 xdebug\n</pre> <pre>.\n\u251c\u2500\u2500 config\n\u251c\u2500\u2500 d8content\n\u251c\u2500\u2500 docs\n\u251c\u2500\u2500 drush\n\u251c\u2500\u2500 nginxconfigford8\n\u251c\u2500\u2500 patches\n\u251c\u2500\u2500 persistent\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 * db *\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiifcache\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiifconfig\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 * miniodata *\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 solrconfig\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 * solrcore *\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 solrlib\n\u251c\u2500\u2500 * private *\n\u251c\u2500\u2500 scripts\n\u251c\u2500\u2500 vendor\n\u251c\u2500\u2500 web\n</pre>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#copying-the-data-into-the-new-structure","title":"Copying the Data into the new Structure","text":"<p>To do so we need to stop Docker again. This is needed because Databases sometimes keep an open Change Log and Locks in place, and if there is any interaction or cron running, your data may end up corrupted.</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#step-1_1","title":"Step 1:","text":"<p>Shut down your <code>docker-compose</code> ensemble. Inside your original <code>archipelago-deployment</code> folder run this:</p> <pre><code>docker-compose down\n</code></pre>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#step-2_1","title":"Step 2:","text":"<p>Verify all containers are actually down:</p> <pre><code>docker ps\n</code></pre>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#step-3_1","title":"Step 3:","text":"<p>We will copy DB, min.io (File and ADO storage as files) and Drupal's private (temporary files, caches) folders to its new place:</p> <pre><code>sudo cp -rpv persistent/db ../archipelago-deployment-live/data_storage/db\nsudo cp -rpv persistent/solrcore ../archipelago-deployment-live/data_storage/solrcore\nsudo cp -rpv persistent/miniodata ../archipelago-deployment-live/data_storage/minio-data\nsudo cp -rpv private ../archipelago-deployment-live/drupal/private\n</code></pre> <p>Running <code>-rpv</code> will copy verbosely and recursively while preserving original permissions.</p> <p>Done!</p> <p>You can now start <code>docker-compose</code> again:</p> <pre><code>docker-compose up -d\n</code></pre>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#the-web","title":"The Web","text":"<p>Collapsing again a few folders to aid in readability, we can now focus on your actual Drupal/Archipelago Code/Web and settings. To be honest (we are), you can easily reinstall and restore all this via <code>composer</code>, but we can also move folders as a learning experience/time and bandwidth experience. Marked with a <code>*</code> are matching folders you want to copy over:</p> archipelago-deployment Live archipelago-deployment <pre>.\n\u251c\u2500\u2500 config_storage\n\u251c\u2500\u2500 data_storage\n\u251c\u2500\u2500 deploy\n\u251c\u2500\u2500 docs\n\u2514\u2500\u2500 drupal\n\u2502   \u251c\u2500\u2500 * config *\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 d8content\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 docs\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 drush\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 patches\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 persistent\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 private\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 scripts\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 * vendor *\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 * web *\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 xdebug\n</pre> <pre>.\n\u251c\u2500\u2500 * config *\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 sync\n\u251c\u2500\u2500 d8content\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 metadatadisplays\n\u251c\u2500\u2500 docs\n\u251c\u2500\u2500 drush\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 Commands\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 sites\n\u251c\u2500\u2500 nginxconfigford8\n\u251c\u2500\u2500 patches\n\u251c\u2500\u2500 persistent\n\u251c\u2500\u2500 private\n\u251c\u2500\u2500 scripts\n\u251c\u2500\u2500 * vendor *\n\u251c\u2500\u2500 * web *\n</pre>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#copying-the-web-into-the-new-structure","title":"Copying the Web into the new Structure","text":"<p>No need to stop Docker again. We can do this while your Archipelago is still running.</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#step-1_2","title":"Step 1:","text":"<p>We will copy all important folders over. From your <code>archipelago-deployment</code> folder run:</p> <pre><code>sudo cp -rpv vendor ../archipelago-deployment-live/drupal/vendor\nsudo cp -rpv web ../archipelago-deployment-live/drupal/web\nsudo cp -rpv config ../archipelago-deployment-live/drupal/config\n</code></pre> <p>And also, selectively, a few files we know you are very fond of!</p> <pre><code>sudo cp -rpv composer.json ../archipelago-deployment-live/drupal/composer.json\nsudo cp -rpv composer.lock ../archipelago-deployment-live/drupal/composer.lock\n</code></pre> <p>Done!</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#ssl-enviromentals-configurations-settings-and-docker","title":"SSL, Enviromentals, Configurations, Settings and Docker","text":"<p>We are almost done, but <code>archipelago-deployment-live</code> has a different, safer way of defining SSL Certs, credentials, and global settings for your Archipelago.  We will start first by copying settings as they are (most likely not very safe), and then we can update passwords/etc. to make your system better-prepared for the world.</p> <p>To learn more about these general settings please read this section of the parent Documentation (who likes duplicated documentation? Nobody.). The gist here is (after reading, please do not skip) that we need to add our service definitions into a <code>.env</code> file.</p> <p>Coming from <code>archipelago-deployment</code> means and assumes that you are running AWS Linux 2 using the suggested locations in this document, that you have a vanilla deployment, and that you followed these instructions) so your values for <code>$HOME/archipelago-deployment-live/deploy/ec2-docker/.env</code> will be the following:</p> <pre><code>ARCHIPELAGO_ROOT=/home/ec2-user/archipelago-deployment-live\nARCHIPELAGO_EMAIL=your@validemail.org\nARCHIPELAGO_DOMAIN=your.domain.org\nMINIO_ACCESS_KEY=minio\nMINIO_SECRET_KEY=minio123\nMYSQL_ROOT_PASSWORD=esmero-db\nMINIO_BUCKET_MEDIA=archipelago\nMINIO_FOLDER_PREFIX_MEDIA=/\nMINIO_BUCKET_CACHE=archipelago\nMINIO_FOLDER_PREFIX_CACHE=/\n</code></pre> <p>If you plan on staying on local storage driven <code>min.io</code>, <code>MINIO_BUCKET_CACHE</code> and <code>MINIO_FOLDER_PREFIX_CACHE</code> are not going to be used. If you are planning on moving your Storage from local to cloud driven please replace with the right values, e.g. AWS IAM keys and Secrets + bucket names and prefixes (folders). Again, refer to the parent Documentation for setting this up.</p> <p>Once you have that in place (Double-check. If something goes wrong here we can always fine-tune and fix again.), we need to decide on a new <code>docker-compose</code> file, and you may need to customize it depending on your choices and current and future needs.</p> <p>If you already have an SSL certificate, and it's provided by <code>CertBot</code> you can either copy the certs from your current system (will totally depend on your setup since <code>archipelago-deployment</code> does not provide out-of-the-box SSL Certs) to <code>$HOME/archipelago-deployment-live/data_storage/letsencrypt</code>.</p> <p>A normal folder structure for that is:</p> <pre><code>.\n\u251c\u2500\u2500 accounts\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 acme-v02.api.letsencrypt.org\n\u2502\u00a0\u00a0     \u2514\u2500\u2500 directory\n\u2502\u00a0\u00a0         \u2514\u2500\u2500 cac9f8218ef18e4f11ec053785bbf648\n\u2502\u00a0\u00a0             \u251c\u2500\u2500 meta.json\n\u2502\u00a0\u00a0             \u251c\u2500\u2500 private_key.json\n\u2502\u00a0\u00a0             \u2514\u2500\u2500 regr.json\n\u251c\u2500\u2500 archive\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 your.domain.org\n\u2502\u00a0\u00a0  \u00a0\u00a0 \u251c\u2500\u2500 cert1.pem\n\u2502\u00a0\u00a0  \u00a0\u00a0 \u251c\u2500\u2500 chain1.pem\n\u2502\u00a0\u00a0  \u00a0\u00a0 \u251c\u2500\u2500 fullchain1.pem\n\u2502\u00a0\u00a0  \u00a0\u00a0 \u2514\u2500\u2500 privkey1.pem\n\u2502\u00a0\n\u251c\u2500\u2500 csr\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 0000_csr-certbot.pem\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 0001_csr-certbot.pem\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 0002_csr-certbot.pem\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 0003_csr-certbot.pem\n\u251c\u2500\u2500 keys\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 0000_key-certbot.pem\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 0001_key-certbot.pem\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 0002_key-certbot.pem\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 0003_key-certbot.pem\n\u251c\u2500\u2500 live\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 README\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 your.domain.org\n\u2502\u00a0\u00a0 \u00a0\u00a0 \u251c\u2500\u2500 cert.pem -&gt; ../../archive/your.domain.org/cert1.pem\n\u2502\u00a0\u00a0 \u00a0\u00a0 \u251c\u2500\u2500 chain.pem -&gt; ../../archive/your.domain.org/chain1.pem\n\u2502\u00a0\u00a0 \u00a0\u00a0 \u251c\u2500\u2500 fullchain.pem -&gt; ../../archive/your.domain.org/fullchain1.pem\n\u2502\u00a0\u00a0 \u00a0\u00a0 \u251c\u2500\u2500 privkey.pem -&gt; ../../archive/your.domain.org/privkey1.pem\n\u2502\u00a0\u00a0 \u00a0\u00a0 \u2514\u2500\u2500 README\n\u251c\u2500\u2500 renewal\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 your.domain.org.conf\n\u2502\u00a0\u00a0 \n\u2514\u2500\u2500 renewal-hooks\n    \u251c\u2500\u2500 deploy\n    \u251c\u2500\u2500 post\n    \u2514\u2500\u2500 pre\n</code></pre> <p>Or if your SSL cert is up for renewal, you can just let Archipelago request it for you. Renewal will happen auto-magically, and you may never ever need to worry about that in the future.</p> <p>Finally, let's adapt the <code>docker-compose</code> file we need to our previous (but still current!) <code>archipelago-deployment</code> reality.</p> <p>For x86/AMD, run (for ARM64/Apple M1 please check the parent Documentation):</p> <pre><code>cp $home/archipelago-deployment-live/deploy/ec2-docker/docker-compose-aws-s3.yml $home/archipelago-deployment-live/deploy/ec2-docker/docker-compose.yml\nnano $home/archipelago-deployment-live/deploy/ec2-docker/docker-compose.yml\n</code></pre> <p>And replace the content with this slightly modified version. Note: we really only changed the lines after this comment: <code># THIS DIFFERS FROM THE NORMAL ONE...</code>.</p> <pre><code># Run docker-compose up -d\n\nversion: '3.5'\nservices:\n  web:\n    container_name: esmero-web\n    image: staticfloat/nginx-certbot\n    restart: always\n    environment:\n      CERTBOT_EMAIL: ${ARCHIPELAGO_EMAIL}\n      ENVSUBST_VARS: FQDN\n      FQDN: ${ARCHIPELAGO_DOMAIN}\n    ports:\n      - \"80:80\"\n      - \"443:443\"\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/config_storage/nginxconfig/conf.d:/etc/nginx/user.conf.d\n      - ${ARCHIPELAGO_ROOT}/config_storage/nginxconfig/certbot_extra_domains:/etc/nginx/certbot/extra_domains:ro\n      - ${ARCHIPELAGO_ROOT}/drupal:/var/www/html:cached\n      - ${ARCHIPELAGO_ROOT}/data_storage/ngnixcache:/var/cache/nginx\n      - ${ARCHIPELAGO_ROOT}/data_storage/letsencrypt:/etc/letsencrypt\n    depends_on:\n      - solr\n      - php\n      - db\n    tty: true\n    networks:\n      - host-net\n      - esmero-net\n  php:\n    container_name: esmero-php\n    restart: always\n    image: \"esmero/php-7.4-fpm:1.0.0-RC2-multiarch\"\n    tty: true\n    networks:\n      - host-net\n      - esmero-net\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/config_storage/php-fpm/www.conf:/usr/local/etc/php-fpm.d/www.conf\n      - ${ARCHIPELAGO_ROOT}/drupal:/var/www/html:cached\n    environment:\n      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}\n      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}\n      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD}\n      MINIO_BUCKET_MEDIA: ${MINIO_BUCKET_MEDIA}\n      MINIO_FOLDER_PREFIX_MEDIA: ${MINIO_FOLDER_PREFIX_MEDIA}\n  solr:\n    container_name: esmero-solr\n    restart: always\n    image: \"solr:8.8.2\"\n    tty: true\n    ports:\n      - \"8983:8983\"\n    networks:\n      - host-net\n      - esmero-net\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/data_storage/solrcore:/var/solr/data\n      - ${ARCHIPELAGO_ROOT}/config_storage/solrconfig:/drupalconfig\n      - ${ARCHIPELAGO_ROOT}/data_storage/solrlib:/opt/solr/contrib/archipelago/lib\n    entrypoint:\n      - docker-entrypoint.sh\n      - solr-precreate\n      - drupal\n      - /drupalconfig\n  db:\n    image: mysql:8.0.22\n    command: mysqld --default-authentication-plugin=mysql_native_password  --max_allowed_packet=256M\n    container_name: esmero-db\n    restart: always\n    environment:\n      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD}\n    networks:\n      - host-net\n      - esmero-net\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/data_storage/db:/var/lib/mysql\n  nlp:\n    container_name: esmero-nlp\n    restart: always\n    image: \"esmero/esmero-nlp:1.0\"\n    ports:\n      - \"6400:6400\"\n    networks:\n      - host-net\n      - esmero-net\n  iiif:\n    container_name: esmero-cantaloupe\n    image: \"esmero/cantaloupe-s3:4.1.9RC\"\n    restart: always\n    ports:\n      - \"8183:8182\"\n    networks:\n      - host-net\n      - esmero-net\n    environment:\n      AWS_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY}\n      AWS_SECRET_ACCESS_KEY: ${MINIO_SECRET_KEY}\n      # THIS DIFFERS FROM THE STANDARD ONE AND ENABLES LOCAL FILESYSTEM CACHE INSTEAD OF AWS S3 one\n      CACHE_SERVER_DERIVATIVE: FilesystemCache\n      S3SOURCE_BASICLOOKUPSTRATEGY_BUCKET_NAME: ${MINIO_BUCKET_MEDIA}\n      S3SOURCE_BASICLOOKUPSTRATEGY_PATH_PREFIX: ${MINIO_FOLDER_PREFIX_MEDIA}\n      S3CACHE_BUCKET_NAME: ${MINIO_BUCKET_CACHE} \n      S3CACHE_OBJECT_KEY_PREFIX: ${MINIO_FOLDER_PREFIX_CACHE} \n      XMS: 2g\n      XMX: 4g\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/config_storage/iiifconfig:/etc/cantaloupe\n      - ${ARCHIPELAGO_ROOT}/data_storage/iiifcache:/var/cache/cantaloupe\n      - ${ARCHIPELAGO_ROOT}/data_storage/iiiftmp:/var/cache/cantaloupe_tmp\n  minio:\n    container_name: esmero-minio\n    restart: always\n    image: minio/minio:latest\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/data_storage/minio-data:/data:cached\n    ports:\n      - \"9000:9000\"\n      - \"9001:9001\"\n    networks:\n      - host-net\n      - esmero-net\n    environment:\n      MINIO_HTTP_TRACE: /tmp/minio-log.txt\n      MINIO_ROOT_USER: ${MINIO_ACCESS_KEY}\n      MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY}\n      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}\n      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}\n      # THIS DIFFERS FROM THE STANDARD ONE AND ENABLES LOCAL MINIO INSTEAD OF AWS S3 one \n    command: server /data --console-address \":9001\"\nnetworks:\n  host-net:\n    driver: bridge\n  esmero-net:\n    driver: bridge\n    internal: true\n</code></pre> <p>Press CNTRL-X, and you are done. Now the final test!!</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#shutdown-the-old-one-start-the-new-one","title":"Shutdown the old one, start the new one","text":"<p>So we are ready. Testing may be a hit-or-miss thing here. Did we cover all the steps? Did a command fail? The good thing is that we can start the new ensemble, and all our old ones will survive. And we can come back over and over until we are ready. Let's try!</p> <p>We will start by shutting down the running Docker ensemble:</p> <pre><code>cd $HOME/archipelago-deployment\ndocker-compose down\n</code></pre> <p>Now let's go to our new deployment. Docker starts here in a different folder:</p> <pre><code>cd $HOME/archipelago-deployment-live/deploy/ec2-docker\ndocker-compose up\n</code></pre> <p>You may notice that we removed the <code>-d</code>. Why? We want to see all the messages and notice/mark/copy any errors, e.g. did the SSL CERT load correctly? Did the MYSQL import work out? To avoid shutting it down while all starts, please open another Terminal and type:</p> <pre><code>docker ps\n</code></pre> <p>And look at the up-times. Do you see any Containers restarting (where Created and the Status differ for a lot and Status keeps resetting to 0?)? A healthy deployment will look similar to this:</p> <pre><code>CONTAINER ID   IMAGE                                    COMMAND                  CREATED          STATUS         PORTS                                      NAMES\nf794c25db64c   esmero/cantaloupe-s3:4.1.9RC2-arm64      \"sh -c 'java -Dcanta\u2026\"   6 seconds ago    Up 3 seconds   0.0.0.0:8183-&gt;8182/tcp                     esmero-cantaloupe\n5b791445720f   jonasal/nginx-certbot                    \"/docker-entrypoint.\u2026\"   6 seconds ago    Up 3 seconds   0.0.0.0:80-&gt;80/tcp, 0.0.0.0:443-&gt;443/tcp   esmero-web\ne38fbbd86edf   esmero/esmero-nlp:1.0.1-RC2-arm64        \"/usr/local/bin/entr\u2026\"   11 seconds ago   Up 6 seconds   0.0.0.0:6400-&gt;6400/tcp                     esmero-nlp\nc84a0a4d43e9   minio/minio:latest                       \"/usr/bin/docker-ent\u2026\"   11 seconds ago   Up 6 seconds   0.0.0.0:9000-9001-&gt;9000-9001/tcp           esmero-minio\n3ec176a960c3   esmero/php-7.4-fpm:1.0.0-RC2-multiarch   \"docker-php-entrypoi\u2026\"   11 seconds ago   Up 6 seconds   9000/tcp                                   esmero-php\ne762ad7ea5e2   solr:8.8.2                               \"docker-entrypoint.s\u2026\"   11 seconds ago   Up 6 seconds   0.0.0.0:8983-&gt;8983/tcp                     esmero-solr\n381166d61f8c   mariadb:10.5.10-focal                    \"docker-entrypoint.s\u2026\"   11 seconds ago   Up 6 seconds   3306/tcp      \n</code></pre> <p>If you feel that all seems to be fine, open a browser window and visit your website. See if you can log in and see ADOs. If not you can momentarily shut down this new Docker ensemble and restart the older one. Nothing is lost! Then with time and tea/coffee and fresh eyes come back and re-trace your steps. 95% of the issues are incorrect values in the <code>.env</code> file. The other 5% may be on us. If you run into any trouble please get in touch!</p> <p>Happy deploying!</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback, or open an ISSUE in this Archipelago Deployment Live Repository.</p> <p>Return to Archipelago Live Deployment.</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/","title":"Archipelago Deployment Live","text":"<p>A Cloud / Local production ready Archipelago Deployment using Docker and soon Kubernetes.</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#what-is-this-repo-for","title":"What is this repo for?","text":"<p>Running Archipelago Commons on a live public instance using SSL with Blob/Object Storage backend </p> <ul> <li>Cloud-based deployment, e.g. AWS/Azure under Linux</li> <li>Self-managed servers running Linux</li> <li>x86/AMD86 or ARM64/v8 CPU architectures</li> </ul>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#what-is-this-repo-not-for","title":"What is this repo not for?","text":"<ul> <li>Running your own local/development Archipelago. For that we suggest using https://github.com/esmero/archipelago-deployment</li> </ul>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#requirements","title":"Requirements","text":"","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#minimal-not-recommended-for-production","title":"Minimal (not recommended for production)","text":"<ul> <li>4 Gbytes of RAM (e.g AWS EC2 t3.medium) 2 CPUs, Single SSD Drive of 100 Gbytes</li> </ul>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#base-line-for-production","title":"Base line for production","text":"<ul> <li>8 Gbytes of RAM (AWS EC2 t3.medium)  2 CPUs, Single SSD Drive of 100 Gbytes, optional: one magnetic Drive of 500 Gbytes for Caches/Temp files/Backups.</li> </ul>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#recommended-for-production","title":"Recommended for production","text":"<ul> <li>16 Gbytes of RAM (AWS EC2 m6g.xlarge - Graviton) 4 CPUs, Single SSD Drive of 200 Gbytes, optional: one magnetic Drive of 1TB for Caches/Temp files/Backups.</li> </ul>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#os","title":"OS:","text":"<ul> <li>Ubuntu 22.04 LTS /Ubuntu 24.04 LTS/Amazon Linux 2023/Debian 10 \"Buster\" matching your CPU architecture (of course)</li> <li>Most recent <code>Docker</code> running as a service and <code>docker-compose</code> or <code>docker compose</code></li> </ul>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#skills-and-the-important-human-aspect","title":"Skills and the important human aspect","text":"<ul> <li>Basic Unix/Linux terminal skills and a root/sudo account</li> <li><code>Docker</code> basics knowledge and how to manage packages in your System</li> <li>Knowledge of how to manage AWS/Azure or any other cloud-based provider for Computing Instances and S3 compatible Object Storage system and the associated permissions credentials to do so.</li> <li>To be patient. Please read everything. Do not skip steps. Do not only copy and paste commands. Explanations give context and might help you troubleshoot issues.</li> </ul> <p>Basically this guide is meant for humans with basic to medium <code>DevOps</code> background or humans with patience that are willing to troubleshoot, ask, and try again when that background is not (yet) enough. And we are here to help.</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#deployment-on-linuxx86amd-system","title":"Deployment on Linux/X86/AMD system","text":"","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#step-1","title":"Step 1:","text":"<p>Deploy your base system</p> <p>Make sure your Firewall/AWS Security group has these ports open for everyone to access</p> <ul> <li>443 (NGINX SSL)</li> <li>80 (NGINX HTTP) And protected/modally open for your own development/testing/administration</li> <li>8183 (Cantaloupe)</li> <li>8983 (Solr)</li> <li>6400 (NLP64)</li> <li>9001 (Minio)</li> <li>22 (so you can ssh into your machine)</li> </ul> <p>Setup your system using your favorite package manager with </p> <ul> <li>Docker</li> <li>git</li> <li>htop</li> <li>tree</li> <li>docker-compose or <code>docker compose</code> (V2) if running the most recent docker service. See https://docs.docker.com/compose/migrate/</li> </ul> <p>e.g. for Amazon Linux 2023 (x86/amd64) these steps are tested:</p> <pre><code>sudo yum update -y\nsudo yum install -y docker\nsudo service docker start\nsudo usermod -a -G docker ec2-user\nsudo chkconfig docker on\nsudo systemctl enable docker\nsudo yum install -y git htop tree jq perl-JSON-PP\nsudo curl -L \"https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)\" -o /usr/local/bin/docker-compose\nsudo chmod +x /usr/local/bin/docker-compose\nsudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose\nsudo reboot\n</code></pre> <p>Reboot is needed to allow Docker to take full control over your OS resources.</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#step-2","title":"Step 2:","text":"<p>In your location of choice clone this repo</p> <pre><code>git clone https://github.com/esmero/archipelago-deployment-live\ncd archipelago-deployment-live\ngit checkout 1.4.0\n</code></pre>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#step-3-setup-your-enviromental-variables-for-dockerservices","title":"Step 3. Setup your enviromental variables for Docker/Services","text":"","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#setup-enviromentals","title":"Setup Enviromentals","text":"<p>Setup your deployment enviromental variables by copying the template</p> <p><pre><code>cp deploy/ec2-docker/.env.template deploy/ec2-docker/.env\n</code></pre> and editing it</p> <pre><code>nano deploy/ec2-docker/.env\n</code></pre> <p>The content of that file would be similar to this.  <code>Note</code>: There are a few extra commented lines at the end only used for: https://docs.archipelago.nyc/1.4.0/security_bots/ if you decide to go that way.</p> <pre><code>ARCHIPELAGO_ROOT=/home/ec2-user/archipelago-deployment-live\nARCHIPELAGO_EMAIL=your@validemail.org\nARCHIPELAGO_DOMAIN=your.domain.org\nMINIO_ACCESS_KEY=THE_S3_AZURE_OR_LOCAL_MINIO_KEY\nMINIO_SECRET_KEY=THE_S3_AZURE_OR_LOCAL_MINIO_SECRET\nMYSQL_ROOT_PASSWORD=YOUR_MYSQL_PASSWORD_FOR_ARCHIPELAGO\nMINIO_BUCKET_MEDIA=THE_NAME_OF_YOUR_S3_BUCKET_FOR_PERSISTEN_STORAGE\nMINIO_FOLDER_PREFIX_MEDIA=media/\nMINIO_BUCKET_CACHE=THE_NAME_OF_YOUR_S3_BUCKET_FOR_IIIF_STORAGE\nMINIO_FOLDER_PREFIX_CACHE=iiifcache/\nREDIS_PASSWORD=YOUR_REDIS_PASSWORD\n</code></pre> <p>What does each key mean?</p> <ul> <li><code>ARCHIPELAGO_ROOT</code>: the absolute path to your <code>archipelago-deployment-live</code> git repo in your host machine.</li> <li><code>ARCHIPELAGO_EMAIL</code>: a valid email, will be used to register your SSL Certificate via Certbot.</li> <li><code>ARCHIPELAGO_DOMAIN</code>: a valid domain name for your repository. This domain will be also used to request your SSL Certificate via Certbot.</li> <li><code>MINIO_ACCESS_KEY</code>: If you are running a Cloud Service backed S3/Azure Storage this needs to be generated there. The user/IAM owner of this ACCESS KEY needs to have access to read/write the bucket you will configure in this same <code>.env</code>. If running local <code>min.io</code> whatever you set will be used.</li> <li><code>MINIO_SECRET_KEY</code>: If you are running a Cloud Service backed S3/Azure Storage this needs to generated there. The user/IAM owner of the matching SECRET_KEY needs to have access to read/write the bucket you will configure in this same <code>.env</code> file. If running local <code>min.io</code> whatever you set will be used.</li> <li><code>MYSQL_ROOT_PASSWORD</code>: The MYSQL 8 or Mariadb 15 password. This password will be used later also during Drupal deployment via <code>drush</code></li> <li><code>MINIO_BUCKET_MEDIA</code>: The name of your Persistant Storage Bucket. If using mini.io local we recommend keeping it simple, e.g. <code>archipelago</code>.</li> <li><code>MINIO_FOLDER_PREFIX_MEDIA</code>: The <code>folder</code> (a prefix really) where your DO Storage and File storage will go inside the <code>MINIO_BUCKET_MEDIA</code> Bucket. <code>media/</code> is a fine name for this one and common in archipelago deployments. IMPORTANT: Always terminate these with a <code>/</code>. </li> <li><code>MINIO_BUCKET_CACHE</code>: The name of your IIIF Cache storage Bucket. May be the same as <code>MINIO_BUCKET_MEDIA</code>. If different make sure your your <code>MINIO_ACCESS_KEY</code> and/or IAM role ACL have permission to read write to this one too.</li> <li><code>MINIO_FOLDER_PREFIX_CACHE</code>:  The <code>folder</code> (a prefix really) where Cantaloupe will/can write its <code>iiif</code> caches. <code>iiifcache/</code> is a lovely name we use a lot. IMPORTANT: Always terminate these with a <code>/</code>.</li> <li><code>REDIS_PASSWORD</code>: Password for your REDIS (Drupal Cache/Queue storage) if you decide to enable the Drupal REDIS module.</li> </ul> <p><code>IMPORTANT NOTE</code>: For AWS EC2. If you selected an <code>IAM role</code> for your server when setting it up/deploying it, <code>min.io</code> will use the AWS EC2-backed internal API to request access to your S3. This means the ROLE itself needs to have read/write access (ACL) to the given Bucket(s) and your key/secrets won't be able to override that. Please do not ignore this note. It will save you a LOT of frustration and coffee. You can also run an EC2 instace without a given IAM and in that case just the ACCESS_KEY/SECRET will matter.</p> <p>Now that you know, you also know that these values should not be shared and this <code>.env</code> file should not be committed/kept in version control. Please be careful.</p> <p><code>docker-compose</code> will read this <code>.env</code> and start all services for you based on its content.</p> <p>Once you have modified this you are ready for your first big decision. </p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#running-a-fully-qualified-domain-you-wish-a-validsigned-certificate-for-amdintel-architecture","title":"Running a fully qualified domain you wish a valid/signed certificate for AMD/INTEL Architecture?","text":"<p>This means you will use the <code>docker-compose-aws-s3.yml</code>. Do the following:</p> <pre><code>cp deploy/ec2-docker/docker-compose-aws-s3.yml deploy/ec2-docker/docker-compose.yml\n</code></pre>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#running-a-fully-qualified-domain-you-wish-a-validsigned-certificate-for-arm64apple-m1-architecture","title":"Running a fully qualified domain you wish a valid/signed certificate for ARM64/Apple M1 Architecture?","text":"<p>This means you will use the <code>docker-compose-aws-s3-arm64.yml</code>. Do the following:</p> <pre><code>cp deploy/ec2-docker/docker-compose-aws-s3-arm64.yml deploy/ec2-docker/docker-compose.yml\n</code></pre>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#or-running-self-signed-optional-and-does-not-apply-to-arm64apple-m1-architecture","title":"OR Running self-signed? (optional and does not apply to ARM64/Apple M1 Architecture):","text":"<p>Only if you are not running a fully qualified domain you wish a valid/signed. We really DO not recommend this route. IF you plan on using this deployment for local testing or running on non SSL please go for https://github.com/esmero/archipelago-deployment which delivers the same experience in less than 20 minutes deployment time.</p> <p>Generate a self signed Cert</p> <pre><code>sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout data_storage/selfcert/private/nginx.key -out data_storage/selfcert/certs/nginx.crt \nsudo openssl dhparam -out data_storage/selfcert/dhparam.pem 4096\ncp deploy/ec2-docker/docker-compose-selfsigned.yml deploy/ec2-docker/docker-compose.yml\n</code></pre> <p>Note: Self signed docker-compose.yml file is setup to use min.io with local storage</p> <pre><code>    volumes:\n      - ${ARCHIPELAGO_ROOT}/data_storage/minio-data:/data:cached\n</code></pre> <p>This folder will be created by min.io. If you are using a secondary Drive (e.g. magnetic) you can modify your <code>deploy/ec2-docker/docker-compose.yml</code> to use a folder there, e.g.</p> <pre><code>    volumes:\n      - /persistentinotherdrive/data_storage/minio-data:/data:cached\n</code></pre> <p>Make sure your logged in user can read/write to it.</p> <p>NOTE: If you want to use AWS S3 storage for the self signed version replace the minio Service yaml block with this Service Block in your new <code>deploy/ec2-docker/docker-compose.yml</code>. You can mix and match services and even remove all <code>:cached</code> statements for improved R/W volumen performance.</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#step-4-first-run","title":"Step 4. First Run","text":"","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#first-permissions","title":"First Permissions","text":"<pre><code>sudo chown 8183:8183 config_storage/iiifconfig/cantaloupe.properties\nsudo chown -R 8183:8183 data_storage/iiifcache\nsudo chown -R 8183:8183 data_storage/iiiftmp\nsudo chown -R 8983:8983 data_storage/solrcore\n</code></pre>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#actual-first-run","title":"Actual first run","text":"<p>Time to spin our docker containers for the first time. We will start all without going into background so log/error checking is easier, especially if you have selected a Valid/Signed Cert choice and also want to be sure S3 keys/access are working.</p> <pre><code>cd deploy/ec2-docker\ndocker-compose up\n</code></pre> <p>You will see a lot of things happening. Check for errors/problems/clear alerts and give all a minute or so to start.  Ok, let's assume your setup managed to request a valid signed SSL cert, you will see a nice message!</p> <pre><code>- Congratulations! Your certificate and chain have been saved at:XXXXX\n   Your certificate will expire on 20XX-XX-XX. To obtain a new or\n   tweaked version of this certificate in the future, simply run\n   certbot again. To non-interactively renew *all* of your\n   certificates, run \"certbot renew\"\n</code></pre> <p>Archipelago will do that for you whenever it's about to expire so no need to deal with this manually, even when <code>docker-compose</code> restarts.</p> <p>Now press CTRL+C. <code>docker-compose</code> will shutdown gracefully. Good!</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#step-5-deploy-drupal-10","title":"Step 5. Deploy Drupal 10","text":"","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#composer-and-drupal","title":"Composer and Drupal","text":"<p>Copy the shipped default composer.default.json to composer.json and composer.default.lock to composer.lock (ONLY if you are installing from scratch):</p> <pre><code>cp ../../drupal/composer.default.json ../../drupal/composer.json\ncp ../../drupal/composer.default.lock ../../drupal/composer.lock\n</code></pre> <p>Start Docker again</p> <pre><code>docker-compose up -d\n</code></pre> <p>Wait a few seconds and run:</p> <pre><code>docker exec -ti esmero-php bash -c \"chown -R www-data:www-data private\"\ndocker exec -ti esmero-php bash -c \"chown -R www-data:www-data web/sites\"\ndocker exec -ti esmero-php bash -c \"composer install\"\n</code></pre> <p>Composer install will take a little while and bring all your PHP libraries.</p> <p>Once done, execute our setup script that will prepare your Drupal <code>settings.php</code> and bring some of the <code>.env</code> enviromental variables to the Drupal environment. </p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/setup.sh'\n</code></pre> <p>And now you can deploy Drupal! </p> <p>IMPORTANT: Make sure you replace in the following command inside <code>root:MYSQL_ROOT_PASSWORD</code> the <code>MYSQL_ROOT_PASSWORD</code> string with the value you used/assigned in your <code>.env</code> file for <code>MYSQL_ROOT_PASSWORD</code>. And replace <code>ADMIN_PASSWORD</code> with a password that is safe and you won't forget! That passwords is for your Drupal super user (uid:0).</p> <pre><code>docker exec -ti -u www-data esmero-php bash -c \"cd web;../vendor/bin/drush -y si --verbose --existing-config --db-url=mysql://root:MYSQL_ROOT_PASSWORD@esmero-db/drupal --account-name=admin --account-pass=ADMIN_PASSWORD -r=/var/www/html/web --sites-subdir=default --notify=false;drush cr;chown -R www-data:www-data sites;\"\n</code></pre>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#step-6-users-and-initial-content","title":"Step 6. Users and initial Content.","text":"<p>After installation is done (may take a few) you can install initial users and assign them roles. Copy each line separately. A missing permission will not let you ingest the initial Metadata Displays and AMI set.</p> <pre><code>docker exec -ti esmero-php bash -c 'drush ucrt demo --password=\"demo\"; drush urol metadata_pro \"demo\"'\n</code></pre> <pre><code>docker exec -ti esmero-php bash -c 'drush ucrt jsonapi --password=\"jsonapi\"; drush urol metadata_api \"jsonapi\"'\n</code></pre> <pre><code>docker exec -ti esmero-php bash -c 'drush urol administrator \"admin\"'\n</code></pre> <p>Before ingesting the base content we need to make sure we can access your <code>JSON-API</code> on for your new domain. That means we need to change internal urls (<code>https://esmero-web</code>) to the new valid SSL driven ones. This is easy:</p> <p>On your host machine (no need to <code>docker exec</code> these ones), replace first in the following command <code>your.domain.org</code> with the domain you setup in your <code>.env</code> file. Go to (cd into) your base git clone folder (Important: YOUR BASE CLONE FOLDER) and then run</p> <pre><code> sed -i 's/http:\\/\\/esmero-web/https:\\/\\/your.domain.org/g' drupal/scripts/archipelago/deploy.sh\n sed -i 's/http:\\/\\/esmero-web/https:\\/\\/your.domain.org/g' drupal/scripts/archipelago/update_deployed.sh\n</code></pre> <p>Now your <code>deploy.sh</code> and <code>update_deployed.sh</code> are update and ready. Let's ingest some Twig Templates, an AMI Set, menus and a Blocks.</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/deploy.sh'\n</code></pre> <p>IMPORTANT: <code>update_deployed.sh</code> is not needed when deploying for the first time and totally discouraged on a customized Archipelago.  If you make modifications to your <code>Twig templates</code>, that command will replace the ones shipped by us with fresh copies overwriting all your modifications. Only run to restore larger errors or when needing to update everything ones with newer versions and you don't care for your own customization. Please read https://docs.archipelago.nyc/1.4.0/utility_scripts/ for more ways of managing exporting/importing Metadata Display Entities (Twig templates).</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#step-7-set-your-public-iiif-server-url-to-your-actual-domain","title":"Step 7. Set your public IIIF server URL to your actual domain","text":"<p>By default archipelago ships with a public facing and an internal facing IIIF Server URLs configured. These urls are used by a number of IIIF enabled viewers and need to be changed to reflect your new reality (a real Domain name and a proxied path!). These settings belong to the <code>strawberryfield/format_strawberryfield</code> module. </p> <p>First check your current settings:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush config-get format_strawberryfield.iiif_settings\"\n</code></pre> <p>You will see the following:</p> <pre><code>pub_server_url: 'http://localhost:8183/iiif/2'\nint_server_url: 'http://esmero-cantaloupe:8182/iiif/2'\n</code></pre> <p>Let's modify <code>pub_server_url</code>. Replace in the following command <code>your.domain.org</code> with the domain you defined in your <code>.env</code> file.  NOTE: We are passing the <code>-y</code> flag to <code>drush</code> avoid that way having to answer \"yes\".</p> <pre><code>docker exec -ti esmero-php bash -c \"drush config-set -y format_strawberryfield.iiif_settings pub_server_url https://your.domain.org/cantaloupe/iiif/2\"\n</code></pre> <p>Finally Done! Now you can log into your new Archipelago using <code>https</code> and start exploring. Thank you for following this guide!</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#deployment-on-arm64v8graviton-apple-m1-system","title":"Deployment on ARM64/v8(Graviton, Apple M1) system:","text":"<p>This applies to AWS <code>m6g</code> and <code>t3g</code> Instances and is documented inline in this guide. Please open an ISSUE in this repository if you run into any problems. Please review https://github.com/esmero/archipelago-deployment-live/blob/1.4.0/deploy/ec2-docker/docker-compose-aws-s3-arm64.yml for more info.</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#how-do-i-know-my-architecture","title":"How do I know my Architecture?","text":"<p>Run </p> <pre><code>uname -m \n</code></pre> <ul> <li>For an <code>x86(64 bit)</code> processor system output will be <code>x86_64</code></li> <li>For an <code>ARM(64 bit)</code> processor system output will be <code>aarch64</code></li> </ul>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Lund</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#acknowledgments","title":"Acknowledgments","text":"<p>This software is a Metropolitan New York Library Council Open-Source initiative and part of the Archipelago Commons project.</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-search_solr_index/","title":"Archipelago-deployment-live: Upgrading Solr","text":"","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#what-is-this-documentation-for","title":"What is this documentation for?","text":"<p>This documentation will help you ugprade your Solr Index from 8.x to 9.x or inside 9.x releases. And is meant to be a guide/helper. There is no simple way of saying this, but because the way a Solr index (sort of a Binary tree) is build, any larger change in the schema, field type definitions requires either a complete reindex but really, most of the time, a wipe, and start fresh situation. There is no perfect way around. There are very complex ways of keeping an old Server running and serving searches while you re-index a new one, but honestly, the how and approach will depend on your existing knowledge of Solr, your skills (even memory!) to execute so, and documenting those hacks are beyond the scope of this documentation. What is proven is what we explain in this document</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#requirements","title":"Requirements","text":"<ul> <li>An archipelago-deployment-live instance (working, tested) deployed using provided instructions via Docker running either Solr 8.x or 9.x</li> <li>Good knowledge, patience and instincts (+ courage and time) on how to run Terminal Commands.</li> <li>Patience(again but also patience from your users since search will be unavailable until you reindex). You can't skip steps here.</li> <li>For shell Commands documented here please copy line by line--not the whole block.</li> <li>You are running already version control and know how to git pull/push/merge.</li> </ul>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#backing-up-and-preparing-for-the-upgrade","title":"Backing up and preparing for the upgrade","text":"<p>Backups are always going to be your best friends. Archipelago's code, database, and settings are mostly self-contained in your current <code>archipelago-deployment-live</code> repo folder, and backing up is simple because of that.</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-1","title":"Step 1:","text":"<p>To make upgrading simpler we will clone archipelago-deployment-live (empty one) into a different folder. That way we can copy complete folders of configs and files instead of fetching them from github one by one.</p> <p>Go to your home folder (for the sake of this documentation it will be <code>/home/ec2-user</code> but you can also use the <code>$HOME</code> environmental variable instead )</p> <pre><code>cd /home/ec2-user\ngit clone https://github.com/esmero/archipelago-deployment-live archipelago-deployment-live-1.4.0\ncd archipelago-deployment-live-1.4.0\ngit switch 1.4.0\n</code></pre> <p>Now, on a terminal, <code>cd</code> into your running <code>archipelago-deployment-live</code> folder, then <code>cd</code> inside the <code>deploy/ec2-docker</code> subfolders and shut down your <code>docker-compose</code> ensemble by running the following:</p> <pre><code>docker-compose down\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-2","title":"Step 2:","text":"<p>Verify that all containers are actually down. The following command should return an empty listing:</p> <pre><code>docker ps\n</code></pre> <p>If anything is still running, wait a little longer and run the command again.</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-3","title":"Step 3:","text":"<p>Now let's <code>tar.gz</code> the whole ensemble with data and configs. We will exclude here the local source caches generated by Cantaloupe. If these or not exist will depend on how custom your deployment is.</p> <p>As an example we will save this into your <code>$HOME</code> folder. As a good practice we append the current date (YEAR-MONTH-DAY) to the filename. Here we assume today is July 7th of 2024.</p> <p>We will <code>cd</code> back to the parent folder of your running <code>archipelago-deployment-live</code> folder, so three levels down, assuming you are right now inside <code>archipelago-deployment-live/deploy/ec2-docker</code></p> <pre><code>cd ../../..\nsudo tar --exclude=archipelago-deployment-live/data_storage/iiifcache --exclude=archipelago-deployment-live/data_storage/iiiftmp -czvpf $HOME/archipelago-deployment-D10-20240707.tar.gz archipelago-deployment-live\n</code></pre> <p>The process may take a few minutes. Now let's verify that all is there and that the <code>tar.gz</code> is not corrupt.</p> <pre><code>tar -tvvf $HOME/archipelago-deployment-D10-20240707.tar.gz\n</code></pre> <p>You will see a listing of files, and at the end you will see something like this: <code>Archive Format: POSIX pax interchange format, Compression: gzip</code>. If corrupt (Do you have enough space? Did your ssh connection drop?) you will see the following:</p> <pre><code>tar: Unrecognized archive format\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-4","title":"Step 4:","text":"<p><code>cd</code> again into your running <code>archipelago-deployment-live</code> folder, then <code>cd</code> inside the <code>deploy/ec2-docker</code> Restart your <code>docker-compose</code> ensemble, and wait a little while for all to start.</p> <pre><code>docker-compose up -d\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-5","title":"Step 5:","text":"<p>Export/backup all of your live Archipelago 1.3.0, Drupal 10 configurations (this allows you to compare/come back in case you lose something custom during the upgrade).</p> <pre><code>docker exec esmero-php mkdir config/backup\ndocker exec esmero-php drush cex --destination=/var/www/html/config/backup\n</code></pre> <p>Good. Now it's safe to begin the upgrade process.</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#upgrading-to-solr-921","title":"Upgrading to Solr 9.2.1","text":"","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-0-get-familiar-with-what-changed","title":"Step 0: Get familiar with what changed.","text":"<p>Running a Production Server requires some informed decision making and thus, we believe, a good pre-step is reviewing what changed between releases.  In specific focus on this folder.</p> <p>https://github.com/esmero/archipelago-deployment-live/tree/1.4.0/config_storage/solrconfig/conf</p> <p>and</p> <p>https://github.com/esmero/archipelago-deployment-live/tree/1.4.0/data_storage/solrlib</p> <p>Also (please) read the official documentation here https://solr.apache.org/guide/8_9/solr-upgrade-notes.html</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-1-edit-docker-composeryml","title":"Step 1: Edit docker-composer.yml","text":"<p>You want to replace your current Solr Service (in its enterity. Please make sure indendation is 1:1). </p> <pre><code> solr:\n    container_name: esmero-solr\n    restart: always\n    image: \"solr:9.2.1\"\n    # If running Docker &lt; 20.10.10 please uncomment the following lines\n    # See https://solr.apache.org/guide/solr/latest/upgrade-notes/major-changes-in-solr-9.html#solr-9-2 \n    #security_opt:\n    #  - seccomp:unconfined\n    tty: true\n    environment:\n      SOLR_HEAP: 1024m\n      SOLR_OPTS: -Dsolr.jetty.request.header.size=65535 -Dsolr.modules=scripting\n    ports:\n      - \"8983:8983\"\n    networks:\n      - host-net\n      - esmero-net\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/data_storage/solrcore:/var/solr/data\n      - ${ARCHIPELAGO_ROOT}/config_storage/solrconfig:/drupalconfig\n      - ${ARCHIPELAGO_ROOT}/data_storage/solrlib:/opt/solr/contrib/archipelago/lib\n    entrypoint:\n      - docker-entrypoint.sh\n      - solr-precreate\n      - drupal\n      - /drupalconfig\n</code></pre> <p>You can also use any of these as reference:</p> <ul> <li>https://github.com/esmero/archipelago-deployment-live/blob/1.4.0/deploy/ec2-docker/docker-compose-aws-s3-arm64.yml</li> <li>https://github.com/esmero/archipelago-deployment-live/blob/1.4.0/deploy/ec2-docker/docker-compose-aws-s3.yml</li> </ul> <p>For this, if you have not already, run:</p> <pre><code>docker-compose down\n</code></pre> <p>Then open your docker-compose.yml file, find the <code>solr:</code> key and replace with this new settings. If for some unknown reason your voluments do not match our defaults, please adapt to your custom edits so they match where the Solr Core and Libraries are saved:</p> <pre><code>nano docker-compose.yml\n</code></pre> <p>Save your changes.</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-2-wipe-clean-get-the-new-configs-get-the-new-ocr-highlight-library","title":"Step 2: Wipe clean. Get the new configs. Get the new OCR Highlight library","text":"<p>Wait! (breath.)</p> <p>This step is only required if you are moving from Solr 8.x to 9.x or inside 9.x you have <code>solr field type</code> definition changes. If you had a stock 1.3.0 with solr 9.1 and want to move to solr 9.2 you can skip deleting everything and can jump to Step 3!</p> <p>This step requires some nerve. Be sure you know where you are inside your terminal (always)</p> <p>Inside your archipelago-deployment-live folder run:</p> <pre><code>cd data_storage/solrcore\npwd\n</code></pre> <p>You should see something like  <pre><code>/home/ec2-user/archipelago-deployment-live/data_storage/solrcore\n</code></pre></p> <p>Which means you are in the correct folder. Now time to clean your index (really think twice here ok? You have a backup. Never run any of these without a backup)</p> <pre><code>sudo rm -rf *\n</code></pre> <p>Now we need the new configurations for your Solr (so then docker container can re-create the index from scratch). Remember we downloaded a reference/empty Archipelago Deployment Live 1.4.0 at <code>/home/ec2-user/archipelago-deployment-live-1.4.0</code>. We are going to use the files there to replace your own configs. <code>cd</code> back to your live deployment assuming here it is (still) <code>/home/ec2-user/archipelago-deployment-live</code></p> <pre><code>cd /home/ec2-user/archipelago-deployment-live\ncp -rpv /home/ec2-user/archipelago-deployment-live-1.4.0/config_storage/solrconfig/conf/* /home/ec2-user/archipelago-deployment-live/config_storage/solrconfig/conf/.\n</code></pre> <p>Now we need to remove the old OCR library and replace with the new one</p> <pre><code>rm /home/ec2-user/archipelago-deployment-live/data_storage/solrlib/solr-ocrhighlighting-0.7.1.jar\ncp -rpv /home/ec2-user/archipelago-deployment-live-1.4.0/data_storage/solrlib/solr-ocrhighlighting-0.8.4-SNAPSHOT.jar /home/ec2-user/archipelago-deployment-live//data_storage/solrlib/.\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-3-docker-pull-and-check","title":"Step 3: docker pull and check","text":"<p>Optional: You might want to review/compare (now) your current Drupal Search API Index against our most current one here:</p> <p>In specific:</p> <ul> <li>https://github.com/esmero/archipelago-deployment-live/blob/1.4.0/drupal/config/sync/search_api.server.esmero_solr.yml</li> <li>https://github.com/esmero/archipelago-deployment-live/blob/1.4.0/drupal/config/sync/search_api.index.default_solr_index.yml</li> </ul> <p>and anything that starts with <code>search_api.</code> too.</p> <p>Time to fetch the latest Solr that will re-create the index:</p> <pre><code>docker compose pull\ndocker compose up -d\n</code></pre> <p>Give all a little time to start. Please be patient. To ensure all is well, run (more than once if necessary) the following:</p> <pre><code>docker ps\n</code></pre> <p>You should see something like this if you synced all containers to the latest (your versions and databse might vary depending on your server's platform):</p> <pre><code>CONTAINER ID   IMAGE                                      COMMAND                  CREATED          STATUS          PORTS                              NAMES\n5b06ee366f58   jonasal/nginx-certbot                      \"/docker-entrypoint.\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8001-&gt;80/tcp               esmero-web\n86b685008158   solr:9.2.1                                 \"docker-entrypoint.s\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8983-&gt;8983/tcp             esmero-solr\na4872b237e17   esmero/cantaloupe-s3:6.0.1-multiarch       \"sh -c 'java -Dcanta\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8183-&gt;8182/tcp             esmero-cantaloupe\nbec0b31f3421   mariadb:10.6.18-focal                      \"docker-entrypoint.s\u2026\"   10 minutes ago   Up 10 minutes   3306/tcp                           esmero-db\n85bedadf9732   redis:6.2-alpine                           \"docker-entrypoint.s\u2026\"   10 minutes ago   10 minutes ago                                     esmero-redis\n6a9e9d8647a9   minio/minio:RELEASE.2022-06-11T19-55-32Z   \"/usr/bin/docker-ent\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:9000-9001-&gt;9000-9001/tcp   esmero-minio\nbc5327680ca7   esmero/php-8.1-fpm:1.2.0-multiarch         \"docker-php-entrypoi\u2026\"   10 minutes ago   Up 10 minutes   9000/tcp                           esmero-php\nd53729be1211   esmero/esmero-nlp:fasttext-multiarch       \"/usr/local/bin/entr\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:6400-&gt;6400/tcp             esmero-nlp\n</code></pre> <p>Important here is the <code>STATUS</code> column. It needs to be a number that goes up in time every time you run <code>docker ps</code> again (and again).</p> <p>Check your Solr logs</p> <pre><code>docker logs -f esmero-solr -n 100 for failure messages.\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-4-re-index-drupal-search-api","title":"Step 4: Re-index Drupal Search API","text":"<p>If you decide/not decide to syncronize Drupal's Search API Server, Index and fields is personal decision (optional). We do recommend it but your Solr Index might have many customizations already, so a better/pro approach would be do <code>diff</code> the .yml files and decide selectively. Once you decided/did that/skipped. Time to reindex</p> <p>Run the following:</p> <p><pre><code>docker exec esmero-php drush search-api-reindex\ndocker exec esmero-php drush search-api-index\n</code></pre> Check your Drupal logs, try some searches.</p> <p>If you made it this far you are done with code/devops (are we ever ready?), and that means you should be able to (hopefully) stay in the Drupal 10.x realm for a few years!</p> <p>Done!</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#need-help-blue-screen-missed-a-step-need-a-hug-or-someone-that-listens-to-you-in-silence","title":"Need help? Blue Screen? Missed a step? Need a hug or someone that listens to you in silence?","text":"<p>If you see any issues or errors or need help with a step, please let us know (ASAP!). You can either open an <code>issue</code> in this repository or use the Google Group. We are here to help.</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Sherrick</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-updatingContainers/","title":"How to update your Docker containers","text":"<p>From time to time you may have a need to update the containers themselves. Primarily this is done for security releases.</p>","tags":["Github","Docker","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-updatingContainers/#1-update-docker-composeyml","title":"1. Update docker-compose.yml","text":"<p>The first thing you need to do is to edit your <code>docker-compose.yml</code> file and replace the version of the container with the new one you wish to use.</p> <p>Navigate to your <code>docker-compose.yml</code> file and open it to edit. On Debian installs it would look like this:</p> <pre><code>  cd /usr/local/archipelago/deploy/archipelago-deployment-live/deploy/ec2-docker\n  vi docker-compose.yml\n</code></pre> <p>You want to change the image line to reflect the name of the new image you wish to use:</p> <pre><code>image: esmero/php-7.4-fpm:1.0.0-RC3-multiarch\n</code></pre> <p>might become:</p> <pre><code>image: esmero/php-8.0-fpm:1.1.0-multiarch\n</code></pre> <p>Save your change. If use vi like in the above it would look like this:</p> <pre><code>  :wq\n</code></pre>","tags":["Github","Docker","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-updatingContainers/#pull-the-new-images","title":"Pull the new image(s)","text":"<p>Docker Compose will now allow us to grab the new image(s) while your current system is running:</p> <pre><code>  docker-compose pull\n</code></pre>","tags":["Github","Docker","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-updatingContainers/#stop-and-restart-the-container","title":"Stop and restart the container","text":"<p>It is necesary to stop and start the container or the current image will continue to be used:</p> <pre><code>  docker-compose stop container-name\n</code></pre> <p>Wait for it to stop. Then bring it back up:</p> <pre><code>docker-compose up -d \n</code></pre> <p>It is important to use the -d flag or you will have your live instance stuck in your terminal. You want it to run in the background. The <code>-d</code> flag stands for detached.</p>","tags":["Github","Docker","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-updatingContainers/#down-and-up","title":"Down and up","text":"<p>If you are more comfortable having the all the containers go down and up you can do that with the following:</p> <pre><code>docker-compose down\ndocker-compose up\n</code></pre> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback, or open an ISSUE in this Archipelago Deployment Live Repository.</p> <p>Return to Archipelago Live Deployment.</p>","tags":["Github","Docker","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/","title":"Archipelago-deployment-live: upgrading Drupal 8 to Drupal 9 (1.0.0-RC2 to 1.0.0-RC3)","text":"","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#what-is-this-documentation-for","title":"What is this documentation for?","text":"<p>If you already have a well-set-up and well-loved Archipelago (RC2 or your own custom version) running Drupal 8 (D8), this documentation will allow you to update to Drupal 9 (D9) without major issues.</p> <p>D8 is no longer supported as of the end of November 2021. D9 has been around for a little while and even if every module is not supported yet, what you need and want for Archipelago has long been D9-ready.</p>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#requirements","title":"Requirements","text":"<ul> <li>An archipelago-deployment-live instance 1.0.0-RC2 (working, tested) deployed using provided instructions via Docker and running Drupal 8.</li> <li>Basic knowledge and instincts (+ courage) on how to run Terminal Commands, <code>composer</code> and <code>drush</code>.</li> <li>Patience. You can't skip steps here.</li> <li>For Shell Commands documented here please copy line by line\u2014not the whole block.</li> </ul>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#backing-up-and-preparing-for-the-upgrade","title":"Backing up and preparing for the upgrade","text":"<p>Backups are always going to be your best friends. Archipelago's code, database and settings are mostly self-contained in your current <code>archipelago-deployment-live</code> repo folder, and backing up is simple because of that.</p>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-1","title":"Step 1:","text":"<p>Shut down your <code>docker-compose</code> ensemble. Move to your <code>archipelago-deployment-live</code> folder and run this:</p> <pre><code>cd deploy/ec2-docker\ndocker-compose down\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-2","title":"Step 2:","text":"<p>Verify that all containers are actually down. The following command should return an empty listing. If anything is still running, wait a little longer, and run the following comman again.</p> <pre><code>docker ps\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-3","title":"Step 3:","text":"<p>Now let's <code>tar.gz</code> the whole ensemble with data and configs. As an example we will save this into your <code>$HOME</code> folder. As a good practice we append the current date (YEAR-MONTH-DAY) to the filename. Here we assume today is December 1st of 2021.</p> <pre><code>sudo tar -czvpf $HOME/archipelago-deployment-live-backup-20211201.tar.gz ../../../archipelago-deployment-live\n</code></pre> <p>The process may take a few minutes. Now let's verify that all is there and that the <code>tar.gz</code> is not corrupt.</p> <pre><code>tar -tvvf $HOME/archipelago-deployment-live-backup-20211201.tar.gz\n</code></pre> <p>You will see a listing of files. If corrupt (Do you have enough space? Did your ssh connection drop?) you will see the following:</p> <pre><code>tar: Unrecognized archive format\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-4","title":"Step 4:","text":"<p>Restart your <code>docker-compose</code> ensemble, and wait a little while for all to start.</p> <pre><code>docker-compose up -d\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-5","title":"Step 5:","text":"<p>Export/backup all of your live Archipelago configurations (this allows you to compare/come back in case you lose something custom during the upgrade).</p> <pre><code>docker exec esmero-php mkdir config/backup\ndocker exec esmero-php drush cex --destination=/var/www/html/config/backup\n</code></pre> <p>Good. Now it's safe to begin the upgrade.</p>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#upgrading-to-100-rc3","title":"Upgrading to 1.0.0-RC3","text":"","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-1_1","title":"Step 1:","text":"<p>First we are going to disable modules that are not part of 1.0.0-RC3 or are not yet compatible with D9. Run the following command:</p> <pre><code>docker exec esmero-php drush pm-uninstall module_missing_message_fixer markdown webprofiler key_value webform_views\n</code></pre> <p>From inside your <code>archipelago-deployment-live</code> repo folder we are going to open up the file <code>permissions</code> for some of your most protected Drupal files.</p> <pre><code>cd ../../\nsudo chmod 777 drupal/web/sites/default\nsudo chmod 666 drupal/web/sites/default/*settings.php\nsudo chmod 666 drupal/web/sites/default/*services.yml\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-2_1","title":"Step 2:","text":"<p>Time to fetch the <code>1.0.0-RC3</code> branch and update our <code>docker-compose</code> and <code>composer</code> dependencies. We are also going to stop the current <code>docker</code> ensemble to update all containers to newer versions:</p> <pre><code>cd deploy/ec2-docker\ndocker-compose down\ngit checkout 1.0.0-RC3\n</code></pre> <p>Then copy the appropriate <code>docker-compose</code> file for your architecture:</p> OSX (macOS)/x86-64 <pre><code>cp docker-compose-osx.yml docker-compose.yml\n</code></pre> Linux/x86-64/AMD64 <pre><code>cp docker-compose-linux.yml docker-compose.yml\n</code></pre> OSX (macOS)/Linux/ARM64 <pre><code>cp docker-compose-arm64.yml docker-compose.yml\n</code></pre> <p>Finally, pull the images, and bring up the ensemble:</p> <pre><code>docker compose pull\ndocker compose up -d\n</code></pre> <p>Give all a little time to start. The latest <code>min.io</code> adds a new console, and your <code>Solr</code> core and <code>Database</code> need to be upgraded. Please be patient. To ensure all is well, run (more than once if necessary) the following:</p> <pre><code>docker ps\n</code></pre> <p>You should see something like this:</p> <pre><code>CONTAINER ID   IMAGE                                    COMMAND                  CREATED          STATUS          PORTS                                                           NAMES\n867fd2a42134   nginx                                    \"/docker-entrypoint.\u2026\"   32 seconds ago   Up 27 seconds   0.0.0.0:8001-&gt;80/tcp, :::8001-&gt;80/tcp                           esmero-web\n8663e84a9b48   solr:8.8.2                               \"docker-entrypoint.s\u2026\"   33 seconds ago   Up 30 seconds   0.0.0.0:8983-&gt;8983/tcp, :::8983-&gt;8983/tcp                       esmero-solr\n9b580fa0088f   minio/minio:latest                       \"/usr/bin/docker-ent\u2026\"   33 seconds ago   Up 28 seconds   0.0.0.0:9000-9001-&gt;9000-9001/tcp, :::9000-9001-&gt;9000-9001/tcp   esmero-minio\n50e2f41c7b60   esmero/esmero-nlp:1.0                    \"/usr/local/bin/entr\u2026\"   33 seconds ago   Up 30 seconds   0.0.0.0:6400-&gt;6400/tcp, :::6400-&gt;6400/tcp                       esmero-nlp\n300810fd6f03   esmero/cantaloupe-s3:4.1.9RC             \"sh -c 'java -Dcanta\u2026\"   33 seconds ago   Up 30 seconds   0.0.0.0:8183-&gt;8182/tcp, :::8183-&gt;8182/tcp                       esmero-cantaloupe\n248e4638ba2a   mysql:8.0.22                             \"docker-entrypoint.s\u2026\"   33 seconds ago   Up 28 seconds   3306/tcp, 33060/tcp                                             esmero-db\n141ace919344   esmero/php-7.4-fpm:1.0.0-RC2-multiarch   \"docker-php-entrypoi\u2026\"   33 seconds ago   Up 28 seconds   9000/tcp                                                        esmero-php\n</code></pre> <p>Important here is the <code>STATUS</code> column. It needs to be a number that goes up in time every time you run <code>docker ps</code> again (and again).</p>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-3_1","title":"Step 3:","text":"<p>Now we are going to tell <code>composer</code> to actually fetch the new code and dependencies using the 1.0.0-RC3 provided <code>composer.lock</code> and update the whole Drupal/PHP/JS environment.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer install\"\n</code></pre> <p>This will fail (sorry!) for a few packages but no worries, they need to be patched and composer is not that smart. So simply run it again:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer install\"\n</code></pre> <p>Well done! If you see no issues and all ends in a Green colored message all is good! Jump to Step 4</p>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#what-if-not-all-is-ok-and-i-see-red-and-a-lot-of-dependency-explanations","title":"What if not all is OK and I see red and a lot of dependency explanations?","text":"<p>If you have manually installed packages via composer in the past that are NO longer Drupal 9 compatible you may see errors. In that case you need to check each package website's (normally https://www.drupal.org/project/the_module_name) and check if there is a Drupal 9 compatible version.</p> <p>If so run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/the_module_name:^VERSION_NUMBER_THAT_WORKS_ON_DRUPAL9_' --update-with-dependencies --no-update\" and run **Step 3 ** again (and again until all is cleared)\n</code></pre> <p>If not: try to find a replacement module that does something simular, but in any case you may end having to remove before proceding. Give us a ping/slack/google group/open a github ISSUE if you find yourself uncertain about this.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer remove drupal/the_module_name --no-update\"\ndocker exec -ti esmero-php bash -c \" drush pm-uninstall the_module_name\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-4_1","title":"Step 4:","text":"<p>We will now ask Drupal to update some internal configs and databases. They will bring you up to date with RC3 settings and D9 particularities.</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/setup.sh'\ndocker exec -ti esmero-php bash -c 'drush urol administrator \"admin\"'\ndocker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-5_1","title":"Step 5:","text":"<p>Previously D8 installations had a \"module/profile\" driven installation. Those are no longer used or even exist as part of core, but a profile can't be changed once installed so you have to do the following to avoid Drupal complaining about our new and simpler way of doing things (a small roll back):</p> <pre><code>docker exec -ti esmero-php bash -c \"sed -i 's/minimal: 1000/standard: 1000/g' config/sync/core.extension.yml\"\ndocker exec -ti esmero-php bash -c \"sed -i 's/profile: minimal/profile: standard/g' config/sync/core.extension.yml\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-6","title":"Step 6:","text":"<p>Now you can Sync your new Archipelago 1.0.0-RC3 and bring all the new configs and settings in. For this you have two options (no worries, remember you made a backup!):</p>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#a-partial-sync-which-will-bring-new-configs-and-update-existing-ones-but-will-not-remove-ones-that-only-exist-in-your-custom-setup-eg-new-webforms-or-view-modes","title":"A Partial Sync, which will bring new configs and update existing ones but will not remove ones that only exist in your custom setup, e.g. new Webforms or View Modes.","text":"<pre><code>docker exec esmero-php drush cim -y --partial\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#a-complete-sync-which-will-bring-new-configs-and-update-existing-ones-but-will-also-remove-all-the-ones-that-are-not-part-of-rc3-its-a-like-clean-factory-reset","title":"A Complete Sync, which will bring new configs and update existing ones but will also remove all the ones that are not part of RC3. It's a like clean factory reset.","text":"<pre><code>docker exec esmero-php drush cim -y\n</code></pre> <p>If all goes well here and you see no errors it's time to reindex <code>Solr</code> because there are new Fields. Run the following:</p> <pre><code>docker exec esmero-php drush search-api-reindex\ndocker exec esmero-php drush search-api-index\n</code></pre> <p>You might see some warnings related to modules dealing with previously non-existent data\u2014no worries, just ignore those.</p> <p>If you made it this far you are done with code/devops (are we ever ready?), and that means you should be able to (hopefully) stay in the Drupal 9 realm for a few years!</p>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-7-update-or-not-your-metadata-display-entities-and-menu-items","title":"Step 7: Update (or not) your Metadata Display Entities and Menu items.","text":"<p>Recommended: If you want to add new templates and menu items 1.0.0-RC3 provides, run this:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/deploy.sh'\n</code></pre> <p>Once that is done, you can choose to update all Metadata Displays (Twig templates) we ship with new 1.0.0-RC3 versions (heavily fixed IIIF manifest, Markdown to HTML for Metadata, better Object descriptions). But before you do this, we really recommend that you first make sure to manually (copy/paste) backup any Twig templates you have modified. If unusure, do not run the command that comes after this warning! You can always manually copy the new templates from the <code>d8content/metadatadisplays</code> folder which contains text versions (again, copy/paste) of each shipped template you can pick and use when you feel ready.</p> <p>If you are sure (like really?) you want to overwrite the ones you modified (sure, just checking?), then you can run this:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/update_deployed.sh'\n</code></pre> <p>Done! (For realz now)</p> <p>Please log into your Archipelago and test/check all is working! Enjoy 1.0.0-RC3 and Drupal 9. Thanks!</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback, or open an ISSUE in this Archipelago Deployment Live Repository.</p> <p>Return to Archipelago Live Deployment.</p>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/","title":"Archipelago-deployment-live: upgrading from 1.0.0-RC3 to 1.0.0","text":"","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#what-is-this-documentation-for","title":"What is this documentation for?","text":"<p>If you already have a well-set-up and well-loved Archipelago (RC3 or your own custom version) running Drupal 9, this documentation will allow you to update to 1.0.0 without major issues.</p>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#requirements","title":"Requirements","text":"<ul> <li>An archipelago-deployment-live instance 1.0.0-RC3 (working, tested) deployed using provided instructions via Docker and running Drupal 9.</li> <li>Basic knowledge and instincts (+ courage) on how to run Terminal Commands, <code>composer</code> and <code>drush</code>.</li> <li>Patience. You can't skip steps here.</li> <li>For Shell Commands documented here please copy line by line\u2014not the whole block.</li> </ul>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#backing-up-and-preparing-for-the-upgrade","title":"Backing up and preparing for the upgrade","text":"<p>Backups are always going to be your best friends. Archipelago's code, database and settings are mostly self-contained in your current <code>archipelago-deployment-live</code> repo folder, and backing up is simple because of that.</p>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-1","title":"Step 1:","text":"<p>Shut down your <code>docker-compose</code> ensemble. Move to your <code>archipelago-deployment-live</code> folder and run this:</p> <pre><code>cd deploy/ec2-docker\ndocker-compose down\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-2","title":"Step 2:","text":"<p>Verify that all containers are actually down. The following command should return an empty listing. If anything is still running, wait a little longer, and run the following comman again.</p> <pre><code>docker ps\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-3","title":"Step 3:","text":"<p>Now let's <code>tar.gz</code> the whole ensemble with data and configs. As an example we will save this into your <code>$HOME</code> folder. As a good practice we append the current date (YEAR-MONTH-DAY) to the filename. Here we assume today is December 1st of 2021.</p> <pre><code>sudo tar -czvpf $HOME/archipelago-deployment-live-backup-20220803.tar.gz ../../../archipelago-deployment-live\n</code></pre> <p>The process may take a few minutes. Now let's verify that all is there and that the <code>tar.gz</code> is not corrupt.</p> <pre><code>tar -tvvf $HOME/archipelago-deployment-live-backup-20220803.tar.gz \n</code></pre> <p>You will see a listing of files. If corrupt (Do you have enough space? Did your ssh connection drop?) you will see the following:</p> <pre><code>tar: Unrecognized archive format\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-4","title":"Step 4:","text":"<p>Restart your <code>docker-compose</code> ensemble, and wait a little while for all to start.</p> <pre><code>docker-compose up -d\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-5","title":"Step 5:","text":"<p>Export/backup all of your live Archipelago configurations (this allows you to compare/come back in case you lose something custom during the upgrade).</p> <pre><code>docker exec esmero-php mkdir config/backup\ndocker exec esmero-php drush cex --destination=/var/www/html/config/backup\n</code></pre> <p>Good. Now it's safe to begin the upgrade.</p>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#upgrading-to-100","title":"Upgrading to 1.0.0","text":"","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-1_1","title":"Step 1:","text":"<p>First we are going to disable modules that are not part of 1.0.0 or are not yet compatible with Drupal 9.4.x or higher . Run the following command:</p> <pre><code>docker exec esmero-php drush pm-uninstall search_api_solr_defaults entity_reference\n</code></pre> <p>From inside your <code>archipelago-deployment-live</code> repo folder we are going to open up the file <code>permissions</code> for some of your most protected Drupal files.</p> <pre><code>cd ../../\nsudo chmod 777 drupal/web/sites/default\nsudo chmod 666 drupal/web/sites/default/*settings.php\nsudo chmod 666 drupal/web/sites/default/*services.yml\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-2_1","title":"Step 2:","text":"<p>First let's back up our current composer.lock:</p> <pre><code>cp drupal/composer.lock drupal/composer.original.lock\n</code></pre> <p>Time to fetch the <code>1.0.0</code> branch and update our <code>docker-compose</code> and <code>composer</code> dependencies. We are also going to stop the current <code>docker</code> ensemble to update all containers to newer versions:</p> <pre><code>cd deploy/ec2-docker\ndocker-compose down\ngit fetch\ngit checkout 1.0.0 \n</code></pre> <p>If you decide to enable the Drupal REDIS module, make sure to add the <code>REDIS_PASSWORD</code> variable to your <code>.env</code> file.</p> <p><code>IMPORTANT NOTE</code>: For AWS EC2. If you selected an <code>IAM role</code> for your server when setting it up/deploying it, <code>min.io</code> will use the AWS EC2-backed internal API to request access to your S3. This means the ROLE itself needs to have read/write access (ACL) to the given Bucket(s) and your key/secrets won't be able to override that. Please do not ignore this note. It will save you a LOT of frustration and coffee. You can also run an EC2 instance without a given IAM and in that case just the ACCESS_KEY/SECRET will matter.</p> <p>Now that you know, you also know that these values should not be shared and this <code>.env</code> file should not be committed/kept in version control. Please be careful.</p> <p>Now let's back up the existing <code>docker-compose</code> file:</p> <pre><code>cp docker-compose.yml docker-compose-original.yml\n</code></pre> <p>Then copy the appropriate <code>docker-compose</code> file for your architecture:</p> Linux/x86-64/AMD64 <pre><code>cp docker-compose-aws-s3.yml docker-compose.yml\n</code></pre> Linux/ARM64/Apple Silicon (M1 and M2) <pre><code>cp docker-compose-aws-s3-arm64.yml docker-compose.yml\n</code></pre> <p>Next, let's review what's changed in case any customizations need to be brought into the new <code>docker-compose</code> configurations:</p> <pre><code>git diff --no-index docker-compose-original.yml docker-compose.yml\n</code></pre> <p>You should encounter something like the following:</p> <pre><code>diff --git a/docker-compose-original.yml b/docker-compose.yml\nindex 6f5b17e..282417f 100644\n--- a/docker-compose-original.yml\n+++ b/docker-compose.yml\n@@ -1,5 +1,5 @@\n # Run docker-compose up -d\n-\n+# Docker file for AMD64/X86 machines\n version: '3.5'\n services:\n   web:\n@@ -23,6 +23,7 @@ services:\n       - solr\n       - php\n       - db\n+      - redis\n     tty: true\n     networks:\n       - host-net\n@@ -30,7 +31,7 @@ services:\n   php:\n     container_name: esmero-php\n     restart: always\n-    image: \"esmero/php-7.4-fpm:1.0.0-RC2-multiarch\"\n+    image: \"esmero/php-8.0-fpm:1.1.0-multiarch\"\n     tty: true\n     networks:\n       - host-net\n@@ -44,10 +45,11 @@ services:\n       MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD}\n       MINIO_BUCKET_MEDIA: ${MINIO_BUCKET_MEDIA}\n       MINIO_FOLDER_PREFIX_MEDIA: ${MINIO_FOLDER_PREFIX_MEDIA}\n+      REDIS_PASSWORD: ${REDIS_PASSWORD}\n   solr:\n     container_name: esmero-solr\n     restart: always\n-    image: \"solr:8.8.2\"\n+    image: \"solr:8.11.2\"\n</code></pre> <p>As you can see, most of the changes in this example are for new images and a new service/container/environment variable (REDIS), but you may have custom settings for your containers. Review any differences carefully and make adjustments as needed.</p> <p>Finally, pull the images:</p> <pre><code>docker compose pull \n</code></pre> <p>1.0.0 provides a new Cantaloupe that uses different permissions so we need to adapt those. From your current folder (../ec2-deploy) run:</p> <pre><code>sudo chown 8183:8183 ../../config_storage/iiifconfig/cantaloupe.properties\nsudo chown -R 8183:8183 ../../data_storage/iiifcache\nsudo chown -R 8183:8183 ../../data_storage/iiiftmp\n</code></pre> <p>Time to start the ensemble again</p> <pre><code>docker compose up -d\n</code></pre> <p>Give all a little time to start. <code>Solr</code> core and <code>Database</code> need to be upgraded, Cantaloupe is new and this brings also Redis for caching. Please be patient. To ensure all is well, run (more than once if necessary) the following:</p> <pre><code>docker ps\n</code></pre> <p>You should see something like this:  e.g if running on ARM64 You should see something like this: </p> <pre><code>CONTAINER ID   IMAGE                                    COMMAND                  CREATED          STATUS          PORTS                                                           NAMES\n4ed2f62e866e   jonasal/nginx-certbot                      \"/docker-entrypoint.\u2026\" 32 seconds ago   Up 27 seconds   0.0.0.0:80-&gt;80/tcp, :::80-&gt;80/tcp, 0.0.0.0:443-&gt;443/tcp, :::443-&gt;443/tcp   esmero-web\ne6b4383039c3   minio/minio:RELEASE.2022-06-11T19-55-32Z   \"/usr/bin/docker-ent\u2026\" 33 seconds ago   Up 30 seconds   0.0.0.0:9000-9001-&gt;9000-9001/tcp, :::9000-9001-&gt;9000-9001/tcp              esmero-minio\nf2b6b173b7e2   solr:8.11.2                                \"docker-entrypoint.s\u2026\" 33 seconds ago   Up 28 seconds   0.0.0.0:8983-&gt;8983/tcp, :::8983-&gt;8983/tcp                                  esmero-solr\na553bf484343   esmero/php-8.0-fpm:1.0.0-multiarch         \"docker-php-entrypoi\u2026\" 33 seconds ago   Up 30 seconds   9000/tcp                                                                   esmero-php\necb47349ae94   esmero/esmero-nlp:fasttext-multiarch       \"/usr/local/bin/entr\u2026\" 33 seconds ago   Up 30 second    0.0.0.0:6400-&gt;6400/tcp, :::6400-&gt;6400/tcp                                  esmero-nlp\n61272dce034a   redis:6.2-alpine                           \"docker-entrypoint.s\u2026\" 33 seconds ago   Up 28 seconds                                                                              esmero-redis\n0ee9869f809b   esmero/cantaloupe-s3:6.0.0-multiarch       \"sh -c 'java -Dcanta\u2026\" 33 seconds ago   Up 28 seconds   0.0.0.0:8183-&gt;8182/tcp, :::8183-&gt;8182/tcp                                  esmero-cantaloupe\n131d072567ce   mariadb:10.6.8-focal                       \"docker-entrypoint.s\u2026\" 33 seconds ago   Up 28 seconds   3306/tcp                                                                   esmero-db                                            esmero-php\n</code></pre> <p>Important here is the <code>STATUS</code> column. It needs to be a number that goes up in time every time you run <code>docker ps</code> again (and again).</p>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-3_1","title":"Step 3:","text":"<p>Instead of using the provided <code>composer.default.lock</code> out of the box we are going to loosen certain dependencies and bring manually Archipelago modules, all this to make update easier and future upgrades less of a pain.</p> <p>First, as a sanity check let's make sure nothing happened to our original <code>composer.lock</code> fileby doing a diff against our backed up file:</p> <pre><code>git diff --no-index ../../drupal/composer.original.lock ../../drupal/composer.lock\n</code></pre> <p>If all is ok, there should be no output. If there's any output, copy your backed up file back to default:</p> <pre><code>cp ../../drupal/composer.original.lock ../../drupal/composer.lock\n</code></pre> <p>Finally, we bring over the modules:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require drupal/core:^9 drupal/core-composer-scaffold:^9 drupal/core-project-message:^9 drupal/core-recommended:^9\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/core-dev:^9 --dev\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/tokenuuid:^2\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/facets:^2.0'\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/moderated_content_bulk_publish:^2\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/queue_ui:^3.1\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/jquery_ui_touch_punch:^1.1\"\ndocker exec -ti esmero-php bash -c \"composer require archipelago/ami:0.4.0.x-dev strawberryfield/format_strawberryfield:1.0.0.x-dev strawberryfield/strawberryfield:1.0.0.x-dev strawberryfield/strawberry_runners:0.4.0.x-dev strawberryfield/webform_strawberryfield:1.0.0.x-dev drupal/views_bulk_operations:^4.1\"\n</code></pre> <p>Now we are going to tell <code>composer</code> to actually fetch the new code and dependencies using <code>composer.lock</code> and update the whole Drupal/PHP/JS environment.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer update -W\"\ndocker exec -ti esmero-php bash -c \"drush cr\"\ndocker exec -ti esmero-php bash -c \"drush en jquery_ui_touch_punch\"\ndocker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre> <p>Well done! If you see no issues and all ends in a Green colored message all is good! Jump to Step 4</p>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#what-if-not-all-is-ok-and-i-see-red-and-a-lot-of-dependency-explanations","title":"What if not all is OK and I see red and a lot of dependency explanations?","text":"<p>If you have manually installed packages via composer in the past that are NO longer Drupal 9 compatible you may see errors.  In that case you need to check each package website's (normally https://www.drupal.org/project/the_module_name) and check if there is a Drupal 9 compatible version. </p> <p>If so run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/the_module_name:^VERSION_NUMBER_THAT_WORKS_ON_DRUPAL9_' --update-with-dependencies --no-update\" and run **Step 3 ** again (and again until all is cleared)\n</code></pre> <p>If not: try to find a replacement module that does something simular, but in any case you may end having to remove before proceding. Give us a ping/slack/google group/open a github ISSUE if you find yourself uncertain about this. </p> <pre><code>docker exec -ti esmero-php bash -c \"composer remove drupal/the_module_name --no-update\"\ndocker exec -ti esmero-php bash -c \" drush pm-uninstall the_module_name\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-4_1","title":"Step 4:","text":"<p>We will now ask Drupal to update some internal configs and databases. They will bring you up to date with 1.0.0 settings and D9 particularities.</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/setup.sh'\ndocker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-5_1","title":"Step 5:","text":"<p>Now you can Sync your new Archipelago 1.0.0 and bring all the new configs and settings in. For this you have two options (no worries, remember you made a backup!):</p>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#a-partial-sync-which-will-bring-new-configs-and-update-existing-ones-but-will-not-remove-ones-that-only-exist-in-your-custom-setup-eg-new-webforms-or-view-modes","title":"A Partial Sync, which will bring new configs and update existing ones but will not remove ones that only exist in your custom setup, e.g. new Webforms or View Modes.","text":"<pre><code>docker exec esmero-php drush cim -y --partial\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#a-complete-sync-which-will-bring-new-configs-and-update-existing-ones-but-will-also-remove-all-the-ones-that-are-not-part-of-rc3-its-a-like-clean-factory-reset","title":"A Complete Sync, which will bring new configs and update existing ones but will also remove all the ones that are not part of RC3. It's a like clean factory reset.","text":"<pre><code>docker exec esmero-php drush cim -y\n</code></pre> <p>If all goes well here and you see no errors it's time to reindex <code>Solr</code> because there are new Fields. Run the following:</p> <pre><code>docker exec esmero-php drush search-api-reindex\ndocker exec esmero-php drush search-api-index\n</code></pre> <p>You might see some warnings related to modules dealing with previously non-existent data\u2014no worries, just ignore those.</p> <p>If you made it this far you are done with code/devops (are we ever ready?), and that means you should be able to (hopefully) stay in the Drupal 9 realm for a few years!</p>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-7-update-or-not-your-metadata-display-entities-and-menu-items","title":"Step 7: Update (or not) your Metadata Display Entities and Menu items.","text":"<p>Recommended: If you want to add new templates and menu items 1.0.0 provides, go to your base Github repo folder, replace in the following commands <code>your.domain.org</code> with the actual domain of your Server and run those individually:</p> <pre><code>sed -i 's/http:\\/\\/esmero-web/https:\\/\\/your.domain.org/g' drupal/scripts/archipelago/deploy.sh\n</code></pre> <pre><code>sed -i 's/http:\\/\\/esmero-web/https:\\/\\/your.domain.org/g' drupal/scripts/archipelago/update_deployed.sh\n</code></pre> <p>Now update your Metadata Display Templates and Blocks</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/deploy.sh'\n</code></pre> <p>Once that is done, you can choose to update all Metadata Displays (Twig templates) we ship with new 1.0.0 versions (heavily fixed IIIF manifest, Markdown to HTML for Metadata, better Object descriptions). But before you do this, we really recommend that you first make sure to manually (copy/paste) backup any Twig templates you have modified. If unusure, do not run the command that comes after this warning! You can always manually copy the new templates from the <code>d8content/metadatadisplays</code> folder which contains text versions (again, copy/paste) of each shipped template you can pick and use when you feel ready.</p> <p>If you are sure (like really?) you want to overwrite the ones you modified (sure, just checking?), then you can run this:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/update_deployed.sh'\n</code></pre> <p>Done! (For realz now)</p> <p>Please log into your Archipelago and test/check all is working! Enjoy 1.0.0. Thanks!</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback, or open an ISSUE in this Archipelago Deployment Live Repository.</p> <p>Return to Archipelago Live Deployment.</p>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-osx/","title":"Installing Archipelago Drupal 10 on OSX (macOS)","text":"","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#about-running-terminal-commands","title":"About running terminal commands","text":"<p>This guide assumes you are comfortable enough running terminal (bash) commands on an OSX Computer.</p> <p>We made sure that you can <code>copy</code> and <code>paste</code> each of these commands from this guide directly into your terminal.</p> <p>You will notice sometimes commands span more than a single line of text. If that is the case, always make sure you copy and paste a single line at a time and press the <code>Enter</code> key afterwards. We suggest also you look at the output.</p> <p>If something fails (and we hope it does not) troubleshooting will be much easier if you can share that output when asking for help.</p> <p>Happy deploying!</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#osx-macos","title":"OSX (macOS):","text":"<ul> <li>Install Docker for Mac</li> <li>For OSX (macOS) <code>Ventura</code> or Higher on Intel (i5/i7) and Apple Silicon Chips (M1/M2/M3) the tested version is: <code>4.31.0(153195)</code> with Docker engine v26.1.4. You may go newer of course.</li> <li>In <code>Preferences</code> -&gt; <code>General</code>: check under <code>Choose file sharing implementation for your containers</code> either the new <code>VirtioFS</code>(faster) or <code>gRPC FUSE</code> and restart. Specially if you are using your <code>$HOME</code> folder for deploying, e.g. <code>/Users/username</code>.</li> <li>In <code>Preferences</code> -&gt; <code>Resources</code>: 4 Gbytes of RAM is the recommended minimun and works; 8 Gbytes is faster and snappier.</li> <li>Install Github Desktop.</li> <li>At least 10 Gbytes of free space (to get started).</li> <li>Being able to open a terminal.</li> </ul> <p>Note: Most Recent Docker Desktop for macOS removed <code>docker-compose</code> and replace it with <code>docker compose</code>. We updated this documentation to reflect the newer version. If you are running an older one, either upgrade or please replace every mention of <code>docker composer</code> with the legacy <code>docker-compose</code> when going through the steps.</p> <p>Note: Recent OSX (macOS) and newer Macs ship with 2 annoying things: Apple Cloud Syncing User Folders and (wait for it) Case insensitive File Systems. If you are happy with your shiny new Mac (like i was) we are aware that it's better to deploy anything mounted outside of the <code>/User</code> folder or even better, in an external drive formatted using a Case Sensitive Unix Filesystem (Mac OS Extended (Case-sensitive, Journaled)).</p> <p>Note 2: \"gRPC FUSE\" experience may vary, recent Docker for Mac does it well. In older RC1 ones it was evil. Changing/Disabling it after having installed Archipelago may affect your S3/Minio storage accessibility. Please let us know what your experience on this is.</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#wait-question-do-you-have-a-previous-version-of-archipelago-running","title":"Wait! Question: Do you have a previous version of Archipelago running?","text":"<p>If so, let's give that hard working repository a break first. If not, skip to Step 1:</p> <ul> <li>Open a terminal (you have that already right?) and go to your previous download/git clone folder and run:</li> </ul> <pre><code>docker compose down\ndocker compose rm\n</code></pre> <ul> <li>Can't remember where you downloaded it? Ok. We can deal with that!</li> </ul> <p>Let's stop the containers gracefully first, run:</p> <pre><code>docker stop esmero-web\ndocker stop esmero-solr\ndocker stop esmero-db\ndocker stop esmero-cantaloupe\ndocker stop esmero-php\ndocker stop esmero-minio\ndocker stop esmero-nlp\n</code></pre> <p>Now we need to remove them, run:</p> <pre><code>docker rm esmero-web\ndocker rm esmero-solr\ndocker rm esmero-db\ndocker rm esmero-cantaloupe\ndocker rm esmero-php\ndocker rm esmero-minio\ndocker rm esmero-nlp\n</code></pre> <p>Ok, now we are ready to start. Depending on what type of Chip your Apple uses you have two options:</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#step-1-intel-docker-deployment-on-intel-chips-apple-machines","title":"Step 1 (Intel): Docker Deployment on Intel Chips Apple Machines","text":"<pre><code>git clone https://github.com/esmero/archipelago-deployment.git archipelago-deployment\ncd archipelago-deployment\ngit checkout 1.4.0\ncp docker-compose-osx.yml docker-compose.yml\ndocker compose pull\ndocker compose up -d\n</code></pre>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#step-1-m1m2m3-docker-deployment-on-apple-silicon-chips-m1m2m3","title":"Step 1 (M1/M2/M3): Docker Deployment on Apple Silicon Chips (M1/M2/M3)","text":"<pre><code>git clone https://github.com/esmero/archipelago-deployment.git archipelago-deployment\ncd archipelago-deployment\ngit checkout 1.4.0\ncp docker-compose-arm64.yml docker-compose.yml\ndocker compose pull\ndocker compose up -d\n</code></pre> <p>Note: If you are running on an Intel Apple Machine from an external Drive or a partition/filesystem that is <code>Case Sensitive</code> and is not syncing automatically to <code>Apple Cloud</code> you can also use <code>docker-compose-linux.yml</code>. Note2: <code>docker-compose.yml</code> is git ignored in case you make local adjustments or changes to it.</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#step-2-set-up-your-minio-s3-bucket","title":"Step 2: Set up your Minio S3 bucket","text":"<p>Once all containers are up and running (you can do a <code>docker ps</code> to check), access the minio console at <code>http://localhost:9001</code> using your most loved Web Browser with the following credentials:</p> <pre><code>user:minio\npass:minio123\n</code></pre> <p>and once logged in, press on \"Buckets\" (left tools column) and then on \"Create Bucket\"  (top right) and under \"Bucket Name\" type <code>archipelago</code>. Leave all other options unchecked for now (you can experiment with those later), and make sure you write <code>archipelago</code> (no spaces, lowercase) and press \"Save\". Done! That is where we will persist all your Files and also your File copies of each Digital Object. You can always go there and explore what Archipelago (well really Strawberryfield does the hard work) has persisted so you can get comfortable with our architecture.</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#step-3-deploy-drupal-10-and-the-awesome-archipelago-modules","title":"Step 3: Deploy Drupal 10 and the awesome Archipelago Modules","text":"<p>The following will run composer inside the esmero-php container to download all dependencies and Drupal Core too:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer install\"\n</code></pre> <p>Once that command finishes run our setup script:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/setup.sh'\n</code></pre> <p>Explanation: That script will append some important configurations to your local <code>web/sites/default/settings.php</code>.</p> <p>Note: We say <code>local</code> because your whole Drupal web root (the one you cloned) is also mounted inside the esmero-php and esmero-web containers. So edits to PHP files, for example, can be done without accessing the container directly from your local folder.</p> <p>If this is the first time you deploy Drupal using the provided Configurations run:</p> <pre><code>docker exec -ti -u www-data esmero-php bash -c \"cd web;../vendor/bin/drush -y si --verbose --existing-config --db-url=mysql://root:esmerodb@esmero-db/drupal --account-name=admin --account-pass=archipelago -r=/var/www/html/web --sites-subdir=default --notify=false;drush cr;chown -R www-data:www-data sites;\"\n</code></pre> <p>Note: You will see these warnings:</p> <p><code>[warning] The \"block_content:9aa72fb1-2817-44a7-8fb5-a3eb51166e83\" was not found</code> <code>[warning] The \"block_content:1cdf7155-eb60-4f27-9e5e-64fffe93127a\" was not found</code> <code>[warning] The \"facets_summary_block:advance\" was not found</code> <code>[warning] The \"facets_summary_block:search_page_facet_summary\" was not found</code> <code>[notice] Missing required data for configuration: role_theme_switcher.settings</code></p> <p>Nothing to worry about. We will provide the missing part in Step 5.</p> <p>Note 2: Please be patient. This step takes since composer 2.0 25-30% longer because of how the most recent Drupal Installation code fetches translations and other resources (see <code>Performed install task</code>). This means progress might look like getting \"stuck\", go and get a coffee/tea and let it run to the end.</p> <p>Once finished, this will give you an <code>admin</code> Drupal user with <code>archipelago</code> as password (Change this if running on a public instance!).</p> <p>Final Note about Steps 2-3: You don't need to, nor should you do this more than once. You can destroy/stop/update, recreate your Docker containers, and start again (<code>git pull</code>), and your Drupal and Data will persist once you're past the <code>Installation complete</code> message. I repeat, all other containers' data is persisted inside the <code>persistent/</code> folder contained in this cloned git repository. Drupal and all its code is visible, editable, and stable inside your <code>web/</code> folder.</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#step-4-create-a-demo-and-a-jsonapi-user-using-drush-and-assign-your-admin-user-the-administrator-role","title":"Step 4: Create a \"demo \"and a \"jsonapi\" user using drush and assign your \"admin\" user the Administrator Role.","text":"<p><pre><code>docker exec -ti esmero-php bash -c 'drush ucrt demo --password=\"demo\"; drush urol metadata_pro \"demo\"'\n</code></pre> <pre><code>docker exec -ti esmero-php bash -c 'drush ucrt jsonapi --password=\"jsonapi\"; drush urol metadata_api \"jsonapi\"'\n</code></pre> <pre><code>docker exec -ti esmero-php bash -c 'drush urol administrator \"admin\"'\n</code></pre></p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#step-5-ingest-some-metadata-displays-to-make-playing-much-more-interactive","title":"Step 5: Ingest some Metadata Displays to make playing much more interactive","text":"<p>Archipelago is more fun without having to start writing Metadata Displays (in Twig) before you know what they actually are. Since you should now have a <code>jsonapi</code> user and jsonapi should be enabled, you can use that awesome functionality of D8 to get that done. We have 4 demo Metadata Display Entities that go well with the demo Webform we provided. To do that execute in your shell (copy and paste):</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/deploy.sh'\n</code></pre> <p>Open your most loved Web Browser and point it to <code>http://localhost:8001</code>.</p> <p>Note: It can take some time to start the first time (Drupal needs some warming up).</p> <p>Also, to make this docker-compose easier to use we are doing something named <code>bind mounting</code> (or similar...) your folders. The good thing is that you can edit files in your machine, and they get updated instantly to docker. The bad thing is that the OSX (macOS) driver runs slower than on Linux. Speed is a huge factor here, but you get the flexibility of changing, backing up, and persisting files without needing a Docker University Degree.</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#step-6-optional-but-more-fun-if-you-add-content","title":"Step 6: Optional but more fun if you add content","text":"<p>One-Step Demo content ingest</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#need-help-blue-screen-missed-a-step-need-a-hug-and-such","title":"Need help? Blue Screen? Missed a step? Need a hug and such?","text":"<p>If you see any issues or errors or need help with a step, please let us know (ASAP!). You can either open an <code>issue</code> in this repository or use the Google Group. We are here to help.</p> <p>If you like this, let us know!</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Sherrick</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-readme/","title":"Archipelago Docker Deployment","text":"<p>Updated: July 1st 2024</p> <p>Previously Updated: January 30th 2024</p> <p>This repository serves as bootstrap for a Archipelago 1.4.0 deployment on a localhost for development/testing/customizing via Docker and provides a more unified experience this time:</p> <ul> <li>minio.io (latest)for local S3 with Console.</li> <li>Apache Solr 9.2.1 with the wizardly Solr OCR Highlight library v0.8.4 built by the Developement Team at the Bavarian State Library. Thanks Johannes Baiter and team.</li> <li>MySQL 8.x (amd64/x86)/MariaDB 10.6.x(Arm64/M1/M2/M3)</li> <li>NGINX 11</li> <li>Custom PHP-FPM 8.1 multi architecture, fine-tuned for Drupal 10 , WARC to WACZ processing, Tesseract 5 with JP2 support, PDFAlto and Composer 2.x, Drush 12, etc</li> <li>Natural Language Processing via NLPWEB64 multi architecture with FastText Language detection (Thanks Mike Bennet!) or alternatively new ML (Image similarity: YOLO,MobileNet,Insightface and Text transformer: SBERT)</li> <li>Cantaloupe 6.0.1 Snapshot multi architecture as IIIF2/3 Server with Video Frame extraction and PDF support (with custom fix for tiled PDF)</li> <li>A Skeleton Project setup to run latest Version of Drupal (10.2.x), our new Bootstrap 5 theme and Strawberry Field modules on 1.4.0 &amp; friends on 0.8.0</li> <li>Complete support for Apple Silicon M1 Machines and in general <code>arm64</code> architecture Chips like Raspberry Pi 4, with specially built arm64 docker containers. The only differences now between deployment strategies is the DB. Blazing fast OCR.</li> </ul> <p>The skeleton project contains all the pieces needed to run a local deployment of a vanilla Archipelago including (YES!) content provided as an optional feature from archipelago-recyclables</p>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-readme/#starting-from-zero","title":"Starting from ZERO","text":"<p>This is the recommended, simplest way for this release. There are a too many, tons of fun new features, Metadata Displays, Webforms, New formatters and Twig extensions, improved viewers, new and improved JS libraries, OpenCV/Face Detection, smarter NLP, File composting, better HUGE import/update capabilities, bug fixes (yes so many) so please try them out. The team has also updated the DEMO AMI set (Content) to showcase metadata/display improvements.</p>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-readme/#macos-intel-or-apple-silicon-m1m2m3","title":"macOS Intel or Apple Silicon M1/M2/M3:","text":"<p>Step by Step deployment on macOS</p>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-readme/#ubuntu-1804-or-2004","title":"Ubuntu 18.04 or 20.04:","text":"<p>Step by Step deployment on Ubuntu</p>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-readme/#windows-10-or-11","title":"Windows 10 or 11:","text":"<p>Step by Step deployment on Windows</p>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-readme/#more-fun-if-you-add-content","title":"More fun if you add content:","text":"<p>One-Step Demo content ingest</p> <p>If you like it (or not), want new features, or want to be part of making this better (documenting, coding and planning) let us know. Make your voice and opinion be heard, this is a community effort.</p>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-readme/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Sherrick</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-readme/#acknowledgments","title":"Acknowledgments","text":"<p>This software is a Metropolitan New York Library Council Open-Source initiative and part of the Archipelago Commons project.</p>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-readme/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-ubuntu/","title":"Installing Archipelago Drupal 10 on Ubuntu 18.04 or 20.04","text":"","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#about-running-terminal-commands","title":"About running terminal commands","text":"<p>This guide assumes you are comfortable enough running terminal (bash) commands on a Linux Computer.</p> <p>We made sure that you can <code>copy</code> and <code>paste</code> each of these commands from this guide directly into your terminal.</p> <p>You will notice sometimes commands span more than a single line of text. If that is the case, always make sure you copy and paste a single line at a time and press the <code>Enter</code> key afterwards. We suggest you also look at the output.</p> <p>If something fails (and we hope it does not) troubleshooting will be much easier if you can share that output when asking for help.</p> <p>Happy deploying!</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#prerequisites","title":"Prerequisites","text":"<ul> <li>At least 10 Gbytes of free space (to get started)</li> <li>Some basic Unix/Terminal Skills</li> <li>2-4 Gbytes of RAM (4 Recommended)</li> <li>Install Docker if you don't have it already by running:</li> </ul> <pre><code>sudo apt install apt-transport-https ca-certificates curl software-properties-common\ncurl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -\nsudo add-apt-repository \"deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable\"\nsudo apt update\nsudo apt-cache policy docker-ce\nsudo apt install docker-ce\nsudo systemctl status docker\n\nsudo usermod -aG docker ${USER}\n</code></pre> <p>Log out, and log in again!</p> <pre><code>sudo apt install docker-compose\n</code></pre> <p>Git tools are included by default in Ubuntu.</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#wait-question-do-you-have-a-previous-version-of-archipelago-running","title":"Wait! Question: Do you have a previous version of Archipelago running?","text":"<p>If so, let's give that hard working repository a break first. If not, Step 1:</p> <ul> <li>Open a terminal (you have that already right?) and go to your previous download/git clone folder and run:</li> </ul> <pre><code>docker-compose down\ndocker-compose rm\n</code></pre> <ul> <li>Can't remember where you downloaded it? Ok. We can deal with that!</li> </ul> <p>Let's stop the containers gracefully first, run:</p> <pre><code>docker stop esmero-web\ndocker stop esmero-solr\ndocker stop esmero-db\ndocker stop esmero-cantaloupe\ndocker stop esmero-php\ndocker stop esmero-minio\ndocker stop esmero-nlp\n</code></pre> <p>Now we need to remove them so we run the following:</p> <pre><code>docker rm esmero-web\ndocker rm esmero-solr\ndocker rm esmero-db\ndocker rm esmero-cantaloupe\ndocker rm esmero-php\ndocker rm esmero-minio\ndocker rm esmero-nlp\n</code></pre> <p>Ok, now we are ready to start.</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#step-1-deployment","title":"Step 1: Deployment","text":"","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#prefer-to-watch-a-video-to-see-what-its-like-to-install-go-to-our-user-contributed-documentation1","title":"Prefer to watch a video to see what it's like to install? Go to our <code>user contributed documentation</code>[^1]!","text":"","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#important","title":"IMPORTANT","text":"<p>If you run <code>docker-compose</code> as root user (using <code>sudo</code>) some enviromental variables, like the current folder used inside the <code>docker-compose.yml</code> to mount the Volumes, will not work and you will see a bunch of errors.</p> <p>There are two possible solutions.</p> <ul> <li>The best is to add your user to the docker group (so no <code>sudo</code> needed).</li> <li>The second option is to replace every <code>{$PWD}</code> inside your <code>docker-compose.yml</code> with either the full path to your current folder, or with a <code>.</code> and wrap that whole line in double quotes, basically making the paths for volumes relatives.</li> </ul> <p>Instead of: <code>- ${PWD}:/var/www/html:cached</code> use: <code>- \".:/var/www/html:cached\"</code></p> <p>Now that you got it, let's deploy:</p> <pre><code>git clone https://github.com/esmero/archipelago-deployment.git archipelago-deployment\ncd archipelago-deployment\ngit checkout 1.4.0\n</code></pre> <pre><code>cp docker-compose-linux.yml docker-compose.yml\ndocker-compose pull\ndocker-compose up -d\n</code></pre> <p>Note: <code>docker-compose.yml</code> is git ignored in case you make local adjustments or changes to it.</p> <p>You need to make sure Docker can read/write to your local Drive, a.k.a mounted volumes (especially if you decided not to run it as <code>root</code> because we told you so!).</p> <p>This means in practice running:</p> <pre><code>sudo chown -R 8183:8183 persistent/iiifcache\nsudo chown -R 8983:8983 persistent/solrcore\n</code></pre> <p>And then:</p> <pre><code>docker exec -ti esmero-php bash -c \"chown -R www-data:www-data private\"\n</code></pre> <p>Question: Why is this last command different? Answer: Just a variation. The long answer is that the internal <code>www-data</code> user in that container (Alpine Linux) has uid:82, but on Ubuntu the <code>www-data</code> user has a different one so we let Docker assign the uid from inside instead. In practice you could also run directly <code>sudo chown -R 82:82 private</code> which would only apply to an Alpine use case, which can differ in the future! Does this make sense? No worries if not.</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#step-2-set-up-your-minio-s3-bucket","title":"Step 2: Set up your Minio S3 bucket","text":"<p>Once all containers are up and running (you can do a <code>docker ps</code> to check), access <code>http://localhost:9001</code> using your most loved Web Browser with the following credentials:</p> <pre><code>user:minio\npass:minio123\n</code></pre> <p>and create a bucket named \"archipelago\". To do so go to the <code>Buckets</code> section in the navigation pane, and click <code>Create Bucket +</code>. Type <code>archipelago</code> under <code>Bucket Name</code> and submit, done! That is where we will persist all your Files and also your File copies of each Digital Object. You can always go there and explore what Archipelago (well really Strawberryfield does the hard work) has persisted so you can get comfortable with our architecture.</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#step-3-deploy-drupal-10-and-the-awesome-archipelago-modules","title":"Step 3: Deploy Drupal 10 and the awesome Archipelago Modules","text":"<p>The following will run composer inside the esmero-php container to download all dependencies and Drupal Core too.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer install\"\n</code></pre> <p>You might see a warning: <code>Do not run Composer as root/super user! See https://getcomposer.org/root for details</code> and the a long list of PHP packages. Don't worry. All is good here. Keep following the instructions! Once that command finishes run our setup script:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/setup.sh'\n</code></pre> <p>Explanation: That script will append some important configurations to your local <code>web/sites/default/settings.php</code>.</p> <p>Note: We say <code>local</code> because your whole Drupal web root (the one you cloned) is also mounted inside the esmero-php and esmero-web containers. So edits to PHP files, for example, can be done without accessing the container directly from your local folder.</p> <p>If this is the first time you're deploying Drupal using the provided Configurations run:</p> <pre><code>docker exec -ti -u www-data esmero-php bash -c \"cd web;../vendor/bin/drush -y si --verbose --existing-config --db-url=mysql://root:esmerodb@esmero-db/drupal --account-name=admin --account-pass=archipelago -r=/var/www/html/web --sites-subdir=default --notify=false;drush cr;chown -R www-data:www-data sites;\"\n</code></pre> <p>Note: You will see these warnings:</p> <p><code>[warning] The \"block_content:9aa72fb1-2817-44a7-8fb5-a3eb51166e83\" was not found</code> <code>[warning] The \"block_content:1cdf7155-eb60-4f27-9e5e-64fffe93127a\" was not found</code> <code>[warning] The \"facets_summary_block:advance\" was not found</code> <code>[warning] The \"facets_summary_block:search_page_facet_summary\" was not found</code></p> <p>Nothing to worry about. We will provide the missing part in Step 5.</p> <p>Note 2: Please be patient. This step takes now 25-30% longer because of how the most recent Drupal Installation code fetches translations and other resources (see <code>Performed install task</code>). This means progress might look like getting \"stuck\", go and get a coffee/tea and let it run to the end.</p> <p>Once finished, this will give you an <code>admin</code> Drupal user with <code>archipelago</code> as password (change this if running on a public instance!) and also set the right Docker Container owner for your Drupal installation files.</p> <p>Final note about Steps 2-3: You don't need to, nor should you do this more than once. You can destroy/stop/update, recreate your Docker containers, and start again (<code>git pull</code>), and your Drupal and Data will persist once you've passed the <code>Installation complete</code> message. I repeat, all other containers' data is persisted inside the <code>persistent/</code> folder contained in this cloned git repository. Drupal and all its code is visible, editable, and stable inside your <code>web/</code> folder.</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#step-4-create-a-demo-and-a-jsonapi-user-using-drush-and-assign-your-admin-user-the-administrator-role","title":"Step 4: Create a \"demo \"and a \"jsonapi\" user using drush and assign your \"admin\" user the Administrator Role.","text":"<p><pre><code>docker exec -ti esmero-php bash -c 'drush ucrt demo --password=\"demo\"; drush urol metadata_pro \"demo\"'\n</code></pre> <pre><code>docker exec -ti esmero-php bash -c 'drush ucrt jsonapi --password=\"jsonapi\"; drush urol metadata_api \"jsonapi\"'\n</code></pre> <pre><code>docker exec -ti esmero-php bash -c 'drush urol administrator \"admin\"'\n</code></pre></p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#step-5-ingest-some-metadata-displays-to-make-playing-much-more-interactive","title":"Step 5: Ingest some Metadata Displays to make playing much more interactive","text":"<p>Archipelago is more fun without having to start writing Metadata Displays (in Twig) before you know what they actually are. Since you should now have a <code>jsonapi</code> user and jsonapi should be enabled, you can use that awesome functionality of D8 to get that done. We have 4 demo Metadata Display Entities that go well with the demo Webform we provided. To do that execute in your shell (copy and paste):</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/deploy.sh'\n</code></pre> <p>You are done! Open your most loved Web Browser and point it to <code>http://localhost:8001</code></p> <p>Note: It can take some time to start the first time (Drupal needs some warming up). The Ubuntu deployment is WAY faster than the OSX deployment because of the way the bind mount volumes are handled by the driver. Our experience is that Archipelago basically reacts instantly!</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#step-6-optional-but-more-fun-if-you-add-content","title":"Step 6: Optional but more fun if you add content","text":"<p>One-Step Demo content ingest</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#need-help-blue-screen-missed-a-step-need-a-hug","title":"Need help? Blue Screen? Missed a step? Need a hug?","text":"<p>If you see any issues or errors or need help with a step, please let us know (ASAP!). You can either open an <code>issue</code> in this repository or use the Google Group. We are here to help.</p> <p>If you like this, let us know!</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#user-contributed-documentation-a-video1","title":"User contributed documentation (A Video!)[^1]:","text":"<p>Installing Archipelago on AWS Ubuntu by Zach Spalding: https://youtu.be/RBy7UMxSmyQ</p> <p>[^1]: You may find this user contributed tutorial video, which was created for an earlier Archipelago release, to be helpful. Please note that there are significant differences between the executed steps and that you need to follow the current release instructions in order to have a successful deployment.</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Sherrick</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-windows/","title":"Installing Archipelago Drupal 10 on Windows 10/11","text":"","tags":["Archipelago-deployment","Drupal 10","Windows","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-windows/#prerequisites","title":"Prerequisites","text":"<ul> <li>Windows 11 64-bit: Home or Pro version 21H2 or higher, or Enterprise or Education version 21H2 or higher (see Docker for Windows link below).</li> <li>Windows 10 64-bit: Home or Pro 21H1 (build 19043) or higher, or Enterprise or Education 20H2 (build 19042) or higher (see Docker for Windows link below).</li> <li>Install Ubuntu on WSL 2</li> <li>Install Docker for Windows</li> <li>Install Github for Desktop</li> <li>At least 10 Gbytes of free space (to get started)</li> <li>Some basic Unix/Terminal Skills</li> <li>4 Gbytes of RAM (8 Recommended)</li> </ul>","tags":["Archipelago-deployment","Drupal 10","Windows","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-windows/#optional","title":"Optional","text":"<ul> <li>Install Github Desktop. Ubuntu comes with Git by default, but for a more full-featured way to work with Github, you can install this application.</li> </ul> <p>Open the Docker Desktop app. The Docker service should start up automatically with a status showing when the service is up and running.</p> <p>Open an Ubuntu Terminal session (type <code>Ubuntu</code> in the Windows Start menu).</p> <p>Bring everything up to date: <code>sudo apt update &amp;&amp; sudo apt upgrade -y</code></p>","tags":["Archipelago-deployment","Drupal 10","Windows","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-windows/#deployment","title":"Deployment","text":"<p>Follow the steps for deployment in Ubuntu.</p>","tags":["Archipelago-deployment","Drupal 10","Windows","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-windows/#acknowledgment","title":"Acknowledgment","text":"<p>Thanks to Corinne Chatnik for documenting these steps!</p>","tags":["Archipelago-deployment","Drupal 10","Windows","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-windows/#need-help-blue-screen-missed-a-step-need-a-hug","title":"Need help? Blue Screen? Missed a step? Need a hug?","text":"<p>If you see any issues or errors or need help with a step, please let us know (ASAP!). You can either open an <code>issue</code> in this repository or use the Google Group. We are here to help.</p> <p>If you like this, let us know!</p>","tags":["Archipelago-deployment","Drupal 10","Windows","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-windows/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Lund</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment","Drupal 10","Windows","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-windows/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment","Drupal 10","Windows","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-glossary/","title":"Archipelago Glossary of Terms","text":"<p>This documentation page is under construction, and will be availble soon. </p> <p>This page will provide a list for quick reference points for some of the most common Archipelago terminology you are likely to encounter.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"createdisplaymodes/","title":"Creating Display Modes for Archipelago Digital Objects","text":"<p>We recommend checking out our primer on Display Modes for a broader overview on Form Modes and View Modes for Archipelago Digital Objects (ADOs).</p> <p>But how do you create and enable these Display Modes in the first place? Let's find out.</p>"},{"location":"createdisplaymodes/#adding-a-new-form-mode","title":"Adding a new Form Mode","text":"<p>Why would you want to create a new form mode? One common reason is to create different data entry experiences for users with different roles. Let's create an example form mode called \"Student Webform\" --  we can imagine a deployment where Students need a simplified form for ADO creation. We are going to create a form mode, enable it for Digital Objects, and give it some custom settings that differentiate it from existing form modes.</p> <ol> <li> <p>Navigate to <code>yoursite/admin/structure/display-modes</code></p> <p></p> </li> <li> <p>Click on Form modes. This image shows the basic Form Modes shipped with Archipelago</p> <p></p> </li> <li> <p>Click the \"Add Form mode\" button at the top of the page. Then select the \"Content\" entity type from the list. In this example, we ultimately want the form mode to be applied to Archipelago Digital Objects, which is a Content entity type.</p> <p></p> </li> <li> <p>Enter the name of your Form Mode and hit save. Here we are entering \"Student Webform\".</p> <p></p> </li> <li> <p>Great. Now you will see your new Form mode in the list! Let's put it to use.</p> <p></p> </li> <li> <p>Head to <code>yoursite/admin/structure/types/manage/digital_object</code> and click the \"Manage Form Display\" tab. As mentioned above, in this example we want to add a new Form Mode for ADOs, so we are dealing with the Digital Object content type. Scroll to the bottom of this page and look for the \"Custom Display Settings\" area, which is collapsed by default. Expand it, and you should see this.</p> <p></p> </li> <li> <p>Enable \"Student Webform\" and hit save! Now scroll back up the page. You'll see it enabled like so.</p> <p></p> </li> <li> <p>Now select our new \"Student Webform\" tab. From here, you have many options and can configure input fields as you see fit! To finish out our specific example though, let's finally add our Student Webform to the display. Click on the settings gear icon next to the Descriptive Metadata field.</p> <p></p> <p>You'll see that the default webform named \"Descriptive Metadata\" is entered. To add custom content to this Field Widget, start typing in the autocomplete. This example assumes you've created a webform called <code>Student Webform</code> in <code>yoursite/admin/structure/webform</code>. For info on how to create a new Webform with proper settings, see our Webforms as input guide.</p> </li> <li> <p>After you've selected your \"Student Webform\" in the Field Widget setting, hit Update, and then Save at the bottom of the page.</p> </li> </ol> <p>All done! So let's recap. We created a new form mode. We added this form mode to the Manage Form Display &gt; Custom Display Settings options for Digital Objects. And finally we configured the Field Widget for Descriptive Metadata in our new Form Mode to use a new Webform. This last step is arbitrary to this example. We could have enabled or disabled fields, or changed other field widget settings depending on our needs. But configuring different Webforms as Field Widgets for Descriptive Metadata is a common use case in Archipelago.</p> <p>Thanks for reading this far! But there is more. We might want to display, in addition to ingest, our ADOs in custom ways. The process for creating new View Modes (the other type of Display Mode) is quite similar to creating new Form Modes, but let's walk through it with another example case.</p>"},{"location":"createdisplaymodes/#adding-a-new-view-mode","title":"Adding a new View Mode","text":"<p>Why would you want to create a new View Mode? Maybe there is a new type of media you are attaching to ADOs that you want to display using the proper player or tool. Or maybe you want to simplify the ADO display, removing fields from the display page. In this example let's create a new View Mode for ADOs that adds some fields to the display to show the Author and Published date of the object.</p> <ol> <li> <p>Navigate to <code>yoursite/admin/structure/display-modes</code></p> <p></p> </li> <li> <p>Select View modes, and click the \"Add View mode\" at the top of the page.</p> <p></p> </li> <li> <p>Select Content as your entity type.</p> <p></p> </li> <li> <p>Enter the name of your new View Mode and save. Ours is \"Digital Object with Publishing Information\"</p> <p></p> </li> <li> <p>Now let's enable this View mode. Go to <code>yoursite/admin/structure/types/manage/digital_object</code> and click the \"Manage Display\" tab.</p> </li> <li> <p>Scroll to the bottom of the page and expand the \"Custom Display Settings\" area. You will see our newly created View Mode. Enable it and hit save.</p> <p></p> </li> <li> <p>Now scroll back to the page top. You will see \"Digital Object with Publishing Information\" in the list of View Modes, so go ahead and select it.</p> <p></p> </li> <li> <p>Scroll down until you see the \"Disabled\" section. This section contains fields that are available to the ADO content type, but are not enabled in this display mode. Let's enable Author and Post date by changing the \"Region\" column dropdown from \"Disabled\" to \"Content\". (To learn more about Regions in Drupal, see here). Basically, this ensures that this field has a home in the page layout. Hit save.</p> <p></p> <p></p> </li> <li> <p>Now, if you want ADOs to use this View Mode for display, there is one last step. You need to select \"Digital Object with Publishing Info\" as the view mode Display Settings when adding new content. This area is located on the right side of the page. See below:</p> <p></p> </li> <li> <p>Now, when we view the individual ADO, these new fields have been added to the display.</p> <p></p> </li> </ol> <p>All done! This was quite a simple example, but now you are aware of how to customize your own ADO display. It can only get more complex and exciting from here.</p> <p>Let's recap. We created a new View Mode. We enabled this View Mode in Manage Display &gt; Custom Display Settings for Digital Objects. We enabled new fields (in this case, just for instruction, the Author and Post date fields) to make our new View Mode unique, and learned about Disabled fields in the process. We selected our new View Mode in the Display Settings area (slightly confusing wording because yes, this is a View Mode, subset of Display Mode) during ADO creation (for more on creating new objects, see this guide).</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"customwebformelements/","title":"Archipelago Custom Webform Elements","text":"<p>In addition to the core elements provided by the Drupal Webform module, Archipelago also deploys a robust set of custom webform elements specific to digital repositories metadata needs and use cases.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"customwebformelements/#linked-data","title":"Linked Data:","text":"<p>(*found under Composite Elements in \"Add Element\" menu)</p> <ul> <li> <p>Library of Congress (LoC) Linked Open data</p> <ul> <li>Provides a form element to reconciliate against LoC Headings and similar LoD Sources</li> <li>Able to configure which LoC Autocomplete Source Provider is used:<ul> <li>Subjects (LCSH)</li> <li>LC Name Authority File (LCNAF)</li> <li>LC Genre/Form Terms (LCGFT</li> <li>Thesaurus of Graphic Materials (TGN)</li> <li>MARC list for Geographic areas</li> <li>Filter Suggest results by RDF Type<ul> <li>Any of the Classes listed here: https://id.loc.gov/ontologies/madsrdf/v1.html</li> </ul> </li> </ul> </li> </ul> </li> <li> <p>Multi LoD Source Agent Items</p> <ul> <li>Provides a form element to reconciliate Agents against Multiple sources of Agents</li> </ul> </li> <li> <p>Wikidata</p> <ul> <li>Provides a form element to reconciliate against Wikidata Items and Agents<ul> <li>via Wikidata Action API</li> </ul> </li> </ul> </li> <li> <p>Getty Vocabulary Term</p> <ul> <li>Provides a form element to reconciliate against the Getty Vocabularies</li> </ul> </li> <li> <p>VIAF</p> <ul> <li>Provides a form element to reconciliate against VIAF (OCLC) Items</li> </ul> </li> <li> <p>Location GEOJSON (Nominatim--Open Street Maps)</p> <ul> <li>Provides a form element to collect valid location information (address, longitude, latitude, geolocation) using Nominatim/Openstreetmap open API</li> </ul> </li> <li> <p>PubMed MeSH Suggest</p> <ul> <li>Provides a form element to reconciliate against PubMed MeSH</li> </ul> </li> <li> <p>SNAC Constellation Linked Open Data</p> <ul> <li>Provides a form element to reconciliate against Social Networks and Archival Context (SNAC)</li> </ul> </li> <li> <p>Europeana Entity Suggest</p> <ul> <li>Provides a form element to reconciliate against Europeana Entity</li> </ul> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"customwebformelements/#customlocal-webform-lod-elements","title":"Custom/Local Webform LoD Elements","text":"<p>(*also found under Composite Elements in \"Add Element\" menu)</p> <ul> <li> <p>Webform Options LoD suggest</p> <ul> <li>Provides a form element autocomplete labels/urls(values) from Webform Options.</li> </ul> </li> <li> <p>Webform LoD from CSV attached to an ADO suggest</p> <ul> <li>Provides a form element autocomplete labels/urls(values) from a CSV attached to a Digital Object.</li> <li>Documentation Guide found here: Using Archipelago's 'Webform LoD from CSV attached to an ADO suggest'</li> </ul> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"customwebformelements/#file-upload-elements","title":"File Upload Elements:","text":"<ul> <li> <p>Enhancements for Audio, Document, Image, Video file uploads</p> <ul> <li>Backend processing for technical metadata (such as pronom, exif extraction)</li> </ul> </li> <li> <p>Import Metadata from a File (such as XML)</p> <ul> <li>Provides a form element for uploading, saving a file and parsing the content as metadata/webform submission data.</li> </ul> </li> <li> <p>Import Metadata in CSV format from a File</p> <ul> <li>Provides a form element for uploading, saving a file and parsing the content as metadata/webform submission data.</li> </ul> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"customwebformelements/#datetime-elements","title":"Date/Time Elements:","text":"<ul> <li>Multi Format Date and Date Range<ul> <li>Provides a form element for setting/reading Dates indifferent formats suitable for metadata.</li> <li>Optional EDTF Validation. </li> </ul> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"customwebformelements/#composite-elements","title":"Composite Elements:","text":"<ul> <li>Panorama Tour Builder<ul> <li>Provides a form element to build multi-scene Panorama Tour Builder with hotspots</li> </ul> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"customwebformelements/#computed-elements","title":"Computed Elements:","text":"<ul> <li> <p>Computed Metadata Transplant</p> <ul> <li>Provides an item that takes source values (not only elements) and distributes them into other places/elements via a twig template</li> </ul> </li> <li> <p>Computed Token</p> <ul> <li>Provides an item to display computed webform submission values using tokens.</li> </ul> </li> <li> <p>Computed Twig  </p> <ul> <li>Provides an item to display computed webform submission values using Twig</li> </ul> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"customwebformelements/#but-wait-theres-more","title":"But wait there's more!","text":"<p>You can review the coding behind these custom elements here: https://github.com/esmero/webform_strawberryfield/tree/1.3.0/src/Element</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"devops/","title":"Archipelago Software Services","text":"<p>At the core of the Archipelago philosophy is our commitment to both simplicity and flexibility.</p>"},{"location":"devops/#under-the-hood-archipelagos-architecture-is","title":"Under the hood, Archipelago's architecture is:","text":"<ul> <li>Drupal (CMS)</li> <li>Solr (Search)</li> <li>Cantaloupe (Image Server)</li> <li>S3 Storage (Mini.io or any other S3 flavor)</li> <li>NLP + ML Server</li> </ul> <p>Installation is entirely Dockerized and scripted with easy-to-follow directions.</p> <ul> <li>Docker containers are as follows:</li> </ul> Container Purpose Description esmero-web NGNIX Routes calls to esmero-php esmero-php PHP-FPM Has all binaries for postprocessing/exif/ocr/etc. Runs PHP code. esmero-db Database AMD and INTEL processors: MYSQL 8ARM processors: MariaDB esmero-minio Storage S3 API compatible Backend file and ADO as file storage. In a local it will do all the S3 stuff, on a live instance it can server as file routing to AWS S3, Azure Blob Storage, etc. esmero-solr Solr Currently version 9.1.1 esmero-nlp Natural Language Processing NLP64 server for entity extraction, language detection, ML, etc. esmero-cantaloupe IIIF Media Server Customized Server used to provide IIIF Image API/Media access <p>_Information related to non-Dockerized installation and configruation can be found here: Traditional Installation Notes</p>"},{"location":"devops/#strawberryfield-modules-at-the-heart-of-every-archipelago","title":"Strawberryfield Modules at the heart of every Archipelago:","text":"<ul> <li>Strawberryfield</li> <li>Format Strawberryfield</li> <li>Webform Strawberryfield</li> <li>Strawberry Runners</li> <li>Archipelago Multi-Importer (AMI)</li> </ul> <p>Documentation related to the Strawberryfield modules can be found here: Strawberryfields Forever</p>"},{"location":"devops/#archipelago-also-extends-these-powerful-tools","title":"Archipelago also extends these powerful tools:","text":"<ul> <li>Annotorius</li> <li>Drupal Webform Module</li> <li>International Image Interoperability Framework (IIIF)</li> <li>Internet Archive BookReader</li> <li>Mirador</li> <li>OpenSeadragon</li> <li>Pannellum</li> <li>Replayweb.page Webarchive Player</li> <li>Solr OCR Highlighting</li> <li>Twig Templating In Symfony / In Drupal</li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"documentation_about/","title":"About This Documentation","text":"<p>Documentation is vital to our community, and contributions are welcome, greatly appreciated, and encouraged. </p>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_about/#how-to-contribute","title":"How to Contribute","text":"<p>Difficulty Level: Moderate\u2013Difficult</p> <ol> <li>Follow the GitHub Workflow.</li> <li>Check out the examples of additional features.</li> </ol>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_features/","title":"Additional Documentation Features","text":"<p>Below are some examples of features that are currently in use on the site. To explore more visit the Material for MkDocs documentation.</p>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_features/#examples","title":"Examples","text":"","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_features/#images","title":"Images","text":"<p>Images are located in the <code>docs/images</code> folder. You can add new ones there and link to them by relative path. For example, if you added <code>strawberries_color.png</code>, you would embed it like so:</p> <p>Image</p> MarkupResult <pre><code>![New Documentation Image](images/strawberries_color.png)\n</code></pre> <p></p>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_features/#admonitions","title":"Admonitions","text":"<p>Question Admonition</p> MarkupResult <pre><code>??? question \"What is a collapsible admonition?\"\n\n    This is a collapsible admonition. It can have a title, and it collapses so as not to interrupt the flow the of the document, but it provides useful information as needed.\n</code></pre> What is a collapsible admonition? <p>This is a collapsible admonition. It can have a title, and it collapses so as not to interrupt the flow the of the document, but it provides useful information as needed.</p> <p>You can read more about admonitions with further examples in the Material for MkDocs documentation.</p>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_features/#code-blocks","title":"Code blocks","text":"<p>Code block with title and highlighted lines</p> MarkupResult <pre><code>```html+twig title=\"HTML in a TWIG template\" hl_lines=\"8 9 10\"\n{% for image in attribute(data, 'as:image')|slice(0,1) %}\n {% if image[\"flv:exif\"] %}\n   {% set height = image[\"flv:exif\"].ImageHeight%}\n {% else  %}\n   {% set width = 1200 %}\n {% endif %}\n   {% set imageurl =  IIIFserverurl ~ image['url']|replace({'s3://':''})|replace({'private://':''})|url_encode %}\n&lt;a href=\"{{ nodeurl }}\" title=\"{{ data.label }}\" alt=\"{{ data.label }}\"&gt;\n&lt;img src=\"{{ imageurl }}/pct:5,5,90,30/,400/0/default.jpg\" class=\"d-block.w-auto\" alt=\"{{ image.name }}\" height=\"400px\" style=\"min-width:1200px\"&gt;\n&lt;/a&gt; \n{% endfor %}\n```\n</code></pre> HTML in a TWIG template<pre><code>{% for image in attribute(data, 'as:image')|slice(0,1) %}\n {% if image[\"flv:exif\"] %}\n   {% set height = image[\"flv:exif\"].ImageHeight%}\n {% else  %}\n   {% set width = 1200 %}\n {% endif %}\n   {% set imageurl =  IIIFserverurl ~ image['url']|replace({'s3://':''})|replace({'private://':''})|url_encode %}\n&lt;a href=\"{{ nodeurl }}\" title=\"{{ data.label }}\" alt=\"{{ data.label }}\"&gt;\n&lt;img src=\"{{ imageurl }}/pct:5,5,90,30/,400/0/default.jpg\" class=\"d-block.w-auto\" alt=\"{{ image.name }}\" height=\"400px\" style=\"min-width:1200px\"&gt;\n&lt;/a&gt; \n{% endfor %}\n</code></pre>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_features/#quirks","title":"Quirks","text":"<p>Because of the use of front matter (the block of YAML at the top that contains settings and data for the file) the markup for a horizontal rule is restricted. To create one you have to use the following:</p> <p>Horizontal Rule</p> MarkupResult <pre><code>___\n</code></pre> <p>Info</p> <p>The above are underscore (<code>_</code>) characters, as opposed to hyphens (<code>-</code>).</p> <p>Some of the documentation that is automatically deployed from the repos have special comments that are converted to theme-specific elements via script.</p> <p>Front Matter</p> Deployment Repo with Front MatterDocumentation Repo with Front Matter <pre><code>&lt;!--documentation\n---\ntitle: \"Adding Demo Archipelago Digital Objects (ADOs) to your Repository\"\ntags:\n  - Archipelago Digital Objects\n  - Demo Content\n---\ndocumentation--&gt;\n</code></pre> <pre><code>---\ntitle: \"Adding Demo Archipelago Digital Objects (ADOs) to your Repository\"\ntags:\n  - Archipelago Digital Objects\n  - Demo Content\n---\n</code></pre> <p>Switching Elements</p> Deployment Repo with Theme-specific MarkupDocumentation Repo with Theme-specific Markup <pre><code>&lt;!--switch_below\n\n??? info \"OSX (macOS)/x86-64\"\n\n    ```shell\n    cp docker-compose-osx.yml docker-compose.yml\n    ```\n\n??? info \"Linux/x86-64/AMD64\"\n\n    ```shell\n    cp docker-compose-linux.yml docker-compose.yml\n    ```\n\n??? info \"OSX (macOS)/Linux/ARM64\"\n\n    ```shell\n    cp docker-compose-arm64.yml docker-compose.yml\n    ```\n\nswitch_below--&gt;\n\n___\n\nOSX (macOS)/x86-64:\n\n```shell\ncp docker-compose-osx.yml docker-compose.yml\n```\n\n___\n\nLinux/x86-64/AMD64:\n\n```shell\ncp docker-compose-linux.yml docker-compose.yml\n```\n\n___\n\nOSX (macOS)/Linux/ARM64:\n\n```shell\ncp docker-compose-arm64.yml docker-compose.yml\n```\n\n___\n\n&lt;!--switch_above\nswitch_above--&gt;\n</code></pre> <pre><code>??? info \"OSX (macOS)/x86-64\"\n\n    ```shell\n    cp docker-compose-osx.yml docker-compose.yml\n    ```\n\n??? info \"Linux/x86-64/AMD64\"\n\n    ```shell\n    cp docker-compose-linux.yml docker-compose.yml\n    ```\n\n??? info \"OSX (macOS)/Linux/ARM64\"\n\n    ```shell\n    cp docker-compose-arm64.yml docker-compose.yml\n    ```\n\n&lt;!--repo_docs\n\n___\n\nOSX (macOS)/x86-64:\n\n```shell\ncp docker-compose-osx.yml docker-compose.yml\n```\n\n___\n\nLinux/x86-64/AMD64:\n\n```shell\ncp docker-compose-linux.yml docker-compose.yml\n```\n\n___\n\nOSX (macOS)/Linux/ARM64:\n\n```shell\ncp docker-compose-arm64.yml docker-compose.yml\n```\n\n___\n\nrepo_docs--&gt;\n</code></pre>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_technical/","title":"Documentation Technical Details","text":"<p>Archipelago documentation is generated using the following open source projects:</p> <ul> <li>MkDocs generates the static site</li> <li>Material for MkDocs provides the site's theme</li> <li>mike generates the routing for versions</li> <li>Github Actions builds and deploys the files to Github Pages</li> </ul> <p>To use any advanced features not mentioned in these pages, you can look through the documentation for each of the above projects.</p> <p>In addition to the pages added directly via this repository, there are some pages automatically deployed here with GitHub Actions from the following repositories:</p> <ul> <li>https://github.com/esmero/archipelago-deployment</li> <li>https://github.com/esmero/archipelago-deployment-live</li> </ul> <p>Both the main READMEs and documentation in the <code>docs</code> folders for those repositories are prepended with <code>archipelago-deployment</code> and <code>archipelago-deployment-live</code> respectively and copied to the <code>docs</code> folder here with the rest of the documentation. In practice that means those pieces of documentation need to be edited in those repositories directly.</p>","tags":["Documentation"]},{"location":"documentation_template/","title":"Example Documentation Page Title","text":"<p>A brief overview of the specific functionality or workflow area that will be covered in your documentation.</p>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_template/#stepsguides","title":"Steps/Guides","text":"<ol> <li>Step one example.</li> <li> <p>Step two example with images:</p> <p></p> </li> <li> <p>Step three example with Details section:</p> Click to open this Details Section <p>More Details in a List</p> <ul> <li>Apples</li> <li>Oranges</li> <li>Peaches</li> <li>Pumpkins</li> </ul> </li> <li> <p>Step four example with a simple code block:</p> <pre><code>{% for image in attribute(data, 'as:image')|slice(0,1) %}\n {% if image[\"flv:exif\"] %}\n   {% set height = image[\"flv:exif\"].ImageHeight%}\n {% else  %}\n   {% set width = 1200 %}\n {% endif %}\n   {% set imageurl =  IIIFserverurl ~ image['url']|replace({'s3://':''})|replace({'private://':''})|url_encode %}\n&lt;a href=\"{{ nodeurl }}\" title=\"{{ data.label }}\" alt=\"{{ data.label }}\"&gt;\n&lt;img src=\"{{ imageurl }}/pct:5,5,90,30/,400/0/default.jpg\" class=\"d-block.w-auto\" alt=\"{{ image.name }}\" height=\"400px\" style=\"min-width:1200px\"&gt;\n&lt;/a&gt; \n{% endfor %}\n</code></pre> </li> <li> <p>Step five example with a code block that has a title and highlighted lines:</p> HTML in a TWIG template<pre><code>{% for image in attribute(data, 'as:image')|slice(0,1) %}\n {% if image[\"flv:exif\"] %}\n   {% set height = image[\"flv:exif\"].ImageHeight%}\n {% else  %}\n   {% set width = 1200 %}\n {% endif %}\n   {% set imageurl =  IIIFserverurl ~ image['url']|replace({'s3://':''})|replace({'private://':''})|url_encode %}\n&lt;a href=\"{{ nodeurl }}\" title=\"{{ data.label }}\" alt=\"{{ data.label }}\"&gt;\n&lt;img src=\"{{ imageurl }}/pct:5,5,90,30/,400/0/default.jpg\" class=\"d-block.w-auto\" alt=\"{{ image.name }}\" height=\"400px\" style=\"min-width:1200px\"&gt;\n&lt;/a&gt; \n{% endfor %}\n</code></pre> </li> <li> <p>Last Step Example. </p> </li> </ol> <p>Congratulations! \ud83c\udf89</p>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_workflow/","title":"GitHub Workflow","text":"<ol> <li>Find an issue to resolve or create a new issue here.</li> <li>Fork the documentation repo. If you're not familiar with forking see this guide.</li> <li>Create an issue branch in your forked repo. For example, if the issue you're resolving is ISSUE-100:    <pre><code>git checkout -b ISSUE-100\n</code></pre></li> <li>Copy this template to create a new piece of documentation:    <pre><code>cp docs/documentation_template.md docs/new_documentation.md\n</code></pre></li> <li>Make your changes to the copied markdown file.</li> <li>If this is new documentation add it to the <code>nav</code> section of the <code>mkdocs.yml</code> configuration file at the root of the repo. For example:    <pre><code>nav:\n  - Home: index.md\n  - About Archipelago:\n    - Archipelago's Philosophy &amp; Guiding Principles: ourtake.md\n    - Strawberryfields Forever: strawberryfields.md\n    - Software Services: devops.md\n    - New Documentation: new_documentation.md\n  - Code of Conduct: CODE_OF_CONDUCT.md\n  - Instructions and Guides:\n    - Archipelago-Deployment:\n      - Start: archipelago-deployment-readme.md\n      - Installing Archipelago Drupal 9 on OSX (macOS): archipelago-deployment-osx.md\n      - Installing Archipelago Drupal 9 on Ubuntu 18.04 or 20.04: archipelago-deployment-ubuntu.md\n      - Installing Archipelago Drupal 9 on Windows 10/11: archipelago-deployment-windows.md\n      - Adding Demo Archipelago Digital Objects (ADOs) to your Repository: archipelago-deployment-democontent.md\n...\n</code></pre></li> <li> <p>To view the changes locally, first install the Python libraries using the Python package manager pip:    <pre><code>pip install mkdocs-material mike git+https://github.com/jldiaz/mkdocs-plugin-tags.git mkdocs-git-revision-date-localized-plugin mkdocs-glightbox\n</code></pre>    You may need to install Python on your machine. Download Python or use your favorite operating system package manager such as Homebrew. </p> </li> <li> <p>Now you can build the site locally, e.g. for the documentation using the 1.4.0 branch:    <pre><code>mike deploy 1.4.0\nmike set-default 1.4.0\n</code></pre>    If you create a new branch to match the issue number as in step 3, you would use your branch instead of 1.4.0. For example, a branch of ISSUE-129.    <pre><code>mike deploy ISSUE-129\nmike set-default ISSUE-129\n</code></pre></p> </li> <li>Start the web server:    <pre><code>mike serve\n</code></pre></li> <li>Check the results in your browser by going to: http://localhost:8000</li> <li>If everything looks good, you can push to your forked repo issue branch:     <pre><code>git add .\ngit commit -m \"Create new docs with useful information.\"\ngit push origin ISSUE-100\n</code></pre></li> <li>Create a pull request and link to the issue by tagging it, e.g. <code>Resolves #100</code>.</li> </ol>","tags":["Documentation","Contributing","Examples"]},{"location":"drupal_core_update/","title":"Updating Drupal Core in Archipelago","text":"<p>Drupal, the project, puts out new core releases on a regular schedule. Your Archipelago site needs to apply the security updates and possibly minor releases between major core updates. Major core updates will typically coincide with an updated Archipelago stable release. </p> <p>Updating core is done via Composer:</p>","tags":["Archipelago-deployment","Archipelago-deployment-live","DevOps","Drupal","Drupal Core"]},{"location":"drupal_core_update/#stepsguides","title":"Steps/Guides","text":"<ol> <li>First you will want to verify Drupal Core will update itself and the dependencies. To do so, you run the following command:     <pre><code>docker exec -ti esmero-php bash -c \"composer update \"drupal/core-*:^9\" --with-all-dependencies --dry-run\n</code></pre>     The <code>--dry-run</code> flag will allow you to see what will be updated. Once you review the updates and are ready to go with the full update, you will run the same command without the <code>dry-run</code> flag.</li> <li>Update Core and the dependencies:     <pre><code>docker exec -ti esmero-php bash -c \"composer update \"drupal/core-*:^9\" --with-all-dependencies\"\n</code></pre></li> <li>Sometimes the database needs to be updated after the update of the files. The following command will prompt you to type yes if there are updates to be done, or it will not return anything if no updates are needed:     <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre></li> <li>Clear the Drupal cache:     <pre><code>docker exec -ti esmero-php bash -c \"drush cache:rebuild\"\n</code></pre> Occasionally there will be other Drupal modules that Archipelago uses, and they need to be updated at the same time you run a Core update. This is an example of updating Drupal Webform, which was required for moving to Drupal 9.5.x:</li> </ol> <p>Updating a Drupal module</p> <ol> <li>Use Composer to update a module to a specific release:     <pre><code>docker exec -ti esmero-php bash -c \"composer update \"drupal/webform:6.1.4\n</code></pre></li> <li>Check for database updates and apply them if necessary:     <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre>     or     <pre><code>docker exec -ti esmero-php bash -c \"drush updb\"\n</code></pre></li> <li>Clear the cache again:     <pre><code>docker exec -ti esmero-php bash -c \"drush cache:rebuild\"\n</code></pre>     or     <pre><code>docker exec -ti esmero-php bash -c \"drush cr\"\n</code></pre></li> </ol>","tags":["Archipelago-deployment","Archipelago-deployment-live","DevOps","Drupal","Drupal Core"]},{"location":"embargo/","title":"Embargo &amp; Access Restrictions in Archipelago","text":"<p>Archipelago features two primary different Metadata Based Embargo / Access Restriction options for limiting access to materials when needed.</p> <p>You can find the main Metadata Based Embargo settings configuration form at:</p> <ul> <li><code>admin/config/archipelago/metadatabased_Embargo</code></li> <li>Through the <code>Configuration</code> menu &gt; <code>Archipelago</code> &gt; <code>Metadata based Embargo settings</code></li> </ul> <p></p> <p>This form allows you to enable/disable Embargo functionality enforced at the Formatter level and configure on which JSON Key/values those will act.</p> <p>For either (or both) Embargo options, you will to select the checkbox for 'Is Embargo checking and enforcing globally active?'</p> <p>Important Note: Your Metadata Keys and Values Matter</p> <p>Please also keep in mind that the Embargo Options noted below need to draw the necessary related metadata values as defined in specific keys. You cannot specify to use either Embargo by Date or Embargo Bypass by IP without also defining the values in your Digital Object/Collection metadata.</p>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#option-1-embargo-by-date","title":"Option 1: Embargo by Date","text":"<p>The first Embargo option you have is to define a \"JSON key present in your metadata that contains an Embargo lift date that will be used to Embargo Metadata and Media.\"</p> <ul> <li>Archipelago recommended default key is 'date_embargo_lift'</li> </ul> <p>Functional mechanism for your specified key: </p> <ul> <li>value for future date in the 'date_embargo_lift' key (or what key you define)</li> <li>set at the individual digital object level</li> </ul> Example of JSON Metadata key and values for 'date_embargo_lift <pre><code>{\n    \"date_embargo_lift\": \"2044-08-23\"\n}\n</code></pre> <p>Functional results for objects with future dates in the 'date_embargo_lift' key: </p> <ul> <li>File media viewer does not display until the date set in the 'date_embargo_lift' key in the Digital Object's JSON metadata has passed.</li> <li>Metadata record displays as usual.</li> </ul>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#option-2-embargo-bypass-by-ip","title":"Option 2: Embargo Bypass by IP","text":"<p>The second Embargo option you have is to define a \"JSON key present in your metadata that contains an allowed to bypass Embargo through a visitor IP or IP range that will be used to Embargo Metadata and Media\".</p> <p>Functional mechanism for your specified key: </p> <ul> <li>value for individual IP address or IP ranges set in the 'embargo_ip_bypass' key (or what key you define)</li> <li>set at the individual digital object level</li> </ul> Example of JSON Metadata key and values for 'embargo_ip_bypass <pre><code>{\n    \"embargo_ip_bypass\": \"[\n    \"123.123.123.123\",\n    \"456.789.0.0\\/11\",\n    \"222.33.444.0\\/56\"\n    ],\n}\n</code></pre> <p>Functional results for objects with IP range/s specified in the 'embargo_ip_bypass' key: </p> <ul> <li>File media viewer does not display for individuals accessing the Digital Object from IP addresses not listed in the 'embargo_ip_bypass' key in </li> <li>Metadata record displays as usual.</li> <li>No caching for any objects with values for this 'embargo_ip_bypass' key</li> </ul>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#additional-configurations-needed-display-mode-formatters-and-twig-templates","title":"Additional Configurations Needed: Display Mode Formatters and Twig Templates","text":"<p>In order to enforce the Embargo options noted above, you have multiple options for configuration. You can use a singular or combination of the options described below, depending on your use cases and desired Embargo application. It is recommended that you at least apply the necessary Display Mode Formatter Configuration Options. You may also optionally wish to apply Twig Template changes noted further below.</p>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#display-mode-formatter-configuration-options","title":"Display Mode Formatter Configuration Options","text":"<p>First, to better understand this functionality area, please first familiarize yourself with Archipelago's Primer on Display Modes and Strawberryfield Formatters.</p> <p>For every Display Mode you have configured for your Digital Objects and Digital Object Collection (Compounds/Creative Work Series), you can configure Embargo settings within the corresponding Strawberryfield Formatter.</p>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#step-1-navigate-to-display-suites-manage-display-tab","title":"Step 1. Navigate to Display Suite's Manage Display Tab","text":"<ul> <li>For Digital Objects, found at <code>/admin/structure/types/manage/digital_object/display</code></li> <li>For Digital Object Collections (Compounds/Creative Work Series), found at <code>/admin/structure/types/manage/digital_object_collection/display</code></li> </ul>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#step-2-edit-the-strawberryfield-formatter-settings","title":"Step 2. Edit the StrawberryField Formatter Settings","text":"<ul> <li>For every Display Mode potentially impacted by Embargo restrictions, select the corresponding StrawberryField Formatter used to render/display the Media Files and Viewer.</li> </ul>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#step-3-setup-your-embargo-configurations","title":"Step 3. Setup your Embargo Configurations","text":"<p>Within the selected StrawberryField Formatter, navigate to the bottom section of the form where you can setup your desired Embargo options.</p> <p></p> <ul> <li> <p>Option to 'Hide the Viewer in the presence of an Embargo'. This is the simplest option for configuring an Embargo restriction. If you leave this option unchecked, acting on an Embargo will be delegated to the IIIF Manifest driving the viewer.</p> </li> <li> <p>Option will be to specify 'When Embargo is used or applied, alternate JSON Key(s) where the files to be used by this formatter where uploaded/store in your JSON.' </p> <ul> <li>Please be careful about providing the same keys used for users that can by pass an Embargo. You can add multiple ones (keys) separated by comma. Some viewers use multiple file types, such as audio files and text-based subtitle files, and in that case please add all of the corresponding file type keys. In case of multiple keys for the same type (e.g. \"audios1\", \"audios2\"), the key names will be also used for grouping. Leave empty to not provide any alternate Embargo option at all.</li> <li>This means you can add conditional logic to your IIIF Manifest (Twig Template) to do things such as \"return an empty manifest and skip the images\", or provide alternative images in case of an Embargo.</li> </ul> </li> <li> <p>Option to 'Use Global IIIF Urls'. This is enabled by default.</p> </li> </ul> <p>Depending on which Display Mode and particular StrawberryField Formatter you are editing, you may see additional options related to the IIIF Manifest.</p> <p></p> <ul> <li>If the particular viewer you are editing is setup to use an Exposed Metadata Endpoint, you can also simplify the response and use the option at the end--'Return 401 in case of an Embargo', removing your need of delegating the decision to twig logic. Archipelago will resolve the Embargo and if the user can't see it, will return an access denied (401 code).</li> <li>You can find the 'Exposed Metadata using Twig Configuration entities' at <code>/admin/config/archipelago/metadataexpose</code>.</li> </ul> <p></p> <ul> <li>If you decide not to code the your response to an Embargo on the IIIF manifest, you should at least enable the Formatter setting noted above to Hide the Viewer in the presence of an Embargo'.</li> </ul>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#twig-template-configuration-option","title":"Twig Template Configuration Option","text":"<p>Please first familiarize yourself with Twig Templates and Archipelago and Working With Twig in Archipelago.</p> <p>To apply either (or both) of the examples noted below, begin by navigating to the Metadata Display List in your Archipelago.</p> <ul> <li>Found at <code>/metadatadisplay/list</code> or <code>Content &gt; Metadata Displays</code></li> </ul> <p>You will most likely only need to apply Embargo-related conditional logic to the primary 'Object Description' template, depending on the particular Metadata Display Usage setup in your Archipelago.</p>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#object-description-template-example-1","title":"Object Description Template Example 1","text":"<p>In your selected Metadata Display Twig Template, such as the primary Object Description, navigate to the section of the template where the main metadata field display output begins. Add the following conditional statement before the output of other elements:</p> <pre><code>  {% if data_embargo.embargoed == true %}\n    &lt;h3&gt;Access to this Object has been restricted.&lt;/h3&gt;\n    &lt;p&gt;Please contact the Repository Administrator for any access questions or concerns.&lt;p&gt;\n  {% endif %}\n</code></pre> <p>Example output when applied within the default Archipelago Object Description Output --and paired with the Formatter Embargo Configuration noted above:</p> <p></p>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#object-description-template-example-2","title":"Object Description Template Example 2","text":"<p>The following example conditional logic is already provided in the default Archipelago Object Description template. To apply within your particular Object Description (or other) template, navigate to the section of the template where the file download section begins. Add the following conditional statement before the output of file download section:</p> <pre><code>{# -- Embargo option -- #} \n{% set file_download_restricted = false %} \n\n{% if data_embargo.embargoed == true %} \n   {# also restrict if Embargoed #} \n   {% set file_download_restricted = true %} \n{% endif %} \n</code></pre> <p>Example output when applied within the default Archipelago Object Description Output --and paired with the Formatter Embargo Configuration noted above (the Download menu tab does not show the media file downloand link):</p> <p></p>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#final-considerations","title":"Final Considerations","text":"<p>There is a specialized Permission found at <code>admin/people/permissions</code> under the Strawberry Metadata and Media Field Formatters section, which allows you to specify that certain User Roles can 'See Embargoed object metadata and assets'. Please note, this Permission is not ACL. Enforced Embargo configured JSON keys will not act on Formatters if a user has this enabled. If an Object is Embargoed this permission will allow any role with this assigned to bypass it.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"experimental_tools/","title":"Experimental Machine Learning (ML) Tools","text":"<p>Archipelago 1.4.0 Local Deployment Release was shipped with a set of experimental Machine Learning (ML) Tools for testing and assessing the applicable use of such tools/workflows within a general repository environment. These tools should not be considered 'final product' integrations, and should not be exposed to end users--there are intentional configurations in place that limit exposure of these endpoints to authenticated users only.</p> <p>Please review Allison &amp; Diego's recent 2024 Conference Presentations related to this important topic in our shared field for a fuller understanding of our team's ethical perspectives and the considerations we keep related to ML tools and applications.</p>","tags":["Experimental ML Tools","Machine Learning","ML"]},{"location":"experimental_tools/#whats-under-the-hood-powering-these-tools","title":"What's under the hood powering these tools?","text":"<ul> <li>New Archipelago ML supporting code (lots of maths!)</li> <li>New Strawberry Runners Post-Processing pipelines for Image and Text Vectorization</li> <li>New Solr Fields for storing the vectorized image or text fragments</li> <li>New Views and contextual and exposed filters for enabling search and results from Solr</li> <li>New UI Interfaces for interacting with these tools</li> <li>New <code>nlp</code> Docker Container (<code>esmero/esmero-nlp:1.4.1-arm64</code> or <code>esmero/esmero-nlp:1.4.1-amd64</code>)</li> </ul> <p>All of the above items, except for the new <code>nlp</code> Docker Container, are available out-of-the-box in default local deployment configurations. However, to use the tools you will first need to switch the <code>nlp</code> Docker Container being used from the default <code>esmero/esmero-nlp:fasttext-multiarch</code> to <code>esmero/esmero-nlp:1.4.1-arm64</code> or <code>esmero/esmero-nlp:1.4.1-amd64</code> in your <code>docker-compose.yml</code> file.</p>","tags":["Experimental ML Tools","Machine Learning","ML"]},{"location":"experimental_tools/#enabling-the-new-nlp-container-triggering-related-post-processing","title":"Enabling the new <code>nlp</code> container &amp; triggering related post-processing","text":"<ol> <li>In your <code>archipelago-deployment</code> folder, navigate to and open the <code>docker-compose.yml</code> file.</li> <li>Comment out (add # at the beggining of) line 76 for <code>image: \"esmero/esmero-nlp:fasttext-multiarch\"</code></li> <li>Remove the comment (# at the begnning) for line 78 for <code>image: \"esmero/esmero-nlp:1.4.1-arm64\"</code> OR <code>image: \"esmero/esmero-nlp:1.4.1-amd64\"</code> (depending on your particular computer platform CPU type).</li> <li>Save your changes.</li> <li> <p>In your terminal, while still in your <code>archipelago-deployment</code> folder, run: <pre><code>docker-compose pull\n</code></pre></p> </li> <li> <p>After the new <code>nlp</code> container finishes pulling and downloading, run: <pre><code>docker-compose up -d\ndocker ps\n</code></pre> Then check that the new <code>nlp</code>container now shows one of <code>esmero/esmero-nlp:1.4.1-arm64</code> or <code>esmero/esmero-nlp:1.4.1-amd64</code> depending on your Platform/CPU under IMAGE.</p> </li> <li> <p>While logged in as an adminsitrator, use Archipelago's Advanced Batch Find and Replace to select all of the objects ingested into your repository and run the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code> Action. If you have not yet ingested any objects into your Archipelago, you can skip this and proceed to the next step. </p> </li> <li>Wait for the Background Post-processing Queues and your Solr Index to finish updating after you have either run the AMI Demo Set and/or ingested objects through the webforms/your own AMI set(s). Feel free to give the Cron a push to start ahead of the default 3 hour schedule. Please be patient with this post-processing and indexing process, as the ML parts of this process (image and text extraction, vector generation) can take some time to complete.</li> </ol> <p>When all of the above steps are completed, you will be ready to start using the following experimental tools.</p>","tags":["Experimental ML Tools","Machine Learning","ML"]},{"location":"experimental_tools/#experimental-image-ml-comparison-integrated","title":"Experimental Image ML comparison (integrated)","text":"<p>As a logged in user, you can find the 'Image KNN Similarity Search' tool:</p> <ul> <li>Through the <code>Tools</code> menu &gt; <code>ML Image Similarity Search</code></li> <li>Directly at <code>/search_ml_images</code> </li> </ul> <p>Uses these pre-trained models:</p> <ul> <li>YOLO v8 (Structural/Layout)</li> <li>MobileNet V3 (Pattern, scale agnostic)</li> <li>InsightFace (Face feature encoding)</li> </ul> <p>Also uses IIIF Content Search API Object detection of Integrated Annotations</p>","tags":["Experimental ML Tools","Machine Learning","ML"]},{"location":"experimental_tools/#how-could-i-test-this","title":"How could I test this?","text":"<p>Conduct a search for an image in your Archipelago repository, then click on one of the images found in the results set. In the section below the top search results, you can review the output of the Image Similarity searches. You could also select a particular image Annotation to use for the Image Similarity search and comparison.</p>","tags":["Experimental ML Tools","Machine Learning","ML"]},{"location":"experimental_tools/#experimental-text-similarity-search","title":"Experimental Text Similarity Search","text":"<p>As a logged in user, you can find the 'Sbert KNN search on OCR-ed Pages Content' tool:</p> <ul> <li>Through the <code>Tools</code> menu &gt; <code>ML Text Similarity Search</code></li> <li>Directly at <code>/search_sbert/</code> </li> </ul> <p>Uses SBert/384 Vector Size embedding with assigned Task on short sentences over uncorrected OCR</p>","tags":["Experimental ML Tools","Machine Learning","ML"]},{"location":"experimental_tools/#how-could-i-test-this_1","title":"How could I test this?","text":"<p>Conduct a search for a particular text fragment, written in plain language style, such as \"Show me resources related to dogs\". In the section below the text-based search, you can review the output of the Text Similarity searches. You will only see matches against documents that have OCR values.</p>","tags":["Experimental ML Tools","Machine Learning","ML"]},{"location":"experimental_tools/#into-the-future","title":"Into the future","text":"<p>We would like to reiterate that these are early explorations of ML tools within the Archipelago repository architecture. Expect changes, enhancements, and even removal of certain aspects of these experimental tools in future releases. </p> <p>__</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Experimental ML Tools","Machine Learning","ML"]},{"location":"export-to-csv/","title":"Export ADOs to CSV Action","text":"<p>Archipelago has a specialty Action, labeled <code>Export Archipelago Digital Objects to CSV content item</code>, which enables you to generate a CSV file for a selected group of Archipelago Digital Objects (ADOs). </p>"},{"location":"export-to-csv/#where-to-find","title":"Where to Find","text":"<p>In default Archipelago deployments, this speciality action is found in the <code>Action</code> menu for the main Content page:</p> <ul> <li>Accessed through the <code>Content</code> menu item</li> <li>Directly at <code>/admin/content</code></li> </ul> <p></p> <p>And in the <code>Action</code> menu for the Advanced Batch Find and Replace view:</p> <ul> <li>Accessed through the <code>Tools</code> menu &gt; <code>Advanced Batch Find and Replace</code> </li> <li>Directly at <code>/search-and-replace</code></li> </ul>"},{"location":"export-to-csv/#export-ados-to-csv-configuration-options","title":"Export ADOs to CSV Configuration Options","text":"<p>After selecting a group of ADOs through either the 'Content' or 'Find and Replace' pages, you will have the following options available for using this speciality Action.</p> <ol> <li>Expand related ADOs to UUIDs: when enabled, all related ADOs (ismemberof, etc) are going to be expanded to their UUIDs. This allows changes to parentship to be made on the CSV.</li> <li>Do not export Media/Files: when enabled, file references and their associated technical Metadata (as:filetype JSON keys, e.g as:image) will be skipped. This allows pure Descriptive Metadata to be exported to CSV.</li> <li>Convert Media to Portable Absolute URLs: when enabled, all File references will be converted to absolute URLs and their associated technical Metadata (as:filetype JSON keys, e.g as:image) will be skipped. This allows CSVs to be used to ingest new ADOs in other repositories.</li> <li>Attach CSV to a new AMI Set: when checked, a new AMI set with the exported data will be created and configured for \"Updating\" existing ADOs.</li> <li>If attaching to a new AMI Set, you will have the option to Please Give your AMI Set a name: If empty, Archipelago will create a (quite) generic one for you (defaults to 'CSV Export/Import AMI Set').</li> </ol> <p></p> <p>Select <code>Apply</code> when you are ready to move forward to the next step of using this Action.</p>"},{"location":"export-to-csv/#executing-the-action-and-downloading-the-csv","title":"Executing the Action and Downloading the CSV","text":"<p>After selecting your Export ADOs to CSV Configuration Options, you will be directed to a page showing a summary list of the 'Items selected' (list may be truncated depending on the amount of items you selected). You will then need to select <code>Execute Action</code> to proceed.</p> <p>Selecting <code>Execute Action</code> will trigger the generation of the CSV in real time, directly in your browser session. This may take a few seconds to a few minutes, depending on the amount of items you selected to include. You will see a progress bar updating while this Action is running.</p> <p>When the Action is finished, you will see a temporary Status Message stating: \"Your source data was saved and is available as CSV at\", followed by a link to the CSV file generated. Be sure to click on the CSV link and download before navigating away from this temporary Status message. No worries if you accidently navigate away from this page before downloading the generated CSV, you can find a temporary copy of the CSV file generated at: - Accessed through the <code>Content</code> menu &gt; <code>Media Files</code> - Directly at <code>/admin/content/files</code></p> <p>Or: you can simply configure and run the Action again to generate another copy of the CSV if needed.</p> <p>If you selected the option to attach to <code>Attach CSV to a new AMI Set</code>, you will also see a status message stating: \"Well Done! New AMI Set was created and you can see it here\" (and 'see it here' will be linked to the corresponding AMI Set). You will be able to access the AMI Set from the <code>AMI Sets</code> tab on the main Content page found at <code>/admin/content</code> or directly at <code>/amiset/list</code>.</p> <p></p>"},{"location":"export-to-csv/#recommendations","title":"Recommendations","text":"<p>This specialty Action is quite useful for generating CSVs to use for reviewing select batches of materials found in your repository, but it is not intended to be used to generate a complete CSV containing all of your Archipelago repository's digital objects and collections. Depending on the size of your repository, generating a CSV for all of your assets may exceed your browser's timeout processing limits/bandwidth (see 'Known Solr-Related Issue' note below) and also result in a CSV file that is unable to be opened by standard spreadsheet software such as Google Sheets or MS Excel.</p> <p>Known Solr-Related Issue</p> <p>Solr has a fixed limits configured for the maxiumum number of results able to retrieved, so if you make a very large query/select a very large number of ADOs to include that exceeds your fixed limits for Solr, you will only retrieve up to the maximum specified in your Solr limits.</p> <p>You can find the related Solr configurations sections the <code>Advanced</code> section of the 'Edit search server' page found at <code>/admin/config/search/search-api/server/esmero_solr/edit</code>. Take caution with making any changes here, and be aware that changing the maximum query settings will impact your Solr query setup sitewide, not just as pertains to the 'Export ADOs to CSV' Action.</p> <p></p> <p>We would recommend executing this specialty Action using Archipelago's Find and Replace to narrow down your selected ADOs more easily using the Facets available for the Find and Replace interface. We would also recommend keeping your selected batches limited to an amount of objects you can reasonably assess and review using your normal spreadsheet review workflows and software.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"find_and_replace/","title":"Advanced Batch Find and Replace","text":"<p>Archipelago's Advanced Batch Find and Replace functionality provides different ways for you to efficiently Find/Search and Replace metadata values found in the raw JSON of your Digital Objects and Collections. Advanced Batch Find and Replace makes use of customized Actions that extend Drupal's VBO module to enable these powerful batch metadata replacement Actions in your Archipelago environment. </p>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace/#where-to-find","title":"Where to Find","text":"<p>In default Archipelagos, you can find Advanced Batch Find and Replace:</p> <ul> <li>Through the <code>Tools</code> menu &gt; <code>Advanced Batch Find and Replace</code> </li> <li>Directly at <code>/search-and-replace</code> </li> </ul> <p></p> Site Administrator Note <ul> <li>The View driving this can be found at <code>/admin/structure/views/view/solr_search_content_with_find_and_replace</code>. The default Facets referenced above can be found at <code>/admin/structure/block/list/archipelago_subtheme</code> in the <code>Sidebar Second</code> section. Please proceed with caution if making any changes to the default configurations for this View or the Facets referenced on this View Page.     </li> </ul>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace/#main-page-overview","title":"Main Page Overview","text":"<p>From the main page (display title 'Search and Replace'), you will see:</p> <ul> <li>A <code>Fulltext Search</code> box</li> <li>Dropdown list of available <code>Actions</code></li> <li>Listing of all the Digital Objects and Collections found in your Archipelago repository<ul> <li>Option to <code>Select/deselect all results in this view (all pages)</code> via toggle switch</li> <li>Same toggle switch option beside each individual Object/Collection to select one-at-a-time</li> <li>Expandable <code>\u25ba Raw Metadata (JSON)</code> section beneath each each individual Object/Collection containing the full Raw JSON metadata record for reference</li> </ul> </li> <li>Expandable section to show all the <code>Selected items in this view</code> (will be 0 items to start).<ul> <li>Individual selections made on different results pages will be preserved in the overall total of <code>Selected items</code> available for preview on this main/top page.</li> </ul> </li> </ul> <p></p>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace/#default-facets-configured","title":"Default Facets Configured","text":"<p>You will also see a listing of a few different default Facets configured to help guide your selection of potential Digital Objects/Collections:</p> <ul> <li>Object Type<ul> <li>the Archipelago Digital Object/Collection Type</li> <li>JSON key: <code>type</code></li> </ul> </li> <li>JSON keys in your metadata<ul> <li>all of the potential JSON keys that are present in your repository</li> <li>includes 'flattened' keys that may not be readily accessible via Webform elements (such as Archipelago-generated technical and administrative keys)</li> </ul> </li> <li>Ingest Method Service URL<ul> <li>The URL of the Digital Object/Collection Webforms and AMI Sets present in your repository that were used to create Digital Objects/Collections.</li> <li>Please note that using the Find and Replace Webform functionality to update ADOs will cause the original AMI Set URL identified in this Facet to be overwritten and replaced with the specified Webform used during the replacement execution.</li> </ul> </li> </ul>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace/#available-actions","title":"Available Actions","text":"<p>The default options available through the Action dropdown menu include:</p> <ul> <li>*<code>Export Archipelago Digital Objects to CSV content item</code></li> <li><code>Text based find and replace Metadata for Archipelago Digital Objects content item</code></li> <li><code>Webform find-and-replace Metadata for Archipelago Digital Objects content item</code></li> <li><code>JSON Patch Metadata for Archipelago Digital Objects content item</code></li> <li>*<code>Publish Digital Object</code></li> <li>*<code>Unpublish Digital Object</code></li> <li>*<code>Change the author of content</code></li> <li>**<code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code></li> <li>*<code>Delete selected entities/translations</code></li> </ul> <p>* denotes Action options that are also shared with the main <code>Content</code> Page Action Menu</p> <p>**You can read more about Strawberry Runners Post-Processing Actions here</p> <p></p>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace/#find-and-replace-specific-actions","title":"Find and Replace Specific Actions","text":"<p>After reviewing the 'Important Notes &amp; Workflow Recommendations' below, please see the following separate pages for detailed examples walking through the usage of the three different Find and Replace specific actions. </p> <ul> <li>Text Based Find and Replace<ul> <li>Enables you to execute simple but powerful text based search and replacement operations.</li> </ul> </li> <li>Webform Find and Replace<ul> <li>Enables you to search against values found within defined Webform elements to apply metadata replacements with targeted care.</li> </ul> </li> <li>JSON Patch Find and Replace<ul> <li>Enables you to carry out advanced JSON Patch operations within your metadata.</li> </ul> </li> </ul>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace/#important-notes-workflow-recommendations","title":"Important Notes &amp; Workflow Recommendations","text":"<p>Important Note</p> <p>The Actions available through Archipelago's Advanced Batch Find and Replace can potentially have repository-wide effects. It is strongly recommended that you proceed with caution when executing any of the available Actions. </p> <p>Adding New Facets</p> <p>The default Facets available through Archipelago's Advanced Batch Find and Replace have an important configuration selection made on each individual Facet. For every new Facet you add for Find and Replace, you need to select the checkboxes for both the 'VBO batch handler' settings to use the <code>VBO Batch Facet processor</code>, and the selection within the 'VBO batch handler settings' to <code>Use URL based facets in VBO Batches</code>. You need to make sure these are selected so that the \"visible\" list/count of objects you filter using a Facet is respected during actual VBO process execution of batch changes you make for any Find and Replace Actions. Also, please be aware that Drupal's VBO does not pass a \"limit\" (except if your VIEW has actually a \"SHOW\" a defined number of results which most users will never use). Because of that, when you run a VBO-based action, the default batch limitation will be set to the Search API/Solr defined Limit. You can view this Limit information at</p> <p>'~yoursite/admin/config/search/search-api/server/esmero_solr/edit', under the Advanced Tab. This all means that if you first set a Limit of 100 in your Search API/SOLR defined Limit, then you see 1000 objects in your Find and Replace results and select all 1000 results for batch change operations, when you run your Find and Replace action only 100 changes will be processed. There is no way Archipelago can work around that VBO related behavior (for now, except open an ISSUE, perhaps a way can be found!).</p>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace/#simulation-mode","title":"Simulation Mode","text":"<p>Before executing any of the available Find and Replace Actions, the best-practice workflow recommendation is to always first run in Simulation Mode:</p> <ul> <li>Before the final 'Execute Action' step of your Find and Replace operation, select the option to '\u2611\ufe0f only simulate and debug affected JSON'. This will run a quick check against your Action specifications and the potentially impacted Digital Objects and Collections.</li> <li>You can then double check that the total effected changes shown reflect your intended amount of changes. <ul> <li>The total number of changes will always be multipled by a factor of 2, as the Actions count a first step of checking against your data, then the second step of applying the change.</li> <li>If your Action specifications do not match against any JSON metadata values in your specified results, you will also see that no matches were applicable.</li> </ul> </li> </ul> <p></p> <ul> <li>Once you have reviewed the results of the Simulation Mode and confirmed the simulated results match your intentend amount of changes, proceed with executing the Action you first simulated.</li> </ul>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace/#checking-your-changes","title":"Checking Your Changes","text":"<p>After applying any of the Find and Replace Actions, you can review the specific changes that were made within the Revision history of the impacted Digital Objects and Collections.</p> <ul> <li>Through the <code>Find and Replace</code> page results listing or the main <code>Content</code> page, navigate to the Digital Object/Collection you wish to review.</li> <li>Open the 'Revision' tab.</li> <li>The details for the specific Action executed will be visible. All of the Find and Replace Actions will result in slightly different operation notes within the the Revision history.</li> </ul> <p></p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace_action_json_patch/","title":"JSON Patch Find and Replace","text":"<p>Enables you to carry out advanced JSON Patch operations within your metadata.</p> <p>Note</p> <p>Please refer to the main Find and Replace documentation page for a general overview of where to find within your Archipelago, a general overview of default options and important notes and workflow recommendations.</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#what-is-a-json-patch-and-when-to-use-it","title":"What is a JSON Patch and when to use it?","text":"<p>Before we dive into the mechanics of doing JSON Patching Batch in Archipelago we need to learn what a JSON Patch is and of course when applying this action is useful, possible (or not).</p> <p>A JSON Patch is a <code>JSON</code> Document containing precise operations that can heavily modify the structure and values of an existing JSON Document, in this case the RAW JSON found inside a strawberryfield of our ADOs. </p> <p>The operations available for modifications of an JSON document are:</p> <ul> <li>\u201cadd\u201d</li> <li>\u201cremove\u201d</li> <li>\u201creplace\u201d,</li> <li>\u201cmove\u201d</li> <li>\u201ccopy\u201d</li> </ul> <p>And there is also one (very important) used to check/validate the existence of values/keys:</p> <ul> <li>\u201ctest\u201d</li> </ul> <p>Even if you can have multiple operations in a single JSON Patch Document, they are always applied as a whole. Means if any of those fail nothing will be applied and in concrete, in our VBO action, no change will be done to your ADO. This in combination with the \u201ctest\u201d operation gives you a lot of safety and a way of discerning/skipping completely a complex set of operations on e.g. a failed \u201dtest\u201d.</p> <p>JSON Patch uses <code>JSON Pointers</code> as arguments in all of these operations to target a specific key/value of your JSON.</p> <p>Using the following JSON Snippet as example:</p> <pre><code>\"subject_lcgft_terms\": [\n    {\n        \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/genreForms\\/gf2017027249\",\n        \"label\": \"Photographs\"\n    }\n]\n</code></pre> <p>These JSON Pointers will resolve in the following manner:</p> <ul> <li><code>/subject_lcgft_terms</code> </li> </ul> <pre><code>[\n    {\n        \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/genreForms\\/gf2017027249\",\n        \"label\": \"Photographs\"\n    }\n]\n</code></pre> <ul> <li><code>/subject_lcgft_terms/0</code></li> </ul> <pre><code>{\n    \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/genreForms\\/gf2017027249\",\n    \"label\": \"Photographs\"\n}\n</code></pre> <ul> <li><code>/subject_lcgft_terms/0/label</code></li> </ul> <pre><code>Photographs\n</code></pre> <p>AS you can see a JSON Pointer is very precise , allowing you to target complete structures and values but it does not allow Wildcard Operations. Means you can not \"search\" or do loosy comparissons. This very fact limits many times the use case. E.g. if you have a list of terms like this:</p> <pre><code>\"terms\": [\n    \"term1\",\n    \"term2\",\n    \"term3\"\n]\n</code></pre> <p>and you want to \"test\" for the existence of <code>\"term3\"</code> before applying a change, you would need to know exactly at what position inside the <code>terms Array</code> (Starting from 0) it will/should be found. And that might not be consistent across every ADO. </p> <p>So how do we use these pointers in an operation inside a JSON Patch document? Using the same fake \"terms\" JSON snippet the previously listed, operations are written like this:</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#add","title":"add","text":"<pre><code>{ \"op\": \"add\", \"path\": \"/terms/0\", \"value\": \"term_another\" }\n</code></pre> <p>This will add before the first term (in this case \"term1\" ) \"term_another\" as a value.</p> At the end <p>you can use a dash (<code>-</code>) (e.g. \"/terms/-\") instead of the numeric position of an array entry to denote that the \"value\" needs to be added at the end. This is needed specially for empty lists. You can not target <code>0</code> position on an empty array.</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#remove","title":"remove","text":"<pre><code>{ \"op\": \"remove\", \"path\": \"/terms/1\"}\n</code></pre> <p>This will remove the second term (in this case \"term2\" )</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#replace","title":"replace","text":"<p><pre><code>{ \"op\": \"replace\", \"path\": \"/terms/1\", \"value\": \"term_again\" }\n</code></pre> This will remove the second term (in this case \"term2\" ) and put in its place \"term_again\" as a value. Basically two operationes, \u201cremove\u201c and \u201cadd \u201c in a single step</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#move","title":"move","text":"<pre><code>{ \"op\": \"move\", \"from\": \"/terms\", \"path\": \"/terms_somewhere_else\"}\n</code></pre> <p>This will copy all values inside top JSON key <code>terms</code> into a new top JSON key named <code>terms_somewhere_else</code> and remove then the old <code>terms</code> key.</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#copy","title":"copy","text":"<pre><code>{ \"op\": \"copy\", \"from\": \"/terms\", \"path\": \"/terms_somewhere_else\"}\n</code></pre> <p>Similar to \u201cmove\u201c, it will copy values inside top JSON key <code>terms</code> into a new top JSON key named <code>terms_somewhere_else</code> without removing then the old <code>terms</code> key or its content!</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#test","title":"test","text":"<pre><code>{ \"op\": \"test\", \"path\": \"/terms/0\", \"value\": \"term1\"}\n</code></pre> <p>Finally, \u201ctest\u201c will check if on position <code>0</code> of terms there is a value of \"term1\". If not, the test will fail. If a single test fails the whole JSON Patch will be cancelled. Tests can not be concatenated via OR bolean operators. So they always act individually. Two tests with one failing is a failed JSON Patch.</p> <p>There is more of course! The Complete documentation can be found here</p> <p>So. When to use JSON Patch? There are a few general rules/suggestions:</p> <ul> <li>When you can generate one or more precise \u201ctest\u201c operations. If you need to test against data that might exists but its position is not clear, a single JSON Patch will not do the trick. That said you could run the same JSON Patch document against the same set of ADOs more than once modifying slightly the \u201ctest\u201c to cover all variations (check for value at positition <code>0</code>, then again at position <code>1</code>, etc)</li> <li>When you need to use data that is already inside the JSON Document and move it around. Sometimes <code>fixing</code> a value, in the sense of putting a static replacement, is not what you need. Other <code>Actions</code> documented will allow you to replace one value with another fixed one. But JSON Patch will allow you to use existing data inside your target JSON and move it/copy it.</li> <li>When you operate on multiple JSON Keys and can target / match complete values. You can not replace a single word inside a value via a JSON Patch. But you can apply multiple precise operations in a single pass.</li> <li>When you need to change data types. e.g. convert a single value into an array.</li> <li>When you need safety. The fact that a single failed \u201ctest\u201c will cancel a whole JSON Patch allows you to operate with safety and ensure the final destination will always be a JSON</li> </ul> <p>Now that we know what it is and when we should do it, we can make a small exercise. The goal:</p> <ul> <li>For all ADO's with json key with value <code>photograph</code> that are member of collection with Node ID <code>16</code><ul> <li>convert the <code>description</code> key from a single value into an array.</li> <li>add an extra LOD entry</li> <li>Copy a Value into a new Key</li> </ul> </li> </ul>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#step-1-select-the-ados-to-be-modified","title":"Step 1: Select the ADOs to be Modified","text":"<p>As with every other VBO action described in our documentation, start by selecting the ADOs you want to modify using the exposed Search Field(s) and/or Facets present in the Search and Replace <code>View</code> found at <code>/search-and-replace</code> . </p> <p>Once you have filtered down the list to manageable size, containing at least the ADOs you plan on modifying (but for JSON Patch operation could be more too because you can \u201ctest\u201c and match ), press either on <code>Select / deselect all results in this view</code> to pass all the result (this includes all pages, not only the currently visible one) or go selectively one by one by checking on the <code>toggle</code> found beside each ADO's title. You will see how the number in <code>Selected 0 item in this view</code> increases. Now press <code>Apply to select Items</code>. We will use for this example <code>Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?]</code>, an ADO we provide in our DEMO sets. </p> <p>Redundant to say, Batch Actions are intended to be used when a modification needs to be applied to more than one item, implying there is a pattern. For single ADOs you can always do this faster directly via the EDIT tab.</p> Keep your JSON Patches (and friends) around <p>Since JSON Patching involves writing a, sometimes complex, JSON Document, please keep around an Application (or Text File) where you can copy/paste and save your JSON Patches for resuse or future references. Archipelago will not store nor remember between runs the JSON Patch document you submitted. It is also very useful to copy and have at hand the <code>RAW JSON</code> of one of the ADOs you plan to modify as a reference/aid while building the given JSON Patch document.</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#step-2-json-patch-metadata-for-archipelago-digital-objects-action-configuration","title":"Step 2: JSON Patch Metadata for Archipelago Digital Objects Action Configuration","text":"<p>The default config JSON Patch form will contain an example JSON Patch set of Commands (Document)</p> <p></p> <p>We are going to replace this one with a valid JSON document. Notice that it does not require a root <code>{}</code> Object wrapper and it is really a list (or array) of operations. </p> <p>A bit of repetition but needed to explain the JSON Patch document. You can see on the following Note box how we copyed the RAW JSON of one ADO to be Patched to have a reference while building the JSON PATCH. for the sake of brevity we remove here the longer Image File technical Metadata.</p> RAW JSON of <code>Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?]</code> Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?] | before Patching<pre><code>{\n    \"note\": \"\",\n    \"type\": \"Photograph\",\n    \"viaf\": \"\",\n    \"label\": \"Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?]\",\n    \"model\": [],\n    \"owner\": \"New-York Historical Society, 170 Central Park West, New York, NY 10024, 212-873-3400.\",\n    \"audios\": [],\n    \"images\": [\n        26\n    ],\n    \"models\": [],\n    \"rights\": \"This digital image may be used for educational or scholarly purposes without restriction. Commercial and other uses of the item are prohibited without prior written permission from the New-York Historical Society. For more information, please visit the New-York Historical Society's Rights and Reproductions Department web page at [http:\\/\\/www.nyhistory.org\\/about\\/rights-reproductions](http:\\/\\/www.nyhistory.org\\/about\\/rights-reproductions).\",\n    \"videos\": [],\n    \"creator\": \"\",\n    \"ap:tasks\": {\n        \"ap:sortfiles\": \"index\"\n    },\n    \"duration\": \"\",\n    \"ispartof\": [],\n    \"language\": \"English\",\n    \"documents\": [],\n    \"edm_agent\": \"\",\n    \"publisher\": \"\",\n    \"ismemberof\": [\n        16\n    ],\n    \"creator_lod\": [\n        {\n            \"name_uri\": \"\",\n            \"role_uri\": \"http:\\/\\/id.loc.gov\\/vocabulary\\/relators\\/pht\",\n            \"agent_type\": \"personal\",\n            \"name_label\": \"Stonebridge, George Ehler\",\n            \"role_label\": \"Photographer\"\n        }\n    ],\n    \"description\": \"George Ehler Stonebridge (d. 1941) was an amateur photographer who lived and worked in the Bronx, New York. He left little record of himself, but an invaluable one of his surroundings and interests. Stonebridge lived at several locations in the Bronx with his wife Bella, and their three children Grace, George, and William. He worked at the Northern Gaslight Company, although the position he held is unknown. In addition to taking photographs, Stonebridge wrote poetry and prose about his love of the Bronx, his children, and in honor of military victories. Some of Stonebridge's photographs appeared in local papers. In 1898, he was an authorized reporter and photographer for the North Side News; in 1905 he was an authorized reporter for the Bronx Borough Record and Times, and probably took photographs for that paper as well. Stonebridge was fascinated with the subject of military preparedness. Training rituals and staged battles were one of his favorite photographic subjects. His 1898 poem, \\\"Remember the Maine,\\\" celebrates the United States' victory in the Spanish-American War. He was especially proud of soldiers from the Bronx, and photographed historical tablets throughout the Borough commemorating previous military victories. Stonebridge also used his photographs to illustrate lectures. In 1907, he gave several lectures on \\\"The Training of War,\\\" using colored lantern slides.\",\n    \"interviewee\": \"\",\n    \"interviewer\": \"\",\n    \"pubmed_mesh\": null,\n    \"sequence_id\": \"\",\n    \"subject_loc\": [\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/subjects\\/sh85038796\",\n            \"label\": \"Dogs\"\n        }\n    ],\n    \"website_url\": \"\",\n    \"as:generator\": {\n        \"type\": \"Update\",\n        \"actor\": {\n            \"url\": \"http:\\/\\/localhost:8001\\/form\\/descriptive-metadata\",\n            \"name\": \"descriptive_metadata\",\n            \"type\": \"Service\"\n        },\n        \"endTime\": \"2022-12-05T09:19:37-05:00\",\n        \"summary\": \"Generator\",\n        \"@context\": \"https:\\/\\/www.w3.org\\/ns\\/activitystreams\"\n    },\n    \"date_created\": \"1910-01-01\",\n    \"issue_number\": null,\n    \"date_published\": \"\",\n    \"subjects_local\": null,\n    \"term_aat_getty\": \"\",\n    \"ap:entitymapping\": {\n        \"entity:file\": [\n            \"model\",\n            \"audios\",\n            \"images\",\n            \"videos\",\n            \"documents\",\n            \"upload_associated_warcs\"\n        ],\n        \"entity:node\": [\n            \"ispartof\",\n            \"ismemberof\"\n        ]\n    },\n    \"europeana_agents\": \"\",\n    \"europeana_places\": \"\",\n    \"local_identifier\": \"nyhs_PR066_6136\",\n    \"subject_wikidata\": [\n        {\n            \"uri\": \"http:\\/\\/www.wikidata.org\\/entity\\/Q3381576\",\n            \"label\": \"black-and-white photography\"\n        },\n        {\n            \"uri\": \"http:\\/\\/www.wikidata.org\\/entity\\/Q60\",\n            \"label\": \"New York City\"\n        }\n    ],\n    \"date_created_edtf\": \"\",\n    \"date_created_free\": null,\n    \"date_embargo_lift\": null,\n    \"physical_location\": null,\n    \"related_item_note\": null,\n    \"rights_statements\": \"In Copyright - Educational Use Permitted\",\n    \"europeana_concepts\": \"\",\n    \"geographic_location\": {\n        \"lat\": \"40.8466508\",\n        \"lng\": \"-73.8785937\",\n        \"city\": \"New York\",\n        \"state\": \"New York\",\n        \"value\": \"The Bronx, Bronx County, New York, United States\",\n        \"county\": \"\",\n        \"osm_id\": \"9691916\",\n        \"country\": \"United States\",\n        \"category\": \"boundary\",\n        \"locality\": \"\",\n        \"osm_type\": \"relation\",\n        \"postcode\": \"\",\n        \"country_code\": \"us\",\n        \"display_name\": \"The Bronx, Bronx County, New York, United States\",\n        \"neighbourhood\": \"Bronx County\",\n        \"state_district\": \"\"\n    },\n    \"note_publishinginfo\": null,\n    \"subject_lcgft_terms\": [\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/genreForms\\/gf2017027249\",\n            \"label\": \"Photographs\"\n        }\n    ],\n    \"upload_associated_warcs\": [],\n    \"physical_description_extent\": \"\",\n    \"subject_lcnaf_personal_names\": \"\",\n    \"subject_lcnaf_corporate_names\": \"\",\n    \"subjects_local_personal_names\": \"\",\n    \"related_item_host_location_url\": null,\n    \"subject_lcnaf_geographic_names\": [\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/names\\/n81059724\",\n            \"label\": \"Bronx (New York, N.Y.)\"\n        },\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/names\\/n79007751\",\n            \"label\": \"New York (N.Y.)\"\n        }\n    ],\n    \"related_item_host_display_label\": null,\n    \"related_item_host_local_identifier\": null,\n    \"related_item_host_title_info_title\": \"\",\n    \"related_item_host_type_of_resource\": null,\n    \"physical_description_note_condition\": null\n}\n</code></pre> <p>Based on our own plan:</p> <ol> <li>Check first if of type <code>photograph</code> and <code>memberof</code> Node ID <code>16</code></li> </ol> <pre><code>{ \"op\": \"test\", \"path\": \"/type\", \"value\": \"Photograph\"},\n{ \"op\": \"test\", \"path\": \"/ismemberof\", \"value\": [16]}\n</code></pre> <p>Notice that to be really sure we also match the data type, an <code>array</code> with a single value <code>16</code> of type integer (makes sense since the Operation is also a JSON and will be evaluated in the same way as the source RAW JSON). This is a precise match. if the ADO belongs to multiple Collections it will fail of course.</p> <ol> <li>Now we are going to convert <code>description</code> key from a single value into an array.</li> </ol> <pre><code>{\"op\": \"add\",\"path\": \"/temp_description_array\",\"value\": []},\n{\"op\": \"move\",\"from\": \"/description\",\"path\": \"/temp_description_array/-\"},\n{\"op\": \"move\",\"from\": \"/temp_description_array\",\"path\": \"/description\"},\n</code></pre> <p>This is a multi step operation. Given that JSON Patch can not \"cast\" types and depends on a given datatype to be present before, e.g. adding a new value to it, we use here a temporary key. Notice that you can not \u201cadd\u201c or \u201cmove\u201c to e.g. the position <code>0</code> because the destination array is indeed empty (that will fail). But by using the <code>dash</code> you can command it to put it at the end, which on an empty list is also at the beginning (we are starting to understand this!). </p> <ol> <li>And the extra LOD entry for wikidata, in this case the Q Item for collie (type of herding dog)</li> </ol> <pre><code>{ \"op\": \"add\", \"path\": \"/subject_wikidata/-\", \"value\": {\n        \"uri\": \"https://www.wikidata.org/wiki/Q1196071\",\n        \"label\": \"collie\"\n    }\n}\n</code></pre> <ol> <li>Finally we are going to copy the <code>geographic_location.state</code> value and put it into <code>subjects_local</code>:</li> </ol> <pre><code>{\"op\": \"remove\", \"path\": \"/subjects_local\"},\n{\"op\": \"add\", \"path\": \"/subjects_local\", \"value\": []},\n{\"op\": \"copy\", \"from\": \"/geographic_location/state\", \"path\": \"/subjects_local/-\"}\n</code></pre> <p>Why so many operations? Because initially <code>\"subjects_local\"</code> had a value of <code>null</code> and it is not suited to generate/add to it as a multivalued key because of that. So we need to remove it first, recreate it as an empty list and then we can copy. Pro Note: you could partially rewrite this as <code>replace</code> operation!</p> what if subjects_local has data? <p>You will lose it! you can add a \u201ctest\u201c or move data instead of recreating. Many choices.</p> <p>The final JSON Patch will look like this. Copy it into the Configuration JSON Patch commands form field: </p> <pre><code>[\n    {\"op\": \"test\", \"path\": \"/type\", \"value\": \"Photograph\"},\n    {\"op\": \"test\", \"path\": \"/ismemberof\", \"value\": [16]},\n    {\"op\": \"add\",\"path\": \"/temp_description_array\",\"value\": []},\n    {\"op\": \"move\",\"from\": \"/description\",\"path\": \"/temp_description_array/-\"},\n    {\"op\": \"move\",\"from\": \"/temp_description_array\",\"path\": \"/description\"},\n    {\"op\": \"add\",\"path\": \"/subject_wikidata/-\",\"value\": \n        {\n        \"uri\": \"https://www.wikidata.org/wiki/Q1196071\",\n         \"label\": \"collie\"\n         }\n    },\n    {\"op\": \"remove\", \"path\": \"/subjects_local\"},\n    {\"op\": \"add\", \"path\": \"/subjects_local\", \"value\": []},\n    {\"op\": \"copy\", \"from\": \"/geographic_location/state\", \"path\": \"/subjects_local/-\"}\n]\n</code></pre> The inverse process online <p>Now that you know (or are starting to understand) the manual process you can also try this online tool that allows you, based on a source and a destination JSON, generate the needed JSON Patch to mutate one JSON into the another. The logic might not be always what you need and most likely it will not take in account that you actually need to move values and will prefer to fix values to be added via an add operation</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#step-3-run-the-json-patch-action-in-simulation-mode","title":"Step 3: Run the JSON Patch Action in simulation mode","text":"<p>Ready. Now check the \"only simulate and debug affected JSON\" checkbox. We want to see if we did well but not yet modify any ADOs. Press <code>Apply</code> button. You will get another confirmation screen. Press <code>Execute Action</code>.</p> <p>It will run quick (on this example) and you will redirected back on the original Drupal Views of Step 1. If your source <code>ADO</code> is actually the one from our Demo Collection you might see a <code>diff</code>, something very similar to this:</p> Text based diff after a successfull Patch<pre><code>129d129 &lt; \"description\": \"George Ehler Stonebridge (d. 1941) was an amateur photographer who lived and worked in the Bronx, New York. He left little record of himself, but an invaluable one of his surroundings and interests. Stonebridge lived at several locations in the Bronx with his wife Bella, and their three children Grace, George, and William. He worked at the Northern Gaslight Company, although the position he held is unknown. In addition to taking photographs, Stonebridge wrote poetry and prose about his love of the Bronx, his children, and in honor of military victories. Some of Stonebridge's photographs appeared in local papers. In 1898, he was an authorized reporter and photographer for the North Side News; in 1905 he was an authorized reporter for the Bronx Borough Record and Times, and probably took photographs for that paper as well. Stonebridge was fascinated with the subject of military preparedness. Training rituals and staged battles were one of his favorite photographic subjects. His 1898 poem, \\\"Remember the Maine,\\\" celebrates the United States' victory in the Spanish-American War. He was especially proud of soldiers from the Bronx, and photographed historical tablets throughout the Borough commemorating previous military victories. Stonebridge also used his photographs to illustrate lectures. In 1907, he gave several lectures on \\\"The Training of War,\\\" using colored lantern slides.\", 155d154 &lt; \"subjects_local\": null, 182a180,183 &gt; }, &gt; { &gt; \"uri\": \"https:\\/\\/www.wikidata.org\\/wiki\\/Q1196071\", &gt; \"label\": \"collie\" 236c238,244 &lt; \"physical_description_note_condition\": null --- &gt; \"physical_description_note_condition\": null, &gt; \"description\": [ &gt; \"George Ehler Stonebridge (d. 1941) was an amateur photographer who lived and worked in the Bronx, New York. He left little record of himself, but an invaluable one of his surroundings and interests. Stonebridge lived at several locations in the Bronx with his wife Bella, and their three children Grace, George, and William. He worked at the Northern Gaslight Company, although the position he held is unknown. In addition to taking photographs, Stonebridge wrote poetry and prose about his love of the Bronx, his children, and in honor of military victories. Some of Stonebridge's photographs appeared in local papers. In 1898, he was an authorized reporter and photographer for the North Side News; in 1905 he was an authorized reporter for the Bronx Borough Record and Times, and probably took photographs for that paper as well. Stonebridge was fascinated with the subject of military preparedness. Training rituals and staged battles were one of his favorite photographic subjects. His 1898 poem, \\\"Remember the Maine,\\\" celebrates the United States' victory in the Spanish-American War. He was especially proud of soldiers from the Bronx, and photographed historical tablets throughout the Borough commemorating previous military victories. Stonebridge also used his photographs to illustrate lectures. In 1907, he gave several lectures on \\\"The Training of War,\\\" using colored lantern slides.\" &gt; ], &gt; \"subjects_local\": [ &gt; \"New York\" &gt; ]\n</code></pre> <p>Which means your Patch would have been applied!</p> <p>In case something went wrong, e.g. any of the operations did not match your source data, you will see a <code>WARNING</code> like this</p> <pre><code>Patch could not be applied for Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?]\n</code></pre>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#step-4-run-the-json-patch-action-but-for-real","title":"Step 4: Run the JSON Patch Action but for real!","text":"<p>Now that actual patching. Repeat from Step 1 to Step 3 but keep \"only simulate and debug affected JSON\" unchecked and follow the steps again. You ADO will be modified and you will get almost no notifications except of an action completed notice (in soothing green). If you check Laddie's ADO RAW json (expand in the same resulting view) it will look now like this</p> Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?] JSON after patching<pre><code>{ \n    \"note\": \"\",\n    \"type\": \"Photograph\",\n    \"viaf\": \"\",\n    \"label\": \"Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?]\",\n    \"model\": [],\n    \"owner\": \"New-York Historical Society, 170 Central Park West, New York, NY 10024, 212-873-3400.\",\n    \"audios\": [],\n    \"images\": [\n        26\n    ],\n    \"models\": [],\n    \"rights\": \"This digital image may be used for educational or scholarly purposes without restriction. Commercial and other uses of the item are prohibited without prior written permission from the New-York Historical Society. For more information, please visit the New-York Historical Society's Rights and Reproductions Department web page at [http:\\/\\/www.nyhistory.org\\/about\\/rights-reproductions](http:\\/\\/www.nyhistory.org\\/about\\/rights-reproductions).\",\n    \"videos\": [],\n    \"creator\": \"\",\n    \"ap:tasks\": {\n        \"ap:sortfiles\": \"index\"\n    },\n    \"duration\": \"\",\n    \"ispartof\": [],\n    \"language\": \"English\",\n    \"documents\": [],\n    \"edm_agent\": \"\",\n    \"publisher\": \"\",\n    \"ismemberof\": [\n        16\n    ],\n    \"creator_lod\": [\n        {\n            \"name_uri\": \"\",\n            \"role_uri\": \"http:\\/\\/id.loc.gov\\/vocabulary\\/relators\\/pht\",\n            \"agent_type\": \"personal\",\n            \"name_label\": \"Stonebridge, George Ehler\",\n            \"role_label\": \"Photographer\"\n        }\n    ],\n    \"description\": [\n        \"George Ehler Stonebridge (d. 1941) was an amateur photographer who lived and worked in the Bronx, New York. He left little record of himself, but an invaluable one of his surroundings and interests. Stonebridge lived at several locations in the Bronx with his wife Bella, and their three children Grace, George, and William. He worked at the Northern Gaslight Company, although the position he held is unknown. In addition to taking photographs, Stonebridge wrote poetry and prose about his love of the Bronx, his children, and in honor of military victories. Some of Stonebridge's photographs appeared in local papers. In 1898, he was an authorized reporter and photographer for the North Side News; in 1905 he was an authorized reporter for the Bronx Borough Record and Times, and probably took photographs for that paper as well. Stonebridge was fascinated with the subject of military preparedness. Training rituals and staged battles were one of his favorite photographic subjects. His 1898 poem, \\\"Remember the Maine,\\\" celebrates the United States' victory in the Spanish-American War. He was especially proud of soldiers from the Bronx, and photographed historical tablets throughout the Borough commemorating previous military victories. Stonebridge also used his photographs to illustrate lectures. In 1907, he gave several lectures on \\\"The Training of War,\\\" using colored lantern slides.\"\n    ],\n    \"interviewee\": \"\",\n    \"interviewer\": \"\",\n    \"pubmed_mesh\": null,\n    \"sequence_id\": \"\",\n    \"subject_loc\": [\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/subjects\\/sh85038796\",\n            \"label\": \"Dogs\"\n        }\n    ],\n    \"website_url\": \"\",\n    \"as:generator\": {\n        \"type\": \"Update\",\n        \"actor\": {\n            \"url\": \"http:\\/\\/localhost:8001\\/form\\/descriptive-metadata\",\n            \"name\": \"descriptive_metadata\",\n            \"type\": \"Service\"\n        },\n        \"endTime\": \"2022-12-05T09:19:37-05:00\",\n        \"summary\": \"Generator\",\n        \"@context\": \"https:\\/\\/www.w3.org\\/ns\\/activitystreams\"\n    },\n    \"date_created\": \"1910-01-01\",\n    \"issue_number\": null,\n    \"date_published\": \"\",\n    \"subjects_local\": [\n        \"New York\"\n    ],\n    \"term_aat_getty\": \"\",\n    \"ap:entitymapping\": {\n        \"entity:file\": [\n            \"model\",\n            \"audios\",\n            \"images\",\n            \"videos\",\n            \"documents\",\n            \"upload_associated_warcs\"\n        ],\n        \"entity:node\": [\n            \"ispartof\",\n            \"ismemberof\"\n        ]\n    },\n    \"europeana_agents\": \"\",\n    \"europeana_places\": \"\",\n    \"local_identifier\": \"nyhs_PR066_6136\",\n    \"subject_wikidata\": [\n        {\n            \"uri\": \"http:\\/\\/www.wikidata.org\\/entity\\/Q3381576\",\n            \"label\": \"black-and-white photography\"\n        },\n        {\n            \"uri\": \"http:\\/\\/www.wikidata.org\\/entity\\/Q60\",\n            \"label\": \"New York City\"\n        },\n        {\n            \"uri\": \"https:\\/\\/www.wikidata.org\\/wiki\\/Q1196071\",\n            \"label\": \"collie\"\n        }\n    ],\n    \"date_created_edtf\": \"\",\n    \"date_created_free\": null,\n    \"date_embargo_lift\": null,\n    \"physical_location\": null,\n    \"related_item_note\": null,\n    \"rights_statements\": \"In Copyright - Educational Use Permitted\",\n    \"europeana_concepts\": \"\",\n    \"geographic_location\": {\n        \"lat\": \"40.8466508\",\n        \"lng\": \"-73.8785937\",\n        \"city\": \"New York\",\n        \"state\": \"New York\",\n        \"value\": \"The Bronx, Bronx County, New York, United States\",\n        \"county\": \"\",\n        \"osm_id\": \"9691916\",\n        \"country\": \"United States\",\n        \"category\": \"boundary\",\n        \"locality\": \"\",\n        \"osm_type\": \"relation\",\n        \"postcode\": \"\",\n        \"country_code\": \"us\",\n        \"display_name\": \"The Bronx, Bronx County, New York, United States\",\n        \"neighbourhood\": \"Bronx County\",\n        \"state_district\": \"\"\n    },\n    \"note_publishinginfo\": null,\n    \"subject_lcgft_terms\": [\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/genreForms\\/gf2017027249\",\n            \"label\": \"Photographs\"\n        }\n    ],\n    \"upload_associated_warcs\": [],\n    \"physical_description_extent\": \"\",\n    \"subject_lcnaf_personal_names\": \"\",\n    \"subject_lcnaf_corporate_names\": \"\",\n    \"subjects_local_personal_names\": \"\",\n    \"related_item_host_location_url\": null,\n    \"subject_lcnaf_geographic_names\": [\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/names\\/n81059724\",\n            \"label\": \"Bronx (New York, N.Y.)\"\n        },\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/names\\/n79007751\",\n            \"label\": \"New York (N.Y.)\"\n        }\n    ],\n    \"related_item_host_display_label\": null,\n    \"related_item_host_local_identifier\": null,\n    \"related_item_host_title_info_title\": \"\",\n    \"related_item_host_type_of_resource\": null,\n    \"physical_description_note_condition\": null\n}\n</code></pre> <p>That is all. Again, keep your JSON Patches safe in a text document, test/try simple things first, look for patterns, look for No-Nos that can become \"tests\" to avoid touching ADOs that do not need to be updated and always remember that the destination type (single value, array or object) of an existing Key might affect your complex logic. Happy Patching!</p> <p>Thank you for reading! Please contact us on our\u00a0Archipelago Commons Google Group\u00a0with any questions or feedback.</p> <p>Return to the main\u00a0Find and Replace documentation page\u00a0or the\u00a0Archipelago Documentation main page.</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_text/","title":"Text Based Find and Replace","text":"<p>The text-based find and replace is case-sensitive and space-sensitive, and while it's the most simple of the actions, it's quite powerful. For this reason it's important to be very precise and target only what's intended. Below are a guide and some examples of use cases for this action.</p> <p>Note</p> <p>Please refer to the main Find and Replace documentation page for a general overview of where to find within your Archipelago, a general overview of default options and important notes and workflow recommendations.</p>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace_action_text/#step-by-step-guide","title":"Step-by-Step Guide","text":"<ol> <li>Go to <code>Tools &gt; Advanced Batch Find and Replace</code>.</li> <li>Identify and/or Filter the Digital Objects (ADOs) that need updates by Searching and/or using the available Facets.</li> <li>Either select all (includes results that appear on additional pages) by toggling <code>Select / deselect all results (all pages, x total)</code> or toggle the buttons for individual objects.</li> <li>Expand the <code>\u25ba Raw Metadata (JSON)</code> for some of the objects and double-check that the text being searched is only targeting what is intended and that the replacement text makes sense.</li> <li>Select <code>Text based find and replace Metadata for Archipelago Digital Objects content item</code> from the <code>Action</code> dropdown.</li> <li>If selecting objects individually, expand the <code>Selected X items</code> and review the list.</li> <li>Press the <code>Apply to selected items</code> button (don't worry, nothing will happen yet).</li> <li>Add the values for search (<code>JSON Search String</code>) and replace (<code>JSON Replacement String</code>).</li> <li> <p>If you're absolutely certain about the replacement you have targeted, uncheck the 'only simulate and debug affected JSON' option and select <code>Apply</code>.</p> <p>only simulate and debug affected JSON</p> <p>This option, which is selected by default, will simulate the action and show the list of objects that would be affected, along with the number of modifications for each object and a total number of results processed. For each JSON key and value affected the modifications will count 2:</p> <p>1 for the deletion of the current key and value</p> <p>+</p> <p>1 for the creation of the modified key and value </p> </li> </ol>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace_action_text/#use-cases-and-examples","title":"Use Cases and Examples","text":"<p>Replacing a JSON key</p> <p>Use Case: A JSON key is currently singular but should be plural.</p> JSON key example with empty array value<pre><code>...\n\"myKey\": [],\n...\n</code></pre> JSON key example with array values<pre><code>...\n\"myKey\": [\"strawberries\",\"blueberries\",\"blackberries\"],\n...\n</code></pre> <p>Follow the steps above and use the following for the search and replace values:</p> Search Value<pre><code>\"myKey\":\n</code></pre> Replace Value<pre><code>\"myKeys\":\n</code></pre> <p>Tip</p> <p>By using quotes and the colon instead of <code>myKey</code> only, we avoid unintentionally replacing other instances of the text within the JSON.</p> <p>After applying the changes, we have the following key:</p> JSON key example with empty array value after update<pre><code>...\n\"myKeys\": [],\n...\n</code></pre> JSON key example with array values after update<pre><code>...\n\"myKeys\": [\"strawberries\",\"blueberries\",\"blackberries\"],\n...\n</code></pre> <p>Replacing a JSON value</p> <p>Use Case: After a batch ingest, it was discovered that JSON values across ADOs in multiple keys contain the same typo: <code>Agnes Meyerhoff</code> (two fs) instead of <code>Agnes Meyerhof</code>.</p> JSON value example with typo<pre><code>...\n\"creator_lod\": [\n    {\n        \"name_uri\": \"\",\n        \"role_uri\": \"http:\\/\\/id.loc.gov\\/vocabulary\\/relators\\/art\",\n        \"agent_type\": \"personal\",\n        \"name_label\": \"Meyerhoff, Agnes\",\n        \"role_label\": \"Artist\"\n    },\n    {\n        \"name_uri\": \"\",\n        \"role_uri\": \"http:\\/\\/id.loc.gov\\/vocabulary\\/relators\\/col\",\n        \"agent_type\": \"personal\",\n        \"name_label\": \"Messenger, Maria, , 1849-1937\",\n        \"role_label\": \"Collector\"\n    }\n],\n\"description\": \"Inscription on mount: \\\"Meyerhoff, Agnes \\\\ Frankfurt - a\\/M. \\\\ Inv el-lith \\\\ Painter.\\\" Inscription on verso: \\\"Agnes Meyerhoff \\\\ Frankfurt a\\/M \\\\ inv. [at?] lith. \\\\ [maker in?]\\\".\",\n...\n</code></pre> <p>Follow the steps above and use the following for the search and replace values:</p> Search Value<pre><code>Meyerhoff, Agnes\n</code></pre> Replace Value<pre><code>Meyerhof, Agnes\n</code></pre> <p>After applying the changes, we have the following values:</p> JSON values after update<pre><code>...\n\"creator_lod\": [\n    {\n        \"name_uri\": \"\",\n        \"role_uri\": \"http:\\/\\/id.loc.gov\\/vocabulary\\/relators\\/art\",\n        \"agent_type\": \"personal\",\n        \"name_label\": \"Meyerhof, Agnes\",\n        \"role_label\": \"Artist\"\n    },\n    {\n        \"name_uri\": \"\",\n        \"role_uri\": \"http:\\/\\/id.loc.gov\\/vocabulary\\/relators\\/col\",\n        \"agent_type\": \"personal\",\n        \"name_label\": \"Messenger, Maria, , 1849-1937\",\n        \"role_label\": \"Collector\"\n    }\n],\n\"description\": \"Inscription on mount: \\\"Meyerhof, Agnes \\\\ Frankfurt - a\\/M. \\\\ Inv el-lith \\\\ Painter.\\\" Inscription on verso: \\\"Agnes Meyerhoff \\\\ Frankfurt a\\/M \\\\ inv. [at?] lith. \\\\ [maker in?]\\\".\",\n...\n</code></pre> <p>Replacing a JSON value with escape characters</p> <p>Use Case: The URL for a website that appears in multiple keys needs to be updated from <code>http://hubblesite.org</code> to <code>https://hubblesite.org</code>.</p> JSON value example with URL after update<pre><code>...\n\"rights\": \"This digital image may be used for educational or scholarly purposes without restriction. Commercial and other uses of the item are prohibited without prior written permission from the NASA and the Space Telescope Science Institute (STScI). For more information, please visit the NASA and the Space Telescope Science Institute's Copyright web page at [http:\\/\\/hubblesite.org\\/copyright](http:\\/\\/hubblesite.org\\/copyright).\",\n...\n\"description\": \"\\\"The largest NASA Hubble Space Telescope image ever assembled, this sweeping bird\u2019s-eye view of a portion of the Andromeda galaxy (M31) is the sharpest large composite image ever taken of our galactic next-door neighbor. Though the galaxy is over 2 million light-years away, The Hubble Space Telescope is powerful enough to resolve individual stars in a 61,000-light-year-long stretch of the galaxy\u2019s pancake-shaped disk. ... The panorama is the product of the Panchromatic Hubble Andromeda Treasury (PHAT) program. Images were obtained from viewing the galaxy in near-ultraviolet, visible, and near-infrared wavelengths, using the Advanced Camera for Surveys and the Wide Field Camera 3 aboard Hubble. This cropped view shows a 48,000-light-year-long stretch of the galaxy in its natural visible-light color, as photographed with Hubble's Advanced Camera for Surveys in red and blue filters July 2010 through October 2013.\\\" -full description available at: [http:\\/\\/hubblesite.org\\/image\\/3476\\/gallery\\/73-phat](http:\\/\\/hubblesite.org\\/image\\/3476\\/gallery\\/73-phat).\",\n...\n</code></pre> <p>Follow the steps above and use the following for the search and replace values:</p> Search Value<pre><code>http://hubblesite.org\n</code></pre> Replace Value<pre><code>https://hubblesite.org\n</code></pre> <p>Note</p> <p>You'll notice that the escape characters for the forward slash (<code>\\/</code>), which appear in the raw JSON, do not need to be included in the search or replace values.</p> <p>After applying the changes, we have the following values:</p> JSON value example with URL after update<pre><code>...\n\"rights\": \"This digital image may be used for educational or scholarly purposes without restriction. Commercial and other uses of the item are prohibited without prior written permission from the NASA and the Space Telescope Science Institute (STScI). For more information, please visit the NASA and the Space Telescope Science Institute's Copyright web page at [https:\\/\\/hubblesite.org\\/copyright](https:\\/\\/hubblesite.org\\/copyright).\",\n...\n\"description\": \"\\\"The largest NASA Hubble Space Telescope image ever assembled, this sweeping bird\u2019s-eye view of a portion of the Andromeda galaxy (M31) is the sharpest large composite image ever taken of our galactic next-door neighbor. Though the galaxy is over 2 million light-years away, The Hubble Space Telescope is powerful enough to resolve individual stars in a 61,000-light-year-long stretch of the galaxy\u2019s pancake-shaped disk. ... The panorama is the product of the Panchromatic Hubble Andromeda Treasury (PHAT) program. Images were obtained from viewing the galaxy in near-ultraviolet, visible, and near-infrared wavelengths, using the Advanced Camera for Surveys and the Wide Field Camera 3 aboard Hubble. This cropped view shows a 48,000-light-year-long stretch of the galaxy in its natural visible-light color, as photographed with Hubble's Advanced Camera for Surveys in red and blue filters July 2010 through October 2013.\\\" -full description available at: [https:\\/\\/hubblesite.org\\/image\\/3476\\/gallery\\/73-phat](https:\\/\\/hubblesite.org\\/image\\/3476\\/gallery\\/73-phat).\",\n...\n</code></pre> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the main Find and Replace documentation page or the Archipelago Documentation main page.</p>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace_action_webform/","title":"Webform Find and Replace","text":"<p>Webform Find and Replace enables you to search against values found within defined Webform elements to apply metadata replacements with targeted care. Below are a guide and an example use case for this Action.</p> <p>Note</p> <p>Please refer to the main Find and Replace documentation page for a general overview of where to find within your Archipelago, a general overview of default options and important notes and workflow recommendations.</p>","tags":["Advanced Batch Find and Replace","Webform Find and Replace","Search and Replace"]},{"location":"find_and_replace_action_webform/#important-notes-about-different-webform-elements","title":"Important Notes about Different Webform Elements","text":"<p>Maximum Length as Defined by your Webform Element Configuration OR Theme Defaults</p> <p>For certain text-based webform element types, the maximum field length (<code>maxlength</code>) defined in your specific webform element configurations will be enforced during Webform Find and Replace operations. If no maximum length is defined, the Admin Theme will enforce a maximum length of 128 characters. Please see our main Webforms documentation for information about configuring webforms in Archipelago.</p> <p>Some Complex Webform Elements Not Available</p> <p>Please note that some complex webform elements are not available for use with Webform Find and Replace. Any webform element that requires user interactions (such as the Nominatim Open Street Maps lookup/query and selection) is not available for usage. The different file upload webform elements are also not available for use with Webform Find and Replace.</p>","tags":["Advanced Batch Find and Replace","Webform Find and Replace","Search and Replace"]},{"location":"find_and_replace_action_webform/#step-by-step-guide","title":"Step by Step Guide","text":"<ol> <li>Go to <code>Tools &gt; Advanced Batch Find and Replace</code>.</li> <li>Identify and/or Filter the Digital Objects (ADOs) that need updates by Searching and/or using the available Facets.</li> <li>If using the 'Ingest Method Service URL' for Faceting, please note that using the Find and Replace Webform functionality to update ADOs will cause the original AMI Set URL identified in this Facet to be overwritten and replaced with the specified Webform used during the replacement execution.</li> <li>Either select all (includes results that appear on additional pages) by toggling <code>Select / deselect all results (all pages, x total)</code> or toggle the buttons for individual objects.</li> <li>Expand the <code>\u25ba Raw Metadata (JSON)</code> for some of the objects and double-check that the metadata field and value you are targeting for replacement is present.</li> <li>Select <code>Webform find-and-replace Metadata for Archipelago Digital Objects content item</code> from the <code>Action</code> dropdown.</li> <li>If selecting objects individually, expand the <code>Selected X items</code> and review the list.</li> <li>Press the <code>Apply to selected items</code> button (don't worry, nothing will happen yet).  </li> <li>On the Webform Find and Replace Action Configuration page, you will need to select the configuration options to be applied for your selected ADOs.</li> <li>From the dropdown:<ol> <li>Select which Webform you want to use</li> <li>Select which Form element you want to use</li> <li>Select which value to search for in the [chosen form element] JSON key</li> <li>Select which value to replace with in the [chosen form element] JSON key</li> </ol> </li> <li>See the Simulation Mode notes here for information about this optional simulation/test mode.</li> <li>If you're absolutely certain about the replacement you have targeted, uncheck the 'only simulate and debug affected JSON' option and select <code>Apply</code>.</li> <li>After the operation is executed, check your changes.</li> </ol>","tags":["Advanced Batch Find and Replace","Webform Find and Replace","Search and Replace"]},{"location":"find_and_replace_action_webform/#use-case-example","title":"Use Case Example","text":"<p>In the following example configuration, for the selected 'Senju no oubashi (Senju great bridge)' object, the Media Type (type JSON key) value of \"Visual Artwork\" will be replaced with the <code>type</code> JSON key value of \"Photograph\".</p> <ul> <li> <p>Selection of Single ADO and <code>Webform find-and-replace</code> Action</p> <p></p> </li> <li> <p>Webform Find and Replace Form</p> <p></p> </li> <li> <p>Confirmation of Successfully Executed Changes</p> <p></p> </li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the main Find and Replace documentation page or the Archipelago Documentation main page.</p>","tags":["Advanced Batch Find and Replace","Webform Find and Replace","Search and Replace"]},{"location":"firstobject/","title":"Your First Digital Object","text":"<p>You followed every Deployment step and you have now a local <code>Archipelago</code> instance. Great!</p> <p>So what now? It is time to give your new repository a try and we feel the best way is to start by ingesting a simple Digital Object.</p> <p>Note</p> <p>This guide will assume Archipelago is running on <code>http://localhost:8001</code>, so if you wizardly deployed everything in a different location, please replace all <code>URIs</code> with your own setup while following this guide.</p>"},{"location":"firstobject/#requirements","title":"Requirements","text":"<ul> <li>Running Archipelago (http://localhost:8001)</li> <li>20 minutes of your time.</li> <li>Open Mind.</li> </ul>"},{"location":"firstobject/#welcome","title":"Welcome!","text":"<p>Start by opening <code>http://locahost:8001</code> in your favourite Web Browser.</p> <p></p> <p>Your Demo deployment will have a fancy Home page with some banners and a small explanation of what Archipelago is and can do. Feel free to read through that now or later.</p> <p>Click on <code>Log in</code> in the top left corner and use your <code>demo</code> credentials from the deployment guide.</p> <ul> <li>user: demo</li> <li>pass: demo</li> </ul> <p>(or whatever password you decided was easy for you to remember during the deployment phase)</p> <p>Press the <code>Log in</code> button.</p> <p></p> <p>Great, welcome <code>demo</code> user! This users has limited credentials and uses the same global theme as any anonymous user would. Still, <code>demo</code> can create content, so let's use those super powers and give that a try.</p> <p>You will see a new <code>Menu item</code> under <code>Tools</code> on the top navigation bar named <code>Add Content</code>. Click it!</p> <p></p>"},{"location":"firstobject/#brief-background","title":"Brief Background","text":"<p>As you already know Archipelago is build on <code>Drupal 8/9</code>, a very extensible <code>CMS</code>. In practice that means you have (at least) the same functionality any Drupal deployment has and that is also true for Content Managment.</p> <p>Drupal ships by default with a very flexible <code>Content Entity Type</code> named <code>Node</code>. <code>Nodes</code> are used for creating Articles and simple Pages but also in Archipelago as <code>Digital Objects</code>. Drupal has a pretty tight integration with <code>Nodes</code> and that means you get a lot of fun and useful functionality by default by using them.</p> Want to know more about Entities? <p>An <code>Article</code> and a <code>Digital Object</code> are both of type <code>Nodes</code>, but each one represents a different <code>Content Type</code>. <code>Content Types</code> are also named <code>Bundles</code>. An individual Content, like \"Black and White photograph of a kind Dog\" is named a <code>Content Entity</code> or more specific in this case a <code>Node</code>.</p> <p>What have <code>Article</code> and <code>Digital Object</code> Content types in common and what puts the apart?</p> <ul> <li>Each Content Type or Bundle has a set of <code>Base Fields</code> and also user configurable set of <code>Fields</code> attached (or bundled together).<ul> <li>E.g. <code>Article</code> has a title, a Body and the option to add an image.</li> <li><code>Digital Object</code> has a title but also a special, very flexible one named <code>Strawberry Field</code> (more about that later).</li> </ul> </li> <li> <p>Fields are where you put your data into and also where your data comes from when you expose it to the world.</p> <ul> <li><code>Nodes</code>, as any other Content entity have Base Fields (which means you can't remove or configure them) that are used all over the place. Good examples are the <code>title</code> and also the owner, named <code>uid</code> (you!).</li> <li>Other Fields, specific to a Content Type, can be added and configured per Bundle.</li> <li>A <code>Field Widget</code> is used to input data into a Field.</li> <li>Each field can have a <code>Field Formatter</code> that allows you to setup how it is displayed to the World.</li> <li>A set of <code>Field Formatters</code> (the way you want to show your content formatted to the world) is named a <code>Display Mode</code>. You can have many, create new ones and remove them, but only use one at the time.</li> <li>A set of <code>Field Widgets</code> (the way you want to Create and Edit a <code>Node</code>) is named a <code>Form Mode</code>. You can also have many, create new ones but only use one at the time.</li> </ul> </li> <li> <p>Each Content Type can have different Permissions (using the build in <code>User Roles</code> system).</p> </li> <li>Each Content Type can have one or more <code>Display Modes</code>. In Practice this means <code>Display Modes</code> are attached to <code>Content Types</code>.<ul> <li>Each display modes can have its own Permissions</li> </ul> </li> <li>Each Content Type can have one or more <code>Form Modes</code>. In Practice this means <code>Form Modes</code> are attached to <code>Content Types</code>.<ul> <li>Each <code>Form Mode</code> can have its own Permissions.</li> </ul> </li> </ul> <p>There is of course a lot more to Nodes, Content Types, Formatters, Widgets and in general Content Entities but this is a good start to understand what will happen next.</p>"},{"location":"firstobject/#adding-content","title":"Adding Content","text":"<p>Below you see all the <code>Content Types</code> defined by default in Archipelago. Let's click on <code>Digital Object</code> to get your first Digital Object Node.</p> <p></p>"},{"location":"firstobject/#my-metadata","title":"My Metadata","text":"<p>What you see below is a <code>Form Mode</code> in action. A multi-step Webform that will ingest metadata into a field of type Strawberry Field (where all the magic happens) attached to that field using a <code>Webform Field Widget</code>, an editorial/advanced Block on the right side, and a <code>Quick Save</code> button at the bottom for saving the session.</p> <p>Let's fill out the form to begin our ingest. We recommend using similar values as the ones shown in the screen capture to make following the tutorial easier.</p> <p>Make sure you select <code>Photograph</code> as <code>Media Type</code> and all the fields with a red <code>*</code> are filled up. Then press <code>Move on to next step</code> at the bottom of the webform to load the next step in line.</p> <p></p>"},{"location":"firstobject/#collections","title":"Collections","text":"<p>Since this is our first digital object we do not yet have a Digital Object Collection for which <code>My First Digital Object</code> could be a member of. In other words, you can leave <code>Collection Membership</code> blank and click <code>Next: Upload Files</code>.</p> <p></p> Why does this look different than Repository X? <p>We assume you come from a world where repositories define different Content types and the shape, the fields and values (Schema) are fixed and set by someone else or at least quite complicated to configure. This is where Archipelago differs and starts to propose its own style. You noticed that there is a single Content Type named <code>Digital Object</code> and you have here a single Web Form. So how does this allow you to have images, sequences, videos, audio, 3D images, etc?</p> <p>There are many ways of answering that, Archipelago works under the idea of an (or many) Open Schema(s), and that notion permeates the whole environment. Practical answer and simplest way to explain based on this demo is:</p> <ul> <li>The <code>Digital Object</code> is a generic container for any shape of metadata. Metadata is generated either via this Webform-based widget you're currently using, manually (power-user need only) or via APIs. Because of this, Metadata can take any shape to express your needs of Digital Objects and therefore we do not recommend making multiple Digital Object types. However, if you ever do need more Digital Object types, the option is available.</li> <li>The Strawberry field Field Widget allows you to attach any <code>Webform</code>, built using the <code>Webform Module</code> and Webforms can be setup in almost infinite ways. Any field, combo, or style can be used. Multi Step, single step - we made sure they always only touch/modify data they know how to touch, so even a single input element webform would ensure any previous metadata to persist even if not readable by itself (See the potential?). And Each Webform can be also quite smart!</li> <li>The Strawberry field Field Widget will take all your Webform input, process any uploaded files, generate a JSON representation, enrich and complement it with Archipelago specific data and save it for you inside the <code>Strawberry field</code>.</li> </ul> <p>We will come back to this later.</p>"},{"location":"firstobject/#linked-data","title":"Linked Data","text":"<p>As the name of this step suggests; you will be adding all your Linked Data elements here. This step showcases some of the autocomplete Linked Data Webform elements we built for Archipelago. We truly believe in Wikidata as an open, honest, source of Linked Open Data and also one where you can contribute back. But we also have LoC autocompletes and Getty.</p> <p>Again, enter all fields with a red <code>*</code> and when you are finished, click <code>Move on to next step</code></p> <p></p> <p>Tip</p> <p>When entering a location, place or address you will need to click on the <code>Search OpenStreet Maps</code> button, which is what that big red arrow is pointing to in the screenshot below.</p> <p></p>"},{"location":"firstobject/#upload-files","title":"Upload Files","text":"<p>Now we will upload our <code>Photograph</code>. Click <code>Choose Files</code> to open your file selector window and choose which file you would like to ingest.</p> <p></p> <p>Once you've uploaded your file, you will see all the Exif data extracted from the image, like so...</p> <p></p> <p>Once you've mentally digested all of that data, let's go ahead and click <code>Save Metadata</code>.</p> Wait! Why only the metadata? I'm ready for this Digital Object to be shared with the world! <p>By clicking <code>Save Metadata</code> we are simply persisting all the metadata in the current webform session. The actual ingest of the Object happens when you click <code>Save</code> on the next and final step, <code>Complete</code>.</p>"},{"location":"firstobject/#complete","title":"Complete","text":"<p>Alright, we've made it. We've added metadata, linked Data, uploaded our files and now... we're ready to save! Go ahead and change the status from Draft to Published and click <code>Save</code>.</p> <p></p> <p>Once you hit save you should see the following green message and your first Archipelago Digital Object!</p> <p></p> <p>Congratulations on creating your first digital object! \ud83c\udf53</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"fragaria/","title":"Fragaria Redirects Module","text":"<p>Archipelago's Fragaria Redirect Module is a Drupal 9/10 module that provides dynamic redirect routes matched against existing Search API field values. This module will also provide future Unique IDs (API) integrations and PURLs.</p>","tags":["Fragaria","Redirects"]},{"location":"fragaria/#prerequisites","title":"Prerequisites","text":"<p>Before proceeding with the following configuration steps, you need to first create the Strawberry Key Name Provider and Solr Field that corresponds to the Field that will be matched against the variable part of the route exists.</p> <p>In other words, if your Digital Objects have a field and value such as:</p> <pre><code>    \"legacy_PID\": [\n        \"mylegacyrepo:1234\"\n    ],  \n</code></pre> <p>You need to make sure the values from the <code>legacy_PID</code> JSON key are indexed (as a Solr/Search API Field) and ready for use as part of the Fragaria Redirects configuration. </p> <p>Best Practice</p> <p>Your new Solr field should to be of field type \"String\" for a perfect match and best results. Using \"Full Text\" or a related variant Solr field type will allow for a partial match, which might lead multiple original URLs redirecting to the first match in Archipelago.</p>","tags":["Fragaria","Redirects"]},{"location":"fragaria/#fragaria-redirect-entity-configuration","title":"Fragaria Redirect Entity Configuration","text":"<ol> <li> <p>Navigate to <code>/admin/config/archipelago/fragariaredirect</code>.</p> </li> <li> <p>Select the <code>Add a Redirect Config</code> button.</p> </li> <li> <p>Enter a label for the Fragaria Redirect Entity you are configuring.</p> </li> <li> <p>Enter the Prefix (that follows your domain) for the Redirect Route.</p> <ul> <li>Do not use <code>node/</code> or <code>do/</code> as a Prefix. Even if these will technically work (redirect), using either of these Prefix paths will override your existing Paths defined by Drupal and Archipelago.</li> </ul> </li> <li> <p>If applicable, enter the the Suffixes (that follow the prefix + the variable part) for the Redirect Route.</p> <ul> <li>Enter one by line. This configuration option is not required.</li> <li>Check/uncheck the option to <code>Instead of fixed Prefixes add a single {catch_all} variable suffix at the end</code> as needed. Checking this will disable any entered static suffixes.</li> </ul> </li> <li> <p>Select the Search API Index where the Field that will be matched against the variable part of the route exists.</p> </li> <li> <p>Select the Search API Field that will be matched against the variable part of the route.</p> </li> <li> <p>If applicable, add static prefixes for to the variable part/argument of the path.</p> <ul> <li>Enter one by line. This is useful when the variable part of the ROUTE does not match 1:1 the actual indexed data. e.g the route is /oldrepo/1 and the indexed value is \"namespace:1\". In that case add \"namespace:\" here.</li> </ul> </li> <li> <p>Add static suffixes for to the variable part/argument of the path.</p> <ul> <li>Enter one by line. This is useful when the variable part of the ROUTE does not match 1:1 the actual indexed data. e.g the route is /oldrepo/namespace:1 and the indexed value is \"namespace:1-page\". In that case add \"-page\" here.</li> </ul> </li> <li> <p>Select the Type of HTTP redirect to perform.</p> <ul> <li>Option 1: Temporary Redirect (forced GET)</li> <li>Option 2: Permanent Redirect</li> </ul> </li> <li> <p>Lastly, select the box next to <code>Is this Fragaria Redirect Route active?</code> to set your Redirect to <code>active</code>, and Save your configuration.</p> </li> </ol>","tags":["Fragaria","Redirects"]},{"location":"fragaria/#example-redirects-configuration","title":"Example Redirects Configuration","text":"<p>The above example configuration would enable a Temporary redirect from a legacy repository site with a URL of https://mylegacyrepo.edu//mylegacyrepo/object/mylegacyrepo:1234 to your new Archipelago PURL of https://mynewarchipelagorepo.edu/do/mynewADOUUID.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Fragaria","Redirects"]},{"location":"generalqa_minio_logging/","title":"Min.io Logging","text":"<p>Q: How can I see my minio (S3) docker container's realtime traffic and requests?</p> <p>A: For standard demo deployments, mini.io storage server runs on the <code>esmero-minio</code> docker container. Steps are:</p> <ol> <li> <p>Install the <code>mc</code> binaries (minio client) for your platform following this instructions. e.g for OSX run on your terminal:</p> <pre><code>brew install minio/stable/mc\nmc alias set esmero-minio http://localhost:9000 user password\n</code></pre> <p>with <code>http://localhost:9000</code> being your current machines mini.io URL and exposed port,  <code>user</code> being your username (defaults to <code>minio</code>) and your original choosen <code>password</code> (defaults to <code>minio123</code>)</p> </li> <li> <p>Run a <code>trace</code> to watch realtime activity on your terminal:</p> <pre><code>mc admin trace -v -a --debug  --insecure --no-color esmero-minio\n</code></pre> </li> </ol> <p>Note: <code>mc</code> client is also AWS S3 compatible and can be used to move/copy/delete files on the local instance and to/from a remote AWS storage.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"generalqa_smtp_configuration/","title":"SMTP Configuration","text":"<p>Q: How can I enable SMTP for Archipelago?</p> <p>A: For standard demo deployments, SMTP is not setup to send emails. To enable SMTP:</p> <ol> <li> <p>Enter the following commands in your terminal. Note: make sure docker is running. Optionally, you can verify that all Archipelago containers are present by entering the <code>docker ps</code> command first.</p> <pre><code>docker exec -ti esmero-php bash -c 'php -dmemory_limit=-1 /usr/bin/composer require drupal/smtp:^1.0'\ndocker exec -ti esmero-php bash -c 'drush en -y smtp'\n</code></pre> </li> <li> <p>Check that the SMTP module has been enabled by navigating (as admin user) to the EXTEND module menu item (<code>localhost:8001/admin/modules</code>). You should see \"SMTP Authentication Support\" listed.</p> </li> <li> <p>Navigate to <code>localhost:8001/admin/config/system/smtp</code> to configure the SMTP settings.</p> This screenshot shows settings if a GMAIL account is used. <p></p> </li> <li> <p>Save your settings, then test by adding a recipient address in the \u201cSEND TEST E-MAIL\u201d field.</p> </li> </ol> <p>Note: Depending on your email provider, you may also need to enable \u201cless secure\u201d applications in your account settings (such as here for Google email accounts: https://myaccount.google.com/lesssecureapps)</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"generalqa_twig_modules_configuration/","title":"Twig Modules Configuration","text":"<p>Q: When attempting to save a Twig template for a Metadata Display, I receive an error message related to an <code>Unknown \"bamboo_load_entity\" function</code>.</p> <p></p> <p>A: You need to enable the necessary Twig modules.</p> <ol> <li> <p>Navigate to: <code>yoursite/admin/modules</code></p> </li> <li> <p>In the \u201cEnter a part of the module name or description\u201d box, enter \u201cbam\u201d to filter for the related Bamboo Twig modules. Alternatively, scroll down to the Bamboo Twig modules section on this page.</p> <p></p> </li> <li> <p>Check the box next to each of the following to enable (some may already be enabled):</p> <ul> <li>Bamboo Twig</li> <li>Bamboo Twig - Loaders</li> <li>Bamboo Twig - Path &amp; Url</li> <li>Bamboo Twig - Token</li> </ul> <p></p> </li> <li> <p>Click <code>Install</code>.</p> </li> <li> <p>After receiving the successful installation confirmation, check to make sure you are now able to save your Twig template without receiving an error message.</p> </li> </ol> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"giveortake/","title":"Archipelago Contribution Guide","text":"<p>Contributing Documentation</p> <p>Looking to contribute documentation? Start here.</p> <p>Archipelago welcomes and appreciates any type of contribution, from use cases and needs, questions, documentation, devops and configuration and -- of course -- code, fixes, or new features. To make the process less painful, we recommend you first to read our documentation and deploy a local instance. After that please follow the guidelines below to help you get started.</p> <p><code>Archipelago</code> welcomes, appreciates, and recognizes any and all types of contribution. This includes input on all use cases and needs, questions or answers, documentation, DevOps, and configurations. We also welcome general ideas, thoughts, and even dreams for the future of our repository! Of course, we also invite you to contribute PHP code, including fixes and new features.</p> <p>We will be helpful, kind, and open. We encourage discussions and always respect one another's opinions, language, gender, style, backgrounds, origins, and destinations, provided they come from the same root values of respect, as stated here. We support conflict resolution using nothing more than basic common sense. We value diversity in all its shapes, forms, colors, epoches, numbers, and kinds, with or without labels, including in-between and evolving. We always assume we can do better and that you have done a lot. Under this very basic social framework, this is how we hope you can contribute:</p>"},{"location":"giveortake/#where-the-wild-things-live","title":"Where The Wild Things Live","text":"<p>Archipelago has 5 active GitHub repositories</p> <ul> <li>Strawberryfield   What? Code: Deals with metadata storage and exposure to Drupal. Events/Subscribers that trigger when content is modified. The way internal pieces of JSON are exposed to the rest of the ecosystem. Core to all of what Archipeleago is as a concept. One of its Drupal forms is a Field.</li> <li>Webform Strawberryfield   What? Code: Deals with UI based ingest of content using webforms. How files and media are attached to JSON, how tech md is extracted, and any interaction that happens during the edit and ingest processes via a form. In its Drupal form it provides a Field Widget for <code>Strawberryfield</code>.</li> <li>Format Strawberryfield   What? Code: Deals with exposing and transforming the JSON when navigating the site. What is displayed, how it is displayed. Provides templating, metadata display entities via Twig, and direct file downloads. In its Drupal form it provides many field formatters for Strawberryfield` and a content entity for the Twig templates.</li> <li>Archipelago Deployment   What? DevOps: Docker-compose deployment strategy, including a full skeleton project with persistence folders for Min.io, DB, Solr, Cantaloupe and Drupal 8. Includes initial deployment configurations, which modules are enabled, how things look in Drupal 8 and some scripts plus the deployment documentation for both OSX and Linux.</li> <li>Archipelago Documentation   What? Documentation: This guide and whatever we manage to write to explain Archipelago goes here.</li> </ul>"},{"location":"giveortake/#questions-answers-and-in-transit-between-both-ends","title":"Questions, Answers and In Transit Between Both Ends","text":"<p>We host a community interaction channel, our google group. This is the best place to ask questions and make suggestions that are not specific to a single module, and/or if you would like to contribute to a larger conversation within our community. Discussions work best in this forum (not excluding GitHub of course), and our official announcements are posted there too.</p>"},{"location":"giveortake/#documentation-workflow","title":"Documentation Workflow","text":"<p>Documentation is an evolving effort, and we need help. This guide lives in GitHub in Archipelago Documentation. Documentation and Development Worklfow both work the same way, so keep reading!</p>"},{"location":"giveortake/#development-workflow","title":"Development Workflow","text":"<p>Start by reading open ISSUES (so you don't end up redoing what someone else is already working on) and looking at our Roadmap for version 1.0.0. If the solution to your problem is not there or if there is an unchecked element in the roadmap, this is a great opportunity to help by creating a new ISSUE.</p> <p>Next, start by opening an GitHub ISSUE in any of the 5 GitHub repositories, depending on what it is you are trying to do.</p> <p>Please be concise with the title of your ISSUE so that it is easy to understand. Use Markdown to explain the what, how, etc, of your contribution. Note: Even if something related is already in the works, you can still contribute. Just add your comments on any open ISSUE. Or, if you think you want to contribute with a totally different perspective, feel free to open a new ISSUE anyway. We can always discuss next steps starting from there. Every community has its rhythm and style and our style is just beginning to develop. We are still figuring out what works best for everyone.</p> <p>Once you are done and you feel comfortable working to make a change yourself, take note of the <code>ISSUE number</code> (lets name it <code>#issuenumber</code>).</p> <p>The gist is:</p> <ul> <li>Fork the GitHub repository where you created the ISSUE, decide which branch you want to change.</li> <li>Make a new branch out of that one and name it ISSUE-#issuenumber. E.g if #issuenumber is 6, name your branch ISSUE-6.</li> <li>Make changes in that branch and send a pull request.</li> </ul> <p>As a best practice, we encourage pull requests to discuss/fix existing code, new code, and documention changes.</p> <p>For the full step-by-step workflow, we will use Archipelago Documentation and the <code>1.0.0</code> branch as example. The same applies to any of the other repositories: just change the remote urls and use the most current branch name.</p>"},{"location":"giveortake/#example-set-up-archipelago-documentation-github-repository","title":"Example: Set Up Archipelago Documentation GitHub Repository","text":"<p>Fork the Archipelago Documentation Upstream source repository to your own personal GitHub account (e.g. YOU). Copy the URL of your Archipelago Documentation fork (you will need it for the <code>git clone</code> command below).</p> <pre><code>$ git clone https://github.com/YOU/archipelago-documentation\n$ cd archipelago-documentation\n$ git checkout 1.0.0\n</code></pre>"},{"location":"giveortake/#set-up-git-remote-as-upstream","title":"Set Up Git Remote As <code>upstream</code>","text":"<pre><code>$ git remote add upstream https://github.com/esmero/archipelago-documentation\n$ git fetch upstream\n$ git merge upstream/1.0.0\n...\n</code></pre>"},{"location":"giveortake/#create-your-issue-branch","title":"Create Your ISSUE Branch","text":"<p>Before making changes, make sure you create a branch using the ISSUE number you created for these contributions.</p> <pre><code>$ git checkout -b ISSUE-6\n</code></pre>"},{"location":"giveortake/#do-some-clean-up-and-test-locally","title":"Do Some Clean Up and Test Locally","text":"<p>After your code changes, make sure</p> <ul> <li>If modifying <code>PHP</code>, run <code>phpcs --standard=Drupal yourchanged.file.php</code>. We (try our best to) use Drupal 8 coding standards.</li> <li>If modifying a <code>MARKDOWN</code> file, make sure it renders well (you can use Textmate, Atom, Textile, etc to preview) and that links are not broken.</li> <li>If writing large pieces of code, add PHP Tests. We can help if you don't know how (tell us in the ISSUE). Tests will be enforced starting with the first stable release.</li> <li>If modifying <code>PHP</code>, please test your changes live on your local instance of Archipelago. All non-documentation modules are already inside <code>web/modules/contrib/</code>.</li> </ul>"},{"location":"giveortake/#commit-changes","title":"Commit Changes","text":"<p>After verification, commit your changes. This is very good post on how to write commit messages.</p> <pre><code>$ git commit -am 'Fix that Strawberry'\n</code></pre>"},{"location":"giveortake/#push-to-the-branch","title":"Push To The Branch","text":"<p>Push your locally committed changes to the remote origin (your fork)</p> <pre><code>$ git push origin ISSUE-6\n</code></pre>"},{"location":"giveortake/#create-a-pull-request","title":"Create A Pull Request","text":"<p>Pull requests can be created via GitHub. This document explains in detail how to create one. After your Pull Request gets peer reviewed and approved, it can be merged. Discussion can happen and peers can ask you for modifications, fixes or more information on how to test. We will be respectful. You will be given credit for all your contributions and shown appreciation. There is no wrong and never too little. There could never be too much!</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"googleapi/","title":"Configuration for Google Sheets API","text":"<p>To allow the Archipelago Multi Importer (AMI) to read from Google spreadsheets, you first need to configure the Google Sheets API as outlined in the following instructions.</p> <p>Please note:</p> <ul> <li>Frequent changes to the Google Sheets API specifications may impact the configurations needed. </li> <li>This set of instructions will only work for individuals using Google accounts affiliated with Organizations.</li> <li>Please contact us on our Archipelago Commons Google Group with any questions/issues.</li> </ul>"},{"location":"googleapi/#generating-google-oauth2-credentials","title":"Generating Google OAuth2 Credentials","text":"<ol> <li> <p>Login to the Google Developer Console. You will see the API &amp; Services Dashboard.</p> <p></p> </li> <li> <p>If you have not created Credentials or a Project before, you will need to first create a Project.  </p> <ul> <li>Recommended Project Name: \"Archipelago Multi Importer\" or \"AMI\".</li> <li>The Organization and Location information should be specific to you and your organization/institution.</li> </ul> <p></p> </li> <li> <p>Next, click the <code>Create credentials</code> select box and select <code>OAuth client ID</code></p> <p></p> </li> <li> <p>You will now need to Configure the Consent Screen.</p> <p></p> </li> <li> <p>On the initial OAuth Consent Screen setup, select <code>Internal</code> for User Type.</p> <p></p> </li> <li> <p>Now enter <code>AMI</code> as the App name, and your email address in the User support email. You may also wish to add Authorized domains (bottom of image below) as well.</p> <p></p> </li> <li> <p>On the Scopes page, select <code>Add or Remove Scopes</code>. Then either search/filter the API table for the Google Sheets API. Or, under <code>Manually add scopes</code> enter: https://www.googleapis.com/auth/spreadsheets.readonly</p> <p></p> </li> <li> <p>After selecting or entering in the Google Sheets API, you should see this listed under <code>Sensitive Scopes</code>.</p> <p></p> </li> <li> <p>Review the information on the <code>Summary</code> page, then Save.</p> <p></p> </li> <li> <p>You will now be able to <code>Create Oauth client ID</code>. Select <code>Web Application</code> as the <code>Application type</code></p> <p></p> </li> <li> <p>Enter \"AMI\" under 'Name' and add any URIs you will be using below.</p> <ul> <li>For using AMI within your local Archipelago environment, enter <code>http://localhost:8001/google_api_client/callback</code></li> <li> <p>All URIs need to include <code>/google_api_client/callback</code></p> <p></p> </li> </ul> </li> <li> <p>After Saving, you will see a message notifying you that the OAuth client was created. You can copy the <code>Client ID</code> and <code>Client Secret</code> directly from this confirmation message into a text editor. You can also access the information from <code>Credentials</code> in the <code>APIs &amp; Services</code> section in the Developer console, where you will have additional options for downloading, copying, and modifying if needed.</p> <p></p> <p></p> </li> <li> <p>On the 'Add Google Api Client account' configuration page, enter the following information using your <code>Client ID</code> and <code>Client Secret</code>. 'Developer Key' is optional. Select <code>Google Sheets API</code> under 'Services' and <code>https://www,googleapis.com/auth/spreadsheets.readonly</code> under 'Scopes'. Check the box for <code>Is Access Type Offline</code>. Select the Save button.</p> <p></p> </li> <li> <p>You will now need to Authenticate your AMI Google API Client. Return to the Google API Client Listing page. Under the Operation menu on the right-hand side of the AMI client listing, select <code>Authenticate</code>.</p> <p></p> </li> <li> <p>You will be directed to the Google Consent Screen. You may need to login to your corresponding Google Account before proceeding. When loged in, you will see the following screen requesting that AMI is allowed to \"View your Google Spreadsheets\".  Click <code>Allow</code>.</p> <p></p> </li> <li> <p>On the Google API Client Listing page, your AMI client listing should now have 'Yes' under 'Is Authenticated'. You are now ready to use Google Sheets with AMI! Return to the main AMI documentation page to get started.</p> </li> </ol> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"iiif-content-search/","title":"IIIF Content Search API Integration","text":"<p>Beginning in release 1.3.0 and now fully mature in 1.4.0, Archipelago features IIIF Content Search API integration with attendant default configurations and settings.</p> <p>Through a non-trifling amount of code and maths, Archipelago speaks the IIIF Content Search API language using data from your Archipelago's Digital Objects, to enable you to search within Mirador (or other supported viewers) for specific hits within OCR, VTT file, or manually created textual annotations. </p> <p>Please also see the related IIIF Server Settings Form, and Strawberry Runners guides for Reviewing and adjusting the <code>pager</code> and <code>ocr</code> Post-Processor operations and Reviewing and Adjusting the <code>subtitle</code> Post-Processor operations.</p>","tags":["IIIF","IIIF Server Settings Form","IIIF Content Search API","Solr","Solr Fields","Solr Index"]},{"location":"iiif-content-search/#1-iiif-manifest-templates","title":"1. IIIF Manifest Templates","text":"<p>First, Archipelago's default IIIF Manifest templates explicitly state that they support the 3 versions of IIIF Content Search APIS in the 'service' key.</p> <pre><code>\"service\": [ \n        { \n            \"id\": \"{{ baseurl }}iiifcontentsearch/v2/do/{{ node.uuid.value }}/metadatadisplayexposed/iiifmanifest/mode/advanced/page/0\", \n            \"type\": \"SearchService2\" \n        }, \n        { \n            \"id\": \"{{ baseurl }}iiifcontentsearch/v1/do/{{ node.uuid.value }}/metadatadisplayexposed/iiifmanifest/mode/advanced/page/0\", \n            \"type\": \"SearchService1\", \n            \"@context\": \"http://iiif.io/api/search/1/context.json\", \n            \"profile\": \"http://iiif.io/api/search/1/search\" \n        }, \n        { \n            \"@id\": \"{{ baseurl }}iiifcontentsearch/v1/do/{{ node.uuid.value }}/metadatadisplayexposed/iiifmanifest/mode/advanced/page/0\", \n            \"@context\": \"http://iiif.io/api/search/0/context.json\", \n            \"profile\": \"http://iiif.io/api/search/0/search\" \n        } \n    ], \n</code></pre>","tags":["IIIF","IIIF Server Settings Form","IIIF Content Search API","Solr","Solr Fields","Solr Index"]},{"location":"iiif-content-search/#2-api-endpoints-exposure","title":"2. API Endpoints Exposure","text":"<p>Next, in the default Exposed Metadata Endpoints API Endpoints (generated from the IIIF templates), Archipelago provides the specific structure needed for the IIIF Content Search API. Archipelago passes the data about \u201cthe template containing it\u201d, the IIIF API version, if simple or advanced, and the Archipelago Digital Object resource UUID we are searching against (the one that contains the RAW data feeding the template, or at least the Top level/parent one of that).</p>","tags":["IIIF","IIIF Server Settings Form","IIIF Content Search API","Solr","Solr Fields","Solr Index"]},{"location":"iiif-content-search/#3-pathway-into-and-out-of-the-solr-index","title":"3. Pathway into and out of the Solr Index","text":"<p>Then, Archipelago's backend recreates an ADO's IIIF manifest using this data (basically repeats what the client did before), but uses JMESPATHs to extract just what is needed, flipping the order of the structure and putting IIIF Image IDs, as \"top keys\" referencing canvases and their #xywh selectors (for the annotation text), if present.</p> <p>Using this transformed data, Archipelago's backend search is able to be limited to OCR generated only by those images (importantly, as Archipelago repositories can contain millions of OCR'd documents). Archipelago's internal search then returns natively, via the Bavarian State Library\u2019s Solr OCR highlight plugin, the relevant hits within a specified ADO. These are then reprocessed to be IIIF compliant (W3C) annotations and then reverted back to results as \u201ccanvases with images\u201d.</p>","tags":["IIIF","IIIF Server Settings Form","IIIF Content Search API","Solr","Solr Fields","Solr Index"]},{"location":"iiif-content-search/#things-to-keep-in-mind","title":"Things to keep in mind","text":"<ul> <li> <p>To make this performant, Archipelago uses two levels of caches that get invalidated automatically on any \"ingredient\" used modification.</p> </li> <li> <p>Archipelago can also tell the backend to use a \"different\" template than the one used at the front (Mirador), allowing you to define which \"canvases\" are searchable. This is not a normal use case, but still a valid one. And you can, per resource, have complex logic and/or different Viewers, even on a one by one basis.</p> </li> </ul>","tags":["IIIF","IIIF Server Settings Form","IIIF Content Search API","Solr","Solr Fields","Solr Index"]},{"location":"iiif-content-search/#acknowledgements","title":"Acknowledgements","text":"<p>Archipelago's developers would like to extend our gratitude to our community, especially to Mike and Johannes for their work and help, and everyone else in the IIIF and repository communities for all the amazing tools, viewers, specs and cookbook examples.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p>","tags":["IIIF","IIIF Server Settings Form","IIIF Content Search API","Solr","Solr Fields","Solr Index"]},{"location":"iiif_server_settings/","title":"IIIF Server Settings Form Default Settings","text":"<p>The IIIF Server Settings Form is used to configure different IIIF related settings used throughout your Archipelago environment. We strongly advise keeping the default settings intact. The necessary Solr Fields listed below should be setup by default.</p> <p>You can find the IIIF Configuration Form:</p> <ul> <li>Through the <code>Manage</code> menu &gt; <code>Configuration</code> &gt; <code>Archipelago</code> &gt; <code>Configure Strawberry Runners Post Processors</code></li> <li>Directly at <code>/admin/config/archipelago/iiif</code> </li> </ul> <p></p> <p>On the IIIF Server Settings Form page, you will see the following:</p> <ol> <li> <p>Note that these 'IIIF Server configuration URLs are used as defaults for field formatters using IIIF, but can be overridden on a one by one basis when setting up your formatters for each Display Mode.'</p> </li> <li> <p>Base URL of your IIIF Media Server public accessible from the Outside World.</p> <ul> <li>Please provide a publicly accessible IIIF server URL. This URL will be used for AJAX and JS calls. Trailing Slashes will be removed.</li> <li>Set to <code>http://localhost:8183/iiif/2</code> by default.</li> <li>We do not recommend changing this selection. </li> </ul> </li> <li> <p>Base URL of your IIIF Media Server accessible from inside this Webserver.</p> <ul> <li>Please provide Internal IIIF server URL. This URL will be used by Internal Server calls and needs to be locally accessible by your server, e.g 127.0.0.1 or an local Docker alias. Trailing Slashes will be removed.</li> <li>Set to <code>http://esmero-cantaloupe:8182/iiif/2</code> by default.</li> <li>We do not recommend changing this selection.</li> </ul> </li> <li> <p>Checkbox to 'Enable IIIF Content Search API V1 and V2 endpoints'.</p> <ul> <li>Checked by default in later (1.4.0+) versions of Archipelago.</li> <li>See the related (and essential) IIIF Manifest snippet shared here</li> <li>APIs are accesible at the following path: \"/iiifcontentsearch/{version}/do/{node_uuid}/metadatadisplayexposed/{metadataexposeconfig_entity}/mode/{mode}/page/{page}\" with:</li> <li>{version} one of [v1,v2]</li> <li>{node_uuid} the UUID of the ADO whose Manifest you want to search inside</li> <li>{metadataexposeconfig_entity} the machine name of the exposed Metadata Display endpoint used to render the Manifest that is calling the API (e.g iiifmanifest)</li> <li>{mode} one of [simple,advanced]. Advanced is the smartest choice. Simple is faster, but requires your Canvas ids to be exactly in this pattern http(s)://domain.ext/do/{node_uuid}/{file_uuid}/canvas/{internal_to_the_file_sequence_order}</li> <li>{page} 0 to N depedening on the Number of results. By default please use 0</li> </ul> </li> <li> <p>Checkbox to 'Only allow searches inside a Manifest If the Manifest itself (for an ADO) defines the Search Endpoints as a Service'</p> <ul> <li>Checked by default in later (1.4.0+) versions of Archipelago.</li> <li>If enabled we will double check if the calling IIIF Manifest defines the Endpoint(s) in the <code>service</code> key. If unchecked any Manifest will be searchable by calling an API URL directly.</li> </ul> </li> <li> <p>IIIF Content Search API: field(s) that holds Parent Nodes</p> <ul> <li>Strawberry Flavor Data Source Search API Fields that can be used to connect a Strawberry Flavor to a Parent AD0.</li> <li>Default specified fields are: <code>Strawberryfield Flavor Datasource &gt;&gt; SBF Parent ID</code> and <code>Strawberryfield Flavor Datasource &gt;&gt; SBF Parent Node &gt;&gt; isPartOf &gt;&gt; ID</code></li> </ul> </li> <li> <p>Strawberry Runner processors that should be searched against for visual highlights.</p> <ul> <li>e.g Strawberry Flavor Data might have been generated by the \"ocr\" strawberry runners processor. A comma separated list of processors (machine names) that generated miniOCR.</li> <li>Default is: <code>ocr</code></li> <li>If you are using the Strawberry Runners <code>pager</code> and <code>ocr</code> post-processors, you should always keep this enabled.</li> </ul> </li> <li> <p>Strawberry Runner processors that should be searched against for time based media.</p> <ul> <li>e.g Strawberry Flavor Data might have been generated by the \"subtitle\" strawberry runners processor. These will have time based fragments and will match IIIF Annotations with motivation supplementing and target the time based media on the parent Canvas. A comma separated list of processors (machine names) that generated time based transcripts encoded as miniOCR.</li> <li>Default is: <code>subtitle</code></li> </ul> </li> <li> <p>Check to 'Target the VTT Supplementing Annotation'</p> <ul> <li>If enabled (aligned with the specs) the target of a hit result will point to the supplementing Annotation containing in its body the VTT file. If not the Canvas containing in its body a Media Resource (less precise but more compatible with Viewers</li> <li>If you are using the Strawberry Runners <code>subtitle</code> post-processor, you should always keep this enabled.</li> </ul> </li> <li> <p>Strawberry Runner processors that should be searched against plain text extractions.</p> <ul> <li>e.g Strawberry Flavor Data might have been generated by the \"text\" strawberry runners processor. These will not have coordinates but will match IIIF Annotations with motivation supplementing and target the whole canvas. A comma separated list of processors (machine names) that generated time based transcripts encoded as miniOCR.</li> <li>Default is: <code>text</code></li> <li>If you are using the Strawberry Runners <code>subtitle</code> post-processor, you should always keep this enabled.</li> </ul> </li> <li> <p>IIIF Content Search API: field(s) that hold the URI of the File that produced the Searchable content</p> <ul> <li>Strawberry Flavor Data Source Search API Fields that hold the URI of the File that generated its content.</li> <li>Default specified fields are: <code>Strawberryfield Flavor Datasource &gt;&gt; Parent File</code>, <code>Strawberryfield Flavor Datasource &gt;&gt; SBF source or related URI/URL</code>, and<code>Strawberryfield Flavor Datasource &gt;&gt; Parent File &gt;&gt; URI</code></li> </ul> </li> <li> <p>IIIF Content Search API: Max Results per Page</p> <ul> <li>Default is: <code>25</code></li> </ul> </li> <li> <p>IIIF Content Search API: Max allowed characters/length for a Search term</p> <ul> <li>Default is: <code>64</code></li> </ul> </li> </ol> <p>Return to the main Strawberry Runners or the Archipelago Documentation main page.</p>","tags":["IIIF","IIIF Server Settings Form","IIIF Content Search API","Solr","Solr Fields","Solr Index"]},{"location":"inthewild/","title":"Archipelagos in the Wild \ud83d\uddfa\ufe0f","text":"<p>Explore Archipelago instances running free across digital realms.</p> <p>Note</p> <p>*Please be aware that some of the following Archipelago instances are still brewing and these links may change.</p>"},{"location":"inthewild/#metro-archipelago","title":"METRO + Archipelago","text":"<p>The Archipelagos listed below are supported by the Digital Services Team at the Metropolitan New York Library Council. \ud83e\uddd1\u200d\ud83c\udf3e \ud83d\udc1d \ud83c\udf53</p> <ul> <li> <p>Archipelago (Early/Legacy) Playground and Studio Site</p> <ul> <li>METRO's public (play..) and internal (studio..) Archipelago playgrounds to experiment, learn, and evaluate.</li> </ul> </li> <li> <p>Barnard College</p> </li> <li> <p>Digital Culture of Metropolitan New York (DCMNY)</p> </li> <li> <p>Empire Archival Discovery Cooperative (EADC) Finding Aid Toolkit</p> <ul> <li>Supported by the Southeastern New York Library Resources Council (SENYLRC) </li> </ul> </li> <li> <p>Empire Immersive Experiences</p> <ul> <li>Supported by the Western New York Library Resources Council (WNYLRC) </li> </ul> </li> <li> <p>Frick Collection and Webrecorder Team Web Archives Collaboration</p> </li> <li> <p>Hamilton College Library &amp; IT Services</p> </li> <li> <p>Olin College Library Phoenix Files</p> <ul> <li>*Early adopter - live since Summer 2020</li> </ul> </li> <li> <p>New York State Archives Finding Aids Discovery Portal</p> <ul> <li>*Migration and development kicked off Spring 2024</li> </ul> </li> <li> <p>New York State COVID-19 Personal History Initiative</p> </li> <li> <p>Rensselaer Polytechnic Institute Libraries</p> </li> <li> <p>San Diego State University Libraries Digital Collections</p> </li> <li> <p>Union College Library</p> </li> <li> <p>Western Washington University</p> </li> </ul>"},{"location":"inthewild/#neighborhood-archipelagos","title":"Neighborhood Archipelagos","text":"<p>From all around our beautiful shared world. \ud83c\udfe1 \ud83c\udfeb \ud83c\udfdb\ufe0f </p> <ul> <li> <p>Amherst College</p> </li> <li> <p>Association Montessori Internationale</p> </li> <li> <p>California Revealed</p> </li> <li> <p>Christian Observatory of the Pro Civitate Christiana</p> </li> <li> <p>Consiglio Nazionale delle Ricerche / National Research Council of Italy</p> <ul> <li>https://dbopen.ba.cnr.it/</li> <li>https://archiplavit.to.cnr.it/</li> <li>https://vitisgrinzane.ipsp.cnr.it/</li> <li>http://archipelago.byterfly.eu/ \ud83e\udd8b<ul> <li>Virtual Tour Santuario Paola</li> </ul> </li> </ul> </li> <li> <p>Universidad Nacional Aut\u00f3noma de M\u00e9xico</p> </li> <li> <p>University of Edinburgh Libraries</p> <ul> <li>*Development of Archipelago environment began late 2022/3</li> </ul> </li> <li> <p>Vipassana Treasures</p> </li> </ul>"},{"location":"inthewild/#we-should-be-here","title":"We should be here","text":"<p>If you have a public Archipelago instance you'd like to share on this page \ud83c\udfdd\ufe0f\ud83d\udccd, please contact us. We would love to add your great work to this list! \ud83d\udc9a </p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"metadata_display_preview/","title":"Metadata Display Preview","text":"<p>Archipelago's Metadata Display Preview is a very handy tool for your repository toolkit that enables you to preview the output of your Metadata Display (Twig) Templates (found at <code>/metadatadisplay/list</code>). You can use the Metadata Display Preview to test and check the results of any type of Template (HTML Display, JSON Ingest, IIIF JSON, XML, etc.) against both Archipelago Digital Objects (ADOs) and AMI Sets (rows within). </p> <p>Prerequisite Note</p> <p>Before diving into Metadata Display (Twig) Template changes, we recommend reading our Twigs in Archipelago documentation overview guide and also our Working with Twig primer.</p>","tags":["Metadata Display","Twig Template","Preview"]},{"location":"metadata_display_preview/#step-by-step","title":"Step-by-Step","text":"<ol> <li> <p>Navigate to the Metadata Display list at <code>/admin/content/metadatadisplay/list</code> (or through the admin menu via  <code>Manage &gt; Content &gt; Metadata Displays</code>). From the main Metadata Display List page, you can access all of the different display, rendering, and processing templates found in your Archipelago.</p> <p>Selecting a Metadata Display Template</p> <p></p> </li> <li> <p>Open and select 'Edit' for the Template you wish to Edit and/or Preview.</p> <p>Editing a Metadata Display Template</p> <p></p> </li> <li> <p>You will now be able to select either an Archipelago Digital Object (ADO) or AMI Set to Preview. Both selection types will use an autocomplete search (make sure the autocomplete matches fully against your selection before proceeding).</p> <p>Archipelago Digital Object (ADO) selection</p> <p></p> <p>AMI Set and Row selection</p> <p>For the Row, you can enter either a (CSV row) number</p> <p></p> <p>AMI Set and Row selection</p> <p>Or a label found within the Source Data CSV:</p> <p></p> </li> <li> <p>After you select your ADO or AMI Set and press the <code>Show Preview</code> button, the fuller Preview section will open up on the right side of the screen. The left side will continue to show the Metadata Display Template you originally selected to Edit.</p> <p>Tip</p> <p>It is strongly recommended to always select the option to \"Show Preview using native Output Format (e.g. HTML)\".</p> <p>Archipelago Digital Object (ADO) selection against an HTML Display template</p> <p></p> <p>AMI Set and Row selection against a JSON Ingest template</p> <p></p> </li> <li> <p>To keep track of the JSON keys used in your template select the <code>Show Preview with JSON keys used in this template</code> option before pressing <code>Show Preview</code>. For more details see below.</p> </li> <li> <p>Within the Preview Section on the right side of the screen:</p> <ul> <li>The top section contains full JSON metadata record for the selected digital object or AMI Set + specified row.</li> <li>If previewing against an AMI Set + specified row, the middle section will show the 'Reconciliated LoD' for the selected row if you have used AMI LoD Reconciliation for the selected AMI Set.</li> <li>The bottom section will show the rendered Output from the Template using the metadata from the selected ADO or AMI Set + specified row.</li> </ul> </li> <li> <p>From the Edit + Preview mode, you can:</p> <ul> <li>Add and edit additional JSON keys to an HTML-output Display Template, such as subjects from LoD sources, found in your digital objects and collections data.</li> <li>Preview your incoming AMI Sets against your Ingest Template to ensure all your Source Data CSV columns and values are being being mapped properly to their Archipelago destination JSON keys; And make adjustments as needed.</li> <li>Enrich a provided schema-based XML template to incorporate more elements found in your Archipelago environment.</li> <li>And more \ud83e\uddd1\u200d\ud83c\udf73\ud83c\udfa8\ud83c\udfc4     </li> </ul> </li> <li> <p>Select the <code>Show Preview</code> button as you make changes to refresh the Preview output and check your work. After saving any changes you may have made to your selected Template, all of the displays/AMI Sets/other outputs that reference this same Template will reflect the changes made.</p> </li> </ol>","tags":["Metadata Display","Twig Template","Preview"]},{"location":"metadata_display_preview/#metadata-display-preview-json-key-variables","title":"Metadata Display Preview JSON Key Variables","text":"<p>Note</p> <p>This feature is available as of <code>strawberryfield/format_strawberryfield:1.2.0.x-dev</code> and <code>archipelago/ami:0.6.0.x-dev</code>. To make use of it before the official 0.6.0/1.2.0 release you can run the following commands:</p> <ol> <li>Alias the next release branches with the current dependency requirement versions:     <pre><code>docker exec -ti esmero-php bash -c \"composer require 'archipelago/ami:0.6.0.x-dev as 0.5.0.x-dev' 'strawberryfield/format_strawberryfield:1.2.0.x-dev as 1.1.0.x-dev'\"\n</code></pre></li> <li>Then run any database updates:     <pre><code>docker exec -ti esmero-php bash -c \"drush updb\"\n</code></pre></li> <li>And finally, clear the cache:     <pre><code>docker exec -ti esmero-php bash -c \"drush cr\"\n</code></pre></li> </ol> <p>When creating or editing a Metadata Display Twig template, you can keep track of the JSON keys being used in the template by enabling the option after selecting an Archipelago Digital Object (ADO) or AMI Set row before pressing <code>Show preview</code>:</p> <p>Enable Metadata Display Preview Variables</p> <p></p> <p>The last two tabs in the Preview section above expand to show two tables listing the JSON keys that are used and unused by the template. The used keys are sorted by first instance line number (from the template) and the unused keys are sorted alphabetically.</p> <p>Metadata Display Preview Variables Used JSON Keys</p> <p></p> <p>Metadata Display Preview Variables Unused JSON Keys</p> <p></p> <p>The JSON Keys that appear in these tables will vary based on changes to the template and the selected ADO or AMI set row.</p>","tags":["Metadata Display","Twig Template","Preview"]},{"location":"metadata_display_preview/#warnings-and-errors","title":"Warnings and Errors","text":"<p>Warnings and Errors encountered during the processing will be shown at the top of the Preview section. A line number (from the template) will be included in the message if available.</p> <p>Warning</p> <p>A Warning will be generated if output can be rendered, and the output will be displayed below it.</p> <p></p> <p>Error</p> <p>An Error will be generated if no output can be rendered, and no output will be displayed.</p> <p></p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Metadata Display","Twig Template","Preview"]},{"location":"metadata_display_usage/","title":"Metadata Display Usage","text":"<p>Archipelago's Metadata Display Usage tab is an extension of Metadata Display Preview that provides a convenient way for you to check where a Metadata Display Template is currently being used across your Archipelago. You can review the direct and indirect usage of each Metadata Display Template (found at <code>/metadatadisplay/list</code>) across your whole system, and quickly jump to corresponding usage areas to make adjustments if needed. </p>","tags":["Metadata Display","Twig Template","Usage"]},{"location":"metadata_display_usage/#step-by-step","title":"Step-by-Step","text":"<ol> <li> <p>Navigate to the Metadata Display list at <code>/admin/content/metadatadisplay/list</code> (or through the admin menu via  <code>Manage &gt; Content &gt; Metadata Displays</code>). From the main Metadata Display List page, you can access all of the different display, rendering, and processing templates found in your Archipelago. You can also quickly review the 'In use' status from this page.</p> <p></p> </li> <li> <p>Click on the Template Name or select 'Edit' for the Template you wish to review and navigate to the 'Usage' tab.</p> </li> <li> <p>On the Usage tab for every Template in your Archipelago, you will be able to see the 'Label' and 'How' (direct or indirect) usage information for the following four areas across your whole system:</p> <ul> <li>Exposed Metadata Display Entities (such as an exposed Dublin Core XML endpoint)</li> <li>AMI sets (Archipelago Multi Importer)</li> <li>Entity View Modes (Primer on View/Display Modes)</li> <li>Views (Templates can be used by Views for rendering outputs, amongst other uses)</li> </ul> <p></p> <p> </p> </li> <li> <p>For each usage section noted above, you will be able to navigate to any corresponding Exposed Metadata Display Entity, AMI Set, Entity View Mode, or View to make changes to the template usage.</p> </li> </ol> <p>Before Making Any Changes</p> <p>Before diving into Metadata Display Usage review and making changes to what/where templates are used, we recommend reading our Twigs in Archipelago documentation overview guide and also our Working with Twig primer.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Metadata Display","Twig Template","Usage"]},{"location":"metadatainarchipelago/","title":"Your JSON, our JSON - RAW Metadata in Archipelago","text":"<p>From the desk of Diego Pino</p> <p></p> <p>Archipelago's RAW metadata is stored as JSON and this is core to our architecture. To avoid writing RAW over and over, this document will refer to RAW Metadata simply as Metadata.</p> <p>Data and Metadata can be extremely complex and extensive. The use cases that define what Data, Media and Metadata to collect, to catalog and expose, to use during search and discovery or to enable interactive functionality, including questions like  \"what public facing schemas, formats and serializations I need or want to be compliant with\" are as diverse and complex as the Metadata driving them.</p> <p>But also Metadata, in specific, is plastic and evolving as are use cases. And more over, some Metadata is descriptive and some Metadata is technical and there are other types of Metadata too, e.g Control Metadata.</p> <p>Finally Metadata is very close to their generators. Means you and your peers will know, better than any Software Development team, what is needed, useful and, many times also, available given what use case you have, end users needs and resources at hand, your In real life workflows and future expectations.</p>"},{"location":"metadatainarchipelago/#reason-behind-using-json","title":"Reason behind using JSON","text":"<ul> <li>Complex hierarchical Metadata needs to be machinable (and fast)</li> <li>All types of Metadata might need to be \"easy to access\" while generating output, visualizations, search and discovery.</li> <li>The shape of this Metadata might need to change and adapt in time. Idea that supports Open Schema v/s a highly structured and fixed Schema, e.g Relational database (RDB) for most of the use cases we think Archipelago can cover.</li> <li>JSON is shareable, portable, easy to validate online and offline.</li> <li>JSON is compact.</li> <li>JSON is searchable/filterable using Query Languages like JSONPATH and JMESPATH.</li> <li>JSON is easy to patch and modify in batches.</li> <li>JSON is platform agnostic and has been around for decades and has not changed since then.</li> <li>JSON is typed and allows ordering/sorting of data easily.</li> </ul>"},{"location":"metadatainarchipelago/#drupal-and-json","title":"Drupal and JSON","text":"<p>Drupal, the OSS CMS system Archipelago uses and extends, is RDB driven. This means that <code>Content Types</code> normally follow the idea of an Entity with Fields attached. Each of these Fields becomes then a Database Table and the sum of all these fields living under a <code>Content Type</code> definition, a fixed schema. </p> <p>For integration and interoperability reasons with the larger Drupal ecosystem, we inherit in Archipelago the idea of an Entity, in specific, a Content Entity (Node) and <code>Content Type</code> (Bundled fields for a Node). But instead of generating (and encouraging) the use of hundreds of fixed fields to describe your Digital Objects we put all Metadata as JSON, means a JSON BLOB, into a single smart Field of type <code>Strawberry Field</code>. \ud83c\udf53 </p> <p>We go a long way of making as much as possible flexible and dynamic. This also implies the definition (and separation) of what an Archipelago Digital Object (ADO) is in our Architecture v/s what a general Drupal Content Type (e.g a static page or a blog post) is defined in code as: \"Any Content type that uses a Strawberry Field is an ADO and will be processed as such\". No configuration is needed. In other words, all is a NODE but any node that uses a Strawberry Field gets a different treatment and will be named in Archipelago an ADO.</p> <p>One of the challenges of our flexible approach is how to allow Drupal to access the JSON in a way, as native as possible, to generate filtered listings via Drupal Views, free text Search and Faceting. To make this happen Strawberry Field uses a JSON Querying and Exposing as \"Native Field Properties\" logic. Through a special type of Plugin system named Strawberry Key Name Providers and associated Configuration Entities (can be found at <code>/admin/structure/strawberry_keynameprovider</code>), you have control on which keys and values of your JSON are going to be exposed as field properties of any Strawberry Field, allowing Drupal through this to access values in a flat manner and expose them to the Search API natively. The access to the values of any JSON is done via JMESPATH expressions and then transformed either to a list of values or even \"cast\" into more complex data Data types, like an Entity Reference (means a connection to another Entity).</p> <p>This gives you a lot of power and control and makes a lot of very heavy operations lighter. You can even plan upfront or evolve these properties in time. </p> <p>In other words, you control how storage is mapped to Discovery and this allows Drupal Views to work that way too. Of course this also means traditional SQL based Drupal Views won't have access to these internals (for filtering) given that your JSON data nor the virtual Properties generated via Strawberry Key Name Providers are not accessible as individual RDB tables to generate SQL joins and that is why we heavily depend on the Search API (Solr).</p>"},{"location":"metadatainarchipelago/#open-schema-what-is-yours-what-is-archipelagos","title":"Open Schema. What is yours, what is Archipelago's","text":"<p>What can you add to an ADO's Strawberry Field? As long as it is valid JSON, Archipelago can store it, display it, transform it and search across it in Archipelago. The way you manage Metadata can be as \"intime\" or \"aligned\" to other schemas as you want. Still, there are a few suggested keys/functional ideas:</p>"},{"location":"metadatainarchipelago/#suggested-json-keys","title":"Suggested JSON keys","text":""},{"location":"metadatainarchipelago/#the-type-key","title":"The <code>type</code> key","text":"<pre><code>{\n  \"type\": \"Photograph\"\n}\n</code></pre> <p>The <code>type</code> JSON key has a semantic and functional importance in Archipelago. Given that we don't use multiple Drupal Content Types to denote the difference between e.g. a Photograph or a Painting (which would also mean you would be stuck with one or other if we did), we use this key's value to allow Archipelago to select/swap View Modes. This approach also allows for your own needs to define what an ADO in real life or digital realm is (the WHAT). This key is also important when doing <code>AMI</code> based batch ingests since many of the mappings and decisions (e.g. what Template to use to transform your CSV or if the Destination Drupal Content Type is going to be a Digital Object or a Digital Object Collection) will depend on this.</p> <p>Note: Archipelago does something extra fun too when using <code>type</code> value for View Mode Selection (and this is also a feature of one of the Key Name provider Plugins). It will flatten the JSON first and then fetch all <code>type</code> keys. How does this in practice work?</p> <pre><code>{\n    \"type\": \"Photograph\",\n    \"subtypes\": [\n        {\n            \"type\": \"125 film\"\n        },\n        {\n            \"type\": \"Instant film\"\n        }\n    ]  \n}\n</code></pre> <p>Means, while doing a View Mode Selection Archipelago will bring all found <code>type</code> key values together and will have ['Photograph', '125 film', 'Instant film'] available as choices, meaning you will be able to make even finer decisions on how to display your ADOs. View Mode selection is based on order or evaluation, means we recommend putting the more specific mappings first.</p>"},{"location":"metadatainarchipelago/#the-label-key","title":"The <code>label</code> key","text":"<pre><code>{\n    \"label\": \"Black and White Photograph of Cataloger working with JSON\"\n}\n</code></pre> <p>Archipelago will use the <code>label</code> key's value to populate the ADO's (Drupal Node) Title. Drupal has a length limit for its native build in Node Entity Title but JSON has not, so in case of more than 255 characters Archipelago will truncate the Title (not the <code>label</code> key's value) adding an ellipsis (...) as suffix.</p>"},{"location":"metadatainarchipelago/#archipelago-drupal-entities-integration-keys","title":"Archipelago Drupal Entities integration keys","text":"<p>Because of the need of having Technical Metadata, Descriptive Metadata and Semantic Metadata while generating different representations of your JSON via Metadata Display Entities (Twig templates) transformations, we store and characterize Files attached to an ADO as part of the JSON. We also use a set of special keys to map and cast JSON keys and values to Drupal's internal Entities system via their Numeric and/or UUID IDs. </p> <p>Through this, Archipelago will also move files between upload locations and permanent storage, execute Technical metadata extraction, keep track of ADO to ADO relationships (e.g ispartof or ismemberof) and emulate what a traditional <code>Drupal Entity Reference field</code> would do without the limitations (speed and immutability) a static RDB definition imposes.</p>"},{"location":"metadatainarchipelago/#the-apentitymapping-key","title":"The <code>ap:entitymapping</code> key","text":"<pre><code>{\n    \"ap:entitymapping\":{\n        \"entity:file\": [\n            \"model\",\n            \"audios\",\n            \"images\",\n            \"videos\",\n            \"documents\",\n            \"upload_associated_warcs\"\n            ],\n        \"entity:node\": [\n            \"ispartof\",\n            \"ismemberof\"\n        ]\n    }\n}\n</code></pre> <p>the <code>ap:entitymapping</code> is a hint for Archipelago. With this key we can treat certain keys and their values as Drupal Numeric Entity IDs instead of semantically unknown values.</p> <p>In the presence of the structure exemplified above the following JSON snipped:</p> <pre><code>    \"images\": [\n        1,\n        2,\n        3\n    ]   \n</code></pre> <p>Will tell Archipelago that the JSON key <code>images</code> should be treated as containing Entity IDs for a Drupal Entity of type (<code>entity:file</code>) File. This has many interessting consequences. Archipelago, on edit/update/ingest will try (hard) to get a hold of Files with ID 1, 2 and 3. If in temporary storage Archipelago will move them to its final Permanent Location, will make sure Drupal knows those files are being used by this ADO, will run multiple Technical Metadata Extractions and classify internally the Files, adding everything it could learn from them. In practice, this means that Archipelago will write for you additional structures into the JSON enriching your Metadata.</p> <p>Without this structure, the <code>images</code> key would not trigger any logic but will of course still exist and can always still be used as a list of numbers while templating.</p> <p>This also implies that for a persisted ADO with those values, if you edit the JSON and delete e.g. the number (<code>integer</code> or <code>string</code> representation of an <code>integer</code>) <code>3</code>, Archipelago will disconnect the File Entity with ID 3 from this ADO, remove the enriched metadata and mark the File as not being anymore used by this ADO. If nobody else is using the File it will become <code>temporary</code> and eventually be automatically removed from the system, if that is setup at the Drupal - Filesystem - level.</p> <p>Using the same example <code>ap:entitymapping</code> structure, the following snippet:</p> <pre><code>    \"ispartof\": [\n        2000\n    ]   \n</code></pre> <p>Will hint to Archipelago on assumed connection between this ADO and another ADO with Drupal Entity ID <code>2000</code>. This will drive other functionality in Archipelago (semantic), allowing for example a Navigation Breadcrumb to be built using all connections found in its hierarchical path.</p> <p>In Archipelago ADO to ADO relationships are normally from Child to Parent and hopefully (but not enforced!) building an Acyclic graph, from leaves to trunk. This will also allow inheritance to happen. This means also that a Parent ADO needs to exist before connecting/relating to it (chicken first). But if it does not, the system will not fail and assume a temporarily broken relationship (egg stays safely intact). </p> <p>Entity mapping key also drives a very special compatibility addition to any ADO. Archipelago will populate Native Computed Drupal fields (attached at run time to each ADO) with these values loading and exposing them as Drupal Entities, processing both Files and Node Entities and making them visible outside the scope of a <code>Strawberry Field</code> to the whole CMS.</p> <p>The following Computed fields are provided:</p> <ul> <li><code>field_file_drop</code>: Computed Entity Reference Field. Needed also for JSON API level upload of Files to an ADO (Drupal need). It will expose all File Entities referenced in an ADO, independently of the type of the File.</li> <li><code>field_sbf_nodetonode</code>: Computed Entity Reference Field. It will expose all Nodes (other ADOs) Entities referenced in an ADO, independently of the Content type and/or the semantic predicate (ismemberof, ispartof, etc) used.</li> </ul> <p>These Fields, because of their native Drupal nature, can be used directly everywhere, e.g. in the Search API to index all related ADOs (or any of their Fields and subproperties, even deeply chained, tree down) without having to specify what predicate is used. Said differently, they act as aggregators, as a generic \"isrelatedto\" property bringing all together.</p>"},{"location":"metadatainarchipelago/#the-asas_file_type-keys","title":"The <code>as:{AS_FILE_TYPE}</code> keys","text":"<p>As explained in the <code>ap:entitymapping</code> section above, when Archipelago gets hold of a File entity it will enrich your JSON with its extracted data. Archipelago will compute and append to your JSON a set of controlled <code>as:{AS_FILE_TYPE}</code> keys containing a classified File's Metadata. The naming will be automatic based on grouping Files by their Mime Types.</p> <p>The possible values for <code>as:{AS_FILE_TYPE}</code> are</p> <ul> <li><code>as:image</code></li> <li><code>as:document</code> </li> <li><code>as:video</code> </li> <li><code>as:audio</code> </li> <li><code>as:application</code></li> <li><code>as:text</code></li> <li><code>as:model</code></li> <li><code>as:multipart</code> </li> <li><code>as:message</code> </li> </ul> <p>An example for an Image attached to an ADO:</p> <pre><code>{\n    \"as:image\": {\n        \"urn:uuid:ef596613-b2e7-444e-865d-efabbf1c59b0\": {\n            \"url\": \"s3:\\/\\/de2\\/image-f6268bde41a39874bc69e57ac70d9764-view-ef596613-b2e7-444e-865d-efabbf1c59b0.jp2\",\n            \"name\": \"f6268bde41a39874bc69e57ac70d9764_view.jp2\",\n            \"tags\": [],\n            \"type\": \"Image\",\n            \"dr:fid\": 7461,\n            \"dr:for\": \"images\",\n            \"dr:uuid\": \"ef596613-b2e7-444e-865d-efabbf1c59b0\",\n            \"checksum\": \"de2862d4accf5165d32cd0c3db7e7123\",\n            \"flv:exif\": {\n                \"FileSize\": \"932 KiB\",\n                \"MIMEType\": \"image\\/jp2\",\n                \"ImageSize\": \"1375x2029\",\n                \"ColorSpace\": \"sRGB\",\n                \"ImageWidth\": 1375,\n                \"ImageHeight\": 2029\n            },\n            \"sequence\": 1,\n            \"flv:pronom\": {\n                \"label\": \"JP2 (JPEG 2000 part 1)\",\n                \"mimetype\": \"image\\/jp2\",\n                \"pronom_id\": \"info:pronom\\/x-fmt\\/392\",\n                \"detection_type\": \"signature\"\n            },\n            \"dr:filesize\": 954064,\n            \"dr:mimetype\": \"image\\/jp2\",\n            \"crypHashFunc\": \"md5\",\n            \"flv:identify\": {\n                \"1\": {\n                    \"width\": \"1375\",\n                    \"format\": \"JP2\",\n                    \"height\": \"2029\",\n                    \"orientation\": \"Undefined\"\n                }\n            }\n        }\n    }\n}\n</code></pre> <p>That is a lot of Metadata! But to understand what is happening here, we need to dissect this into more readable chunks. Let's start with the basics from root to leaves of this hierarchy.</p>"},{"location":"metadatainarchipelago/#direct-file-level-metadata","title":"Direct File level Metadata","text":"<p>Every Classified File inside the <code>as:{AS_FILE_TYPE}</code> key will be contained in a unique URN JSON Object property:</p> <pre><code>\"urn:uuid:ef596613-b2e7-444e-865d-efabbf1c59b0\": {}\n</code></pre> <p>We use a Property instead of a \"List or Array\" of Technical Metadata because this allows us (at code level) to access quickly from e.g. <code>as:image</code> structure all the data for a File Entity with UUID <code>ef596613-b2e7-444e-865d-efabbf1c59b0</code> without iterating. (Also now you know what urn:uuid:ef596613-b2e7-444e-865d-efabbf1c59b0 means.)</p> <p>Next, inside that property, the following Data provides basic Information about the File so you can access/make decisions when Templating. Notice the duplication of similar data at different levels. Duplication is on purpose and again, allows you to access certain JSON values (or filter) quicker without having to go to other keys or hierarchies to make decisions.</p> <pre><code>{\n    \"url\": \"s3:\\/\\/de2\\/image-f6268bde41a39874bc69e57ac70d9764-view-ef596613-b2e7-444e-865d-efabbf1c59b0.jp2\",\n    \"name\": \"Original Name of my Image.jp2\",\n    \"tags\": [],\n    \"type\": \"Image\",\n    \"dr:fid\": 3,\n    \"dr:for\": \"images\",\n    \"dr:uuid\": \"ef596613-b2e7-444e-865d-efabbf1c59b0\",\n    \"crypHashFunc\": \"md5\",\n    \"checksum\": \"de2862d4accf5165d32cd0c3db7e7123\",\n    \"dr:filesize\": 954064,\n    \"dr:mimetype\": \"image\\/jp2\",\n    \"sequence\": 1\n</code></pre> <ul> <li><code>\"url\"</code>: Contains the Final Storage location/URI of the File. It's prefixed with the configured Streamwrapper, a functional symbolic link to the underlying complexities of the backend storage. e.g <code>s3://</code> implies an S3 API backend with a (hidden/abstracted) set of credentials, Bucket and Prefixes inside the bucket. This value is also used in Archipelago's IIIF Cantaloupe Service as the Image <code>id</code> when building a IIIF Image API URL.</li> <li><code>\"name\"</code>: The Original Name of the File. Can be used to give a Download a human readable name or as an internal hint/preservation for you.</li> <li><code>\"tags\"</code>: Unused by default.  You can use this for your own logic if needed.</li> <li><code>\"type\"</code>: A redundant (contextual, at this level) key whose value will match <code>{AS_FILE_TYPE}</code> already found at 2 levels before. Allows you to know what File type this is when iterating over this File's data (without having to look back, or on our Code, when dealing with Flattened JSON).</li> <li><code>\"dr:fid\"</code>:  The Drupal Entity Numeric ID.</li> <li><code>\"dr:for\"</code>:  Where in your JSON (top level key) this File ID was stored (or in other words where you can find the value of <code>\"dr:fid\"</code>. All this will match / was be driven of course by <code>ap:entitymapping</code>. Sometimes (try uploading a WARC file and run the queue) this key might contain  <code>flv:{ACTIVE_STRAWBERRY_RUNNERS_CONFIG_ID}</code>. This means the File will have been generated by an active Strawberry Runners Processor and not uploaded by you. <code>ACTIVE_STRAWBERRY_RUNNERS_CONFIG_ID</code> will be the Machine name (or ID) of a given Strawberry Runners Processor Configuration Entity.</li> <li><code>\"dr:uuid\"</code>: A redundant (contextual, at this level) key whose value will match the Drupal File entity UUID for this File.</li> <li><code>\"crypHashFunc\"</code>: What Cryptographic function was used for generating the checksum. By default Archipelago will do MD5 (faster but also because S3 APIs use that to ensure upload consistency and E-tag). In the future others can be enabled and made configurable</li> <li><code>\"checksum\"</code>: The Checksum (calculated) of this File via <code>\"crypHashFunc\"</code></li> <li><code>\"dr:filesize\"</code>:  The File size in Bytes.</li> <li><code>\"dr:mimetype\"</code>:  The Drupal level infered Mime Type. Archipelago extends this list. This is based on the File Extension.</li> <li><code>\"sequence\"</code>:  A number (integer) denoting order of this file relative to other files of the same type inside the JSON. Which default type ordering is used will depend on how the ADO was created/edited, but can be overriden using Control Metadata.</li> </ul>"},{"location":"metadatainarchipelago/#technical-file-level-metadata","title":"Technical File level Metadata","text":"<p>Deeper inside this structure Archipelago will produce Extracted Technical Metadata. Some of this Metadata will be common to every File Type, some will be specific to a subset, like Moving Media or PDFs. What runs and how it runs can be configured at the File Persister Service Settings configuration form found at<code>/admin/config/archipelago/filepersisting</code>. Why there? These are service that run syncroniusly on ADO save (Create/Edit) and in while doing  File persistance.</p> <p><code>\"flv:exif\"</code>: EXIF Tool extraction for a file. The number of elements that come out might vary, for an Image file it might be normally short, but a PDF might have a very extensive and long list. The above mentioned File Persister Service Settings form allows you to also set a Files Cap Number, that will, once reached, limit and reduce the EXIF. This is very useful if you want to control the size of your complete JSON for any reason you feel that is needed (performance, readability, etc).</p> <pre><code>{\n    \"flv:exif\": {\n                \"FileSize\": \"932 KiB\",\n                \"MIMEType\": \"image\\/jp2\",\n                \"ImageSize\": \"1375x2029\",\n                \"ColorSpace\": \"sRGB\",\n                \"ImageWidth\": 1375,\n                \"ImageHeight\": 2029\n            },\n}\n</code></pre> <p><code>\"flv:identify\"</code>: Graphics Magic Identity binary will run on every file and format it knows how to run (and will try even on the ones it does not). Will give you data similar to EXIF but processed based on the actual File and not just extracted from the EXIF data found at the header. Notice that the details will be inside a \"1\", \"2\", etc property.  This is because Identify might also go deeper and for e.g a Multi Layer Tiff extract different sequences on the same File.</p> <pre><code>{\n    \"flv:identify\": {\n                \"1\": {\n                    \"width\": \"1375\",\n                    \"format\": \"JP2\",\n                    \"height\": \"2029\",\n                    \"orientation\": \"Undefined\"\n                }\n            }\n}\n</code></pre> <p><code>\"flv:pronom\"</code>: Droid, a File Signature detection tool will find a  matching <code>pronom_id</code>for your File based on  https://www.nationalarchives.gov.uk/aboutapps/pronom/droid-signature-files.htm. This detection type is deeper that EXIF or the mime type based on extension, reading from binary data. It allows you to get small differences between formats (even if e.g both are JP2) and thus make decisions like \"Will <code>Cantaloupe IIIF Image Server</code> be able to handle this type?\". This has also positive Digital Preservation consequences.</p> <pre><code>{\n    \"flv:pronom\": {\n                \"label\": \"JP2 (JPEG 2000 part 1)\",\n                \"mimetype\": \"image\\/jp2\",\n                \"pronom_id\": \"info:pronom\\/x-fmt\\/392\",\n                \"detection_type\": \"signature\"\n            },\n}\n</code></pre> <p><code>\"flv:mediainfo\"</code>: Media Info works on Video and Audio. It goes very detailed into <code>codecs</code> and<code>streams</code> and the output added to your JSON might look massive. This is also very needed when working with IIIF Manifests and deciding if a certain Video will be able to play natively on a browser or if <code>Cantaloupe IIIF Image Server</code> will be able to extract individual frames as images. This again has positive Digital Preservation consequences. The Following is an example of an MP4 file generated via Quicktime on an Apple MacOS computer.</p> <pre><code>{\n    \"flv:mediainfo\": {\n                \"menus\": [],\n                \"audios\": [\n                    {\n                        \"id\": {\n                            \"fullName\": \"1\",\n                            \"shortName\": \"1\"\n                        },\n                        \"count\": \"282\",\n                        \"title\": \"Core Media Audio\",\n                        \"format\": {\n                            \"fullName\": \"AAC LC\",\n                            \"shortName\": \"AAC\"\n                        },\n                        \"bit_rate\": {\n                            \"textValue\": \"85.3 kb\\/s\",\n                            \"absoluteValue\": 85264\n                        },\n                        \"codec_id\": \"mp4a-40-2\",\n                        \"duration\": {\n                            \"milliseconds\": 21215\n                        },\n                        \"channel_s\": {\n                            \"textValue\": \"1 channel\",\n                            \"absoluteValue\": 1\n                        },\n                        \"frame_rate\": {\n                            \"textValue\": \"43.066 FPS (1024 SPF)\",\n                            \"absoluteValue\": 43\n                        },\n                        \"format_info\": \"Advanced Audio Codec Low Complexity\",\n                        \"frame_count\": \"914\",\n                        \"stream_size\": {\n                            \"bit\": 226109\n                        },\n                        \"streamorder\": \"0\",\n                        \"tagged_date\": \"UTC 2017-12-05 17:14:10\",\n                        \"encoded_date\": \"UTC 2017-12-05 17:14:07\",\n                        \"source_delay\": \"-0\",\n                        \"bit_rate_mode\": {\n                            \"fullName\": \"Variable\",\n                            \"shortName\": \"VBR\"\n                        },\n                        \"samples_count\": \"935582\",\n                        \"sampling_rate\": {\n                            \"textValue\": \"44.1 kHz\",\n                            \"absoluteValue\": 44100\n                        },\n                        \"channel_layout\": \"C\",\n                        \"kind_of_stream\": {\n                            \"fullName\": \"Audio\",\n                            \"shortName\": \"Audio\"\n                        },\n                        \"commercial_name\": \"AAC\",\n                        \"source_duration\": [\n                            \"21269\",\n                            \"21 s 269 ms\",\n                            \"21 s 269 ms\",\n                            \"21 s 269 ms\",\n                            \"00:00:21.269\"\n                        ],\n                        \"compression_mode\": {\n                            \"fullName\": \"Lossy\",\n                            \"shortName\": \"Lossy\"\n                        },\n                        \"channel_positions\": {\n                            \"fullName\": \"1\\/0\\/0\",\n                            \"shortName\": \"Front: C\"\n                        },\n                        \"samples_per_frame\": \"1024\",\n                        \"stream_identifier\": \"0\",\n                        \"source_frame_count\": \"916\",\n                        \"source_stream_size\": [\n                            \"226460\",\n                            \"221 KiB (1%)\",\n                            \"221 KiB\",\n                            \"221 KiB\",\n                            \"221 KiB\",\n                            \"221.2 KiB\",\n                            \"221 KiB (1%)\"\n                        ],\n                        \"source_delay_source\": \"Container\",\n                        \"format_additionalfeatures\": \"LC\",\n                        \"proportion_of_this_stream\": \"0.01178\",\n                        \"count_of_stream_of_this_kind\": \"1\",\n                        \"source_streamsize_proportion\": \"0.01180\"\n                    }\n                ],\n                \"images\": [],\n                \"others\": [\n                    {\n                        \"type\": \"meta\",\n                        \"count\": \"188\",\n                        \"duration\": {\n                            \"milliseconds\": 21215\n                        },\n                        \"frame_count\": \"1\",\n                        \"kind_of_stream\": {\n                            \"fullName\": \"Other\",\n                            \"shortName\": \"Other\"\n                        },\n                        \"stream_identifier\": [\n                            \"0\",\n                            \"1\"\n                        ],\n                        \"count_of_stream_of_this_kind\": \"2\"\n                    },\n                    {\n                        \"type\": \"meta\",\n                        \"count\": \"188\",\n                        \"duration\": {\n                            \"milliseconds\": 21215\n                        },\n                        \"frame_count\": \"1\",\n                        \"kind_of_stream\": {\n                            \"fullName\": \"Other\",\n                            \"shortName\": \"Other\"\n                        },\n                        \"stream_identifier\": [\n                            \"1\",\n                            \"2\"\n                        ],\n                        \"count_of_stream_of_this_kind\": \"2\"\n                    }\n                ],\n                \"videos\": [\n                    {\n                        \"id\": {\n                            \"fullName\": \"2\",\n                            \"shortName\": \"2\"\n                        },\n                        \"count\": \"380\",\n                        \"title\": \"Core Media Video\",\n                        \"width\": {\n                            \"textValue\": \"1 280 pixels\",\n                            \"absoluteValue\": 1280\n                        },\n                        \"format\": {\n                            \"fullName\": \"AVC\",\n                            \"shortName\": \"AVC\"\n                        },\n                        \"height\": {\n                            \"textValue\": \"720 pixels\",\n                            \"absoluteValue\": 720\n                        },\n                        \"bit_rate\": {\n                            \"textValue\": \"7 144 kb\\/s\",\n                            \"absoluteValue\": 7144261\n                        },\n                        \"codec_id\": \"avc1\",\n                        \"duration\": {\n                            \"milliseconds\": 21215\n                        },\n                        \"rotation\": \"0.000\",\n                        \"bit_depth\": {\n                            \"textValue\": \"8 bits\",\n                            \"absoluteValue\": 8\n                        },\n                        \"scan_type\": {\n                            \"fullName\": \"Progressive\",\n                            \"shortName\": \"Progressive\"\n                        },\n                        \"format_url\": \"http:\\/\\/developers.videolan.org\\/x264.html\",\n                        \"frame_rate\": {\n                            \"textValue\": \"29.970 (29970\\/1000) FPS\",\n                            \"absoluteValue\": 29\n                        },\n                        \"buffer_size\": \"768000\",\n                        \"color_range\": \"Limited\",\n                        \"color_space\": \"YUV\",\n                        \"format_info\": \"Advanced Video Codec\",\n                        \"frame_count\": \"636\",\n                        \"stream_size\": {\n                            \"bit\": 18951244\n                        },\n                        \"streamorder\": \"1\",\n                        \"tagged_date\": \"UTC 2017-12-05 17:14:10\",\n                        \"encoded_date\": \"UTC 2017-12-05 17:14:07\",\n                        \"bit_rate_mode\": {\n                            \"fullName\": \"Variable\",\n                            \"shortName\": \"VBR\"\n                        },\n                        \"codec_id_info\": \"Advanced Video Coding\",\n                        \"framerate_den\": \"1000\",\n                        \"framerate_num\": \"29970\",\n                        \"sampled_width\": \"1280\",\n                        \"format_profile\": \"Main@L3.1\",\n                        \"kind_of_stream\": {\n                            \"fullName\": \"Video\",\n                            \"shortName\": \"Video\"\n                        },\n                        \"sampled_height\": \"720\",\n                        \"color_primaries\": \"BT.709\",\n                        \"commercial_name\": \"AVC\",\n                        \"format_settings\": \"CABAC \\/ 2 Ref Frames\",\n                        \"frame_rate_mode\": {\n                            \"fullName\": \"Variable\",\n                            \"shortName\": \"VFR\"\n                        },\n                        \"bits_pixel_frame\": \"0.259\",\n                        \"maximum_bit_rate\": {\n                            \"textValue\": \"768 kb\\/s\",\n                            \"absoluteValue\": 768000\n                        },\n                        \"stream_identifier\": \"0\",\n                        \"chroma_subsampling\": [\n                            \"4:2:0\",\n                            \"4:2:0\"\n                        ],\n                        \"maximum_frame_rate\": [\n                            \"30.000\",\n                            \"30.000 FPS\"\n                        ],\n                        \"minimum_frame_rate\": [\n                            \"28.571\",\n                            \"28.571 FPS\"\n                        ],\n                        \"pixel_aspect_ratio\": \"1.000\",\n                        \"colour_range_source\": \"Stream\",\n                        \"format_settings_gop\": \"M=2, N=30\",\n                        \"internet_media_type\": \"video\\/H264\",\n                        \"matrix_coefficients\": \"BT.709\",\n                        \"original_frame_rate\": [\n                            \"25.000\",\n                            \"25.000 FPS\"\n                        ],\n                        \"display_aspect_ratio\": {\n                            \"textValue\": \"16:9\",\n                            \"absoluteValue\": 1.778\n                        },\n                        \"format_settings_cabac\": {\n                            \"fullName\": \"Yes\",\n                            \"shortName\": \"Yes\"\n                        },\n                        \"codec_configuration_box\": \"avcC\",\n                        \"colour_primaries_source\": \"Container \\/ Stream\",\n                        \"transfer_characteristics\": \"BT.709\",\n                        \"proportion_of_this_stream\": \"0.98734\",\n                        \"colour_description_present\": \"Yes\",\n                        \"matrix_coefficients_source\": \"Container \\/ Stream\",\n                        \"count_of_stream_of_this_kind\": \"1\",\n                        \"transfer_characteristics_source\": \"Container \\/ Stream\",\n                        \"format_settings_reference_frames\": [\n                            \"2\",\n                            \"2 frames\"\n                        ],\n                        \"colour_description_present_source\": \"Container \\/ Stream\"\n                    }\n                ],\n                \"general\": {\n                    \"count\": \"336\",\n                    \"format\": {\n                        \"fullName\": \"MPEG-4\",\n                        \"shortName\": \"MPEG-4\"\n                    },\n                    \"codec_id\": [\n                        \"qt  \",\n                        \"qt   0000.00 (qt  )\"\n                    ],\n                    \"datasize\": \"19177730\",\n                    \"duration\": {\n                        \"milliseconds\": 21215\n                    },\n                    \"file_name\": \"c98e7bc52e4bd3fe5681a746f2d9c76f_diego4\",\n                    \"file_size\": {\n                        \"bit\": 19194157\n                    },\n                    \"footersize\": \"0\",\n                    \"frame_rate\": {\n                        \"textValue\": \"29.970 FPS\",\n                        \"absoluteValue\": 29\n                    },\n                    \"headersize\": \"16427\",\n                    \"othercount\": \"2\",\n                    \"folder_name\": \"\\/tmp\\/ami\\/setfiles\\/cb606b13b823eaea784dc77c460f3baf\",\n                    \"frame_count\": \"636\",\n                    \"stream_size\": {\n                        \"bit\": 16804\n                    },\n                    \"tagged_date\": \"UTC 2017-12-05 17:14:10\",\n                    \"audio_codecs\": \"AAC LC\",\n                    \"codec_id_url\": \"http:\\/\\/www.apple.com\\/quicktime\\/download\\/standalone.html\",\n                    \"codecs_video\": \"AVC\",\n                    \"encoded_date\": \"UTC 2017-12-05 17:14:07\",\n                    \"isstreamable\": \"Yes\",\n                    \"complete_name\": \"\\/tmp\\/ami\\/setfiles\\/cb606b13b823eaea784dc77c460f3baf\\/c98e7bc52e4bd3fe5681a746f2d9c76f_diego4.m4v\",\n                    \"file_extension\": \"m4v\",\n                    \"format_profile\": \"QuickTime\",\n                    \"kind_of_stream\": {\n                        \"fullName\": \"General\",\n                        \"shortName\": \"General\"\n                    },\n                    \"codecid_version\": \"0000.00\",\n                    \"commercial_name\": \"MPEG-4\",\n                    \"writing_library\": {\n                        \"fullName\": \"Apple QuickTime\",\n                        \"shortName\": \"Apple QuickTime\"\n                    },\n                    \"overall_bit_rate\": {\n                        \"fullName\": \"7 238 kb\\/s\",\n                        \"shortName\": \"7237957\"\n                    },\n                    \"audio_format_list\": \"AAC LC\",\n                    \"stream_identifier\": \"0\",\n                    \"video_format_list\": \"AVC\",\n                    \"codecid_compatible\": \"qt  \",\n                    \"file_name_extension\": \"c98e7bc52e4bd3fe5681a746f2d9c76f_diego4.m4v\",\n                    \"internet_media_type\": \"video\\/mp4\",\n                    \"encoded_library_name\": \"Apple QuickTime\",\n                    \"comapplequicktimemake\": \"Apple\",\n                    \"overall_bit_rate_mode\": {\n                        \"fullName\": \"Variable\",\n                        \"shortName\": \"VBR\"\n                    },\n                    \"comapplequicktimemodel\": \"iPhone SE\",\n                    \"count_of_audio_streams\": \"1\",\n                    \"count_of_video_streams\": \"1\",\n                    \"comapplequicktimesoftware\": \"10.3.2\",\n                    \"proportion_of_this_stream\": \"0.00088\",\n                    \"audio_format_withhint_list\": \"AAC LC\",\n                    \"video_format_withhint_list\": \"AVC\",\n                    \"file_last_modification_date\": {\n                        \"date\": \"2022-10-19 20:02:32.000000\",\n                        \"timezone\": \"UTC\",\n                        \"timezone_type\": 3\n                    },\n                    \"count_of_stream_of_this_kind\": \"1\",\n                    \"comapplequicktimecreationdate\": \"2017-10-25T16:58:17-0400\",\n                    \"format_extensions_usually_used\": \"braw mov mp4 m4v m4a m4b m4p m4r 3ga 3gpa 3gpp 3gp 3gpp2 3g2 k3g jpm jpx mqv ismv isma ismt f4a f4b f4v\",\n                    \"comapplequicktimelocationiso6709\": \"+40.6145-074.2678+020.977\\/\",\n                    \"file_last_modification_date_local\": {\n                        \"date\": \"2022-10-19 20:02:32.000000\",\n                        \"timezone\": \"America\\/New_York\",\n                        \"timezone_type\": 3\n                    }\n                },\n                \"version\": \"21.09\",\n                \"subtitles\": []\n            }\n        }\n}\n</code></pre> <p><code>\"flv:pdfinfo\"</code>: PDF Info will get Page level Information for a PDF or Ghostscript document. The dimensions displayed in the following example are not in pixels but points (resolution independent) and are also used for IIIF generation when deciding at what rasterized pixel size a given PDF document page will be rendered. Same as with <code>flv:identify</code>, the technical metadata will be contained inside a keyed (string but semantically an integer) property. In this particular case each number is a page sequence in the original PDF order.</p> <pre><code>{\n    \"flv:pdfinfo\": {\n                \"1\": {\n                    \"width\": \"612\",\n                    \"height\": \"792\",\n                    \"rotation\": \"0\",\n                    \"orientation\": \"TopLeft\"\n                }\n            }\n}\n</code></pre> <p>@TODO: add the extra special key used by Strawberry Runners when it attaches a file. e.g WARC to WACZ</p>"},{"location":"metadatainarchipelago/#did-you-know","title":"Did you #know?","text":"<p>If you delete a whole <code>as:{AS_FILE_TYPE}</code> structure or one of the File level structures (a <code>urn:uuid:{uuid}</code> key and its children), Archipelago will recreate it. If you modify any internal value contained in it, Archipelago will do nothing and will trust you (and if you do strange things like modifying the <code>url</code> something might even fail e.g in a IIIF Metadata Display Entity Twig Template). No data edit there will trigger a modification/moving/deletion of a File (or e.g write back EXIF to be binary). You will have time to revert to a previous revision (version) of the ADO if any manual change was done. So, should you modify/delete this structures? Almost never. Ever. But you might find needs for that someday. Also to be noted. Producing this structure for a large file in S3:// is intensive. It needs to be downloaded to a local path and if the File is a few Gigabytes in size Archipelago might even run out of PHP processing time. If that ever happens you can also copy/paste from a previous revision of the ADO the relevant piece. If archipelago finds it (implied in the previous explanation) it will not have to regenerate it. The AMI module does this in an async/enqueued way to avoid time out issues and can reuse a cached metadata extraction between runs, but when working directly on an ADO via e.g a webform or via RAW edit, take that in account.  More work is being done to allow also one on one async File operations and larger uploads via the web.</p>"},{"location":"metadatainarchipelago/#the-aptasks-keys","title":"The <code>ap:tasks</code> keys","text":"<p>As mentioned briefly before, there is also Control Metadata. What do we mean with that? Control metadata in Archipelago's way of allowing you to give, through metadata, (that you might want to preserve or not) instructions to Archipelago that relate to processing. Let's start with the basic one:</p> <pre><code>{\n    \"ap:tasks\": {\n        \"ap:sortfiles\": \"index\"\n    }\n}\n</code></pre> <p><code>\"ap:sortfiles\"</code> key will instruct Archipelago to sort (create a <code>sequence</code> key and a sequential number (integer value) inside each Metadata File entry of a <code>as:{AS_FILE_TYPE}</code> structure. Values can be one of <code>['natural', 'index', 'manual']</code> defaulting, if absent or has an invalid value, to <code>natural</code>.  - <code>natural</code>: files will be sorted by File Name, the <code>filename</code> key found at the same level of <code>sequence</code> in the previously mentioned <code>as:{AS_FILE_TYPE}</code> structure.  a <code>Photograph_1.jpeg</code> will come before a <code>Photograph_10.jpeg</code>. The way a human being naturally would order by name. - <code>index</code> files will be sorted by the order in which they appear inside the upload JSON key (the <code>dr:for</code> key, one of the keys mapped in the <code>ap:entitymapping</code> structure under <code>entity:file</code> explained before. e.g. <code>images</code>:[5, 10 , 1 ], would imply the File Entity with Drupal ID 5 would get <code>\"sequence\": 1</code>, the one with Drupal ID 10 will get <code>\"sequence\": 1</code>, etc. This is the default when ingesting via the AMI module  given the need to preserve file order that has/might have unknown names or names you don't have control of (thus natural won't work) coming from e.g a Remote, HTTP/HTTPS location - <code>manual</code>: You can modify the values manually for any <code>sequence</code> key inside <code>as:{AS_FILE_TYPE}</code> structure and those values will stick.</p> <p>What do we mean with stick? Well, everytime archipelago gets a change in this <code>\"ap:sortfiles\"</code>, e.g a new File is added, a File is deleted, automatic re-sorting will happen. </p> <pre><code>{\n    \"ap:tasks\": {\n        \"ap:forcepost\": true\n    }\n}\n</code></pre> <p><code>\"ap:forcepost\"</code>: A boolean. The functionality of this key is provided by the Strawberry Runners Module. Will force Strawberry Runners Post processing for this ADO. </p> <p>Each Configured and active Postprocessor provided by the <code>Strawberry Runners</code> module might or not <code>kick in</code> by evaluating a set of rules. If rule evaluates to TRUE, the PostProcessor will generate a certain output., e.g a Solr Indexed <code>Strawberry Flavor</code> Data Source containing OCR, HOCR and NLP metadata for one or more pages of a PDF. </p> <p>Everytime a Create or Update operation on an ADO happens, these rules will be evaluated and the Processor will be enqueued as a future task. But at the moment of executing, when the queue workers take one item, a check will be made and if the result of a previous run (e.g HOCR) is already present in the system (e.g in Solr) and it's veryfied to be belonging to the same source, the actual heavyload of processing the PDF will be skipped. </p> <p>While testing, coding, doing complex changes in the system (like modifying largely the settings for one processor) or even in the case of an ISSUE (e.g HOCR was wrongly run with a setting that made all look like garbage!) you can instruct Archipelago run again without checks. And again. And again. basically everytime you Save an ADO by setting <code>ap:forcepost</code> to <code>true</code>. This can also be used batch and is already implied (means it does it for you but only once, without modifying the JSON) in the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects</code> VBO action we provide.</p> <p>In the absence of <code>\"ap:forcepost\"</code> the  value is implicitly <code>false</code>, same as setting it explicitly  to <code>false</code>.</p> <pre><code>{\n    \"ap:tasks\": {\n        \"ap:nopost\": [\n            'pager'\n        ]\n    }\n}\n</code></pre> <p><code>\"ap:nopost\"</code>:  an array or list of <code>ACTIVE_STRAWBERRY_RUNNERS_CONFIG_ID</code> entries, means Machine names (or IDs) Strawberry Runners Processor Configuration Entities. The functionality of this key is provided by the Strawberry Runners. If not an array it will be ignored. Any value present not matching an active Strawberry Runners Processor Configuration Entity ID will also be ignored. Effectively, any post processors in this list will be skipped for this ADO. This allows a finer grained avoidance of some expensive processing that might lead to unsuable data. E.g a particular Manuscript ADO that Tesseract won't be able to OCR correctly.  Adding this key to an ADO that was already processed won't remove existing generated/stored processing.</p> <pre><code>{\n    \"ap:tasks\": {\n        \"ap:ami\": {\n            \"metadata_display\": 7\n\n        }\n    }\n}\n</code></pre> <p><code>\"ap:ami\"</code> is a newer key ( as of Archipelago 1.1.0 and AMI 0.5.0) and for now  can only contain another single key named <code>\"metadata_display\"</code>. The value of this one can be either a single Integer, the Drupal ID of a Metadata Display Entity or a string, the <code>UUID</code> of a Metadata Display Entity.  The functionality triggered by this key is provided by the AMI module and will do something extremely powerfull: it will take the complete JSON and process through the Twig or Metadata Display Entity refererenced in its value, IF, and only IF, the output of that template is JSON. This runs before any other event (Archipelago runs a ton of events that validate, enrich, check, etc your ADOs from the moment you SAVE or Create it) and because of that allows you to totally pivot, transform, change RAW data coming into Archipelago, e.g via the JSON:API into the structure you need/want.  Said differently, you could push JSON from a totally different system and if the referenced  Metadata Display Entity is well written, end with a perfectly aligned JSON matching your internal structure without modifying the INPUT manually. Because Twig is very powerful you can also do cleanups, complex logic, etc. More over, you can transform any existing ADO via Batch by adding this key(s) and values using the JSON Patch VBO action. Once processed and if all went well, meaning the output of the Template is valid JSON, the key itself will be removed. This, to avoid running over and over (invisibly to you) on further operations/edits/etc.  This is a one time operation that does not stick. What happens if it does not run well, fails, errors out or the Template referenced does not exist? You get a second change (everyone deserves one), the Original ingested JSON, without transformations is kept. All this is very similar to what the AMI module does via a CSV but in this case its atomic. We know what you are thinking. You can process data twice, via AMI and then at the end pass it again through another template based on a certain logic coming from the first? yes. you can!</p> <p>In the future <code>\"ap:ami\"</code> might contain more keys to do more advanced File level actions. Archipelago is being constantly enhanced!</p>"},{"location":"metadatainarchipelago/#activity-stream","title":"Activity Stream","text":"<p>Archipelago also keeps information about who/how a certain JSON was generated. Depending on how the Ingest/Edit of an ADO happened, this can be automatically generated or added manually (the case for AMI ingests).</p> <p>The structure is simple and not accumulative because there is also versioning at the ADO (Drupal) level that allows you to look back if needed.</p> <pre><code>{\n     \"as:generator\": {\n        \"type\": \"Update\",\n        \"actor\": {\n            \"url\": \"http:\\/\\/localhost:8001\\/form\\/default-descriptive-metadata-ami\",\n            \"name\": \"default_descriptive_metadata_ami\",\n            \"type\": \"Service\"\n        },\n        \"endTime\": \"2022-03-16T15:51:24-04:00\",\n        \"summary\": \"Generator\",\n        \"@context\": \"https:\\/\\/www.w3.org\\/ns\\/activitystreams\"\n    },\n}\n</code></pre> <p>The <code>\"as:generator\"</code> Conforms to the Activity Stream Vocabulary Version 2.0 and keeps track of the last Operation executed on an ADO. Edits and Ingests via the Webform Widget will create this automatically using the Canonical URL of the Webform That generated the content and <code>\"type\"</code> might be either <code>\"Update\"</code> or \"<code>Create</code>\". ADOs that were processed via <code>\"ap:ami\"</code> will have automatically one generated to express the fact that the Original JSON was modified by a Metadata Display Entity.  Objects created via AMI and using a <code>Metadata Display Entity</code> can also add via the Twig template syntax the AMI Set ID used to generate the ADO (or Update) allowing the Service URL (<code>\"url\"</code>) to be faceted/searched for (e.g show me all objects ingested via AMI Set with ID 4).</p>"},{"location":"metadatainarchipelago/#all-the-other-keys-lists-objects-your-metadata","title":"All the other keys, lists, objects. Your Metadata","text":"<p>Anything or everything else (including unknow data, future data, upcoming data) belongs to you. How you name it, how you structure, how it evolves is up to you and the functionality and integration you want. That said, as someone (the writer) that enjoys cooking and had to learn the hardway (experimenting and failing sometmes) the basics before doing proper meals for others to enjoy, we suggest you plan on this before inventing the next Open Schema. </p> <p>Note: Why the out-of-context Cooking Analogy?</p> <p>This idea is deeply embedded in our Architecture. We see Metadata as ingredients. Your JSON is your fridge (or pantry or both). Metadata Display Entities, and their Twig Templates, recipes that allow you to pick and choose Ingredients and your Twig coding skills (and filters, functions, loops and conditionals) your basic cooking skills. This analogy has many consequences:</p> <ul> <li>You can plan upfront and have more ingredients (metadata keys and JSON structures) than what you need/know how to cook, your current Recipes allow. Planning for \"your future\" needs and skills (and imagination) is a big part of this</li> <li>Reading, understanding and testing small changes on the existing recipes will give provide you with future skills and the ability to do similar, incremental better, meals before going for something totally new and complex</li> <li>Keeping your Ingredientes RAW and unprocessed is a good idea. You might be tempted to store canned refried beans in the fridge, but that might limit your future options. Maybe try both until you feel more secure, the canned ones AND the dried, single beans too.</li> <li>Your use cases might dictate where you start. You cook visually, starting with a very concrete Meal in mind? (means your plan is to make a certain Caserolle - in a less metaphoricall way, a certain well defined output Schema like MODS 3.7). So you need to be sure you have the ingredients that Schema at least requires and know how (or look and copy, MODS is provided already) to process the data. Or you are into a certain type of food? And want to have the most common ingredientes needed around, you might not know how to make all the different dishes but you for sure know Onions (and cutting those) is needed. This are just 2. So many ways of approaching Cooking.</li> </ul> <p>The Open Schema you will get from a Vanilla Archipelago already covers many many uses cases and was developed by a caring team of metadata professionals and practitioners working with Archipelago for a while already. It covers LoD and most Description needs for your Whys, Wheres, When, Who/Whom. Some tips:</p> <ul> <li>Start by adding new Keys instead of removing existing ones. Existing keys might already be used in your Vanilla Archipelago (in their Processed form) in, e.g the Search API, as Key Name providers (means the target of a JMESPATH query that will be exposed to Drupal as a Field Property), in Views (Filters, Sorting of data). Or in Twig templates (as part of Recipees) that Provide data for Viewers, IIIF V2 and V3, or in your HTML Object Descriptions. Or a Webform configured to Edit/Create new ADOs. All these can be of course modified and adapted, but before spending too much time doing that experiment with adding a new Ingredient and incorporating it to either one of the many Outputs of Archipelago or writing a simple new Recipee that uses it.</li> <li>Not all Metadata needs to be used. Want to keep track of your Workflows? Contextual cataloging data? Notes for other metadata professionals in your team? The command line used to capture a WACZ file? You can add that to your JSON. Might all come handy in the future. You don't need to display it at all if you don't want to.</li> <li>Experiment with the Existing Webforms and Webform Elements. If you need to allow Metadata that is very unique in structure and values (and validation), try first generating the simplest structure via the shipped Webforms and their elements via the UI. If you feel Webform can not handle then maybe you are structuring it in a too complex way. Test adding and editing. Check the \"What if/what ifs\". Is your value correctly a string? Would it be better as a boolean or an integer? Think of the \"i want many values\" v/s this is a single value differences. Because you are in the presence of an OpenSchema you can always change your mind, still better to start on the correct track.</li> <li>Give your keys meaningfull names for you/use case/others: <code>property_1</code> might be hard to document for you. But <code>original_artifact_in_collection</code> might be better (and denotes semantically the value might be a boolean, true of false). Use plural and singular in your naming to denote that something might contain more than one entry. Try to be generic but assertive. <code>mods_modsinfo_namepart</code> is tempting but is already hinting a single original fixed schema. And you might end using the same value (the who) in Dublin Core, IIIF, schema.org, etc outputs. So mybe <code>author</code> instead? This also leads to: sometimes multiple keys are better than many deeply nested ones where understanding. You can keep authors and contributors in separate keys.</li> <li>Document your keys in the Metadata Display Templates (JSON does not allow comments but also why would you want to document the same in every ADO, would be like writing notes on your Potatoes. The most obvious place to document are your recipees. If adding a new Key add a small note using <code>{# #}</code> explaining why/what it holds. You can also add Help/Extra info when designing your schema via a the Webform. Each element has extra properties to do so and that way you can also explain others (the ones using the Webform to add/edit) what the purpose of your metadata is.</li> <li>Use a (or many) local Archipelago Deployment as your experimental Kitchen</li> </ul> <p>Do you have your own Kitchen/cooking tips you want to share? We hope you enjoy the learning process and the many choices Archipelago provides.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"metadatatwigs/","title":"Twig Templates and Archipelago","text":"<p>Archipelago uses a fast, cached templating system that is core to Drupal, called <code>Twig</code>. In its guts (or its heart?) Archipelago uses this system to transform the close to your needs open schema metadata that lives in every <code>strawberryfield</code> as JSON into close to other one's fixed schema needs metadata. This is quite simple, but it is an essential component of our vision of how a repository should manage metadata.</p>"},{"location":"metadatatwigs/#what-is-twig","title":"What is Twig?","text":"<p>Twig is a template engine for PHP part of Symfony framework.</p> <ul> <li>Twig in Symfony</li> <li>Twig in Drupal</li> <li>A template engine is a processor. It allows you to mix and process templates with data to generate an output document.<ul> <li>Template: Some type of static Document (we name this a \u201cFrame\u201d)</li> <li>Data: Your Archipelago Digital Object (ADO) info and your Metadata</li> <li>Processor: Allows you to use a rich and expressive language to pick, check, iterate, transform and output your data inside the Template. We refer to this as \u201ccasting\u201d.</li> </ul> </li> </ul>"},{"location":"metadatatwigs/#where-is-twig-used-in-archipelago","title":"Where is Twig used in Archipelago?","text":"<p>This templating system is exposed to Archipelago users through the UI, and is stored in the repository as content. This setup empowers users to fully control how metadata is transformed and published without touching their individual sources or needing to manage hard-coded configurations. We named these readily accessible and powerful templates <code>Metadata Display entities</code>, but they serve more than just display needs.</p> <p>Twig drives every Page in a Drupal 8/9/10 environment.</p> <ul> <li>Twig templates are normally files (.twig.html) that live in your Code.</li> <li>Modules provide Templates, Themes provide Templates</li> </ul> <p>Twig drives every aspect of your ADO exposure to the world in Archipelago and even batch Ingest.</p> <ul> <li>Strawberryfield Metadata (JSON, your Data) is passed through a Metadata Display Entity which holds:<ul> <li>A Twig template (so you do not need to edit Files)</li> <li>A desired output serialization format (the Output Document)</li> </ul> </li> </ul>"},{"location":"metadatatwigs/#twig-templates-as-metadata-display-entities","title":"Twig Templates as Metadata Display Entities","text":"<p>Templates or recipes can be shared, exported, ingested, updated, and adapted in many ways. This means you can make changes quickly without having to wait for the next major release of Archipelago or your favorite Metadata Schema Specs Committee\u2019s agreement to implement the next or the last version. This module not only handles metadata but media assets as well. It will extract local or remote URIs and files from your metadata and render them as media viewers: books, 3D models, images, panoramas, A/V, all with IIIF in its soul.</p> <p>Metadata Display Entities are used for:</p> <ul> <li>Display:<ul> <li>ADO landing pages (via Drupal Field Formatter)</li> <li>IIIF or JSON driven viewers (via Drupal Field Formatter and using Exposed Metadata Endpoints)</li> <li>Map Formatter (Drupal Field Formatter)</li> <li>Custom Blocks (Drupal Views)</li> <li>Search Result Displays (Drupal Views)</li> <li>Collection and Creative Work Series (old compound) displays (Drupal Views)</li> </ul> </li> <li>Machinable Output<ul> <li>Exposed Metadata Endpoints (Standalone URLs to access metadata)</li> </ul> </li> <li>Batch Ingest<ul> <li>AMI Ingest: To transform your CSV data (one row == DATA) to Strawberry field JSON to generate an ADO</li> </ul> </li> </ul>"},{"location":"metadatatwigs/#twig-templates-shipped-with-archipelago","title":"Twig Templates Shipped with Archipelago","text":"<p>Archipelago Ships with:</p> <ul> <li>IIIF Manifest V3 for Images (JSON-LD) Metadata Display</li> <li>IIIF Manifest V2 for Images and Documents (JSON-LD) Metadata Display</li> <li>IIIF Manifest V3 for Collections (JSON-LD) Metadata Display</li> <li>IIIF Manifest V3 for Creative Work Series/Compound Objects Parent and Children (JSON-LD) Metadata Displays</li> <li>A General ADO Description (HTML) Metadata Display</li> <li>A Linked Data Display (HTML) Metadata Display</li> <li>GEOJSON (JSON) Metadata Display</li> <li>An AMI (JSON) Ingest Template</li> <li>A Multiple Thumbnails via IIIF and Fontawesome (HTML) Metadata Display</li> <li>A Metadata Abstract for Search Results (HTML) Metadata Display</li> <li>A Simple Dublin Core (XML) Metadata Display</li> <li>MODS 3.7 (XML) Metadata Display</li> <li>A Schema.org (JSON-LD) Metadata Display</li> <li>Carousel (in Bootstrap) for Images (HTML) Metadata Display </li> </ul> <p>You can find these templates here:</p> <ul> <li>On Github:<ul> <li>Local Deployment</li> <li>Live/Production Deployment</li> </ul> </li> <li>In your local instance: http://localhost:8001/metadatadisplay/list</li> <li>In your live instance: https://yourdomain.org/metadatadisplay/list</li> </ul> <p>Archipelago (the humans) will keep adding and refining these with every release.</p>"},{"location":"metadatatwigs/#instructions-and-examples","title":"Instructions and Examples","text":"<p>While a lot of core needs and use cases are covered with the Twig Templates shipped with Archipelago, you may want to add more Input elements to your Webforms, which in turn will generate new JSON Values, which in turn you may want to show/expose to end users.</p> <p>Knowing (even if you do not plan to) how to edit or create your own Twig templates is important.</p> <ul> <li>This guide covers the Basics of Working With Twig in Archipelago</li> <li>This section contains Full Examples of Common Use Cases</li> <li>This section covers a Recommended Workflow</li> <li>You may also want learn more about what <code>format_strawberryfield</code> can do and what many other possibilities are exposed through our templating system in this guide: Strawberryfield Formatters.</li> <li>Metadata Display Preview: use Preview to check your template outputs against Archipelago Digital Objects or AMI Sets.</li> <li>Metadata Display Usage: use the Usage tab to check where a Metadata Display Template is currently being used across your Archipelago.</li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"modifyingfileextensionsinwebform/","title":"Modifying allowable file extensions","text":"<ul> <li>Webform</li> <li>Webform Elements</li> <li>File Extensions</li> </ul>"},{"location":"modifyingfileextensionsinwebform/#customizing-webforms-modifying-allowable-file-extensions","title":"Customizing Webforms: Modifying allowable file extensions","text":"<p>A guide to walk users through how to modify the Webform <code>Descriptive Metadata</code> to allow additional file extensions to be ingested into Archipelago. This is the default Webform with Archipelago by following archipelago-deployment.</p>"},{"location":"modifyingfileextensionsinwebform/#context","title":"Context","text":"<p>When creating an Archipelago Digital Object (ADO), on Step 4 of the ingest, <code>Attach Files</code>, there is a step during the ingest to upload the files associated with your ADO. There will be a section on the Webform outlining the maximum number of files allowed, the maximum file size allowed, and the allowed file extensions that can be uploaded.</p> <p>Let's say we are creating an ADO with the media type <code>DigitalDocument</code> and this ADO contains a data set saved as a <code>csv</code> file, but when we get to Step 4 of the ingest workflow we find that <code>csv</code> is not an allowed file extension. Fortunately, Archipelago has no restrictions on what file extensions can be uploaded, but some use cases will require a little configuring to fit a specific need. This guide will walk users through the steps to modify the default Webform, <code>Descriptive Metadata</code>, to allow additional file extensions to be included during an ingest.</p> <p>Prerequisites for following this guide:</p> <ul> <li>Running instance of Archipelago (on http://localhost:8001 if you followed the deployment guide verbatim)</li> <li>Admin credentials</li> </ul>"},{"location":"modifyingfileextensionsinwebform/#lets-begin","title":"Let's begin!","text":""},{"location":"modifyingfileextensionsinwebform/#managing-webforms","title":"Managing Webforms","text":"<p>Once logged in as <code>admin</code>, the first thing we need to do is navigate to the Webforms page so we can edit the Webform <code>Descriptive Metadata.</code> Click on <code>Manage</code>, then <code>Structure</code> and when the page loads, scroll down and click <code>Webforms</code>.</p> <p></p> <p>This is where all of the Webforms inside your Archipelago live. For this guide we're going to edit the Webform <code>Descriptive Metadata</code>. Go ahead and click <code>Build</code> under the <code>OPERATIONS</code> column for <code>Descriptive Metadata</code>.</p> <p></p>"},{"location":"modifyingfileextensionsinwebform/#step-3-editing-elements","title":"Step 3: Editing Elements","text":"<p>Here we see all of the elements in <code>Descriptive Metadata</code>; Title, Media type, Description, Linked Data elements, etc. The element that we want to edit is <code>Upload Associated Documents</code> as this is the field you will use to upload <code>pdf</code>, <code>doc</code>, <code>rtf</code>, <code>txt</code>, etc. files during the ingest workflow. Click on <code>Edit</code> under the <code>OPERATIONS</code> column.</p> <p></p> <p>A new screen will pop up named <code>Edit Upload Associated Documents element</code>. This is where you can configure the maximum number of values (under <code>ELEMENT SETTINGS</code>), the maximum file size and also edit the allowed file extensions for this element, which is what we'll be doing. The latter both exist under <code>FILE SETTINGS</code> section, highlighted in the screenshot below.</p> <p></p> <p>When you scroll down you'll see the <code>Allowed file extensions</code> field. This is where we will add the <code>csv</code> file extension. Please note: All file extensions are separated by a space; no <code>,</code> or <code>.</code> between the values.</p> <p>Once you've added all the file extensions your project needs, scroll down to the bottom of <code>Edit Upload Associated Documents element</code> and click <code>Save</code>.</p> <p></p> <p>This next step is imperative for saving your changes, scroll to the bottom of your elements list page and click <code>Save elements</code> in order to persist all changes made.</p> <p></p>"},{"location":"modifyingfileextensionsinwebform/#complete","title":"Complete","text":"<p>Woohoo! Now when you are ingesting a <code>DigitalDocument</code> object, you will be able to add <code>csv</code> files! \ud83c\udf53</p> <p></p>"},{"location":"modifyingfileextensionsinwebform/#recap","title":"Recap","text":"<p>When logged in as an admin, we go to Manage &gt; Structure &gt; Webforms and click on <code>Build</code> under the <code>OPERATIONS</code> column of <code>Descriptive Metadata</code> (shortcut: /admin/structure/webform/manage/descriptive_metadata). Then we click on <code>Upload Associated Documents</code> to edit the element, scroll down to the Allowed file extensions field and add <code>csv</code> without <code>.</code> or <code>,</code> separating the values. Click <code>Save</code> at the bottom of the <code>Edit Upload Associated Documents element</code> page and then <code>Save elements</code> at the bottom of the Webform page.</p>"},{"location":"modifyingfileextensionsinwebform/#that-was-helpful-but","title":"That was helpful, but...","text":"How do I upload a <code>wav</code> or <code>aiff</code> file for \"MusicRecording\" or an <code>mov</code> file for a \\\"Movie\\\"? <p>The steps are virtually the same as what is outlined in this guide! The difference here is that instead of editing <code>Upload Associated Documents</code>, you will need to edit the field element that is associated with your ADO's media type. For example, with Media type <code>MusicRecording</code>, you will edit <code>Upload Audio File</code>, for <code>Movie</code>, will edit <code>Videos</code>.</p> How do I know which element in Descriptive Metadata to edit per media type? <p>When editing an element inside <code>Descriptive Metadata</code>, at the top of the window <code>Edit Upload Associated Documents element</code> (see Step 3 for a recap on how to get here) there is a tab next to <code>General</code> titled <code>Conditions</code>. Inside of <code>Conditions</code> we have <code>CONDITIONAL LOGIC</code> which is where the Webform is told which <code>Media type</code> needs this element to be visible in the Webform. In the example below, we know that the field element <code>Upload Associated Documents</code> will be visible when <code>DigitalDocument</code>, <code>Thesis</code> and <code>Book</code> are the selected <code>Media type</code>.</p> <p>This is also the place you can add new logic or delete present logic by clicking the <code>+</code> or <code>-</code> next to the <code>TRIGGER/VALUE</code> to create new conditionals.</p> <p></p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"open_api/","title":"Metadata API Module","text":"<p>Archipelago's Metadata API Module is a special Strawberryfield Formatter Module that implements Metadata API Endpoints and uses Views and Metadata Display Entities as a configuration option to provide dynamic APIs based on Metadata present in each Archipelago Digital Object (or Node) that contains a Strawberryfield type of field (JSON). </p> <p>Beginning with Archipelago 1.4.0, Archipelago is shipped with a standard implementation of a set of OAI-PMH functions. This includes a full Open API Configuration for OAI-PMH in the Metadata API Module, a corresponding OAI-PMH View, and two corresponding OAI-PMH Metadata Display Templates (Wrapper and Item to output object data into Dublin Core XML). All of these components are necessary to enable OAI-PMH functionality in your Archipelago. Please see the notes below that further explain the specific OAI-PMH <code>verbs</code> (request and response types) supported by Archipelago's default configuration. Please see the OAI Protocol for Metadata Harvesting Specifications for more information about OAI-PMH functions.</p> <p>This module could be extended to be used with other Open APIs, in addition to the default OAI-PMH configurations.</p>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"open_api/#quick-local-test-drive-for-oai-pmh","title":"Quick Local Test Drive for OAI-PMH","text":"<p>The default configurations (specifically the View, see more details below) limits the OAI-PMH queries to members of the 'Biodiversity Heritage Library Collection' (Node #25 as ingested via the default AMI Demo Set).</p> <p>To give this a quick test drive in your running local 1.4.0 Archipelago deployment, copy and paste the following query into your browser:</p> <pre><code>http://localhost:8001/ap/api/oai_pmh/oai?verb=ListRecords&amp;set=c80703db-4644-4f9c-99d4-46a12c500870&amp;metadataPrefix=oai_dc\n</code></pre> Click to see the Example Test Drive Results <p></p>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"open_api/#1-default-oai-pmh-configuration-form","title":"1. Default OAI-PMH Configuration Form","text":"<p>You can find the Metadata API Module Configuration Form:</p> <ul> <li>Through the <code>Manage</code> menu &gt; <code>Configuration</code> &gt; <code>Archipelago</code> &gt; <code>Configure Metadata APIs</code></li> <li>Directly at <code>/admin/config/archipelago/metadataapi</code> </li> </ul> <p></p>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"open_api/#default-supported-oai-pmh-verbs-configuration","title":"Default Supported OAI-PMH Verbs &amp; Configuration","text":"<p>In Archipelago's default <code>oai_pmh</code> Metadata API Endpoint, the following OAI-PMH verbs (requests and responses) are supported:</p> <ul> <li>GetRecord</li> <li>Identify</li> <li>ListIdentifiers</li> <li>ListMetadataFormats</li> <li>ListRecords</li> <li>ListSets</li> </ul> Click to see the Full Default OAI-PMH Open API Config JSON of Default OAI-PMH Open API Config<pre><code>{\n    \"openapi\": \"3.0.2\",\n    \"info\": {\n        \"title\": \"Test API\",\n        \"version\": \"1.0.0\"\n    },\n    \"paths\": {\n        \"http://localhost:8001/ap/api/oai_pmh/{oai}\": {\n            \"get\": {\n                \"parameters\": [\n                    {\n                        \"name\": \"verb\",\n                        \"in\": \"query\",\n                        \"description\": \"\",\n                        \"required\": true,\n                        \"deprecated\": false,\n                        \"style\": \"form\",\n                        \"explode\": true,\n                        \"schema\": {\n                            \"enum\": [\n                                \"GetRecord\",\n                                \"Identify\",\n                                \"ListIdentifiers\",\n                                \"ListMetadataFormats\",\n                                \"ListRecords\",\n                                \"ListSets\"\n                            ],\n                            \"type\": \"string\"\n                        }\n                    },\n                    {\n                        \"name\": \"set\",\n                        \"in\": \"query\",\n                        \"description\": \"\",\n                        \"required\": false,\n                        \"deprecated\": false,\n                        \"style\": \"form\",\n                        \"explode\": true,\n                        \"schema\": {\n                            \"type\": \"string\",\n                            \"format\": \"uuid\"\n                        }\n                    },\n                    {\n                        \"name\": \"identifier\",\n                        \"in\": \"query\",\n                        \"description\": \"\",\n                        \"required\": false,\n                        \"deprecated\": false,\n                        \"style\": \"form\",\n                        \"explode\": true,\n                        \"schema\": {\n                            \"type\": \"string\",\n                            \"format\": \"uuid\"\n                        }\n                    },\n                    {\n                        \"name\": \"oai\",\n                        \"in\": \"path\",\n                        \"description\": \"\",\n                        \"required\": true,\n                        \"deprecated\": false,\n                        \"style\": \"simple\",\n                        \"explode\": false,\n                        \"schema\": {\n                            \"type\": \"string\"\n                        }\n                    },\n                    {\n                        \"name\": \"page\",\n                        \"in\": \"query\",\n                        \"description\": \"\",\n                        \"required\": false,\n                        \"deprecated\": false,\n                        \"style\": \"form\",\n                        \"explode\": true,\n                        \"schema\": {\n                            \"type\": \"integer\"\n                        }\n                    },\n                    {\n                        \"name\": \"metadataPrefix\",\n                        \"in\": \"query\",\n                        \"description\": \"\",\n                        \"required\": false,\n                        \"deprecated\": false,\n                        \"style\": \"form\",\n                        \"explode\": true,\n                        \"schema\": {\n                            \"enum\": [\n                                \"oai_dc\",\n                                \"mods\"\n                            ],\n                            \"type\": \"string\"\n                        }\n                    },\n                    {\n                        \"name\": \"from\",\n                        \"in\": \"query\",\n                        \"description\": \"\",\n                        \"required\": false,\n                        \"deprecated\": false,\n                        \"style\": \"form\",\n                        \"explode\": true,\n                        \"schema\": {\n                            \"type\": \"string\",\n                            \"format\": \"date\"\n                        }\n                    },\n                    {\n                        \"name\": \"until\",\n                        \"in\": \"query\",\n                        \"description\": \"\",\n                        \"required\": false,\n                        \"deprecated\": false,\n                        \"style\": \"form\",\n                        \"explode\": true,\n                        \"schema\": {\n                            \"type\": \"string\",\n                            \"format\": \"date\"\n                        }\n                    }\n                ]\n            }\n        }\n    }\n}\n</code></pre>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"open_api/#reviewing-and-editing-the-default-oai-pmh-configuration-form","title":"Reviewing and Editing the Default OAI-PMH Configuration Form","text":"<p>You can make changes to the above configurations by selecting <code>Edit</code> on the right-hand side of the configuration overview.</p> <p></p> <p>Caution with making changes</p> <p>Proceed with caution before making changes to the default OAI-PMH Configuration Form. This functionality relies on the interconnected components described in this documentation. Before making changes to the configuration form itself, first backup your configurations so you can revert your changes if needed. It is also recommended to test your changes in a local environment before implementing in a production instance.</p>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"open_api/#2-default-oai-pmh-view","title":"2. Default OAI-PMH View","text":"<ul> <li>Through the <code>Manage</code> menu &gt; <code>Structure</code> &gt; <code>Views</code> &gt; <code>OAI_exposed_entity_reference</code></li> <li>Directly at <code>/admin/structure/views/view/oai_exposed_entity_reference</code> </li> </ul> <p>As stated above, the Default OAI-PMH View is configured to limit the OAI-PMH queries to members of the 'Biodiversity Heritage Library Collection' (Node #25 as ingested via the default AMI Demo Set). To change the collection this View limits queries and results for, change the Node ID listed in the <code>Content datasource: # Strawberry_(Descriptive Metadata source) \u00bb entity_sbf_entity_reference_ismemberof</code> under the Filter Criteria section.</p> <p></p> <ul> <li>Please note: this View is using the Full Pager, and this is required for properly outputting the fuller lists of results for OAI-PMH queries.</li> </ul>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"open_api/#3-default-oai-pmh-templates","title":"3. Default OAI-PMH Templates","text":"<p>The default OAI-PMH functionality relies on using two Metadata Display Templates, one for wrapping or setting the query results within a standard structure, and one for outputting individual query results (such as the Dublin Core XML for a specific Archipelago Digital Object). The two different templates are specified at the bottom of the OAI-PMH Configuration Form described above.</p> <p></p>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"open_api/#oai-pmh-wrapper-template","title":"OAI-PMH Wrapper Template","text":"<ul> <li>Found at: http://localhost:8001/metadatadisplay/17</li> <li>The Metadata display Entity (Twig) used to generate data for the API wrapper response.</li> </ul>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"open_api/#oai-pmh-item-with-dc-template","title":"OAI-PMH Item with DC Template","text":"<ul> <li>Found at: http://localhost:8001/metadatadisplay/18</li> <li>The Metadata display Entity (Twig) to be used to generate data at this endpoint.</li> </ul> <p>Caution with making changes</p> <p>Please also proceed with caution before making changes to the default OAI-PMH Metadata Display Templates. Before saving any changes you make to the templates, mark to save as a 'Revision' at the bottom of the template form, so you can revert your template to a previous version if needed.</p> <p>__</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"ourtake/","title":"Archipelago's Philosophy &amp; Guiding Principles","text":"<p>Archipelago operates under a  different concept than the one we all have become used to in recent times. We like to think this is not done by re-inventing the wheel, but by making sure the road is clean, level, and with fewer obstacles than before. We do this by removing some heavy weight from the top, some unneeded ballast, plus, of course, some well positioned innovations to make the ride enjoyable.</p> <p>We also like to say that Archipelago is like a Metadata Synthetizer (LFO anyone?) and we want to give you all the knobs, parameters, inputs and outputs to make the best out of it. Still, you can make \"music\" by just tapping the keyboard.</p> <p>To get here we had to do a full stop first. Look around. Questioning everything we knew. Research and test (repeat) and then re-architect slowly on new and old assumptions, and especially new community values.</p>"},{"location":"ourtake/#whys-and-whats-of-archipelago","title":"Whys and Whats of Archipelago","text":"<p>Because this topic is near and dear to our hearts, we are taking extra care with writing this important document. Please stay tuned for the full, verbose, heartfelt, and detailed long story of Archipelago's origins, development, future hopes and dreams.</p> <p>In the meantime, please consider reviewing this presentation created by Archipelago's Lead Architect Diego Pino which captures the essence of Archipelago's philosophy and guiding principles:</p> <ul> <li>Archipelago : an empathic Digital Repository Architecture</li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"presentations_events/","title":"Archipelago Presentations, Events, and Additional Resources","text":"<p>Important General &amp; Internal Recordings Notes</p> <p>Please be aware that some of the presentation documents shared above may contain links to older documentation resources that have since changed or are no longer available. We recommend referring to the latest documentation versions available on this site whenever needed.</p> <p>METRO's Digital Services Team facilitated many different internal training sessions throughout 2020-2022. If you and your team need access to any of these sessions that were recorded, please contact us. Thank you!</p>"},{"location":"presentations_events/#2024","title":"2024","text":"<ul> <li> <p>Archipelago Summer Workshop Series (August 2024) : registration details here</p> <ul> <li>Session 1: Archipelago 1.4.0 Local Deployment &amp; Features Tour<ul> <li>\ud83c\udfa5 Recording available (registration required)</li> </ul> </li> <li>Session 2: Review of Display/View Modes and Strawberryfield Formatters<ul> <li>\ud83c\udfa5 Recording available (registration required)</li> </ul> </li> <li>Session 3: Search &amp; Solr Overview<ul> <li>\ud83c\udfa5 Recording available (registration required)</li> </ul> </li> </ul> </li> <li> <p>IIIF AI/ML Community Group (July 2024)</p> <ul> <li>Creating a Better Balance: Respectful Reuse &amp; ML/AI Tags in IIIF Manifests. Allison Sherrick.</li> </ul> </li> <li> <p>IIIF Annual Conference (June 2024)</p> <ul> <li>Creating a Better Balance: Respectful Reuse &amp; ML/AI Tags in IIIF Manifests. Allison Sherrick, Diego Pino Navarro.</li> <li>Transdimensional mutations for audio/moving media annotations in IIIF Content Search. Allison Sherrick, Diego Pino Navarro.</li> <li>\ud83d\udcfa Recordings made available on the IIIF Youtube channel.</li> </ul> </li> <li> <p>Open Repositories Annual Conference (June 2024):</p> <ul> <li>Hybrid ML/AI driven search as cataloging aid in Archipelago Commons. Diego Pino Navarro.</li> <li>Creating a better balance: the need for tools and practices to combat AI harvests and resource flooding in repository environments. Diego Pino Navarro, Allison Sherrick.</li> <li>Collaboration Across Borders. Jessica Barlow, Lisa Lamont, Matt Ferrill, Hilario Castillo Castillo, Kristofer Patr\u00f3n Soberano.</li> </ul> </li> <li> <p>Public Release of IOI's InfraFinder Tool (Spring 2024): Archipelago Commons on Infrafinder</p> </li> </ul>"},{"location":"presentations_events/#2023","title":"2023","text":"<ul> <li> <p>IIIF Search API and Dynamic/evolving Manifest Generation: Facing the Unknown. Diego Alberto Pino Navarro, Allison Sherrick.</p> <ul> <li>\ud83d\udcfa Recording available</li> </ul> </li> <li> <p>DLF Forum (November 2023)</p> <ul> <li>Working and Learning the IIIF Search API in Archipelago. Diego Pino Navarro, Allison Sherrick.</li> <li>Working with Open-Schema JSON in Archipelago. Allison Sherrick, Diego Pino Navarro, Martha Tenney, Joanna DiPasquale, Corinne Chatnik.</li> <li>Slaying the Migration Dragon: Approaches to Navigating an Open Source System Migration. Lisa McFall, Sarah Walden McGowan, Brenden McCarthy, Shay Foley.</li> </ul> </li> <li> <p>Archipelago 1.3.0 Release Announcement (October 31, 2023) </p> </li> <li> <p>IIIF Annual Conference (June 2023)</p> <ul> <li>Experimental IIIF Kitchen using Archipelago. Pino Navarro, Diego; Sherrick, Allison.</li> <li>Mapping an Engineer Through IIIF. Monger, Jenifer J.; McCarthy, Brenden; Pino Navarro, Diego; Sherrick, Allison.</li> </ul> </li> <li> <p>Into Archipelago Commons: Access, Innovation and Community in Modern Archives.  Monger, Jenifer J.; McCarthy, Brenden; Corinne Chatnik. (June 2023)</p> </li> <li>Implementing Archipelago: An Innovative, Community Driven, Open-Source Repository. Corinne Chatnik, Union College; Martha Tenney, Barnard College. (June 2023)</li> <li>For the Love of Data and Ourselves: The Bumpy, Technical Road to Modern Archives. Monger, Jenifer J.; McCarthy, Brenden. (January/February 2023)</li> </ul>"},{"location":"presentations_events/#2022","title":"2022","text":"<ul> <li> <p>Archipelago Late 2022 Workshop Series:</p> <ul> <li>Session 1 : AMI Essentials and Tricks of the Trade. Pino Navarro, Diego; Sherrick (Lund), Allison; Romabiles, Katie. (November 2022)<ul> <li>\ud83c\udfa5 Recording available (registration required) </li> </ul> </li> <li>Session 2: Twig Templating and Metadata Display Preview for AMI Ingest. Pino Navarro, Diego; Sherrick (Lund), Allison; Romabiles, Katie; Min, Albert. (December 2022)<ul> <li>\ud83c\udfa5 Recording available (registration required)</li> </ul> </li> <li>Session 3: AMI Set Processing and Advanced Find + Replace (December 2022)<ul> <li>\ud83c\udfa5 Recording available (registration required)</li> </ul> </li> </ul> </li> <li> <p>McCarthy, B. J. (2022). Archipelago Commons: Using the Archipelago and AMI software to provide access to Rensselaer Polytechnic Institute's engineering drawings, a pilot project. Issues in Science and Technology Librarianship, 101. https://doi.org/10.29173/istl2717</p> </li> <li> <p>Open Perspectives Forum. Monger, Jenifer J.; McCarthy, Brenden. (November 2022) </p> </li> <li> <p>Migration, Collaboration and Innovation with Archipelago Commons. Monger, Jenifer J. (September 2022)</p> </li> <li> <p>\ud83c\udf53 Archipelago 1.0.0 - August 2022 Release Announcement (August 2022) and updated Specs and Features List </p> </li> <li> <p>Open Repositories June 2022</p> <ul> <li>Collaborative W3C Web Annotations using Annotorious in Archipelago and computer vision explorations as cataloger aids. Pino Navarro, Diego; Simon, Rainer.</li> <li>Modern Web Archiving with Archipelago and Webrecorder : WACZ, Replay.web and Deep Discovery in Digital Repositories. Pino Navarro, Diego; Kreymer, Ilya.</li> <li>Working with IIIF Manifests in Archipelago. Sherrick (Lund), Allison.</li> </ul> </li> <li> <p>Formation of the Archipelago Working Group (April 2022)</p> <ul> <li>In the Spring of 2022, METRO supported the creation of a select group of both early adopters and longtime members of the Archipelago community to provide a dedicated space for Archipelago power users to build upon their demonstrated use-explorations, contribute further to the platform and have a direct influence on roadmap code, direction, and timeline. This group will also work on documentation needs, use cases and outreach (including public showcases, trainings/workshops, and other events).</li> </ul> </li> <li> <p>Toward Empathetic Digital Repositories: An Interview with Diego Pino Navarro (January 2022)</p> </li> </ul>"},{"location":"presentations_events/#2021","title":"2021","text":"<ul> <li> <p>\ud83c\udf53 Archipelago 1.0.0-RC3 and 1.0.0 Release Announcement - November 2021</p> </li> <li> <p>AMIA Conference Workshop: Building a Web Archive-Capable Digital Repository with Webrecorder and Archipelago. Kreymer, Ilya; Ramirez-Lopez, Lorena; Dickson, Emma; Pino Navarro, Diego; Sherrick (Lund), Allison. (November 2021)</p> </li> <li> <p>Solr Importer AMI Migrations, Showcase and Roundtable. Pino Navarro, Diego; Sherrick (Lund), Allison. (July 2021)</p> <ul> <li>Please see latest I7 Solr Importer and AMI LoD Reconciliation documentation.</li> </ul> </li> <li> <p>IIIF Annual 2021 Conference:</p> <ul> <li>Learning &amp; Working with IIIF in Archipelago - 2021 IIIF Annual Conference. Pino Navarro, Diego; Sherrick (Lund), Allison. (June 2021)</li> <li>Editing a IIIF Manifest in Archipelago. Pino Navarro, Diego; Sherrick (Lund), Allison.</li> </ul> </li> <li> <p>June 2021 Open Repositories Conference:</p> <ul> <li>Europeana Data Model (EDM) Workflows in Archipelago. Pino Navarro, Diego; Sherrick (Lund), Allison.<ul> <li>\ud83d\udcfa Recording available</li> </ul> </li> <li>'Broken for All' Persistent Identifiers Panel Discussion. Kunze, John; Holmes-Wong, Deborah; Rafique, Zahid; Lohnash, Megan; Sherrick (Lund), Allison; Turner, Adrian; McKinley, Matthew.<ul> <li>\ud83d\udcfa Recording available </li> </ul> </li> </ul> </li> <li> <p>WebRecorder + Archipelago Workshop. Pino Navarro, Diego; Sherrick (Lund), Allison; Kreymer, Ilya; Ramirez-Lopez, Lorena; Dickson, Emma. (May 2021)</p> <ul> <li>\ud83d\udcfa Recording available</li> </ul> </li> <li> <p>Twig Templates and Archipelago. Pino Navarro, Diego; Sherrick (Lund), Allison. (May 2021)</p> <ul> <li>Review of the core role Twig templating plays in Archipelago, introduction to the basics of Twig, and demonstration of editing Twig templates in Archipelago to refine metadata displays and AMI ingests.</li> <li>\ud83c\udfa5 Recording available (registration required)</li> </ul> </li> <li> <p>\ud83c\udf53Archipelago 1.0.0-RC2 Release Announcement (May 2021) and Archipelago RC2 Specs and Features List</p> </li> <li> <p>Working with Archipelago Multi-Importer (AMI). Pino Navarro, Diego; Sherrick (Lund), Allison. (April 2021)</p> <ul> <li>Introduction to AMI, discussion of ingest strategies and options, and demonstration of an AMI ingest.</li> <li>\ud83d\udcfa Recording available</li> </ul> </li> <li> <p>Archipelago Digital Objects Repository (an) architecture to last. Pino Navarro, Diego. (DrupalCon North America 2021)</p> <ul> <li>\ud83d\udcfa Recording available</li> </ul> </li> <li> <p>Metadata, Schemas and Media in Archipelago. Pino Navarro, Diego; Sherrick (Lund), Allison (February 2021)</p> <ul> <li>Exploration of the flexible and extensible ways Archipelago manages Metadata, Schemas, and Media.</li> <li>\ud83c\udfa5 Recording available (registration required)</li> </ul> </li> <li> <p>Deploying Archipelago 1.0.0-RC1. Pino Navarro, Diego; Sherrick (Lund), Allison. (February 2021)</p> <ul> <li>Demonstration of a complete walkthrough of a local Archipelago 1.0.0-RC1 deployment.</li> <li>\ud83c\udfa5 Recording available (registration required)</li> </ul> </li> </ul>"},{"location":"presentations_events/#2020","title":"2020","text":"<ul> <li> <p>\ud83c\udf53 Archipelago 1.0.0-RC1 Release Announcement (December 2020)</p> </li> <li> <p>Webforms in Archipelago. Pino Navarro, Diego; Sherrick (Lund), Allison; Palmentiero, Jennifer. (December 2020)</p> </li> <li> <p>IIIF and Archipelago - Community Call. Pino Navarro, Diego. (October 2020)</p> </li> <li> <p>Archipelago : an empathic Digital Repository Architecture (September 2020)</p> </li> <li> <p>\ud83c\udf53 Archipelago 8.x-1.0-beta3 Release Announcement (July 2020)</p> </li> </ul>"},{"location":"presentations_events/#we-should-be-here","title":"We should be here","text":"<p>If you have a public Archipelago presentation, recording, or other resource you'd like to share on this page \ud83c\udfdd\ufe0f\ud83d\udccd, please contact us. We would love to add your great work to this list! \ud83d\udc9a </p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"search-within-collection/","title":"How to Add a 'Search Within Collection' Block","text":"<p>This guide covers how to add a 'Search Within Collection' Exposed Form Block to the default Archipelago Collection Display Page. </p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#preamble-prerequisites","title":"Preamble + prerequisites","text":"<p>Before diving into any Search and Solr configuration related changes, we strongly recommend that you read our Metadata in Archipelago overview documentation, which provides important context for understanding how the shape of your Archipelago Digital Objects/Collections (ADOs) metadata will inform your Search and Solr options and outcomes. If you don't have the bandwidth to read the (stellar) Metadata in Archipelago documentation, we recommend you read through our in-a-nutshell overview.</p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#step-1-open-the-collection-membership-view","title":"Step 1:  Open the Collection Membership View","text":"<p>Navigate to the Collection Membership view found at:</p> <ul> <li><code>/admin/structure/views/view/collection_membership</code></li> <li>Through the <code>Structure</code> menu &gt; <code>Views</code> &gt; <code>Collection Membership</code></li> </ul> <p>This View is setup to list the member Digital Objects of a Collection and is driven by Solr.</p> <p></p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#step-2-adjust-the-views-advanced-tab-settings","title":"Step 2. Adjust the View's Advanced tab settings","text":"<p>Open the 'Advanced' tab settings and change the 'Exposed form in block' to 'Yes'.</p> <p></p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#step-3-add-the-fulltext-search-filter-criteria-for-the-view","title":"Step 3. Add the Fulltext Search Filter Criteria for the View","text":"<p>On the left-hand side of the Collection Membership View form, under the 'Filter criteria' section, add and configure the 'Fulltext search' Criteria.</p> <p></p> <p>At the top of the 'Configure filter criterion: Search: Fulltext search' form that opens:</p> <ul> <li>Select the option to 'Expose this filter to visitors, to allow them to change it'.</li> </ul> <p>On the lower section of the form, adust the configuration options as follows:</p> <ul> <li>Remove any text from 'Label' or 'Description</li> <li>Operator: select/check 'Contains all of these words'</li> <li>Allow multiple selections : leave unchecked</li> <li>Remember the last selection : leave unchecked</li> <li>Filter identifier: leave default of 'search_api_fulltext'</li> <li>Placeholder: enter 'Search within Collection' (or your preferred text)</li> <li>Search field character limit: set to '128'</li> <li>Expose searched fields : leave unchecked</li> <li>Searched fields : leave all unselected, so \"If no fields are selected, all available fulltext fields will be searched.\"</li> <li>Minimum keyword length : set to '1'</li> </ul> <p>Select 'Apply' and continue to the next Step.</p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#step-4-reviewadjust-options-and-save-the-updated-view","title":"Step 4. Review/adjust options and Save the Updated View","text":"<p>Review the changes you made to the Collection Membership View. Optionally further adjust the options if desired. For example, you may choose to change the <code>Submit button text</code> to 'Search' instead of 'Apply'). If you make any further changes select, 'Apply' before exiting.</p> <p></p> <p>'Save' your changes made to the View before proceeding.</p> <p>Your updated Collection Membership View should look like the following now:</p> <p></p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#step-5-place-the-exposed-form-block","title":"Step 5. Place the Exposed Form Block","text":"<p>Navigate to the Block Layout found at:</p> <ul> <li><code>/admin/structure/block</code></li> <li>Through the <code>Structure</code> menu &gt; <code>Block layout</code></li> </ul> <p>Select the <code>Archipelago Base Theme</code> (or whatever theme you are using):</p> <ul> <li><code>admin/structure/block/list/archipelago_subtheme</code></li> </ul> <p>Navigate to the 'Content' section of the theme and select 'Place Block'.</p> <p></p> <p>Search for the 'Exposed form: collection_membership-block_1' and select 'Place Block'.</p> <p></p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#step-6-configure-the-blocks-settings","title":"Step 6. Configure the Block's Settings:","text":"<p>Configure the different settings for the Block. </p> <p>You will need to specify both of the following options under the <code>Visibility</code> section:</p> <ul> <li>ADO Type: specify Collection (and any other Collection types you may also have such as 'Newspaper')</li> </ul> <p></p> <ul> <li>Content type: select 'Digital Object Collection'</li> </ul> <p></p> <p>In the top section of the form, we recommend the following settings:</p> <ul> <li>Deselect 'Display title'</li> <li>Under <code>Exposed Form element and component Visibility</code>, deselect the following:<ul> <li>Show filter components of type select if exposed</li> <li>Show filter components of type checkbox/options if exposed</li> <li>Override Submit button Label</li> </ul> </li> </ul> <p></p> <p>Select 'Save block' when you are finished configuring the Block's Settings.</p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#step-7-position-the-block-and-save","title":"Step 7. Position the Block and Save.","text":"<p>Drag to re-order and position the Exposed Form Block to sit above the Collection Membership block in the <code>Content</code> section. This will position the Exposed Form Block above the list of Collection Member Objects on the display pages for Collections.</p> <p>After you have positioned the blocks, scroll down to the bottom of the <code>Block layout</code> page and select 'Save blocks'.</p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#step-8","title":"Step 8.","text":"<p>Navigate to a Collection and test out a Search in the search box that is now in place above the 'Objects in this Collection' listing.</p> <p>In this screenshot, you can see a demonstrative Search for 'map' within one of the Archipelago Demo Collections.</p> <p></p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#additional-considerations","title":"Additional Considerations","text":"<p>You may also wish to pair this 'Search Within Collection' Exposed Form Block with related Facets (setup on the same corresponding collection membership View). You can find follow the step-by-step instructions in our Strawberry Key Name Providers, Solr Field, and Facet Configuration documentation.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search_advanced/","title":"Advanced Search","text":"<p>To provide Advanced Search functionality, Archipelago extends a custom Search API based Fulltext Filter for (Drupal) Views. This special 'Search: Advanced Fulltext search' filter is capable of handling multiple inputs with a combination of Boolean Operators, and features additional customizable options for tailoring your Archipelago's Advanced Search setup.</p>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#default-advanced-search","title":"Default Advanced Search:","text":"<p>The default Advanced Search setup shipped with standard Archipelago deployments is intended to serve as an initial blueprint for your own learning and further customizations. Your Archipelago instance, whether you are working in a modified local or production deployment, may not have the same corresponding Solr Fields and/or Facets as present in the default Advanced Search configuration. This guide covers the default Advanced Search Setup, and offers general guidance for customizing to match your desired Advanced Search setup for your unique environment variables (metadata elements/schema found in your JSON data, Keyname combinations, Solr fields, etc.).</p>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#where-to-find-and-corresponding-configurations","title":"Where to Find and Corresponding Configurations","text":"<p>In default Archipelagos, you can find Archipelago's default Advanced Search page:</p> <ul> <li>Through the top navigation menu &gt; <code>Advanced Search</code></li> <li>Directly at <code>/advanced-search</code></li> </ul> <p></p> <p>The default Advanced Search Page includes the following:</p> <ul> <li>Advanced Fulltext Search form (more information on the out-of-the-box View settings associated with this form below)</li> <li>Five Facets for narrowing your results (Date of Original, Digital Object Type, Collection Membership, Subjects, and Agents)</li> <li>A Facet Summary section (visible only after selecting Facets)</li> <li>Custom Block containing a Default Advanced Search Note</li> </ul> <p></p> <p>The five Facets associated with the default Advanced Search can be found at <code>Administration &gt; Configuration &gt; Search and metadata &gt; Facets</code> under the <code>Facet source - search_api:views_page__advanced_search__page_1</code>.</p> <p>The Custom Block containing the Default Advanced Search Note can be found at <code>/admin/content/block</code> or as a Tab on the main Content page.</p> <p>The corresponding Facet Blocks and Custom Block can be found at <code>/admin/structure/block/list/archipelago_subtheme</code> in the <code>Sidebar Second</code> section. Please see the this documentation for more information about Keyname Providers, Solr Fields, and Facet Configurations</p>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#advanced-search-view","title":"Advanced Search View","text":"<p>The default Advanced Search View drives the Advanced Search Form. This View can be found at:</p> <ul> <li><code>/admin/structure/views/view/advanced_search</code></li> <li>Through the <code>Structure</code> menu &gt; <code>Views</code> &gt; <code>Advanced Search</code></li> </ul> <p></p>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#configuring-the-filter-criteria-for-the-search-advanced-fulltext-search","title":"Configuring the filter criteria for the 'Search: Advanced Fulltext search'","text":"<p>To view or adjust the configurations click on 'Search: Advanced Fulltext search (and)' in the 'Filter criteria section in the View. You will then see the first part of the following configuration form:</p> <p></p>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#part-1-of-the-advanced-search-filter-form","title":"Part 1 of the Advanced Search Filter Form","text":"<p>Beginning at the top of the form, you will see:</p> <ul> <li>Brief note explaining that the Advanced Fulltext search filter can 'Search several or all fulltext fields at once allowing also multiple operator separated entries'</li> <li>Option to 'Expose this filter to visitors, to allow them to change it'<ul> <li>checked in default</li> </ul> </li> <li>Option to marked as 'Required'<ul> <li>unchecked in default</li> </ul> </li> <li>Label: Text field to enter Label for this filter<ul> <li>set to 'Advanced Fulltext search' in default</li> </ul> </li> <li>Description: Text field to enter brief description for this filter.<ul> <li>Tokens are allowed in this field. Replacement options can be found in the \"Global replacement patterns\" section, below.</li> <li>empty in default</li> </ul> </li> </ul> <p>You will then see a section for configuring the starting Operator to use for the form:</p> <ul> <li>Contains all of these words<ul> <li>checked in default</li> </ul> </li> <li>Contains any of these words</li> <li>Contains none of these words</li> <li>A note that 'Depending on the parse mode set, some of these options might not work as expected. Please either use \"Multiple words\" as the parse mode or make sure that the filter behaves as expected for multiple words.'</li> <li>An optional starting <code>Value</code><ul> <li>empty in default</li> </ul> </li> </ul> <p>Next is the beginning of the fuller Operator and Additional Options:</p> <ul> <li>Option to 'Expose operator' <ul> <li>Allow the user to choose the operator</li> <li>checked in default</li> </ul> </li> <li>Option to 'Limit the available operators'<ul> <li>Limit the available operators to be shown on the exposed filter</li> <li>unchecked checked in default</li> <li>recommended to enable for Classic Mode (see further notes related to Classic Mode below)</li> </ul> </li> <li>Operator identifier: Text field to specify 'the URL after the ? to identify this operator'<ul> <li>set to 'sbf_advanced_search_api_fulltext_op' in default</li> </ul> </li> <li>Option to 'Allow multiple selections' (Enable to allow users to select multiple items)</li> <li>Option to 'Remember the last selection' (Enable to remember the last selection made by the user)</li> <li>Parse mode : dropdown menu to choose how the search keys will be parsed<ul> <li>Direct query</li> <li>Single phrase</li> <li>Multiple words *selected by default and strongly recommended to keep</li> <li>Multiple words with EDisMax</li> <li>Multiple words with fuzziness</li> <li>Phrase search with sloppiness Multiple words with sloppiness</li> </ul> </li> <li>Filter identifier : Text field to specify what will appear in the URL after the ? to identify this filter<ul> <li>Cannot be blank. Only letters, digits and the dot (\".\"), hyphen (\"-\"), underscore (\"_\"), and tilde (\"~\") characters are allowed.</li> <li>set to 'sbf_advanced_search_api_fulltext' in default</li> </ul> </li> </ul>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#part-2-of-the-advanced-search-filter-form","title":"Part 2 of the Advanced Search Filter Form","text":"<ul> <li>Placeholder : Text field to enter hint text that appears inside the field when empty<ul> <li>empty in default</li> </ul> </li> <li>Note about the 'Multiple words' parse mode (selected earlier in the form)<ul> <li>The query is interpreted as multiple keywords separated by spaces. Keywords containing spaces may be \"quoted\". Quoted keywords must still be separated by spaces. Keywords can be negated by prepending a minus sign (-) to them.</li> </ul> </li> <li>Search field character limit : specify the maximum number of characters to allow as keywords input.<ul> <li>set to 128 in default</li> </ul> </li> <li>Option to 'Expose searched fields', which allows users to narrow the search to the desired fields.<ul> <li>checked in default</li> </ul> </li> <li>Searched fields identifier : text field to specify the URL after the ? to identify this searched fields form element.<ul> <li>set to 'sbf_advanced_search_api_fulltext_searched_fields' in default</li> </ul> </li> <li>Option to have 'Multiple/add more Search Fields'<ul> <li>This allows users to add more Search Fields with the same general exposed settings. All identifiers passed by this filter in the URL after the ? will get an incremental suffix.</li> <li>checked in default</li> </ul> </li> <li> <p>Max number of Multiple/add more Search Fields the user can expose.</p> <ul> <li>The number of additional search Fields with the same general exposed settings the user will be able to expose.</li> <li>set to a value of '4' in default</li> </ul> </li> <li> <p>Searched fields</p> <ul> <li>Select the fields that will be searched. If no fields are selected, all available fulltext fields will be searched.</li> <li>The options exposed here will be dependent on your configured Solr Fields.</li> <li>Fields selected by default include: <ol> <li>Content &gt; Strawberry (Descriptive Metadata source) &gt; local_identifier</li> <li>Content &gt; Strawberry (Descriptive Metadata source) &gt; name</li> <li>Rendered HTML output</li> <li>Content &gt; Title FullText</li> </ol> </li> </ul> </li> <li> <p>Min number of Multiple/add more Search Fields the user will see.</p> <ul> <li>Number must be less or equal to the max. If not it will cap automatically.</li> <li>The number of search Fields with the same general exposed settings the user will see by default.</li> <li>set to a value of '1' in default</li> </ul> </li> <li> <p>Minimum keyword length</p> <ul> <li>Minimum length of each word in the search keys. Leave empty to allow all words.</li> <li>empty in default</li> </ul> </li> </ul>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#part-3-of-the-advanced-search-filter-form","title":"Part 3 of the Advanced Search Filter Form","text":"<p>In the last part of the form, you will see:</p> <ul> <li> <p>Label to be used for the \"add one\" button</p> <ul> <li>\"Label to be used for the \"add more\" button. By default it is \"add one\" if left empty</li> <li>empty in default</li> </ul> </li> <li> <p>Replacement pattern for user facing Fields.</p> <ul> <li>In this text area, you will have the option to supply your preferred user-facing label for your selected search fields.</li> <li>You will see all of the fields available in your Solr.</li> <li>You supply your preferred labels and set the user-facing dropdown ordering for your targeted fields being used.</li> <li>Use a Pipe (|) to separate value from desired label. One per line</li> <li>To adjust the label for a field, replace the text following the pipe (|) symbol in the string to your preferred label.</li> <li>Field label adjustments and order specified selected by default include:<ol> <li>rendered_item|Rendered HTML Output</li> <li>title|Title (full text)</li> <li>name_1|Names/Agents</li> <li>local_identifier|Local Identifier</li> </ol> </li> </ul> </li> </ul> <p></p> <ul> <li>Label to be used for the \"remove one\" button<ul> <li>\"Label to be used for the \"add one\" button. By default it is \"remove one\" if left empty</li> <li>set to 'remove' in default</li> </ul> </li> <li>Option to 'Expose between search fields operator'<ul> <li>Allow the user to choose the operator between Multiple Search Fields (AND/OR)</li> <li>checked in default    </li> </ul> </li> </ul>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#classic-mode","title":"Classic Mode","text":"<ul> <li>Option to 'Use Classic Mode' : When you select the 'Classic Mode', your Advanced Search form will interact in a way that mimics older catalog search interfaces. This means that:<ul> <li>when you add/remove fields, this does not trigger automatic refreshing of the Search results</li> <li>search only triggers on the default Form Filter submit button</li> <li>this mode fully depends on Javascript to get around Views Exposed Filters in forms always submitting on any interaction.</li> <li>unchecked in default</li> </ul> </li> </ul> <p>If you specify to 'Use Classic Mode', you will also see:</p> <ul> <li> <p>Option to  'Add a Remove button to every Advanced Search Field combo.'</p> <ul> <li>This will allow a user to remove a specific Advanced Search Field/And/or/Text. Only works on Classic Mode.</li> <li>only appears on form when 'Use Classic Mode' is enabled, hidden and unchecked in default</li> <li>strongly recommended to enable for Classic Mode setups</li> </ul> </li> <li> <p>Please see above note in Part 1 of the Advanced Search Filter Form to also select the option to 'Limit the available operators' when using Classic Mode.  </p> </li> </ul>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#end-of-advanced-search-filter-form","title":"End of Advanced Search Filter Form","text":"<p>For both the normal, modern Advanced Search setup and also with Classic Mode enabled, you will also see:</p> <ul> <li> <p>Multiple/add more Search Fields operator (AND/OR) fields identifier</p> <ul> <li>Text field to specify what will appear in the URL after the ? to identify the operator used in the filter when multiple Search Fields and their extra options are exposed.</li> <li>set to 'sbf_advanced_search_api_fulltext_group_operator' in default</li> </ul> </li> <li> <p>You will also see a section for 'Global replacement patterns (for description field only), where you can Brose available tokens for this purpose.</p> <ul> <li>No replacement patterns specified for the View Description field in default. Not recommended.</li> </ul> </li> </ul>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#example-query-test","title":"Example Query Test","text":"<p>In your local or fresh live Archipelago deployment, populated with the Archipelago Demo Set Objects and Collections, test your Advanced Search setup using the following query: - 'Contains all of these words' + the term 'Art' in the 'Rendered HTML Output' Field - 'OR' operator + 'Contains all of these words' + the term 'Diego' in the 'Names/Agents' field - Select the 'Archipelago Demo Collection' from the Collection Membership Facet</p> <p>You should see the following results:</p> <p></p> <p>The corresponding advanced search url should be: - http://localhost:8001/advanced-search?sbf_advanced_search_api_fulltext_op=and&amp;sbf_advanced_search_api_fulltext=Art&amp;sbf_advanced_search_api_fulltext_searched_fields=rendered_item&amp;sbf_advanced_search_api_fulltext_advanced_search_fields_count=2&amp;sbf_advanced_search_api_fulltext_group_operator_1=or&amp;sbf_advanced_search_api_fulltext_op_1=and&amp;sbf_advanced_search_api_fulltext_1=Diego&amp;sbf_advanced_search_api_fulltext_searched_fields_1=name_1&amp;f%5B0%5D=is_member_of_content_title%3AArchipelago+Demo+Collection&amp;op=Search</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_solr_index/","title":"Search and Solr","text":"<p>Archipelago's default Search and Solr configurations are intended to cover the most common needs for a typical repository search experience.</p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#preamble-prerequisites","title":"Preamble + prerequisites","text":"<p>Before diving into any Search and Solr configuration changes, we strongly recommend that you read our Metadata in Archipelago overview documentation, which provides important context for understanding how the shape of your Archipelago Digital Objects/Collections (ADOs) metadata will inform your Search and Solr options and outcomes.</p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#archipelago-and-solr","title":"Archipelago and Solr","text":"<p>Archipelago's latest Release (1.4.0) uses Apache Solr 9.2, which incorporates some major improvements and changes from Solr 8. Please refer to the primary Solr documentation for the most comprehensive and in-depth information about Solr's wide breadth of functionality and configuration options.</p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#ocr-highlights","title":"OCR Highlights","text":"<p>Archipelago uses solr-ocrhighlighting v0.8.4, built by the Development Team at the Bavarian State Library. See our Strawberry Runners Post-Processing documentation for more information about configuring page-based HOCR/OCR extraction for image and pdf-based ADOs and options for sending that output to the Search API.</p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#in-a-nutshell-json-data-to-strawberry-keyname-providers-to-solr","title":"In-a-nutshell : JSON data to Strawberry Keyname Providers to Solr","text":"<p>If you don't have the bandwidth to read the (stellar) Metadata in Archipelago documentation, focus on the following in-a-nutshell understanding of the way Archipelago's Search and Solr is crafted. Then follow the step-by-step instructions found in Strawberry Key Name Providers, Solr Field, and Facet Configuration to get started customizing your Search setup.</p> <p></p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#1-json-data-for-your-archipelago-digital-objects","title":"1. JSON Data for your Archipelago Digital Objects","text":"<p>You need to have your descriptive metadata in JSON keys and values for ADOs and Collections in your Archipelago. Your JSON metadata and (extracted technical) data is the crucial source for Search and Solr.</p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#2-strawberry-keyname-providers","title":"2. Strawberry Keyname Providers","text":"<p>You need to have corresponding Strawberry Keyname Providers configured to feed the values from your desired JSON keys to Solr fields. One of the most powerful tools in Archipelago's Search functionality comes from the extensibility of Strawberry Keyname Providers, which can be used to source specific data points from a variety of JSON keys found in your repository.</p> More about Strawberry Keyname Providers","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#excerpted-from-metadata-in-archipelago","title":"Excerpted from Metadata in Archipelago","text":"<p>One of the challenges of our flexible approach is how to allow Drupal to access the JSON in a way, as native as possible, to generate filtered listings via Drupal Views, free text Search and Faceting. To make this happen Strawberry Field uses a JSON Querying and Exposing as \"Native Field Properties\" logic. Through a special type of Plugin system named Strawberry Key Name Providers and associated Configuration Entities (can be found at /admin/structure/strawberry_keynameprovider), you have control on which keys and values of your JSON are going to be exposed as field properties of any Strawberry Field, allowing Drupal through this to access values in a flat manner and expose them to the Search API natively. The access to the values of any JSON is done via JMESPATH expressions and then transformed either to a list of values or even \"cast\" into more complex data Data types, like an Entity Reference (means a connection to another Entity). This gives you a lot of power and control and makes a lot of very heavy operations lighter. You can even plan upfront or evolve these properties in time.</p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#3-solr-fields","title":"3. Solr Fields","text":"<p>You need to configure your desired Solr Fields to source from the Strawberry Keynames you have configured. By default, Archipelago also provider Solr Fields sourced from your HOCR data and the Rendered HTML output of your ADOs.</p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#4-drupal-views","title":"4. Drupal Views","text":"<p>For your regular Search, Advanced Search, and potentially other specialized Views, you can configure to search within specific and/or a variety of Solr Fields. </p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#5-search-results-facets","title":"5. Search Results &amp; Facets","text":"<p>Your metadata and data, as configured through Keyname Providers and Fields, Facets and Facet Blocks, indexed into your Solr. </p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#instructions-and-guides","title":"Instructions and Guides","text":"<ul> <li>Strawberry Key Name Providers, Solr Field, and Facet Configuration </li> <li>Advanced Search</li> <li>How to Add a 'Search Within Collection' Block</li> <li>IIIF Content Search</li> <li>For Live Archipelago Deployment Site Administrators: Upgrading Solr</li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"security_bots/","title":"Managing Bots","text":"<p>A public-facing production instance will likely encounter bad bots and other malicious traffic that will consume resources. There are many solutions available that address a variety of different needs, but we provide basic configurations and a Docker image for integrating the NGINX Ultimate Bad Bot &amp; Referrer Blocker.</p> <p>Warning</p> <p>Before proceeding, please be sure to familiarize yourself with the NGINX Ultimate Bad Bot &amp; Referrer Blocker README.</p>","tags":["Security","Bots"]},{"location":"security_bots/#deployment","title":"Deployment","text":"<ol> <li>Uncomment or add the following docker-compose environment variables, replacing any appropriate values with your own and leaving the blocker and cron disabled to start (see highlighted lines):    .env<pre><code>MSMTP_ACCOUNT=SMTP_ACCOUNT_NAME\nMSMTP_EMAIL=repositorysupport@metro.org\nMSMTP_HOST=smtp.metro.org\nMSMTP_PASSWORD=YOUR_SMTP_PASSWORD\nMSMTP_PORT=SMTP_PORT\nMSMTP_STARTTLS=on\nNGXBLOCKER_ENABLE=false\nNGXBLOCKER_CRON=00 22 * * *\nNGXBLOCKER_CRON_COMMAND=/usr/local/sbin/update-ngxblocker -x\nNGXBLOCKER_CRON_START=false\n</code></pre></li> <li>Uncomment or add the following lines and comment out the line for the original NGINX image:    docker-compose.yml<pre><code># Run docker-compose up -d\n# Docker file for Arm64 and Apple M1 machines\nversion: '3.5'\nservices:\n  web:\n    container_name: esmero-web\n    # image: jonasal/nginx-certbot\n    image: esmero/nginx-bot-blocker:1.1.0-multiarch\n    restart: always\n    environment:\n      CERTBOT_EMAIL: ${ARCHIPELAGO_EMAIL}\n      ENVSUBST_VARS: FQDN\n      FQDN: ${ARCHIPELAGO_DOMAIN}\n      NGINX_ENVSUBST_OUTPUT_DIR: /etc/nginx/user_conf.d\n      MSMTP_ACCOUNT: ${MSMTP_ACCOUNT}\n      MSMTP_EMAIL: ${MSMTP_EMAIL}\n      MSMTP_HOST: ${MSMTP_HOST}\n      MSMTP_PASSWORD: ${MSMTP_PASSWORD}\n      MSMTP_PORT: ${MSMTP_PORT}\n      MSMTP_STARTTLS: ${MSMTP_STARTTLS}\n      NGXBLOCKER_CRON: ${NGXBLOCKER_CRON}\n      NGXBLOCKER_CRON_COMMAND: ${NGXBLOCKER_CRON_COMMAND}\n      NGXBLOCKER_CRON_START: ${NGXBLOCKER_CRON_START}\n      NGXBLOCKER_ENABLE: ${NGXBLOCKER_ENABLE}\n    ports:\n      - \"80:80\"\n      - \"443:443\"\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/config_storage/nginxconfig/template:/etc/nginx/templates\n      - ${ARCHIPELAGO_ROOT}/drupal:/var/www/html:cached\n      - ${ARCHIPELAGO_ROOT}/data_storage/ngnixcache:/var/cache/nginx\n      - ${ARCHIPELAGO_ROOT}/data_storage/letsencrypt:/etc/letsencrypt\n      - ${ARCHIPELAGO_ROOT}/config_storage/nginxconfig/bots.d:/etc/nginx/bots.d\n</code></pre></li> <li> <p>First pull the new image:    <pre><code>docker compose pull\n</code></pre></p> <p>Note</p> <p>If using an older version of docker, don't forget the hyphen: <pre><code>docker-compose pull\n</code></pre></p> </li> <li> <p>Now bring the Docker ensemble down and up again:    <pre><code>docker compose down &amp;&amp; docker compose up -d\n</code></pre></p> <p>Note</p> <p>If using an older version of docker, don't forget the hyphen: <pre><code>docker-compose down &amp;&amp; docker-compose up -d\n</code></pre></p> </li> <li> <p>Run the install script for the bot blocker in the default dry run mode and review the output:    <pre><code>docker exec -ti esmero-web bash -c \"/usr/local/sbin/install-ngxblocker\"\n</code></pre></p> </li> <li>The script will output the changes that are going to be made. Review them carefully and ensure that they are ok to make. Then run the command with the execute flag:    <pre><code>docker exec -ti esmero-web bash -c \"/usr/local/sbin/install-ngxblocker -x\"\n</code></pre></li> <li>Run the setup script for the bot blocker in the default dry run mode and review the output:    <pre><code>docker exec -ti esmero-web bash -c \"/usr/local/sbin/setup-ngxblocker -v /etc/nginx/templates -e .copy\"\n</code></pre></li> <li>The script will output the NGINX configuration changes that are going to be made. Review them carefully and ensure that they are ok to make. Then run the command with the execute flag:    <pre><code>docker exec -ti esmero-web bash -c \"/usr/local/sbin/setup-ngxblocker -v /etc/nginx/templates -e .copy -x\"\n</code></pre></li> <li> <p>Enable the bot blocker and cron (if applicable):    .env<pre><code>MSMTP_ACCOUNT=SMTP_ACCOUNT_NAME\nMSMTP_EMAIL=repositorysupport@metro.org\nMSMTP_HOST=smtp.metro.org\nMSMTP_PASSWORD=YOUR_SMTP_PASSWORD\nMSMTP_PORT=SMTP_PORT\nMSMTP_STARTTLS=on\nNGXBLOCKER_ENABLE=true\nNGXBLOCKER_CRON=00 22 * * *\nNGXBLOCKER_CRON_COMMAND=/usr/local/sbin/update-ngxblocker -x\nNGXBLOCKER_CRON_START=true\n</code></pre></p> <p>Note</p> <p>If <code>MSMTP_EMAIL</code> is blank and cron is enabled the flag for sending email notifications will be skipped.</p> </li> <li> <p>Bring the Docker ensemble down and back up again:    <pre><code>docker compose down &amp;&amp; docker compose up -d\n</code></pre></p> <p>Note</p> <p>If using an older version of docker, don't forget the hyphen: <pre><code>docker-compose down &amp;&amp; docker-compose up -d\n</code></pre></p> </li> <li> <p>Test that it is working by following the \"TESTING\" section (STEP 10) in the official documentation: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker</p> </li> </ol>","tags":["Security","Bots"]},{"location":"security_bots/#deployment-with-upgrade","title":"Deployment with Upgrade","text":"<p>If looking to use this solution as part of an upgrade (from 1.0.0 to 1.1.0, for example) we recommend coming back to the above steps after successfully completing the upgrade. After the upgrade, you will only need to add the environment variables and docker compose configurations and follow the steps as detailed above.</p>","tags":["Security","Bots"]},{"location":"security_bots/#advanced-configuration","title":"Advanced Configuration","text":"<p>Because our Docker containers only persist our mounted files and folders, any advanced configurations may require overriding the files generated by our esmero-web container on boot. For example, the above <code>setup-ngxblocker</code> script is normally responsible for writing the following include lines:</p> <pre><code>include /etc/nginx/bots.d/blockbots.conf;\ninclude /etc/nginx/bots.d/ddos.conf;\n</code></pre> <p>Because the script is unable to place them in the correct part of our <code>nginx.conf.template</code> file, which in turn generates our <code>nginx.conf</code> file (see Using environment variables in nginx configuration), our own script adds (when <code>NGINXBLOCKER_ENABLE=true</code>) or removes (when <code>NGINXBLOCKER_ENABLE=false</code>) the lines to an empty file, which in turn is statically included in our main <code>nginx.conf.template</code> file. One option provided by <code>setup-ngxblocker</code> is to exclude (<code>-d</code>) the DDOS rule. In our case, we need to manually override the lines in our template file to reproduce this behavior:</p> <p>Example</p> nginx.conf.template<pre><code>upstream cantaloupe {\n  server esmero-cantaloupe:8182;\n}\n\nserver {\n    listen              443 ssl;\n    server_name         ${FQDN};\n    ssl_certificate     /etc/letsencrypt/live/${FQDN}/fullchain.pem;\n    ssl_certificate_key /etc/letsencrypt/live/${FQDN}/privkey.pem;\n    client_max_body_size 1536M; ## Match with PHP from FPM container\n\n    root /var/www/html/web; ## &lt;-- Your only path reference.\n\n    fastcgi_send_timeout 120s;\n    fastcgi_read_timeout 120s;\n    fastcgi_pass_request_headers on;\n\n    fastcgi_buffers 16 16k;\n    fastcgi_buffer_size 32k;\n\n    # Please adapt to your needs\n    proxy_buffers 16 16k;  \n    proxy_buffer_size 16k;\n\n    #include /etc/nginx/conf.d/bots.include;\n    include /etc/nginx/bots.d/blockbots.conf;\n</code></pre> <p>Note</p> <p>Keep in mind that from this point, when disabling/enabling the bot blocker via the environment variable, you'll also need to uncomment/comment the added line. </p> <p>Another more generally applicable approach is to override files that are part of the docker image:</p> <p>Example</p> <p>Our bash script (setup_bot_blocker.sh) is triggered by and runs just before the startup script (start_nginx_certbot.sh) for the NGINX Certbot image. For any advanced needs involving custom startup behavior, our script can be modified and overwridden:</p> <ol> <li>First, we'll copy the script from the running Docker container onto our host:    <pre><code>docker cp esmero-web:/scripts/setup_bot_blocker.sh drupal/scripts/archipelago/\n</code></pre></li> <li>Then we mount the file to override the container's file:    docker-compose.yml<pre><code># Run docker-compose up -d\n# Docker file for Arm64 and Apple M1 machines\nversion: '3.5'\nservices:\n  web:\n    container_name: esmero-web\n    # image: jonasal/nginx-certbot\n    image: esmero/nginx-bot-blocker:1.1.0-multiarch\n    restart: always\n    environment:\n      CERTBOT_EMAIL: ${ARCHIPELAGO_EMAIL}\n      ENVSUBST_VARS: FQDN\n      FQDN: ${ARCHIPELAGO_DOMAIN}\n      NGINX_ENVSUBST_OUTPUT_DIR: /etc/nginx/user_conf.d\n      MSMTP_ACCOUNT: ${MSMTP_ACCOUNT}\n      MSMTP_EMAIL: ${MSMTP_EMAIL}\n      MSMTP_HOST: ${MSMTP_HOST}\n      MSMTP_PASSWORD: ${MSMTP_PASSWORD}\n      MSMTP_PORT: ${MSMTP_PORT}\n      MSMTP_STARTTLS: ${MSMTP_STARTTLS}\n      NGXBLOCKER_CRON: ${NGXBLOCKER_CRON}\n      NGXBLOCKER_CRON_COMMAND: ${NGXBLOCKER_CRON_COMMAND}\n      NGXBLOCKER_CRON_START: ${NGXBLOCKER_CRON_START}\n      NGXBLOCKER_ENABLE: ${NGXBLOCKER_ENABLE}\n    ports:\n      - \"80:80\"\n      - \"443:443\"\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/config_storage/nginxconfig/template:/etc/nginx/templates\n      - ${ARCHIPELAGO_ROOT}/drupal:/var/www/html:cached\n      - ${ARCHIPELAGO_ROOT}/data_storage/ngnixcache:/var/cache/nginx\n      - ${ARCHIPELAGO_ROOT}/data_storage/letsencrypt:/etc/letsencrypt\n      - ${ARCHIPELAGO_ROOT}/config_storage/nginxconfig/bots.d:/etc/nginx/bots.d\n      - ${ARCHIPELAGO_ROOT}/drupal/scripts/archipelago/setup_bot_blocker.sh:/scripts/setup_bot_blocker.sh\n</code></pre></li> <li>Then we modify our script copy (we'll reproduce the same behavior from the previous example but incorporate it into our script): drupal/scripts/archipelago/setup_bot_blocker.sh<pre><code>#!/bin/bash\n\nset -e\n\nif [ ! -z \"${MSMTP_EMAIL}\" ]; then\n    envsubst &lt; /root/.msmtprc.template &gt; /root/.msmtprc\nfi\n\nif [ \"${NGXBLOCKER_CRON_START}\" = true ]; then\n    if [ ! -z \"${MSMTP_EMAIL}\" ]; then\n      CRON_COMMAND=\"${NGXBLOCKER_CRON} ${NGXBLOCKER_CRON_COMMAND} -e ${MSMTP_EMAIL}\"\n    else\n      CRON_COMMAND=\"${NGXBLOCKER_CRON} ${NGXBLOCKER_CRON_COMMAND} -n\"\n    fi\n    echo \"${CRON_COMMAND}\" | crontab - &amp;&amp;\n    /etc/init.d/cron start\nfi\n\nif [ ! -f /etc/nginx/templates/bots.include.copy ]; then\n    touch /etc/nginx/templates/bots.include.copy\nfi\nif [ ! -f /etc/nginx/templates/bots.include.template ]; then\n    touch /etc/nginx/templates/bots.include.template\nfi\n\nif [ \"${NGXBLOCKER_ENABLE}\" = true ]; then\n    if [ ! -L /etc/nginx/conf.d/botblocker-nginx-settings.conf ]; then\n        ln -s /etc/nginx/bots_settings_conf.d/botblocker-nginx-settings.conf /etc/nginx/conf.d/botblocker-nginx-settings.conf\n    fi\n    if [ ! -L /etc/nginx/conf.d/globalblacklist.conf ]; then\n        ln -s /etc/nginx/bots_settings_conf.d/globalblacklist.conf /etc/nginx/conf.d/globalblacklist.conf\n    fi\n    if ! grep -q blockbots.conf /etc/nginx/templates/bots.include.copy; then\n        echo \"include /etc/nginx/bots.d/blockbots.conf;\" &gt;&gt; /etc/nginx/templates/bots.include.copy\n    fi\n    #if ! grep -q ddos.conf /etc/nginx/templates/bots.include.copy; then\n    #    echo \"include /etc/nginx/bots.d/ddos.conf;\" &gt;&gt; /etc/nginx/templates/bots.include.copy\n    #fi\n    if ! grep -q blockbots.conf /etc/nginx/user_conf.d/bots.include; then\n        echo \"include /etc/nginx/bots.d/blockbots.conf;\" &gt;&gt; /etc/nginx/user_conf.d/bots.include\n    fi\n    #if ! grep -q ddos.conf /etc/nginx/user_conf.d/bots.include; then\n    #    echo \"include /etc/nginx/bots.d/ddos.conf;\" &gt;&gt; /etc/nginx/user_conf.d/bots.include\n    #fi\n    cp /etc/nginx/templates/bots.include.copy /etc/nginx/templates/bots.include.template\nelse\n    &gt;|/etc/nginx/templates/bots.include.template\n    &gt;|/etc/nginx/user_conf.d/bots.include\n    if [ -L /etc/nginx/conf.d/botblocker-nginx-settings.conf ]; then\n        rm /etc/nginx/conf.d/botblocker-nginx-settings.conf\n    fi\n    if [ -L /etc/nginx/conf.d/globalblacklist.conf ]; then\n        rm /etc/nginx/conf.d/globalblacklist.conf\n    fi\nfi\n</code></pre></li> <li>Next we remove the <code>include</code> line from the existing files:    <pre><code>docker exec -ti esmero-web bash -c \"sed -i '/include \\/etc\\/nginx\\/bots.d\\/ddos.conf/d' /etc/nginx/templates/bots.include.copy\"\n</code></pre> <pre><code>docker exec -ti esmero-web bash -c \"sed -i '/include \\/etc\\/nginx\\/bots.d\\/ddos.conf/d' /etc/nginx/templates/bots.include.template\"\n</code></pre> <pre><code>docker exec -ti esmero-web bash -c \"sed -i '/include \\/etc\\/nginx\\/bots.d\\/ddos.conf/d' /etc/nginx/user_conf.d/bots.include\"\n</code></pre></li> <li> <p>Finally we bring the Docker ensemble down and back up again to propagate the changes in our container:    <pre><code>docker compose down &amp;&amp; docker compose up -d\n</code></pre></p> <p>Note</p> <p>If using an older version of docker, don't forget the hyphen: <pre><code>docker-compose down &amp;&amp; docker-compose up -d\n</code></pre></p> </li> </ol> <p>The above is an example of a more complicated customization, but it's a pattern that can be used more generally throughout the Docker containers, i.e.:</p> <ol> <li>Copy the file that needs to be overwridden from the Docker container to the host and make custom changes.</li> <li>Mount the file from the host to the location within the docker container, e.g.:    <pre><code>- ${ARCHIPELAGO_ROOT}/LOCATION_ON_HOST/CUSTOMIZED_FILE_ON_HOST:/LOCATION_IN_DOCKER_CONTAINER/FILE_IN_DOCKER_CONTAINER\n</code></pre></li> <li>Bring the Docker ensemble down and bring it up again.</li> </ol>","tags":["Security","Bots"]},{"location":"sslsetup/","title":"How to Setup SSL for Docker/Archipelago","text":"<p>Work-In-Progress Note This documentation page is still under construction and content may change with future updates. Please use caution when implementing any instructions referenced herein, as there may be missing steps or corresponding configuration files. Thank you for your patience as we continue to update Archipelago's documentation.</p> <p>The steps found below describe one potential manual SSL configuration for Archipelago deployments. A <code>git clone</code> deployment option will be available for future releases.</p>"},{"location":"sslsetup/#manual-configuration-steps-for-an-ec2-aws-server","title":"Manual Configuration Steps for an EC2 AWS Server","text":"<p>This process takes less than 10 minutes of reading YML files and editing a few files (described below) to get SSL running and setup with auto-renewal.</p> <ol> <li> <p>First, configure Certbot, following the instructions found on https://certbot.eff.org.</p> </li> <li> <p>Inside a /persistent partition, establish the following folder structure. Note: you can keep the existing folder structure if you so choose. A benefit of the following structure is that it decouples the git clone of archipelago-deployment, which is made to be self sustainable and good for coding or smaller deployments.</p> <pre><code>[ec2-user@ip-17x-xx-x-xxx persistent]$ ls -lah\ntotal 64K\ndrwxr-xr-x 14 root           root  4.0K Oct  5 23:11 .\ndr-xr-xr-x 19 root           root  275 Dec 15  2019 ..\ndrwxr-xr-x  8       999  999   4096 Oct 13 20:07 db\ndrwxr-xr-x 13 root           root  4.0K Oct  5 23:03 drupal8\ndrwxr-xr-x  5           8183  8183   4.0K Feb 23  2020 iiifcache\ndrwxr-xr-x  2 root           root  4.0K Feb 23  2020 iiifconfig\ndrwxr-xr-x  4 root           root  4.0K Oct  5 22:45 nginx_conf\ndrwxr-xr-x  3 root           root  4.0K Feb 26  2019 solrconfig\ndrwxr-xr-x  3           8983 8983  4.0K Feb 26  2019 solrcore\n</code></pre> <p>To get to this point, create a git clone of archipelago deployment and then copy the content of the /persistent out of the repo folder into this structure. The original (or what is left) archipelago-deployment ends inside a drupal8 folder here.</p> </li> <li> <p>Copy and paste the following to create a local copy of this file:</p> docker-compose.yml <p>**Be sure to replace youremail@gmail.com with your email address.</p> <pre><code> version: '3.5'\n services:\n   web:\n     container_name: esmero-web\n     image: staticfloat/nginx-certbot\n     restart: always\n     environment:\n       CERTBOT_EMAIL: \"youremail@gmail.com\"\n     ports:\n       - \"80:80\"\n       - \"443:443\"\n     volumes:\n       - /persistent/nginx_conf/conf.d:/etc/nginx/user.conf.d:ro\n       - /persistent/nginx_conf/certbot_extra_domains:/etc/nginx/certbot/extra_domains:ro\n       - /persistent/drupal8:/var/www/html:cached\n     depends_on:\n       - solr\n       - php\n     tty: true\n     networks:\n       - host-net\n       - esmero-net\n   php:\n     container_name: esmero-php\n     restart: always\n     image: \"esmero/php-7.3-fpm:latest\"\n     tty: true\n     networks:\n       - host-net\n       - esmero-net\n     volumes:\n       - ${PWD}:/var/www/html:cached\n   solr:\n     container_name: esmero-solr\n     restart: always\n     image: \"solr:7.5.0\"\n     tty: true\n     ports:\n       - \"8983:8983\"\n     networks:\n       - host-net\n       - esmero-net\n     volumes:\n       - /persistent/solrcore:/opt/solr/server/solr/mycores:cached\n       - /persistent/solrconfig:/drupalconfig:cached\n     entrypoint:\n       - docker-entrypoint.sh\n       - solr-precreate\n       - drupal\n       - /drupalconfig\n   # see https://hub.docker.com/_/mysql/\n   db:\n     image: mysql:5.7\n     command: --max_allowed_packet=256M\n     container_name: esmero-db\n     restart: always\n     environment:\n       MYSQL_ROOT_PASSWORD: esmerodb\n     networks:\n       - host-net\n       - esmero-net\n     volumes:\n       - /persistent/db:/var/lib/mysql:cached\n   iiif:\n     container_name: esmero-cantaloupe\n     image: \"esmero/cantaloupe-s3:4.1.6\"\n     restart: always\n     ports:\n       - \"8183:8182\"\n     networks:\n       - host-net\n       - esmero-net\n     volumes:\n       - /persistent/iiifconfig:/etc/cantaloupe\n       - /persistent/iiifcache:/var/cache/cantaloupe\n networks:\n   host-net:\n     driver: bridge\n   esmero-net:\n     driver: bridge\n     internal: true\n</code></pre> <p>Note: This file shows how the folders in Step 1 are being used, and how SSL is being automatically deployed and renewed (without any human interaction other than starting the docker-compose and watching the logs).</p> </li> <li> <p>Now copy and paste the following to create a local copy of this file:</p> ngnix.conf <p>**Be sure to replace all instances of yoursite.org with your own domain.</p> <pre><code> # goes into /persistent/nginx_conf/conf.d/nginx.conf\n upstream cantaloupe {\n  server  esmero-cantaloupe:8182;\n  }\n\n  server {\n    listen              443 ssl;\n    server_name         yoursite.org;\n    ssl_certificate     /etc/letsencrypt/live/yourstie.org/fullchain.pem;\n    ssl_certificate_key /etc/letsencrypt/live/yoursite.org/privkey.pem;\n\n     client_max_body_size 512M; ## Match with PHP from FPM container\n\n    root /var/www/html/web; ## &lt;-- Your only path reference.\n\n    fastcgi_send_timeout 120s;\n    fastcgi_read_timeout 120s;\n    fastcgi_pass_request_headers on;\n\n    fastcgi_buffers 16 16k;\n    fastcgi_buffer_size 32k;\n\n    # Cantaloupe proxypass\n    location /cantaloupe/ {\n       proxy_set_header X-Forwarded-Proto $scheme;\n       proxy_set_header X-Forwarded-Host $host;\n       proxy_set_header X-Forwarded-Port $server_port;\n       proxy_set_header X-Forwarded-Path /cantaloupe/;\n       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n       if ($request_uri ~* \"/cantaloupe/(.*)\") {\n         proxy_pass http://cantaloupe/$1;\n       }\n    }\n\n    location = /favicon.ico {\n        log_not_found off;\n        access_log off;\n    }\n\n    location = /robots.txt {\n        allow all;\n        log_not_found off;\n        access_log off;\n    }\n\n    # Very rarely should these ever be accessed outside of your lan\n    location ~* \\.(txt|log)$ {\n        deny all;\n    }\n\n    location ~ \\..*/.*\\.php$ {\n        return 403;\n    }\n\n    location ~ ^/sites/.*/private/ {\n        return 403;\n    }\n\n    # Allow \"Well-Known URIs\" as per RFC 5785\n    location ~* ^/.well-known/ {\n        allow all;\n    }\n\n    # Block access to \"hidden\" files and directories whose names begin with a\n    # period. This includes directories used by version control systems such\n    # as Subversion or Git to store control files.\n    location ~ (^|/)\\. {\n        return 403;\n    }\n\n    location / {\n        try_files $uri /index.php?$query_string; # For Drupal &gt;= 7\n    }\n\n    location @rewrite {\n        rewrite ^/(.*)$ /index.php?q=$1;\n    }\n\n    # Don't allow direct access to PHP files in the vendor directory.\n    location ~ /vendor/.*\\.php$ {\n        deny all;\n        return 404;\n    }\n\n    # Allow Modules to be updated via UI (still we believe composer is the way)    \n    rewrite ^/core/authorize.php/core/authorize.php(.*)$ /core/authorize.php$1;\n\n    # In Drupal 8, we must also match new paths where the '.php' appears in\n    # the middle, such as update.php/selection. The rule we use is strict,\n    # and only allows this pattern with the update.php front controller.\n    # This allows legacy path aliases in the form of\n    # blog/index.php/legacy-path to continue to route to Drupal nodes. If\n    # you do not have any paths like that, then you might prefer to use a\n    # laxer rule, such as:\n    #   location ~ \\.php(/|$) {\n    # The laxer rule will continue to work if Drupal uses this new URL\n    # pattern with front controllers other than update.php in a future\n    # release.\n    location ~ '\\.php$|^/update.php' {\n        fastcgi_split_path_info ^(.+?\\.php)(|/.*)$;\n        include fastcgi_params;\n        # Block httpoxy attacks. See https://httpoxy.org/.\n        fastcgi_param HTTP_PROXY \"\";\n        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;\n        fastcgi_param PATH_INFO $fastcgi_path_info;\n        fastcgi_param PHP_VALUE \"upload_max_filesize=512M \\n post_max_size=512M\";\n        proxy_read_timeout 900s;\n        fastcgi_intercept_errors on;\n        fastcgi_pass esmero-php:9000;\n    }\n\n     # Fighting with Styles? This little gem is amazing.\n    location ~ ^/sites/.*/files/styles/ { # For Drupal &gt;= 7\n        try_files $uri @rewrite;\n    }\n\n    # Handle private files through Drupal.\n    location ~ ^/system/files/ { # For Drupal &gt;= 7\n        try_files $uri /index.php?$query_string;\n    }\n}\n</code></pre> </li> <li> <p>Create the following folder:</p> <pre><code>/persistent/nginx_conf/conf.d/\n</code></pre> </li> <li> <p>Place the ngnix.conf file inside the <code>/conf.d/</code> folder.</p> </li> <li> <p>Create also this other folder:</p> <pre><code>/persistent/nginx_conf/certbot_extra_domains/\n</code></pre> </li> <li> <p>Inside the <code>/certbot_extra_domains/</code> folder, create a text file named the same way as your domain (which can/or not contain additional subdomains but needs to exist).</p> <pre><code>cat  /persistent/nginx_conf/certbot_extra_domains/yoursite.org\n</code></pre> <pre><code>drwxr-xr-x 2 root root 4.0K Oct  5 22:46 .\ndrwxr-xr-x 4 root root 4.0K Oct  5 22:45 ..\n-rw-r--r-- 1 root root   48 Oct  5 22:46 yoursite.org\n</code></pre> <p>Optionally, create additional subdomains if needed.</p> <pre><code>cat  /persistent/nginx_conf/certbot_extra_domains/yoursite.org\nsubdomain.yoursite.org\nanothersub.yoursite.org\n</code></pre> </li> <li> <p>Make sure you have edited the <code>docker-compose.yml</code> and <code>ngnix.conf</code> files you created to match your own information. Also make sure to also adjust the paths if you do not want the /persistent approach described in Step 1.</p> </li> <li> <p>Run the following commands:</p> <pre><code>docker -compose up -d\ndocker ps\n</code></pre> <p>You should see this:</p> <pre><code>b5a04747ee06        staticfloat/nginx-certbot    \"/bin/bash /scripts/\u2026\"   8 days ago          Up 8 days           0.0.0.0:80-&gt;80/tcp, 0.0.0.0:443-&gt;443/tcp   esmero-web\n84afae094b57        esmero/php-7.3-fpm:latest    \"docker-php-entrypoi\u2026\"   8 days ago          Up 8 days           9000/tcp                                   esmero-php\n13a9214acfd0        esmero/cantaloupe-s3:4.1.6   \"sh -c 'java -Dcanta\u2026\"   8 days ago          Up 8 days           0.0.0.0:8183-&gt;8182/tcp                     esmero-cantaloupe\n044dd5bc7245        mysql:5.7                    \"docker-entrypoint.s\u2026\"   8 days ago          Up 8 days           3306/tcp, 33060/tcp                        esmero-db\n31f4f0f45acc        solr:7.5.0                   \"docker-entrypoint.s\u2026\"   8 days ago          Up 8 days           0.0.0.0:8983-&gt;8983/tcp                     esmero-solr\n</code></pre> </li> <li> <p>SSL has now been configured for your Archipelago instance.</p> </li> </ol>"},{"location":"sslsetup/#user-contributed-documentation","title":"User contributed documentation:","text":"<p>Adding SSL to Archipelago running docker by Zachary Spalding: https://youtu.be/rfH5TLzIRIQ</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"strawberry_key_name_providers/","title":"Strawberry Key Name Providers, Solr Field, and Facet Configuration","text":"<p>For an overview of how Strawberry Key Name Providers fit within the context of the rest of Archipelago, please see the Drupal and JSON section in our Metadata in Archipelago overview documentation.</p> <p>In order to expose the Strawberry Field JSON keys (and values) for Archipelago Digital Objects (ADOs) to Search/Solr, Views, and Facets, we need to make use of a plugin system called Strawberry Key Name Providers. The following guide covers:</p> <ul> <li>Creating first a Strawberry Key Name Provider</li> <li>Creating a corresponding Solr Field necessary for Search and Views exposure</li> <li>Creating of Facets</li> <li>Creating of Facet blocks on your theme as needed </li> </ul>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#creating-a-strawberry-key-name-provider","title":"Creating a Strawberry Key Name Provider","text":"<ol> <li> <p>First, we'll start with an example of a Strawberry Field JSON key that we would like to expose:</p> <p>date_created_edtf</p> <pre><code>...\n\"subject_wikidata\": [\n    {\n        \"uri\": \"http:\\/\\/www.wikidata.org\\/entity\\/Q55488\",\n        \"label\": \"railway station\"\n    }\n],\n\"date_created_edtf\": {\n    \"date_to\": \"\",\n    \"date_free\": \"2016~\\/2017~\",\n    \"date_from\": \"\",\n    \"date_type\": \"date_free\"\n},\n\"date_created_free\": null,\n...\n</code></pre> </li> <li> <p>Next, we are going to create a new Strawberry Key Name Provider by going to <code>Administration &gt; Structure &gt; Strawberry Key Name Providers</code>, pressing the <code>+ Add Strawberry Key Name Provider</code> button, filling in the fields as follows, and saving:</p> <ul> <li><code>Label</code>: <code>Date Created EDTF</code></li> <li><code>Strawberry Key Name Provider Plugin</code>: <code>JmesPath Strawberry Field Key Name Provider</code></li> <li><code>One or more comma separated valid JMESPaths</code>: <code>date_created_edtf.date_free</code></li> <li>Confirm that the value for <code>Exposed Strawberry Field Property</code> (under the <code>One or more comma separated valid JMESPaths</code> field) is set to <code>date_created_edtf_date_free</code>. This is the <code>Strawberry Field Property</code> that will hold the data coming from the JMESPath Query when evaluated against and ADO's JSON and will be visible as a Strawberry Field Property to Drupal and the Search API. When doing this in a production environment, you might want to change the automatically generated value and assign a simpler one to remember. You can always do this by pressing <code>Edit</code>. But for the purpose of this documentation please keep <code>date_created_edtf_date_free</code>.</li> <li><code>Is Date?</code>: <code>\u2611</code></li> </ul> </li> </ol>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#strawberry-key-name-provider-plugins","title":"Strawberry Key Name Provider Plugins","text":"<p>You'll notice that there are four plugins, each with different options, available for different use cases. Below you'll find each plugin with examples from the providers that come with a default deployment.</p>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#entity-reference-jmespath-strawberry-field-key-name-provider","title":"Entity Reference JmesPath Strawberry Field Key Name Provider","text":"<p>ismemberof</p> <p>One or more comma separated valid JMESPaths: <code>ismemberof</code></p> <p>Entity type: <code>node</code></p>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#flavorembedded-json-service-strawberry-field-key-name-provider","title":"Flavor/Embedded JSON Service Strawberry Field Key Name Provider","text":"<p>hoCR Service</p> <p>Source JSON Key used to read the Service/Flavour: <code>ap:hocr</code></p>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#jmespath-strawberry-field-key-name-provider","title":"JmesPath Strawberry Field Key Name Provider","text":"<p>Subject Labels</p> <p>One or more comma separated valid JMESPaths: <code>subject_loc[*].label, subject_wikidata[*].label, subject_lcnaf_geographic_names[*].label,subject_temporal[*].label, subject_lcgft_terms[*].label, term_aat_getty[*].label, pubmed_mesh[*].label</code></p>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#jsonld-strawberry-field-key-name-provider","title":"JSONLD Strawberry Field Key Name Provider","text":"<p>Best Practice</p> <p>As in the example below, if there are a group of flat and unique keys that you want to expose, we recommend creating one provider with this plugin and using a list of keys instead of creating multiple providers. This Provider will also auto assign Lists of Properties from an external JSON-LD ontology/vocabulary (e.g Schema.org). It uses direct access approach, e.g. <code>type</code> will get all values for any JSON Key named <code>type</code> at any hierarchy level (across the whole JSON document) and it will also use the same exact name (<code>type</code>) for the <code>Exposed Strawberry Field Property</code>.</p> <p>schema.org</p> <p>Additional keys separated by commas: <code>ismemberof,type,hocr,city,category,country,state,display_name,author,license</code></p>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#creating-a-solr-field","title":"Creating a Solr Field","text":"<ol> <li>Go to <code>Administration &gt; Configuration &gt; Search and metadata &gt; Search API &gt; Drupal Content to Solr 9 &gt; Fields</code>.</li> <li>Press the <code>Add fields</code> button.</li> <li>Search for the field created above (expand the <code>\ud83c\udf53 Strawberry (Descriptive Metadata source) (field_descriptive_metadata)</code>, e.g. for the key mapped above, look for <code>field_descriptive_metadata:date_created_edtf_date_free</code>.</li> <li>Scroll down after adding to make sure the <code>Type</code> for the field is correct (<code>date</code> for the example in this guide).</li> <li>Reindex Solr.<ol> <li>Go to <code>Administration &gt; Configuration &gt; Search and metadata &gt; Search API</code> and click on the link to the index for your Drupal data.</li> <li>Press the <code>Queue all items for reindexing</code> button.</li> <li>Let cron reindex or press the <code>Index now</code> button.</li> </ol> </li> </ol> <p>Important Note About Facets and Facet Blocks</p> <p>You cannot reuse the same Facets or Facet Blocks for different Views display outputs. You need to create a unique Facet and corresponding Facet Block for every distinct <code>Facet Source</code> you will be using in your Archipelago. For example, if you have both <code>List</code> and <code>Grid</code> style Views setup for your Search display outputs, you will need to create Facets for your desired Subjects, etc. for both the <code>List</code> and <code>Grid</code> View Sources. You will then need to place your Facet Blocks in the appropriate region and make sure each Facet Block corresponds to the correct underlying <code>View</code> using the <code>View inclusion</code> settings for the Facet Block.</p>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#creating-a-facet","title":"Creating a Facet","text":"<ol> <li>Go to <code>Administration &gt; Configuration &gt; Search and metadata &gt; Facets</code>.</li> <li>Press the <code>+ Add facet</code> button.</li> <li>Select your facet settings. For the example in this guide, we'll select the following:<ul> <li><code>Facet source</code>: <code>View Solr search content, display Page</code></li> <li><code>Field</code>: <code>\ud83c\udf53 Strawberry (Descriptive Metadata source) &gt;&gt; date_created_edtf_date_free (field_descriptive_metadata:date_created_edtf_date_free)</code></li> <li><code>Name</code>: <code>\ud83c\udf53 Strawberry (Descriptive Metadata source) &gt;&gt; date_created_edtf_date_free</code></li> </ul> </li> <li>Save.</li> <li>Continue with the facet configuration by pressing <code>Edit</code> for the facet we just created and adjusting the many options available as needed. For the example in this guide, we'll adjust the below from the default settings:<ul> <li><code>Facet settings</code><ul> <li><code>\u2611</code> <code>Date item processor</code><ul> <li><code>Date display</code><ul> <li><code>\ud83d\udd18</code> <code>Actual date with granularity</code></li> </ul> </li> <li><code>Granularity</code><ul> <li><code>\ud83d\udd18</code> <code>Year</code></li> </ul> </li> </ul> </li> <li><code>URL alias</code>: <code>sbf_date_created_edtf</code></li> </ul> </li> </ul> </li> <li>Save.</li> </ol>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#creating-a-block-for-the-facet","title":"Creating a Block for the Facet","text":"<ol> <li>Go to <code>Administration &gt; Structure &gt; Block layout</code>.</li> <li>Select the appropriate theme. For the example in this guide, we'll select <code>Archipelago Base Theme</code>.</li> <li>Press the <code>Place block</code> button next to the appropriate region. For the example in this guide, we'll be placing the block in the <code>Sidebar second</code> region.</li> <li>Select your facet from the list. For the example in this guide, we'll select <code>\ud83c\udf53 Strawberry (Descriptive Metadata source) &gt;&gt; date_created_edtf_date_free</code></li> <li>Press the <code>Place block</code> button next to the facet. Once the block is added, you can drag and drop it to change its position among the existing blocks and saving.</li> </ol> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberryfield-formatters/","title":"Strawberryfield Formatters","text":"<p>This documentation will give a brief overview of Archipelago's Strawberryfield Formatters and how they work using the default View mode <code>Digital Object Full View</code> as an example.</p>"},{"location":"strawberryfield-formatters/#at-a-glance","title":"At a glance","text":"<p>When taking a look at your First Digital Object note that multiple formatters are working together to create this <code>Display</code> ( or <code>View mode</code>). Since \"My First Digital Object\" is a <code>Photograph</code> the <code>Display</code> being used is <code>Digital Object Full View</code> which, by default, uses formatters to:</p> <ul> <li>(Red) Create the image viewer where users can zoom in, zoom out, fullscreen and rotate all the images associated with the ADO.</li> <li>(Blue) Display the <code>Object Description</code> and <code>Type of Resource</code>.</li> <li>(Green) Display the Raw JSON Metadata and IIIF Presentation Manifest.</li> </ul> <p></p>"},{"location":"strawberryfield-formatters/#in-greater-detail","title":"In Greater Detail","text":"<p>When editing an ADO, at the top of the Webform page there is a tab titled <code>Manage display</code> which will take us to where all the Formatters live. Take note that the <code>DISPLAY SETTINGS</code> shown in the screenshot below are using the Default View mode.</p> <p></p> <p>Once the page loads the <code>Default</code> View mode is automatically selected. However, because we are editing an object with the <code>Media type</code> <code>Photograph</code>, we need to edit the View mode <code>Digital Object Full View</code> since it is the Default View mode for this <code>Media type</code>.</p>"},{"location":"strawberryfield-formatters/#how-to-find-and-configure-which-view-mode-is-default-per-media-type","title":"How to find and configure which View mode is Default per Media type","text":"<p>The ADO Type to View Mode Mapping page tells the ADOs which View mode to use by default per Media type. You can find the ADO Type to View Mode Mapping Form at <code>~yoursite/admin/config/archipelago/viewmode_mapping</code></p> <p></p> Strawberryfield Formatters Shipped and Configured with Archipelago <ol> <li>Default</li> <li>Collection listing</li> <li>Digital Object Full View</li> <li>Digital Object Image Only for Carousel</li> <li>Digital Object with 3D Viewer</li> <li>Digital Object with Audio Player</li> <li>Digital Object with Book Reader</li> <li>Digital Object with Mirador Viewer</li> <li>Digital Object with PDF Viewer</li> <li>Digital Object with Pannellum Panorama</li> <li>Digital Object with Video Player</li> <li>Digital Object with Replay.web WARC Replay.web Widget</li> <li>Digital Object with thumbnail and abstract</li> <li>Digital Object with thumbnail for Grid For Digital Object Collections/Compound Objects/CreativeWorkSeries</li> <li>Digital Object Collection with Mirador Viewer</li> <li>Digital Object Creative Work Series with Mirador Viewer</li> </ol> Default View Modes <p>You can see the full list of Default View Modes included in Archipelago here, and you can access your Archipelago's ADO Type to View Mode Mapping at <code>~yoursite/admin/config/archipelago/viewmode_mapping</code>.</p> <p></p> <p>There are two sections in <code>Manage display</code> for <code>Digital Object Full View</code>: 1) Content and 2) Disabled. Moving a field into Content means this formatter will be used to the display the ADO in some way. The formatters moved to Disabled are inactive and are subsequently not being used for displaying the ADO.</p> <p>There are four fields named <code>\ud83c\udf53Strawberry</code> and each one is a copy of the field <code>\ud83c\udf53Strawberry (Descriptive Metadata source)</code>. Since the names of the fields do not imply their function, they have been named Strawberry in four different ways (Italiano, Deutsch, Din\u00e9 Bizaad, and English) in order to organize and help users visually remember which field is doing what for the <code>Display</code>.</p> <p>Recall My First Digital Object at beginning of this document where there were 3 sections highlighted in Red, Blue, and Green.</p> <ul> <li>In Red (<code>\ud83c\udf53Fragola</code>) there is the Strawberry Field Formatter for IIIF media which takes the image stored in S3 to display the photograph with the image viewer.</li> <li>In Blue (<code>\ud83c\udf53Erdbeere</code>) there is the Strawberry Field Formatter for Custom Metadata Templates which displays the raw JSON metadata using configurable Twig templates. In this example, the default Twig template uses the JSON key <code>type</code> to display the <code>Type of Resource</code>.</li> <li>In Green (<code>\ud83c\udf53Strawberry (Descriptive Metadata)</code>) there is the Strawberry Default Formatter which is used to display the Raw JSON Metadata.</li> </ul> <p></p>"},{"location":"strawberryfield-formatters/#at-the-end-of-the-day","title":"At the end of the day","text":"<p>The decision for how your metadata is displayed is totally in your control.</p> <p>Under the <code>WIDGET</code> column, there is a quick description/overview of what the formatter is doing.</p> <p></p> <p>And by clicking on the gear icon under the <code>OPERATIONS</code> column, all of the options for configuring the formatter are revealed. To use <code>\ud83c\udf53Fragola</code> as an example (the Formatter for IIIF media), we can choose which JSON Key is being used to fetch the IIIF Media URLs (found inside the raw JSON being played with <code>Strawberry Default Formatter</code>), the maximum height and width of the viewer, etc.</p> <p></p> <p>And then with <code>\ud83c\udf53Erdbeere</code> (the Formatter for Custom Metadata Templates) there is the option, among many others, to configure which Twig template the formatter will use for displaying your Metadata.</p> <p></p> <p>More information about Managing Metadata Displays with Twig Templates can be found here.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"strawberryfields/","title":"Strawberryfields Forever","text":""},{"location":"strawberryfields/#what-strawberry-fields-does-why-we-built-it-and-what-issues-it-addresses","title":"What Strawberry fields does, why we built it, and what issues it addresses","text":"<p>Archipelago integrates transparently into the Drupal 8 ecosystem using its Core Content Entity System (Nodes), Discovery (Search API) and in general all its Core Components plus a few well maintained external ones.</p> <p>By design (and because we think its imperative), Archipelago takes full charge of the metadata layer and associated media assets by implementing a highly configurable, smart Drupal field written in JSON named <code>Strawberryfield</code> that attaches to any content.</p> <p>All of JSON's internals, keys, paths, and values are dynamically exposed to the rest of the ecosystem. Strawberryfield even remembers its structure as data evolves by storing JSON paths of every little detail.</p>"},{"location":"strawberryfields/#nothing-is-real","title":"Nothing Is Real","text":"<p>Archipelago includes additional companion modules, <code>Webform_strawberryfield</code> and <code>Format_strawberryfield</code> that extend the core metadata capabilities of the main <code>Strawberryfield</code> module and allow the same flexibility to be exposed during ingest and viewing of digital objects.</p> <p>The in-development <code>Strawberry Runners</code> and <code>AMI</code> modules further extend Archipelago's capabilities. Additional information related to these modules will be made available following initial public releases.</p>"},{"location":"strawberryfields/#ingesting","title":"Ingesting","text":"<p><code>Webform Strawberryfield</code> (we had a better name) extends and integrates into the <code>amazing Drupal Webform module</code> to allow Archipelago users to build any possible metadata and media, ingest and edit, workflows directly via the UI using webforms.</p> <p>By not having a hardcoded ingest method, Archipelago can be used outside the GLAM community too, as a pure data repository in biological sciences, digital humanities, archives, or even as a mixed, multidisciplinary/cross-domain system.</p> <p>We also added <code>WIKIDATA</code>, <code>LoC</code>, <code>Getty</code>, and <code>VIAF</code> authority querying elements to aid in linking to external Linked Open Data sources.</p> <p>All these integrations are made to help local needs and community identities to survive the never-ending race for the next metadata schema. They are made to prototype, plan, and grow independently of how metadata will need to be exposed yesterday or tomorrow. And we plan to add more.</p> <p>Explore what other features <code>webform_strawberryfield</code> provides to help with ingesting, reading, and interacting with your metadata during that process.</p>"},{"location":"strawberryfields/#exposing","title":"Exposing","text":"<p><code>Format Strawberryfield</code> (we had even a better name but...) deals with taking your JSON based metadata and <code>casting</code>, mashing, mixing, exposing, displaying, and transforming it to allow rich interaction for users and other systems with your digital objects.</p> <p>In its guts (or heart?), Archipelago does something quite simple but core to our concept of repository: it transforms in realtime the close to your needs open schema metadata that lives in strawberryfield as JSON into close to other one's fixed schema needs metadata; any destination format, using a fast, cached templating system. A templating system that is core to Drupal, called <code>Twig</code>:</p> <ul> <li>Twig in Symfony</li> <li>Twig in Drupal</li> </ul> <p>This templating system is exposed to Archipelago users through the UI and stored side by side in the repository as content (we named them <code>Metadata Display entities</code>, but they not only serve display needs!) so users can fully control how metadata is transformed and published without touching their individual sources.</p> <p>Templates or recipes can be shared, exported, ingested, updated, and adapted in many ways. Fast changes are possible without having to wait for the next mayor release of Archipelago or your favorited Metadata Schema Specs Committee agreeing on the next or the last version. Of course, this module not only handles metadata but media assets too, extracting local or remote URIs and files from your metadata and rendering them as media viewers: books, 3D models, images, panoramas, A/V with IIIF in its soul.</p> <p>You can learn more about what format_strawberryfield can do and what many other possibilities are exposed through our templating system.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"strawberryrunners/","title":"Strawberry Runners Post-Processing Configuration","text":"<p>Archipelago's Strawberry Runners (SBR) module provides provides a set of post-processing capabilities for the JSON based metadata, files and entities that comprise your Archipelago Digital Objects (ADOs). These post-processing actions are based on dispatched events, direct http calls, and invoked webhooks from partner services (such as Min.io, AWS S3 or self-invoked). </p> <p>The default Archipelago SBR post-processor configurations include operations that: - perform page-based HOCR/OCR for image and pdf-based ADOs, send the output to the Search API, and use Natural Language Processing to extract entities from the output - extract text from pages within a Webarchives File and send the output to the Search API - convert WARC format Webarchives Files into WACZ format and attach the new WACZ file to the original source ADO to complement the WARC original - extract textual values from subtitle/transcript VTT files and generates time/space transmuted OCR</p> <p>SBR actions can be chained and nested to enable ordered operations, such as first extract individual pages in an ordered sequence and then run HOCR/OCR across the individual pages.</p>","tags":["Strawberry Runners","HOCR","OCR","Post-processing","Background Processing"]},{"location":"strawberryrunners/#strawberry-runners-settings-overview","title":"Strawberry Runners Settings Overview","text":"<p>You can access the Strawberry Runners Settings:</p> <ul> <li>Through the <code>Manage</code> menu &gt; <code>Configuration</code> &gt; <code>Archipelago</code> &gt; <code>Configure Strawberry Runners Post Processors</code></li> <li>Directly at <code>/admin/config/archipelago/strawberry_runners</code> </li> </ul> <p>On the Strawberry Runners Settings page, you will see the Archipelago default post processor configurations (unless modified).</p> <p></p> <ol> <li>The <code>pager</code> action uses the 'Post processor that extracts/generates Ordered Sequences of files/pages/children using Files present in an ADO' plugin.</li> <li>Nested one level in, the <code>ocr</code> action uses the 'Post processor that Runs OCR/HORC against files' plugin. The <code>ocr</code> operations will be executed after the completion of the <code>pager</code> operations.</li> <li>The <code>wacz_page_extractor</code> action uses the 'Post processor that extracts/generates Indexed Page Content from WACZ files in an ADO' plugin.</li> <li>Nested one level in, the <code>webpage</code> action uses the 'Post processor that Indexes WACZ Frictionless data Search Index to Search API' plugin. The <code>webpage</code> operations will be executed after the completion of the <code>wacz_page_extractor</code> operations.</li> <li>The <code>warc_to_wacz</code> action uses the 'Post processor that uses a System Binary to process * files' operations.</li> <li>The <code>subtitle</code> action extracts textual values from subtitle/transcript VTT files and generates time/space transmuted OCR. This transmuted OCR can be used to search within a time-based video or audio file's corresponding subtitle/transcript VTT file(s), then navigate to the matching time of the video or audio file within a media viewer.</li> </ol>","tags":["Strawberry Runners","HOCR","OCR","Post-processing","Background Processing"]},{"location":"strawberryrunners/#reviewing-and-adjusting-the-default-post-processors","title":"Reviewing and Adjusting the default Post-Processors","text":"<p>From the main Strawberry Runner Settings page, you can review and adjust the settings for the default Archipelago configurations by selecting <code>Edit</code> from the `Operations`` menu. </p> <p>Please see the following guides for:</p> <ul> <li>Adjusting the <code>pager</code> and <code>ocr</code> operations</li> <li>Adjusting the <code>wacz_page_extractor</code> and <code>webpage</code> operations</li> <li>Adjusting the <code>warc_to_wacz</code> operation</li> <li>Adjusting the <code>subtitle</code> operation</li> </ul>","tags":["Strawberry Runners","HOCR","OCR","Post-processing","Background Processing"]},{"location":"strawberryrunners/#triggering-post-processing-actions-manually","title":"Triggering Post-Processing Actions Manually","text":"<p>After making adjustments to Strawberry Runners Post-Processing configurations, you may want to trigger/re-trigger a particular action manually.</p> <p>You can use Archipelago's Find and Replace to first select a specific group of Digital Objects you wish to target for Post-Processing, then select the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code> from the Find and Replace <code>Actions menu</code>.</p>","tags":["Strawberry Runners","HOCR","OCR","Post-processing","Background Processing"]},{"location":"strawberryrunners/#additional-post-processor-operations","title":"Additional Post Processor Operations","text":"<p>Archipelago also includes the <code>Post processor that writes/reads Frictionless Data Packages</code> plugin. Please keep a lookout for future documentation related to using this plugin.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Strawberry Runners","HOCR","OCR","Post-processing","Background Processing"]},{"location":"strawberryrunners_pager_ocr/","title":"Reviewing and adjusting the <code>pager</code> and <code>ocr</code> Post-Processor operations","text":"<p>The <code>pager</code> and <code>ocr</code> Post-processor operations are likely the most important pair of Strawberry Runners in your Archipelago.</p> <p>As stated on the Strawberry Runners overview page, the <code>pager</code> action uses the 'Post processor that extracts/generates Ordered Sequences of files/pages/children using Files present in an ADO' plugin. Nested one level in, the <code>ocr</code> action uses the 'Post processor that Runs OCR/HORC against files' plugin. The <code>ocr</code> operations will be executed after the completion of the <code>pager</code> operations.</p> <p>Common changes you may wish to make for the <code>pager</code> and/or <code>ocr</code> operations include adding or removing particular types of Archipelago Digital Objects to apply these operations to.</p>","tags":["Strawberry Runners","HOCR","OCR","Pager","Post-processing","Background Processing"]},{"location":"strawberryrunners_pager_ocr/#pager-settings","title":"Pager Settings","text":"<p>To review or adjust the configurations for the <code>pager</code> operation, select <code>Edit</code> from the <code>Operations</code> menu.</p> <p>In the <code>pager</code> settings, you will see the following configuration options:</p> <p></p> <ol> <li> <p>Label: </p> <ul> <li>Label for this Processor; which should be a unique machine-readable name</li> <li>Can only contain lowercase letters, numbers, and underscores</li> <li>We do not recommend changing this Label from the default <code>pager</code>. </li> </ul> </li> <li> <p>Strawberry Runner Post Processor Plugin:</p> <ul> <li>The <code>Post processor that extracts/generates Ordered Sequences of files/pages/children using Files present in an ADO</code> should be selected.</li> <li>We do not recommend changing this Plugin selection.</li> </ul> </li> <li> <p>Checkbox to mark this processor plugin as active </p> <ul> <li>We recommend keeping this checked as <code>active</code> at all times, but you may wish to temporarily disable this if you are performing certain types of administrative review tasks such as running large test ingests where you plan on deleting the ADOs before a final ingest.</li> <li>If you accidentally uncheck this and need to re-trigger the <code>pager</code> (and corresponding nested <code>ocr</code> action), you can use Archipelago's Find and Replace to first select a specific group of Digital Objects you wish to target for Post-Processing, then select the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code> from the Find and Replace <code>Actions menu</code>.</li> </ul> </li> <li> <p>ADO type(s) to limit this processor to:</p> <ul> <li>A single ADO type or a comma delimited list of ado types that qualify to be Processed.</li> <li>Leave empty to apply to all ADOs. If you do not provide any specific ADO types here, the processor will be applied for all ADOs with the JSON keys selected in the next step.</li> <li>Default ADO types specified are: 'Document,Book,Article'  </li> <li>You may wish to add additional types of document/multiple-paged type of ADOs to this list that are custom to you Archipelago environment. </li> </ul> </li> <li> <p>The JSON key that contains the desired source files:</p> <ul> <li>By default, the <code>as:image</code> and <code>as:document</code> keys are selected.</li> <li>We do not recommend changing this selection.</li> </ul> </li> <li> <p>Mimetypes(s) to limit this Processor to:</p> <ul> <li>A single Mimetype type or a comma separated list of mimetypes that qualify to be Processed.</li> <li>Leave empty to apply any file.</li> <li>Default mimetypes are: 'application/pdf,image/tiff,image/jpeg,image/jp2'  </li> </ul> </li> <li> <p>Within the ADO's metadata, the JSON key that contains the language in ISO639-3 (3 letter) format to be used for OCR/NLP processing via Tesseract.</p> <ul> <li>Default JSON key specified is: 'language_iso639_3'</li> </ul> </li> <li> <p>Please provide a default language in ISO639-3 (3 letter) format. If none is provided we will use 'eng'.</p> <ul> <li>Default language specified is: 'eng'</li> </ul> </li> <li> <p>Timeout in seconds for this process.</p> <ul> <li>If the process runs out of time it can still be processed again</li> <li>Default selection is: 10</li> </ul> </li> <li> <p>Order or execution in the global chain.</p> <ul> <li>Default selection is: 0</li> </ul> </li> </ol>","tags":["Strawberry Runners","HOCR","OCR","Pager","Post-processing","Background Processing"]},{"location":"strawberryrunners_pager_ocr/#ocr-hocr-settings","title":"OCR / HOCR Settings","text":"<p>To review or adjust the configurations for the <code>ocr</code> operation, select <code>Edit</code> from the <code>Operations</code> menu.</p> <p>In the <code>pager</code> settings, you will see several different configuration options.</p> <p></p> <ol> <li> <p>Label: </p> <ul> <li>Label for this Processor; which should be a unique machine-readable name</li> <li>Can only contain lowercase letters, numbers, and underscores</li> <li>We do not recommend changing this Label from the default <code>ocr</code>. </li> </ul> </li> <li> <p>Strawberry Runner Post Processor Plugin:</p> <ul> <li>The <code>Post processor that Runs OCR/HORC against files</code> should be selected.</li> <li>We do not recommend changing this Plugin selection.</li> </ul> </li> <li> <p>Checkbox to mark this processor plugin as active </p> <ul> <li>We recommend keeping this checked as <code>active</code> at all times, but you may wish to temporarily disable this if you are performing certain types of administrative review tasks such as running large test ingests where you plan on deleting the ADOs before a final ingest.</li> <li>If you accidentally uncheck this and need to re-trigger the <code>pager</code> (and corresponding nested <code>ocr</code> action), you can use Archipelago's Find and Replace to first select a specific group of Digital Objects you wish to target for Post-Processing, then select the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code> from the Find and Replace <code>Actions menu</code>.</li> </ul> </li> <li> <p>The type of source data this processor works on:</p> <ul> <li>Select from where the source file this processor needs is fetched.</li> <li>Default selection of 'File entities referenced in the as:filetype JSON structure'.</li> <li>You also have the option of selecting 'Full file paths passed by another processor', but we do not recommend using this option as it is less granular in its application.      </li> </ul> </li> <li> <p>ADO type(s) to limit this processor to:</p> <ul> <li>A single ADO type or a comma delimited list of ado types that qualify to be Processed</li> <li>Leave empty to apply to all ADOs. If you do not provide any specific ADO types here, the processor will be applied for all ADOs with the JSON keys selected in the next step.</li> <li>Default ADO types specified are: 'Document,Book,Article'  </li> <li>You may wish to add additional types of document/multiple-paged type of ADOs to this list that are custom to you Archipelago environment. </li> </ul> </li> <li> <p>The JSON key that contains the desired source files:</p> <ul> <li>By default, the <code>as:image</code> and <code>as:document</code> keys are selected.</li> <li>We do not recommend changing this selection.</li> </ul> </li> <li> <p>Mimetypes(s) to limit this Processor to:</p> <ul> <li>A single Mimetype type or a comma separated list of mimetypes that qualify to be Processed.</li> <li>Leave empty to apply any file.</li> <li>Default mimetypes are: 'application/pdf,image/tiff,image/jpeg,image/jp2'  </li> </ul> </li> </ol> <p>Advanced OCR/HOCR Settings</p> <p>We do not recommend making changes to the follow settings unless you are the System Administrator.</p> <ol> <li> <p>The system path to the ghostscript (gs) binary that will be executed by this processor.</p> <ul> <li>A full system path to the gs binary present in the same environment your PHP runs</li> <li>Default path specified is: '/usr/bin/gs'</li> </ul> </li> <li> <p>Any additional argument your executable binary requires.</p> <ul> <li>Any arguments your ghostscript (gs) binary requires to run. Use %file as replacement for the file if the executable requires the filename to be passed under a specific argument. We recommend testing with -r150 (150dpi image extraction) for better performance but -r300 can be also used if source Images in a PDF are small</li> <li>Default argument specified is: -r150 %file</li> </ul> </li> <li> <p>The system path to the Tesseract binary that will be executed by this processor.</p> <ul> <li>A full system path to the Tesseract binary present in the same environment your PHP runs</li> <li>Default path specified is: '/usr/bin/tesseract'</li> </ul> </li> <li> <p>Within the ADO's metadata, the JSON key that contains the language in ISO639-3 (3 letter) format to be used for OCR/NLP processing via Tesseract.</p> <ul> <li>Default JSON key specified is: 'language_iso639_3'</li> </ul> </li> </ol> <p></p> <ol> <li> <p>Please provide a default language in ISO639-3 (3 letter) format. If none is provided we will use 'eng' </p> <ul> <li>Default language specified is: 'eng'</li> </ul> </li> <li> <p>Any additional argument for your tesseract binary.</p> <ul> <li>Any arguments your binary requires to run. Use %file as replacement for the file that is output by the GS binary. Use %language as replacement for the chosen language.</li> <li>Default arguments specified are: '%file stdout -l %language hocr'</li> </ul> </li> <li> <p>The data/languages folder for Tesseract</p> <ul> <li>Absolute path where the Languages are stored in the Server. This will be used in --tessdata-dir</li> <li>Default path specified is: '/usr/share/tessdata'  </li> </ul> </li> <li> <p>The system path to the pdfalto binary that will be executed by this processor.</p> <ul> <li>A full system path to the pdfalto binary present in the same environment your PHP runs, e.g /usr/local/bin/pdfalto</li> <li>Default path specified is: '/usr/bin/pdfalto'</li> </ul> </li> <li> <p>Any additional argument for your pdfalto binary.</p> <ul> <li>Any arguments your binary requires to run. Use %file as replacement for the file that is output by the pdfalto binary.</li> <li>Default arguments specified are: '%file -q</li> </ul> </li> <li> <p>The expected and desired output of this processor.</p> <ul> <li>If the output is just data and \"One or more Files\" is selected all data will be dumped into a file and handled as such.</li> <li>Default selection is: 'Data/Values that can be serialized to JSON'</li> <li>Additional optional is to select 'One or more Files', but it is not recommended unless to use this for the default <code>ocr</code> operation since this will alter how the data is incorporated in the Search API (Solr index).</li> </ul> </li> <li> <p>Where and how the output will be used.</p> <ul> <li>Default select is: 'In a Search API Document using the Strawberryfield Flavor Data Source (e.g used for HOCR highlight)'  </li> <li>Additional option to select 'As Input for another processor Plugin' --which will only have an effect if another Processor is setup to consume this output.</li> </ul> </li> <li> <p>The queue to use for this processor.</p> <ul> <li>The primary queue will be execute in realtime while the Secondary will be execute in background</li> <li>Default selection is for the 'Secondary queue in background'</li> </ul> </li> <li> <p>Checkbox to Use NLP (Natural Language Processing) to extract entities from Text</p> <ul> <li>If checked Full text will be processed for Natural language Entity extraction using Polyglot.</li> <li>Default option is to have the option checked. </li> </ul> </li> <li> <p>The URL location of your NLP64 server.</p> <ul> <li>Defaults to http://esmero-nlp:6400</li> </ul> </li> <li> <p>Which method(NER) to use</p> <ul> <li>The NER NLP method to use to extract Agents, Places and Sentiment.</li> <li>Default selection: 'Polyglot (faster)'</li> <li>Alternation selection: 'spaCy (more accurate)'</li> </ul> </li> <li> <p>Timeout in seconds for this process.</p> <ul> <li>900</li> <li>If the process runs out of time it can still be processed again.</li> </ul> </li> <li> <p>Order or execution in the global chain.</p> <ul> <li>0</li> </ul> </li> </ol> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the main Strawberry Runners or the Archipelago Documentation main page.</p>","tags":["Strawberry Runners","HOCR","OCR","Pager","Post-processing","Background Processing"]},{"location":"strawberryrunners_subtitle/","title":"Reviewing and adjusting the <code>subtitle</code> Post-Processor operations","text":"<p>As stated on the Strawberry Runners overview page, the <code>subtitle</code> action extracts textual values from subtitle/transcript VTT files and generates time/space transmuted OCR. This transmuted OCR can be used to search within a time-based video or audio file's corresponding subtitle/transcript VTT file(s), then navigate to the matching time of the video or audio file within a media viewer.</p> <p>We strongly recommend using caution when making any adjustments to the default <code>subtitle</code> configurations as this may result in unexpected issues with the transmuted OCR values in your Solr Index. Also, the <code>subtitle</code> Strawberry Runner Post-Processor needs to used with corresponding related default IIIF Configuration Form Settings. </p>","tags":["Strawberry Runners","Subtitles","VTT","HOCR","OCR","Post-processing","Background Processing"]},{"location":"strawberryrunners_subtitle/#subtitle-settings","title":"Subtitle Settings","text":"<p>To review or adjust the configurations for the <code>subtitle</code> operation, select <code>Edit</code> from the <code>Operations</code> menu.</p> <p>In the <code>subtitle</code> settings, you will see the following configuration options:</p> <p></p> <ol> <li> <p>Label: </p> <ul> <li>Label for this Processor; which should be a unique machine-readable name</li> <li>Can only contain lowercase letters, numbers, and underscores</li> <li>We do not recommend changing this Label from the default <code>pager</code>. </li> </ul> </li> <li> <p>Strawberry Runner Post Processor Plugin:</p> <ul> <li>The <code>Post processor that extracts/generates Ordered Sequences of files/pages/children using Files present in an ADO</code> should be selected.</li> <li>We do not recommend changing this Plugin selection.</li> </ul> </li> <li> <p>Checkbox to mark this processor plugin as active </p> <ul> <li>We recommend keeping this checked as <code>active</code> at all times, but you may wish to temporarily disable this if you are performing certain types of administrative review tasks such as running large test ingests where you plan on deleting the ADOs before a final ingest.</li> <li>If you accidentally uncheck this and need to re-trigger the <code>pager</code> (and corresponding nested <code>ocr</code> action), you can use Archipelago's Find and Replace to first select a specific group of Digital Objects you wish to target for Post-Processing, then select the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code> from the Find and Replace <code>Actions menu</code>.</li> </ul> </li> <li> <p>ADO type(s) to limit this processor to:</p> <ul> <li>A single ADO type or a comma delimited list of ado types that qualify to be Processed.</li> <li>Leave empty to apply to all ADOs. If you do not provide any specific ADO types here, the processor will be applied for all ADOs with the JSON keys selected in the next step.</li> <li>Default ADO types specified are: 'Document,Book,Article'  </li> <li>You may wish to add additional types of document/multiple-paged type of ADOs to this list that are custom to you Archipelago environment. </li> </ul> </li> <li> <p>The JSON key that contains the desired source files:</p> <ul> <li>By default, the <code>as:image</code> and <code>as:document</code> keys are selected.</li> <li>We do not recommend changing this selection.</li> </ul> </li> <li> <p>Mimetypes(s) to limit this Processor to:</p> <ul> <li>A single Mimetype type or a comma separated list of mimetypes that qualify to be Processed.</li> <li>Leave empty to apply any file.</li> <li>Default mimetypes are: 'application/pdf,image/tiff,image/jpeg,image/jp2'  </li> </ul> </li> <li> <p>Within the ADO's metadata, the JSON key that contains the language in ISO639-3 (3 letter) format to be used for OCR/NLP processing via Tesseract.</p> <ul> <li>Default JSON key specified is: 'language_iso639_3'</li> </ul> </li> <li> <p>Please provide a default language in ISO639-3 (3 letter) format. If none is provided we will use 'eng'.</p> <ul> <li>Default language specified is: 'eng'</li> </ul> </li> <li> <p>Timeout in seconds for this process.</p> <ul> <li>If the process runs out of time it can still be processed again</li> <li>Default selection is: 10</li> </ul> </li> <li> <p>Order or execution in the global chain.</p> <ul> <li>Default selection is: 0</li> </ul> </li> </ol>","tags":["Strawberry Runners","Subtitles","VTT","HOCR","OCR","Post-processing","Background Processing"]},{"location":"strawberryrunners_subtitle/#related-iiif-configuration-form-and-iiif-content-search-guides","title":"Related IIIF Configuration Form and IIIF Content Search Guides","text":"<ul> <li>IIIF Configuration Form</li> <li>IIIF Content Search API Overview</li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the main Strawberry Runners or the Archipelago Documentation main page.</p>","tags":["Strawberry Runners","Subtitles","VTT","HOCR","OCR","Post-processing","Background Processing"]},{"location":"strawberryrunners_wacz_binary/","title":"Reviewing and adjusting the <code>warc_to_wacz</code> Post-Processor operation","text":"<p>As stated on the Strawberry Runners overview page, the <code>warc_to_wacz</code> action uses the 'Post processor that uses a System Binary to process * files' operations.</p>","tags":["Strawberry Runners","WACZ","WARC","Binary","Post-processing","Background Processing"]},{"location":"strawberryrunners_wacz_binary/#wacz-page-extractor-settings","title":"Wacz Page Extractor Settings","text":"<p>To review or adjust the configurations for the <code>wacz_page_extractor</code> operation, select <code>Edit</code> from the <code>Operations</code> menu.</p> <p>In the <code>wacz_page_extractor</code> settings, you will see the following configuration options:</p> <ol> <li> <p>Label: </p> <ul> <li>Label for this Processor; which should be a unique machine-readable name</li> <li>Can only contain lowercase letters, numbers, and underscores</li> <li>We do not recommend changing this Label from the default <code>pager</code>. </li> </ul> </li> <li> <p>Strawberry Runner Post Processor Plugin:</p> <ul> <li>The <code>Post processor that extracts/generates Indexed Page Content from WACZ in an ADO</code> should be selected.</li> <li>We do not recommend changing this Plugin selection.</li> </ul> </li> <li> <p>Checkbox to mark this processor plugin as active </p> <ul> <li>We recommend keeping this checked as <code>active</code> at all times, but you may wish to temporarily disable this if you are performing certain types of administrative review tasks such as running large test ingests where you plan on deleting the ADOs before a final ingest.</li> <li>If you accidentally uncheck this and need to re-trigger the <code>pager</code> (and corresponding nested <code>ocr</code> action), you can use Archipelago's Find and Replace to first select a specific group of Digital Objects you wish to target for Post-Processing, then select the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code> from the Find and Replace <code>Actions menu</code>.</li> </ul> </li> <li> <p>The JSON key that contains the desired source files:</p> <ul> <li>By default, only the <code>as:document</code> key is selected.</li> <li>We do not recommend changing this selection.</li> </ul> </li> <li> <p>Mimetypes(s) to limit this Processor to:</p> <ul> <li>A single Mimetype type or a comma separated list of mimetypes that qualify to be Processed.</li> <li>Default mimetype for this Processor is: 'application/warc'      </li> </ul> </li> </ol> <p>Advanced OCR/HOCR Settings</p> <p>We do not recommend making changes to the follow settings unless you are the System Administrator.</p> <ol> <li> <p>The system path to the binary that will be executed by this processor.</p> <ul> <li>A full system path to a binary present in the same environment your PHP runs</li> <li>Default is: '/usr/bin/wacz'</li> </ul> </li> <li> <p>Any additional argument your executable binary requires.</p> <ul> <li>Any arguments your binary requires to run. Use %file as replacement for the file if the executable requires the filename to be passed under a specific argument. Use %outfile if the binary is intended to generate a new file and the output is going to be a file entity. If you know the extension please add it in the form of %outfile.extension</li> <li>Default is: 'create %file -t -o %outfile.wacz'</li> </ul> </li> <li> <p>The expected and desired output of this processor.</p> <ul> <li>If the output is just data and \"One or more Files\" is selected all data will be dumped into a file and handled as such.</li> <li>Default is set to 'One or more Files'</li> </ul> </li> <li> <p>Where and how the output will be used.</p> <ul> <li>Default is set to 'A new file to be attached to the source ADO'</li> <li>The 'As Input for another processor Plugin' selection will only have an effect if another Processor is setup to consume this output.</li> <li>Other optional selections include:</li> <li>In the same Source Metadata, as a child structure of each Processed file</li> <li>In the same Source Metadata but inside its own, top level, \"as:flavour\" subkey based on the given machine name of the current plugin</li> <li>A new file to be attached to the source ADO</li> <li>As Input for another processor Plugin</li> <li>In a Search API Document using the Strawberryfield Flavor Data Source (e.g used for HOCR highlight)</li> </ul> </li> <li> <p>Timeout in seconds for this process.</p> <ul> <li>Default is set to: 99</li> <li>If the process runs out of time it can still be processed again.</li> </ul> </li> <li> <p>Order or execution in the global chain.</p> <ul> <li>Default is set to: 0</li> </ul> </li> </ol> <p>Return to the main Strawberry Runners or the Archipelago Documentation main page.</p>","tags":["Strawberry Runners","WACZ","WARC","Binary","Post-processing","Background Processing"]},{"location":"strawberryrunners_webpage_text/","title":"Reviewing and adjusting the <code>wacz_page_extractor</code> and <code>webpage</code> Post-Processor operations","text":"<p>As stated on the Strawberry Runners overview page, the <code>wacz_page_extractor</code> action uses the 'Post processor that extracts/generates Indexed Page Content from WACZ files in an ADO' plugin. Nested one level in, the <code>webpage</code> action uses the 'Post processor that Indexes WACZ Frictionless data Search Index to Search API' plugin. The webpage operations will be executed after the completion of the wacz_page_extractor operations. The <code>ocr</code> operations will be executed after the completion of the <code>pager</code> operations.</p>","tags":["Strawberry Runners","Fulltext Search","WACZ","Webpage","Post-processing","Background Processing"]},{"location":"strawberryrunners_webpage_text/#wacz-page-extractor-settings","title":"Wacz Page Extractor Settings","text":"<p>To review or adjust the configurations for the <code>wacz_page_extractor</code> operation, select <code>Edit</code> from the <code>Operations</code> menu.</p> <p>In the <code>wacz_page_extractor</code> settings, you will see the following configuration options:</p> <ol> <li> <p>Label: </p> <ul> <li>Label for this Processor; which should be a unique machine-readable name</li> <li>Can only contain lowercase letters, numbers, and underscores</li> <li>We do not recommend changing this Label from the default <code>pager</code>. </li> </ul> </li> <li> <p>Strawberry Runner Post Processor Plugin:</p> <ul> <li>The <code>Post processor that extracts/generates Indexed Page Content from WACZ in an ADO</code> should be selected.</li> <li>We do not recommend changing this Plugin selection.</li> </ul> </li> <li> <p>Checkbox to mark this processor plugin as active </p> <ul> <li>We recommend keeping this checked as <code>active</code> at all times, but you may wish to temporarily disable this if you are performing certain types of administrative review tasks such as running large test ingests where you plan on deleting the ADOs before a final ingest.</li> <li>If you accidentally uncheck this and need to re-trigger the <code>pager</code> (and corresponding nested <code>ocr</code> action), you can use Archipelago's Find and Replace to first select a specific group of Digital Objects you wish to target for Post-Processing, then select the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code> from the Find and Replace <code>Actions menu</code>.</li> </ul> </li> <li> <p>The JSON key that contains the desired source files:</p> <ul> <li>By default, only the <code>as:document</code> key is selected.</li> <li>We do not recommend changing this selection.</li> </ul> </li> <li> <p>Mimetypes(s) to limit this Processor to:</p> <ul> <li>A single Mimetype type or a comma separated list of mimetypes that qualify to be Processed.</li> <li>Default mimetype for this Processor is: 'application/vnd.datapackage+zip'   </li> </ul> </li> <li> <p>ADO type(s) to limit this processor to:</p> <ul> <li>A single ADO type or a comma delimited list of ado types that qualify to be Processed.</li> <li>Leave empty to apply to all ADOs. If you do not provide any specific ADO types here, the processor will be applied for all ADOs with the JSON keys selected in the next step.</li> <li>Default ADO type specified for this Processor: 'WebPage'  </li> </ul> </li> <li> <p>The expected and desired output of this processor:</p> <ul> <li>Only option for this Processor is JSON output</li> <li>Default is set to: <code>Data/Values that can be serialized to JSON</code></li> </ul> </li> <li> <p>The queue to use this processor:</p> <ul> <li>The primary queue will be execute in realtime while the Secondary will be execute in background</li> <li>Default selection is for the 'Secondary queue in background' </li> </ul> </li> <li> <p>Timeout in seconds for this process.</p> <ul> <li>Default is set to: 300</li> <li>If the process runs out of time it can still be processed again.</li> </ul> </li> <li> <p>Order or execution in the global chain.</p> <ul> <li>Default is set to: 7</li> </ul> </li> </ol>","tags":["Strawberry Runners","Fulltext Search","WACZ","Webpage","Post-processing","Background Processing"]},{"location":"strawberryrunners_webpage_text/#webpage-settings","title":"Webpage  Settings","text":"<p>To review or adjust the configurations for the <code>webpage</code> operation, select <code>Edit</code> from the <code>Operations</code> menu.</p> <p>In the <code>webpage</code> settings, you will see the following configuration options:</p> <ol> <li> <p>Label: </p> <ul> <li>Label for this Processor; which should be a unique machine-readable name</li> <li>Can only contain lowercase letters, numbers, and underscores</li> <li>We do not recommend changing this Label from the default <code>webpage</code>. </li> </ul> </li> <li> <p>Strawberry Runner Post Processor Plugin:</p> <ul> <li>The <code>Post processor that extracts/generates Indexed Page Content from WACZ in an ADO</code> should be selected.</li> <li>We do not recommend changing this Plugin selection.</li> </ul> </li> <li> <p>Checkbox to mark this processor plugin as active </p> <ul> <li>We recommend keeping this checked as <code>active</code> at all times, but you may wish to temporarily disable this if you are performing certain types of administrative review tasks such as running large test ingests where you plan on deleting the ADOs before a final ingest.</li> <li>If you accidentally uncheck this and need to re-trigger the <code>pager</code> (and corresponding nested <code>ocr</code> action), you can use Archipelago's Find and Replace to first select a specific group of Digital Objects you wish to target for Post-Processing, then select the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code> from the Find and Replace <code>Actions menu</code>.</li> </ul> </li> <li> <p>ADO type(s) to limit this processor to:</p> <ul> <li>A single ADO type or a comma delimited list of ado types that qualify to be Processed.</li> <li>Leave empty to apply to all ADOs. If you do not provide any specific ADO types here, the processor will be applied for all ADOs with the JSON keys selected in the next step.</li> <li>Default ADO type specified for this Processor: 'WebPage'</li> </ul> </li> <li> <p>The expected and desired output of this processor:</p> <ul> <li>Only option for this Processor is JSON output</li> <li>If the output is just data and \"One or more Files\" is selected all data will be dumped into a file and handled as such.</li> <li>Default is set to: <code>Data/Values that can be serialized to JSON</code></li> </ul> </li> <li> <p>Where and how the output will be used.</p> <ul> <li>'As Input for another processor Plugin' selection will only have an effect if another Processor is setup to consume this ouput.</li> <li>Default selection is: 'In a Search API Document using the Strawberryfield Flavor Data Source (e.g used for HOCR highlight)'</li> </ul> </li> <li> <p>The queue to use this processor:</p> <ul> <li>The primary queue will be execute in realtime while the Secondary will be execute in background</li> <li>Default selection is for the 'Secondary queue in background' The queue to use for this processor.</li> </ul> </li> <li> <p>Checkbox option to 'Use NLP to extract entities from Text'</p> <ul> <li>If checked Full text will be processed for Natural language Entity extraction using Polyglot</li> <li>Default is have this option checked</li> </ul> </li> <li> <p>The URL location of your NLP64 server.</p> <ul> <li>Defaults to http://esmero-nlp:6400</li> </ul> </li> <li> <p>Which method(NER) to use</p> <ul> <li>The NER NLP method to use to extract Agents, Places and Sentiment.</li> <li>Default selection: 'Polyglot (faster)'</li> <li>Alternation selection: 'spaCy (more accurate)'</li> </ul> </li> <li> <p>Timeout in seconds for this process.</p> <ul> <li>Default is set to: 25</li> <li>If the process runs out of time it can still be processed again.</li> </ul> </li> <li> <p>Order or execution in the global chain.</p> <ul> <li>Default is set to: 0</li> </ul> </li> </ol> <p>Return to the main Strawberry Runners or the Archipelago Documentation main page.</p>","tags":["Strawberry Runners","Fulltext Search","WACZ","Webpage","Post-processing","Background Processing"]},{"location":"traditional-install/","title":"Traditional install","text":""},{"location":"traditional-install/#traditional-installation-notes","title":"Traditional Installation Notes","text":"<p>For those who prefer classic approaches to system installation and configuration (instead of Dockerized deployment), this page is reserved for notes, recommendations, and guides.</p> <ul> <li>Giancarlo Birello is maintaining and sharing the following documentation:<ul> <li>Dev DBOpen: developer site of the DBOPen project</li> <li>Includes an Architecture overview and Step by Step instructions</li> </ul> </li> </ul> <p>Please stay tuned for additional future updates. Thank you!</p> <p>Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"twig_extensions/","title":"Twig Extensions","text":"<p>One advantage of Drupal's integration of the Twig template engine is the availability of extensions (filters and functions).</p>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON","Markdown","HTML"]},{"location":"twig_extensions/#default-twig-extensions-from-symfony","title":"Default Twig Extensions from Symfony","text":"<p>The Symfony PHP framework, which is integrated into Drupal Core, provides extensions, which we use in our default templates:</p> <ul> <li>Twig Filters from Symfony</li> <li>Twig Functions from Symfony</li> </ul>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON","Markdown","HTML"]},{"location":"twig_extensions/#default-twig-extensions-from-drupal","title":"Default Twig Extensions from Drupal","text":"<p>Additionally, we have some very handy Drupal-specific extensions:</p> <ul> <li>Twig Filters from Drupal</li> <li>Twig Functions from Drupal</li> </ul>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON","Markdown","HTML"]},{"location":"twig_extensions/#default-twig-extensions-from-archipelago","title":"Default Twig Extensions from Archipelago","text":"<p>Finally, we have a growing list of extensions that apply to our own specific use cases:</p>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON","Markdown","HTML"]},{"location":"twig_extensions/#twig-filters-from-archipelago","title":"Twig Filters from Archipelago","text":"<p>edtf_2_human_date</p> <p>The <code>edtf_2_human_date</code> filter takes an EDTF date and an optional language code (defaults to English), and converts it to a human-readable format using the EDTF PHP library. The list of language codes is available here.</p> <p>Let's start with the following metadata fragment: Metadata Fragment<pre><code>...\n\"subject_wikidata\": \"\",\n\"date_created_edtf\": {\n    \"date_to\": \"\",\n    \"date_free\": \"~1899\",\n    \"date_from\": \"\",\n    \"date_type\": \"date_free\"\n},\n\"date_created_free\": null,\n...\n</code></pre></p> <p>Then we pass the <code>date_free</code> field through the <code>trim</code> filter (as a precaution, in case there's any accidental whitespace), and then we finally hand off the field to our <code>edtf_2_human_date</code> filter: edtf_2_human_date<pre><code>{{ data.date_created_edtf.date_free|trim|edtf_2_human_date('en') }}\n\n{# Output: Circa 1899 #}\n</code></pre></p> <p>html_2_markdown</p> <p>The <code>html_2_markdown</code> filter, as the name suggests, converts HTML to Markdown.</p> <p>We start with this string of HTML: HTML string<pre><code>{% set html_string = \"\n  &lt;ul&gt;\n    &lt;li&gt;One thing&lt;/li&gt;\n    &lt;li&gt;Another thing&lt;/li&gt;\n    &lt;li&gt;The last thing&lt;/li&gt;\n  &lt;/ul&gt;\n\" %}\n</code></pre></p> <p>Then we pass it to the filter: html_2_markdown<pre><code>{{ html_string | html_2_markdown }}\n\n{# Output:\n  - One thing\n  - Another thing\n  - The last thing\n#}\n</code></pre></p> <p>markdown_2_html</p> <p>The <code>markdown_2_html</code> filter, as the name suggests, is the reverse of the above and converts Markdown to HTML.</p> <p>We start with this string of Markdown: Markdown string<pre><code>{% set markdown_string = \"\n  - One thing\n  - Another thing\n  - The last thing\n\" %}\n</code></pre></p> <p>Then we pass it to the filter: markdown_2_html<pre><code>{{ markdown_string | markdown_2_html }}\n\n{# Output:\n  &lt;ul&gt;\n    &lt;li&gt;One thing&lt;/li&gt;\n    &lt;li&gt;Another thing&lt;/li&gt;\n    &lt;li&gt;The last thing&lt;/li&gt;\n  &lt;/ul&gt;\n#}\n</code></pre></p> <p>sbf_json_decode</p> <p>The <code>sbf_json_decode</code> filter decodes a JSON-encoded string.</p> <p>We start with this string of JSON string: JSON string<pre><code>{% set json_string = \"\n  {\n    \\\"date_to\\\": \\\"\\\",\n    \\\"date_free\\\": \\\"~1899\\\",\n    \\\"date_from\\\": \\\"\\\",\n    \\\"date_type\\\": \\\"date_free\\\"\n  }\n\" %}\n</code></pre></p> <p>Then we pass it to the filter: sbf_json_decode<pre><code>{% json_decoded = json_string | sbf_json_decode %}\n\n{{ json_decoded.date_free }}\n{# Output:\n  ~1899\n#}\n</code></pre></p>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON","Markdown","HTML"]},{"location":"twig_extensions/#twig-functions-from-archipelago","title":"Twig Functions from Archipelago","text":"<p>clipboard_copy</p> <p>The <code>clipboard_copy</code> function, using the clipboard-copy-element library, takes a provided CSS class for the element(s) whose text we'd like to copy, and targets the CSS class of an existing HTML element on the page or generates an HTML element that can be clicked to copy the text to the user's clipboard.</p> <p>Usage</p> clipboard_copy usage<pre><code>{{ clipboard_copy('CSS CLASS','OPTIONAL CSS CLASS(ES)','OPTIONAL TEXT') }}\n</code></pre> <p>This function takes three arguments:</p> <ul> <li>a CSS class for the element to copy</li> <li>an optional CSS class (the default is <code>clipboard-copy-button</code>) or classes (space-separated) for the copy button if auto-generating or a single, unique class if using your own existing button(s) </li> <li>optional text (the default is <code>Copy to Clipboard</code>) for the copy button if auto-generating</li> </ul> <p>In the examples below, we want users to be able to copy the text from three different kinds of HTML elements: a <code>div</code>, an <code>input</code>, and an <code>a</code> hyperlink href.</p> Copying div element text with auto-generated button <p>First we start by giving the div element(s) we'd like to copy a unique class:</p> div element text<pre><code>&lt;div class=\"csl-bib-body-container chicago-fullnote-bibliography\"&gt;\n  &lt;div id=\"copy-csl\" class=\"csl-bib-body\"&gt;\n    &lt;div class=\"csl-entry\"&gt;\n      New York Botanical Garden. \u201cDescriptive Guide to the Grounds, Buildings and Collections.\u201d\n    &lt;/div&gt;\n  &lt;/div&gt;\n&lt;/div&gt;\n</code></pre> <p>Then we pass the class to the function:</p> clipboard_copy for div element text<pre><code>{{ clipboard_copy('csl-bib-body','','Copy Bibliography Entry') }}\n</code></pre> <p>Note</p> <p>The class can be attached to parent elements of the element we are ultimately targeting if needed, but any intermediate characters may get caught up in the copied text.</p> <p>Or to give the generated button multiple classes (in case they need additional styling):</p> clipboard_copy for div element text<pre><code>{{ clipboard_copy('csl-bib-body','custom custom-button','Copy Bibliography Entry') }}\n</code></pre> <p>The result for the above <code>div</code> example looks as follows:</p> <p></p> <p>The following is the HTML for the auto-generated button with no provided CSS class:</p> <pre><code>&lt;button class=\"clipboard-copy-button\"&gt;\n  &lt;clipboard-copy for=\"copy-csl\" tabindex=\"0\" role=\"button\"&gt;Copy Bibliography Entry&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n</code></pre> <p>And the following is HTML for the auto-generated button with multiple CSS classes provided:</p> <pre><code>&lt;button class=\"custom custom-button\"&gt;\n  &lt;clipboard-copy for=\"copy-csl\" tabindex=\"0\" role=\"button\"&gt;Copy Bibliography Entry&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n</code></pre> <p>Note</p> <p>The clipboard-copy-element library requires an element ID. If the element being copied does not have an ID, one will automatically generated and assigned. </p> Copying input element value with auto-generated button <p>First we start by giving the input element(s) we'd like to copy a unique class:</p> input element value<pre><code>{% if attribute(data, 'as:image')|length &gt; 0  or attribute(data, 'as:document')|length &gt; 0  %}\n  &lt;h2&gt;\n    &lt;span class=\"align-middle\"&gt;Direct Link to Digital Object's IIIF Presentation Manifest V3 &lt;/span&gt;\n    &lt;img src=\"https://iiif.io/img/logo-iiif-34x30.png\"&gt;\n  &lt;/h2&gt;\n  {% set iiifmanifest = nodeurl|render ~ \"/metadata/iiifmanifest/default.jsonld\" %}\n  &lt;input type=\"text\" value=\"{{ iiifmanifest }}\" id=\"iiifmanifest_copy\" size=\"{{ iiifmanifest|length }}\" class=\"col-xs-3 copy-content\"&gt;\n{% endif %}\n</code></pre> <p>Then we pass the class to the function:</p> clipboard_copy for input element value<pre><code>{{ clipboard_copy('copy-content','',\"Copy Link to Digital Object's IIIF Presentation Manifest V3\") }}\n</code></pre> <p>Or to give the generated button multiple classes (in case they need additional styling):</p> clipboard_copy for input element text<pre><code>{{ clipboard_copy('copy-content','custom custom-button',\"Copy Link to Digital Object's IIIF Presentation Manifest V3\") }}\n</code></pre> <p>The result for the above <code>input</code> example looks as follows:</p> <p></p> <p>The following is the HTML for the auto-generated button with no provided CSS class:</p> <pre><code>&lt;button class=\"clipboard-copy-button\"&gt;\n  &lt;clipboard-copy for=\"iiifmanifest_copy\" tabindex=\"0\" role=\"button\"&gt;Copy Link to Digital Object's IIIF Presentation Manifest V3&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n</code></pre> <p>And the following is HTML for the auto-generated button with multiple CSS classes provided:</p> <pre><code>&lt;button class=\"custom custom-button\"&gt;\n  &lt;clipboard-copy for=\"iiifmanifest_copy\" tabindex=\"0\" role=\"button\"&gt;Copy Link to Digital Object's IIIF Presentation Manifest V3&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n</code></pre> <p>Note</p> <p>The clipboard-copy-element library requires an element ID. If the element being copied does not have an ID, one will automatically generated and assigned. </p> Copying anchor element hyperlink href with auto-generated button <p>First we start by giving the <code>a</code> element(s) we'd like to copy a unique class:</p> anchor element hyperlink href<pre><code>&lt;a id=\"copy-documentation-id\" class=\"copy-documentation-class row\" href=\"https://docs.archipelago.nyc\"&gt;Archipelago Documentation&lt;/a&gt;\n</code></pre> <p>Then we pass the class to the function:</p> clipboard_copy for anchor element hyperlink href<pre><code>{{ clipboard_copy('copy-documentation-class','',\"Copy Link to Documentation\") }}\n</code></pre> <p>Or to give the generated button multiple classes (in case they need additional styling):</p> clipboard_copy for anchor element text<pre><code>{{ clipboard_copy('copy-documentation-class','custom custom-button',\"Copy Link to Documentation\") }}\n</code></pre> <p>The result for the above anchor example looks as follows:</p> <p></p> <p>The following is the HTML for the auto-generated button with no provided CSS class:</p> <pre><code>&lt;button class=\"clipboard-copy-button\"&gt;\n  &lt;clipboard-copy for=\"copy-documentation-id\" tabindex=\"0\" role=\"button\"&gt;Copy Link to Documentation&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n</code></pre> <p>And the following is HTML for the auto-generated button with multiple CSS classes provided:</p> <pre><code>&lt;button class=\"custom custom-button\"&gt;\n  &lt;clipboard-copy for=\"copy-documentation-id\" tabindex=\"0\" role=\"button\"&gt;Copy Link to Documentation&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n</code></pre> <p>Note</p> <p>The clipboard-copy-element library requires an element ID. If the element being copied does not have an ID, one will automatically generated and assigned. </p> <p>The above examples automatically generate <code>copy</code> buttons. They can be styled, but if we need more control over the button placement and styling, we can use our own button(s) by ensuring that they meet the following requirements:</p> <ol> <li>A <code>&lt;copy-clipboard&gt;</code> element (this can be hidden) with a <code>for</code> attribute, whose value is the ID of the source element, attached to the element acting as the button.</li> <li>A class on the existing button that can be targeted. The class must either be unique (if a single button) or the number of elements with the class must match the number of source elements.</li> <li>A separate class for the copy source(s) with the same requirements as the previous step.</li> </ol> Copying div element text with custom button <p>First we start by giving the div element(s) we'd like to copy a unique class:</p> div element text<pre><code>&lt;div class=\"csl-bib-body-container chicago-fullnote-bibliography\"&gt;\n  &lt;div id=\"copy-csl\" class=\"csl-bib-body\"&gt;\n    &lt;div class=\"csl-entry\"&gt;\n      New York Botanical Garden. \u201cDescriptive Guide to the Grounds, Buildings and Collections.\u201d\n    &lt;/div&gt;\n  &lt;/div&gt;\n&lt;/div&gt;\n</code></pre> <p>Then we generate the button and pass the class to the function:</p> clipboard_copy custom button for div element text<pre><code>&lt;button class=\"custom-button btn btn-primary btn-sm\"&gt;\n  &lt;clipboard-copy for=\"copy-csl\"&gt;Copy Text&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n\n{{ clipboard_copy('csl-bib-body','custom-button','') }}\n</code></pre> <p>Note</p> <p>The clipboard-copy-element library requires an element ID. If the element being copied does not have an ID, one will automatically generated and assigned. </p> Copying input element value with custom button <p>First we start by giving the input element(s) we'd like to copy a unique class:</p> input element value<pre><code>{% if attribute(data, 'as:image')|length &gt; 0  or attribute(data, 'as:document')|length &gt; 0  %}\n  &lt;h2&gt;\n    &lt;span class=\"align-middle\"&gt;Direct Link to Digital Object's IIIF Presentation Manifest V3 &lt;/span&gt;\n    &lt;img src=\"https://iiif.io/img/logo-iiif-34x30.png\"&gt;\n  &lt;/h2&gt;\n  {% set iiifmanifest = nodeurl|render ~ \"/metadata/iiifmanifest/default.jsonld\" %}\n  &lt;input type=\"text\" value=\"{{ iiifmanifest }}\" id=\"iiifmanifest_copy\" size=\"{{ iiifmanifest|length }}\" class=\"col-xs-3 copy-content\"&gt;\n{% endif %}\n</code></pre> <p>Then we generate the button and pass the class to the function:</p> clipboard_copy custom button for input element value<pre><code>&lt;button class=\"custom-button btn btn-primary btn-sm\"&gt;\n  &lt;clipboard-copy for=\"iiifmanifest_copy\"&gt;Copy Input&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n\n{{ clipboard_copy('copy-content','custom-button','') }}\n</code></pre> <p>Note</p> <p>The clipboard-copy-element library requires an element ID. If the element being copied does not have an ID, one will automatically generated and assigned. </p> Copying anchor element with custom button <p>First we start by giving the <code>a</code> element(s) we'd like to copy a unique class:</p> anchor element hyperlink href<pre><code>&lt;a id=\"copy-documentation-id\" class=\"copy-documentation-class row\" href=\"https://docs.archipelago.nyc\"&gt;Archipelago Documentation&lt;/a&gt;\n</code></pre> <p>Then we generate the button and pass the class to the function:</p> clipboard_copy custom button for anchor element hyperlink href<pre><code>&lt;button class=\"custom-button btn btn-primary btn-sm\"&gt;\n  &lt;clipboard-copy for=\"copy-documentation-id\"&gt;Copy Link&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n\n{{ clipboard_copy('copy-documentation-class','custom-button','') }}\n</code></pre> <p>Note</p> <p>The clipboard-copy-element library requires an element ID. If the element being copied does not have an ID, one will automatically generated and assigned. </p> <p>sbf_entity_ids_by_label</p> <p>The <code>sbf_entity_ids_by_label</code> function, as the name suggests, provides a Drupal entity ID for the following Drupal entity types:</p> <ul> <li>node</li> <li>taxonomy_term</li> <li>group</li> <li>user </li> </ul> <p>If we start with the user entity <code>jsonapi</code>, we can do the following: sbf_entity_ids_by_label<pre><code>{% set jsonapi_user_ids=sbf_entity_ids_by_label('jsonapi','user','') %}\n\n{% for jsonapi_user_id in jsonapi_user_ids %}\n  {{ jsonapi_user_id }}\n{% endfor %}\n\n{# Output:\n  3\n#}\n</code></pre></p> <p>As you can see above, the <code>sbf_entity_ids_by_label</code> function takes three arguments:</p> <ul> <li>the entity label</li> <li>the entity type (see above for supported types)</li> <li>an optional entity bundle</li> </ul> <p>We then loop through the returned result, which is an array of IDs (in this case, just a single one).</p> <p>sbf_search_api</p> <p>The <code>sbf_search_api</code> function executes a search API query against a specified index.</p> sbf_search_api<pre><code>{% set search_results=sbf_search_api('default_solr_index','strawberry',[],{'status':1},[]) %}\n{% set labels=search_results['results']['13']['fields']['label_2'] %}\n&lt;ul&gt;\n  {% for label in labels %}\n    &lt;li&gt;{{ label }}&lt;/li&gt;\n  {% endfor %}\n&lt;/ul&gt;\n</code></pre> <p>As you can see above, the <code>sbf_search_api</code> function takes eight arguments:</p> <ul> <li>The machine name of the Search API index to search against (string)</li> <li>A full text term to search (string)</li> <li>An array of full text fields to search the term against. If empty all will be used. (array)</li> <li>The fields =&gt; filters to match against (associative array)</li> <li>The fields to facet (array)</li> <li>The fields to sort against (associative array)</li> <li>Offset for the results (int)</li> </ul> <p>For this example we end up with the following output:</p> <ul> <li>JPEG File Interchange Format</li> <li>Organic farming--United States</li> <li>Strawberries</li> <li>Strawberry Field at Thorpes Organic Family Farm</li> <li>organic agriculture</li> <li>strawberries</li> </ul>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON","Markdown","HTML"]},{"location":"twig_recipe_cards/","title":"Twig Recipe Cards for Common Use Cases","text":"<p>The Twig Recipe Cards below reference common Metadata transformation, display, or other use cases/needs you may have in your own Archipelago repository.</p>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON"]},{"location":"twig_recipe_cards/#getting-started-working-with-twig-in-archipelago","title":"Getting Started Working with Twig in Archipelago","text":"<p>We recommend reading through our main Metadata Display Preview and Twigs in Archipelago documentation overview guides, and also our Working with Twig primer before diving into applying any of these recipes in your own Archipelago.</p>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON"]},{"location":"twig_recipe_cards/#ami-ingest-template-adaptations-common-use-cases-and-twig-recipe-cards","title":"AMI Ingest Template Adaptations -- Common Use Cases and Twig Recipe Cards:","text":"<p>Use Case #1: I used AMI LoD Reconciliation to reconciliate the values in my AMI Set Source CSV <code>mods_subject_topic</code> column against both LCSH and Wikidata. I would like to map the reconciliated values into the Archipelago default <code>subject_loc</code> and <code>subject_wikidata</code> JSON keys.</p> <p>Twig Recipe Card for Use Case #1:</p> <pre><code>  {#- LCSH -#}\n    {% if data.mods_subject_topic|length &gt; 0 %}\n    \"subject_loc\": {{ data_lod.mods_subject_topic.loc_subjects_thing|json_encode|raw }},\n    {% endif %}\n  {#- Wikidata -#}     \n    {% set subject_wikidata = [] %}\n    {% for source, reconciliated in data_lod %}\n      {% if (('subject' in source) or ('genre' in source)) and reconciliated.wikidata_subjects_thing and reconciliated.wikidata_subjects_thing|length &gt; 0 %}\n         {% set subject_wikidata = subject_wikidata|merge(reconciliated.wikidata_subjects_thing) %}\n      {% endif %}\n    {% endfor %}  \n</code></pre> <p>Use Case #2: I have both columns containing a <code>mods_subject_authority_lcsh_topic</code> (labels) and corresponding <code>mods_subject_authority_lcsh_valueuri</code> (URIs) data in my AMI Set Source Data CSV that I would like to pair and map into the Archipelago default <code>subject_loc</code> JSON key.</p> <p>Twig Recipe Card for Use Case #2:</p> <pre><code>{%- if data['mods_subject_authority_lcsh_topic'] is defined and not empty -%}\n    {% set subjects = data[\"mods_subject_authority_lcsh_topic\"] is iterable ? data[\"mods_subject_authority_lcsh_topic\"] : data[\"mods_subject_authority_lcsh_topic\"]|split('|@|') %} \n    {% set subject_uris = data[\"mods_subject_authority_lcsh_valueuri\"] is defined ? data[\"mods_subject_authority_lcsh_valueuri\"] : '' %} \n    {% set subject_uris_list = subject_uris is iterable ? subject_uris : subject_uris|split('|@|') %}\n    \"subject_loc\": [\n        {% for subject in subjects %}\n        {\n            \"uri\": {{ subject_uris_list[loop.index0]|default('')|json_encode|raw }},\n            \"label\": {{ subject|json_encode|raw }}\n        }\n        {{ not loop.last ? ',' : '' }}\n        {% endfor %}\n    ],\n{%- endif -%}\n</code></pre> <p>Use case #3: I have <code>dc.creator</code> and <code>dc.contributor</code> columns in my AMI Set Source Data CSV with simple JSON-encoded values (e.g. source column cells contain <code>[\"Name 1, Name 2\"]</code>) that I would like to map to the Archipelago default <code>creator_lod</code> JSON key.</p> <p>Twig Recipe Card for Use Case #3:</p> <p><pre><code>{% if data['dc.creator']|length &gt; 0 or data['dc.contributor']|length &gt; 0 %}\n    {% set total_creators = (data[\"dc.creator\"]|length) + (data[\"dc.contributor\"]|length) %}\n    {% set current_creator = 0 %}                \n    \"creator_lod\": [\n        {% for creator in data[\"dc.creator\"] %}\n            {% set current_creator = current_creator + 1 %}\n            {% set creator_source = data[\"dc.creator\"][loop.index0] %}\n        {\n            \"name_uri\": null,\n            \"agent_type\": null,\n            \"name_label\": {{ creator|json_encode|raw }},\n            \"role_label\": \"Creator\",\n            \"role_uri\": \"http://id.loc.gov/vocabulary/relators/cre\"\n        }\n        {{ current_creator != total_creators ? ',' : '' }}\n        {% endfor %}\n        {% for creator in data[\"dc.contributor\"] %}\n            {% set current_creator = current_creator + 1 %}\n            {% set creator_source = data[\"dc.contributor\"][loop.index0] %}\n        {\n            \"name_uri\": null,\n            \"agent_type\": null,\n            \"name_label\": {{ creator|json_encode|raw }},\n            \"role_label\": \"Contributor\",\n            \"role_uri\": \"http://id.loc.gov/vocabulary/relators/ctb\"\n        }\n        {{ current_creator != total_creators ? ',' : '' }}\n        {% endfor %}   \n ],\n{% endif %}  \n</code></pre> Use Case #4: I have a mix of different columns containing Creator/Contributor/Other-Role-Types Name values with or without corresponding URI values that I would like to map to the default Archipelago <code>creator_lod</code> JSON key.</p> <p>Twig Recipe Card for Use Case #4:</p> Click to view the full Recipe Card <pre><code>{#- START Names from LoD and MODS CSV with/without URIS. -#} \n    {# Updated August 26th 2022, by Diego Pino. New checks/logic for mods_name_type_role_composed_or_more_namepart\n    - Check first IF for a given namepart there is already reconciliaton. \n    - IF not i check if there is a matching valueuri, \n    - If not leave the URL empty and use the value in the namepart (label) only?\n    - Only check/use mods_name_corporate/personal_namepart field IF there are no other fields\n    - That specify Roles. Since normally in ISLANDORA that field (no role) is a Catch all names one\n    - And in that case USE creator as the default ROLE\n    #}\n    {%~ set creator_lod = [] -%} \n    {# Used to keep track of parts after the type (corporate, etc) that are no roles\n     but authority properties. Add more if you find them #}\n    {%  set roles_that_are_no_roles = ['authority_naf','authority_marcrelator',''] %}\n    {# Used to keep track of the ones that are reconciled already #}\n    {%- set name_has_creator_lod = [] -%}\n    {%- for key,value in data_lod -%}\n      {%- if key starts with 'mods_name_' and key ends with '_namepart' -%}\n      {# If there is mods_name_SOMETHING_namepart in data_lod we keep track so we \n      do not try afterwards to use that Sources KEY from the CSV.\n      #}\n        {%- set name_has_creator_lod = name_has_creator_lod|merge([key]) -%}\n      {# Now we remove 'mods_name_' and '_namepart' #}\n        {%- set name_type_and_role = key|replace({'mods_name_':'', '_namepart':''}) -%}\n      {# We will only target personal or corporate. If any of those are missing we skip? #}\n        {% set name_type = null %}\n        {%- if name_type_and_role starts with 'personal_' -%}\n           {% set name_type = 'personal' %}\n        {%- elseif name_type_and_role starts with 'corporate_' -%}\n           {%- set name_type = 'corporate' -%}\n        {%- endif -%}\n        {%- if name_type is not empty -%}\n        {#- Now we remove 'type', e.g 'corporate_' -#}\n          {%- set name_role = name_type_and_role|replace({(name_type ~ '_'):''}) -%}\n          {# in case the name_role contains one of roles_that_are_no_roles, e.g\n          something like `creator_authority_marcrelator` we remove that #}\n          {% for role_that_is_no_role in roles_that_are_no_roles %}\n             {%- set name_role = name_role|replace({(role_that_is_no_role):''}) -%}\n          {% endfor %}\n          {# After removing all what can not be a role if we end with an empty #}\n          {% if name_role|trim|length == 0 %}\n             {%- set name_role = \"creator\" %}\n          {% else %}\n            {%- set name_role = name_role|replace({'\\\\/':'//' , '_':' '})|trim -%}\n          {% endif %}\n        {#- we iterate over all possible vocabularies and fetch the reconciliated names from them (if any) -#}\n          {%- for approach, names in value -%} \n        {#- if there are actually name pairs (name and uri) that were reconciliated we use them -#}\n            {%- if names|length &gt; 0 -%}\n              {#- we call the ami_lod_reconcile twig extension with the role label using the LoC Relators endpoint in english and get 1 result -#}\n              {%- set role_uri = ami_lod_reconcile(name_role|lower|capitalize,'loc;relators;thing','en',1) -%}\n              {#- for each found name pair in a list of possible LoD reconciliated elements we generate the final structure that goes into \"creator_lod\" json key -#}\n              {%- for name in names -%} \n                {%- set creator_lod = creator_lod|merge([{'role_label': name_role|lower|capitalize, 'role_uri': role_uri[0].uri, \"agent_type\": name_type, \"name_label\": name.label, \"name_uri\": name.uri}]) -%}     \n             {%- endfor -%}\n            {%- endif -%}\n          {%- endfor -%}\n        {% endif -%}\n      {%- endif -%} \n    {%- endfor -%}\n    {# Now go for the RAW CSV data for names #}\n    {%- for key,value in data -%}\n    {# here we skip values previoulsy fetched from LoD and stored in name_has_creator_lod #}\n      {%- if key not in name_has_creator_lod and key starts with 'mods_name_' and key ends with '_namepart' -%}\n      {# If there is mods_name_SOMETHING_namepart in data_lod we keep track so we \n      do not try afterwards to use that Sources KEY from the CSV.\n      #}\n        {%- set name_has_creator_lod = name_has_creator_lod|merge([key]) -%}\n      {# Now we remove 'mods_name_' and '_namepart' #}\n        {%- set name_type_and_role = key|replace({'mods_name_':'', '_namepart':''}) -%}\n      {# We will only target personal or corporate. If any of those are missing we skip? #}\n        {%- set name_type = null -%}\n        {%- if name_type_and_role starts with 'personal_' -%}\n           {%- set name_type = 'personal' -%}\n        {%- elseif name_type_and_role starts with 'corporate_' -%}\n           {%- set name_type = 'corporate' -%}\n        {%- endif -%}\n        {% if name_type is not empty %}\n        {# Now we remove 'type', e.g 'corporate_' #}\n          {%- set name_role = name_type_and_role|replace({(name_type ~ '_'):''}) -%}\n          {# in case the name_role contains one of roles_that_are_no_roles, e.g\n          something like `creator_authority_marcrelator` we remove that #}\n          {% for role_that_is_no_role in roles_that_are_no_roles %}\n             {%- set name_role = name_role|replace({(role_that_is_no_role):''}) -%}\n          {% endfor %}\n          {# After removing all what can not be a role if we end with an empty #}\n          {% if name_role|trim|length == 0 %}\n             {%- set name_role = \"creator\" %}\n          {% else %}\n            {%- set name_role = name_role|replace({'\\\\/':'//' , '_':' '})|trim -%}\n          {% endif %}\n          {# Now we check if there is a corresponding _valueuri for this #}\n          {% set name_uris = [] %}\n          {%- if data[('mods_name_' ~ name_type_and_role ~ '_valueuri')] is not empty \n          and data[('mods_name_' ~ name_type_and_role ~ '_valueuri')] != '' -%}\n            {%- set name_uris = data[('mods_name_' ~ name_type_and_role ~ '_valueuri')]|split('|@|') -%}\n          {%- endif -%}\n          {%- set role_uri = ami_lod_reconcile(name_role|lower|capitalize,'loc;relators;thing','en',1) -%}\n        {#- we split and iterate over the value of the mods_name key -#}\n        {# NOTE. THIS IS TARGETING Anything after the year 1000, or 2000 #}\n          {%- for index,name in value|replace({'|@|1':', 1', '|@|2':', 2', '|@|-':', -'})|split('|@|') -%}\n            {%- if name is not empty and name|trim != '' -%}\n              {%- set name_uri = null -%}\n              {# Here we can check if one of the names IS not a name (e.g a year? #}\n              {#- we call the ami_lod_reconcile twig extension with the role label using the LoC Relators endpoint in english and get 1 result -#}\n              {%- if name_uris[index] is defined and name_uris[index] is not empty -%}\n                 {%- set name_uri = name_uris[index] -%}\n              {%- endif -%}\n              {%- set creator_lod = creator_lod|merge([{'role_label': name_role|lower|capitalize, 'role_uri': role_uri[0].uri, \"agent_type\": name_type, \"name_label\": name, \"name_uri\": name_uri}]) -%}\n            {%- endif -%}\n          {%- endfor -%}\n        {%- endif -%}\n      {%- endif -%}\n    {%- endfor ~%}\n    {# Use reduce filter + other logic for depulicating #}\n    {% set creator_lod = creator_lod|reduce((unique, item) =&gt; item in unique ? unique : unique|merge([item]), []) %}\n    \"creator_lod\": {{ creator_lod|json_encode|raw -}},\n    {#- END Names from LoD and MODS CSV with/without URIS. -#}\n</code></pre> <p>Use Case #5: I have geographic location information that I would like to reconciliate against Nominatim and map into the default Archipelago 'geographic_location' key. I have AMI Source Data CSVs which contain values/labels and some which contain coordinates.</p> <p>Twig Recipe Card for Use Case #5 with variation notes:</p> <pre><code>{#- &lt;-- Geographic Info and terms:\n  Includes options for geographic info for:\n  - Nominatim lookup by value/label\n  - Nominatim lookup by coordinates \n  -#}\n    {#- use value for Nominatim search -#}\n    {% if data.mods_subject_geographic|length &gt; 0 %}\n      {% set nominatim_from_label = ami_lod_reconcile(data.mods_subject_geographic,'nominatim;thing;search','en') -%}\n    \"geographic_location\": {{ nominatim_from_label|json_encode|raw }},\n    {% endif %}\n    {#- use coordinates for Nominatim search, if provided -#}\n    {% if data.mods_subject_cartographics_coordinates|length &gt; 0 %}\n      {% set nominatim_from_coordinates = ami_lod_reconcile(data.mods_subject_cartographics_coordinates,'nominatim;thing;reverse','en') -%}\n    \"geographic_location\": {{ nominatim_from_coordinates|json_encode|raw }},\n    {% endif %}\n{#- Geographic Info and terms --&gt; #}  \n</code></pre> <p>Use Case #6: I have date values in a <code>dc.date</code> column that contain instances of 'circa' or 'Circa' where I would like to replace with the EDTF-friendly '~' instead and map to the Archipelago default 'date_created_edtf' JSON key.</p> <p>Twig Recipe Card for Use Case #6: </p> <pre><code>    {% if data['dc.date'] is defined %}\n        {% set datecleaned = data['dc.date']|replace({\"circa \":\"~\", \"Circa \":\"~\"}) %}\n        \"date_created_edtf\": {\n        \"date_to\": \"\",\n        \"date_free\": {{ datecleaned|json_encode|raw }},\n        \"date_from\": \"\",\n        \"date_type\": \"date_free\"\n        },        \n    {% endif %}\n</code></pre> <p>More recipe cards will be added over time. Please see our Archipelago Contribution Guide to learn about contributing your own recipe card or other documentation.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON"]},{"location":"utility_scripts/","title":"Utility Scripts","text":"<p>If you've already followed deployment guides for archipelago-deployment and archipelago-deployment-live, you may have used some shell scripts that archipelago provides. The scripts are available in the <code>scripts/archipelago/</code> and <code>drupal/scripts/archipelago/</code> folders respectively.</p>","tags":["Bash","Scripts","DevOps"]},{"location":"utility_scripts/#importexport-metadata-display-twig-templates","title":"Import/Export Metadata Display Twig Templates","text":"<p>Metadata Display Entity Twig Templates can be exported out of and imported into both local and remote deployments with the following script: <code>import_export.sh</code>. The script can be run interactively or non-interactively.</p> <p>Docker host vs. Docker container</p> <p>Because the script uses the Docker <code>.env</code> file for the JSONAPI user and URL by default, we recommend running this directly on the host.</p>","tags":["Bash","Scripts","DevOps"]},{"location":"utility_scripts/#interactive-mode","title":"Interactive Mode","text":"<p>Running the script interactively will guide you through a number of prompts to configure and import or export to an existing folder or to one which will be created.</p> <pre><code>./import_export.sh -n\n</code></pre>","tags":["Bash","Scripts","DevOps"]},{"location":"utility_scripts/#non-interactive-mode","title":"Non-interactive Mode","text":"<p>To run the command non-interactively provide the required and optional parameters with the necessary arguments as needed.</p> <p>Options for Non-interactive Mode</p> <p><code>-i</code> or <code>-e</code> (required)</p> <p>\u00a0\u00a0\u00a0\u00a0Import or export, respectively, Metadata Entity Display Twig Templates from a local folder.</p> <p><code>-s path</code> (required)</p> <p>\u00a0\u00a0\u00a0\u00a0The absolute path of the local folder to export to or import from.</p> <p><code>-j path/filename</code> (only required if the <code>.env</code> file containing the JSONAPI user and password is in a non-standard location)</p> <p>\u00a0\u00a0\u00a0\u00a0The absolute path to the <code>.env</code> file containing the JSONAPI user and password.</p> <p><code>-d url</code> (required if URL is not in <code>.env</code> file or importing to or exporting from a remote deployment)</p> <p>\u00a0\u00a0\u00a0\u00a0The URL of the archipelago deployment.</p> <p><code>-k</code> (optional)</p> <p>\u00a0\u00a0\u00a0\u00a0Keep any existing files ending with <code>.json</code> in the specified folder (the default is to delete) before exporting.</p> <p>JSONAPI User</p> <p>The JSONAPI user credentials, by default, will be read from the <code>.env</code> files in the following locations (relative to the root of the deployment):</p> Deployment File Location archipelago-deployment-live <code>./deploy/ec2-user/.env</code> archipelago-deployment <code>./.env</code> <p>A separate file can also be passed as an argument using the <code>-j</code> option.</p> .env<pre><code>JSONAPI_USER=jsonapi\nJSONAPI_PASSWORD=jsonapi\n</code></pre> <p>Exporting from local archipelago-deployment-live</p> <p><pre><code>./import_export.sh -e -s /home/ec2-user/metadatadisplay_export\n</code></pre> After logging into the archipelago-deployment-live host, the above command will delete any files with the <code>.json</code> extension if the destination folder exists. Otherwise, the folder will be created. The JSON user credentials and domain from the <code>.env</code> file will then be used to download the files so please make sure these are set.</p> <p>Exporting from local archipelago-deployment</p> <p><pre><code>./import_export.sh -e -s /home/user/metadatadisplay_export -d http://localhost:8001\n</code></pre> This will work the same way as the above example, but the URL is passed as an argument in this case since the <code>.env</code> file will not contain (in most cases) the domain. As above, the JSON user credentials will have to be set in the <code>.env</code> file.</p> <p>Exporting from remote archipelago-deployment-live</p> <p><pre><code>./import_export.sh -e -s /home/user/metadatadisplay_export -d https://archipelago.nyc\n</code></pre> This is essentially the same as the example directly above, except that in this case the JSON user credentials in the <code>.env</code> file will have to be set to the ones used to access the remote instance.</p> <p>Importing locally into archipelago-deployment-live</p> <p><pre><code>./import_export.sh -i -s /home/user/metadatadisplay_import\n</code></pre> This is essentially the same as the first example above, except that the import option (<code>-i</code>) is used. The folder name is changed for the sake of example, but you can use the same folder that was used for exporting.</p> <p>Importing locally into archipelago-deployment</p> <p><pre><code>./import_export.sh -i -s /home/user/metadatadisplay_import -d http://localhost:8001\n</code></pre> As in the example directly above, this corresponds to the example for exporting with a local archipelago-deployment instance, except that the import option (<code>-i</code>) is used.</p> <p>Importing from local instance into remote archipelago-deployment-live</p> <p><pre><code>./import_export.sh -i -s /home/user/metadatadisplay_import -d https://archipelago.nyc\n</code></pre> In this example, the locally exported files are being imported into a remote instance. As in the above examples with remote instances, the JSON user credentials need to be set in the <code>.env</code> file to those with access to the remote instance.</p>","tags":["Bash","Scripts","DevOps"]},{"location":"utility_scripts/#automatic-deployment-script","title":"Automatic Deployment Script","text":"<p>If you're frequently deploying locally with archipelago-deployment, you may want to use the automated deployment script available at <code>scripts/archipelago/devops/auto_deploy.sh</code>. The script is interactive and can be called from the root of the deployment, e.g. <code>/home/user/archipelago-deployment/</code>:</p> <p>Automatic Deployment</p> <p>scripts/archipelago/devops/auto_deploy.sh</p> <p>Follow the prompts and select your options to complete the deployment.</p>","tags":["Bash","Scripts","DevOps"]},{"location":"webformLoDfromCSV/","title":"Using Archipelago's 'Webform LoD from CSV attached to an ADO suggest'","text":"<p>Archipelago's custom webform element 'Webform LoD from CSV attached to an ADO suggest' provides a form element autocomplete labels/urls(values) from a CSV attached to a Digital Object. Using this element affords a way for you to utilize a custom local vocabulary, a subset of labels and URIs from a wider LoD authority source, or an LoD vocabulary that does not have an accessible API or public query service.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#step-by-step","title":"Step-by-step","text":"","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#1-prepare-your-csv-file","title":"1. Prepare your CSV File","text":"<p>To use this element, you need to first have prepared a CSV file containing two columns only:</p> <ul> <li>one column for 'label' containing the labels for your vocabulary</li> <li>one column for 'uris' containing the corresponding uris for your vocabulary</li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#2-new-ado-using-your-csv-file","title":"2. New ADO Using Your CSV File","text":"<p>Create a new Digital Object, and attached the prepared CSV file to the new Digital Object.</p> <ul> <li>Be sure to provide a unique label to help you identify this Object in future steps.</li> <li>It is recommended that you do not Publish this Object.</li> </ul> <p>Note</p> <p>If you are not yet familiar with how to create a Digtial Object, please refer to this guide.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#3-edit-your-webform","title":"3. Edit your Webform","text":"<p>Go to <code>Admin &gt; Structure &gt; Webforms</code> and select the 'Build' button beside the Default Descriptive Metadata Webform.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#4-add-new-custom-webform-element","title":"4. Add New Custom Webform Element","text":"<p>Scroll down to the 'Subjects and Other Classifications' page of the Webform and select 'Add Element'.</p> <p></p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#5-webform-from-lod-from-csv-attached-to-an-ado-suggest","title":"5. Webform from LoD from CSV attached to an ADO suggest","text":"<p>In the 'Select an element to add ..' popup that opens:</p> <ul> <li>Either scroll down to select the 'Composite Element' section or search for the 'Webform LoD from CSV attached to an ADO suggest.' </li> <li> <p>Select 'Add Element'.</p> <p></p> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#6-adjust-element-settings","title":"6. Adjust Element Settings","text":"<p>In the Edit tab that opens for your newly added element, you will need to review the following sections.</p> <ul> <li> <p>Element Settings</p> <ul> <li>provide a Title for element</li> <li>check that the Key generated from the Title you supply is well formed and make changes if needed</li> <li>specify the 'Allowed number of values'</li> </ul> <p></p> </li> <li> <p>Webform LoD from CSV attached to an ADO suggest settings:</p> <ul> <li>It is recommended to keept both 'label' and 'uri' checked as Visible.</li> <li>You may also wish to mark both elements as 'Required'</li> </ul> <p></p> </li> <li> <p>Autocomplete settings</p> <ul> <li>In the 'Choose an ADO' box, begin typing to search for the Digital Object that holds a CSV containing the Vocabulary you want to autocomplete.</li> <li>Under 'The CSV column(header name) that will be used for autocompleting', enter 'label'</li> <li>Under 'The CSV column(header name) that will be used for the URL value', enter 'uri'</li> <li>'Autocomplete limit'<ul> <li>determines the maximum number of matches to be displayed</li> <li>recommended that you set to '10'.</li> </ul> </li> <li>'Autocomplete minimum number of characters'<ul> <li>determines the minimum number of characters a user must type before a search is performed</li> <li>recommended that you set to '3'.</li> </ul> </li> <li>'Autocomplete matching operator'<ul> <li>determines the method used to collect autocomplete suggestions</li> <li>recommended to use 'Starts with'          </li> </ul> </li> </ul> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#7-advanced-tab-for-webform-element","title":"7. Advanced Tab for Webform Element","text":"<p>Navigate to the 'Advanced' tab for this Webform element.</p> <ul> <li>Open the 'Multiple settings' section</li> <li> <p>Deselect the options to 'Allow users to sort elements' and 'Allow users to add more items'</p> <p></p> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#8-save-your-work","title":"8. Save Your Work","text":"<p>Save your new form element settings. Then Save your updated Webform.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#9-use-your-new-webform-element","title":"9. Use Your New Webform Element","text":"<p>Navigate to a Digital Object in your repository that you would like to use this new custom vocabulary element with. </p> <ul> <li>Select 'Edit' for that Digital Object and navigate to the 'Subjects and Other Classifications' page of the webform.</li> <li> <p>Begin typing a label found in your prepared CSV associated with the webform element.</p> <p></p> </li> </ul> <p>You can now begin using this custom vocab vocabulary element when using the corresponding webform (where you added this element) to Edit and Update your Digital Objects. You may also wish to add this same element to the Default Digital Object Collection/Creative Work Series webform.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webforms/","title":"Webforms in Archipelago","text":"<p>The Webform Strawberryfield module provides Drupal Webform ( == awesome piece of code) integrations for StrawberryField so you can really have control over your Metadata ingests. These custom elements provide Drupal Webform integrations for Archipelago\u2019s StrawberryField so you can have fine grained and detailed control over your Metadata ingests and edits.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webforms/#where-to-find","title":"Where to Find","text":"<p>You can access Webforms in Archipelago at: - <code>Admin &gt; Structure &gt; Webforms</code> - <code>/admin/structure/webform/</code></p> <p></p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webforms/#instructions-and-guides","title":"Instructions and Guides","text":"<ul> <li>How to Create a Webform as an Input Method for Archipelago Digital Objects (ADO)</li> <li>Customizing Webforms: Modifying allowable file extensions</li> <li>Archipelago Custom Webform Elements</li> <li>Using Archipelago's 'Webform LoD from CSV attached to an ADO suggest'</li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webforms/#archipelago-default-example-webforms","title":"Archipelago Default Example Webforms","text":"<p>Use these webforms or their elements to create a custom webform for your own repository/project needs:</p> <ul> <li> <p>Archipelago Default Deployment Webforms</p> <ul> <li> <p>Descriptive Metadata</p> <ul> <li>Corresponding Schema.org Type Options</li> </ul> </li> <li> <p>Digital Object Collection</p> <ul> <li>Corresponding Schema.org Type Options</li> </ul> </li> </ul> </li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformsasinput/","title":"Primer on Display Modes &amp; How to Create a Webform as an Input Method for Archipelago Digital Objects (ADO)","text":"<p>Drupal 9/10 provides a lot of out-of-the-box functionality to setup the way Content Entities (Nodes or in our case ADOs) are exposed to users with the proper credentials. That functionality lives under the \"Display Modes\" &gt;&gt; and can be accessed at <code>yoursite/admin/structure/display-modes</code>.</p> <p>In a few quick words, The Display Mode Concept covers: formatting your Content Entities and their associated Fields so when a user lands on a Content Page, they are displayed in a certain, hopefully pleasing, way and also how users with proper Credentials can fill inputs/edit values for each <code>field</code> a Content Entity provides.</p> <p>First, formatting output (basically building the front facing page for each content entity) is done by a <code>View Mode</code>. Second, defining how/what input method you are going to use to create or edit Content entities, is handled by a <code>Form Mode</code>. Both Modes, are, in Drupal Lingo, Configuration Entities, they provide things you can configure, you can name them and reuse them and those configurations can all be exported and reimported using YAML files. Also both Modes the following in common:</p> <ul> <li>Drupal always provides a \"default\" one that can not be deleted.</li> <li>You can create new ones.</li> <li>You can apply permissions to them.</li> <li>All Modes work on \"fields\", means the tiny little input/output pieces that are either part of a Content Entity or attached to them (the title, the Body, and in our case a Strawberryfield (SBF),</li> <li>They Provide Config/setting options for each Field.</li> <li>They are always associated to Content Types/Bundles. Means all Nodes of the same Content type will share the same modes.</li> </ul> <p>The main difference, other than their purpose (Output v/s Input) is that, on View Modes, the settings you apply to each field are associated to \"Formatters\" and on Form Modes, the settings you apply to each field are connected to \"Widgets\".</p> <p>*Please note that this guides features some older screenshots using earlier versions of Archipelago/Drupal Adminsitrative Theming. Please pardon any jumps between themes.</p> <p>So, resuming, this is what lives under the Concept of a \"Display Mode\"...</p>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#view-mode","title":"View Mode","text":"<ul> <li>Each field attached to a Content Entity can have a Formatter applied and most of them have configuration options.</li> <li>Formatters do one thing right: they take the raw, stored value and make it \"visible\" inside Drupal.</li> <li>Which formatters are available will depend on the \"type\" of field the Content Entity has.<ul> <li>E.g A Node title/Label will have a Title formatter with the option of just displaying a text or a text with a link to the entity.</li> <li>More Complex and fun Fields, like the ones of type <code>SBF</code> will provide a large list of possible <code>Formatters</code>, like IIIF driven viewers, Video formatters, Metadata Display (Twig template driven) ones, etc. This is because a SBF type of field has much more than just a text value, it contains a full graph of metadata and properties, inclusive links to Files and provenance metadata.</li> </ul> </li> </ul>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#all-of-the-default-view-modes-bundled-in-archipelago","title":"All of the Default View Modes Bundled in Archipelago","text":"Click to see the full list of Default View Modes <ul> <li>Collection listing        </li> <li>Digital Object Collection with Mirador Viewer     </li> <li>Digital Object Creative Work Series with Mirador Viewer       </li> <li>Digital Object Full View      </li> <li>Digital Object Image Only for Carousel        </li> <li>Digital Object Oral History with Multiple Media       </li> <li>Digital Object with 3D Viewer     </li> <li>Digital Object with Audio Player      </li> <li>Digital Object with Book Reader       </li> <li>Digital Object with Mirador Viewer        </li> <li>Digital Object with Pannellum Panorama        </li> <li>Digital Object with PDF Viewer        </li> <li>Digital Object with thumbnail and abstract        </li> <li>Digital Object with thumbnail for Grid        </li> <li>Digital Object with Video Player      </li> <li>Digital Object with WARC Replay.web Widget        </li> <li>Search index      </li> <li>Search result highlighting input      </li> <li>Plus Drupal default View Modes: RSS, Teaser, and Token</li> </ul>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#form-mode","title":"Form Mode","text":"<ul> <li>Each field attached to a Content Entity can have a Widget applied and most of them have configuration options.</li> <li>Widgets do one thing right: they expose some type of Form/UI interaction that allows a user to input data into the Entity, under that specific field. And of course they make sure that what you input is validated and saved (if good) correctly.</li> <li> <p>Which Widgets are available will depend on the \"type\" of field the Content Entity has.</p> <ul> <li>Example: A <code>Node</code> Title will have a single Text Input with some options, like the size of the Textfield used to feed it.</li> <li>More Complex and fun fields, like the ones of type <code>SBF</code> (strawberryfield), will provide a larger list of possible Widgets, ranging from raw JSON input (which you could select if your data was already in the right format) to the reason we are reading this: <code>Webform driven Widgets</code>. These Widgets include:<ul> <li>ones the webform_strawberryfield Drupal module provides</li> <li>ones that use an existing Webform (which are also Entitites!) which either 1) you created or 2) we provided as a setting</li> </ul> </li> </ul> <p>If you chose a widget other than the raw JSON, the widget will take the raw JSON to build, massage and enrich the data so that it can be presented in a visual format by the SBF. This is because a SBF type of field has much more than just a text value. It contains a full graph of metadata and properties, inclusive links to Files and provenance metadata, which for example allows us to use an Upload field directly in the attached/configured webform. - Form modes also have an additional benefit. Each one can have fine grained permissions. That way you can have many different Form Modes, but allow only certain ones to be visible, or usable by users of a given Drupal Role.</p> </li> </ul>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#i-think-i-get-thisbut-how-can-i-use-this-knowledge-now","title":"I think I get this...but how can I use this knowledge now?","text":"<p>Good question! So, to enable, configure, and customize these Display Modes you have to navigate to your <code>Content Type</code> Configuration page in your running Archipelago. This is found at <code>/admin/structure/types</code>. Note: the way things are named in Drupal can be confusing to even the most deeply committed Drupal user, so bear in mind some terms will change. Feel free to read and re-read.</p> <p></p> <p>You can see that for every existent Content Type, there is a drop down menu with options:</p> <ul> <li>Manage Display: will lead you to configuration page where you can setup each View/Display Mode and its settings for a given Content Type</li> <li>Manage Form Display: will lead you to configuration page where you can setup each Form Mode and its settings</li> </ul>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#manage-display","title":"Manage Display","text":"<p>For Digital Objects: </p> <p>For Digital Object Collections and Compounds/Creative Work Series </p> <p>On the top you will see all your View Modes Listed, with the <code>Default</code> one selected and expanded. The Table that follows has one row per Field attached/part of this Content Type. Some of the fields are part of the Content Type itself, in this case Digital Object (bundled) and some other ones are common to every Content Entity derived from a Node. The \"Field\" column contains each field name (not their type, reason why you don't see Strawberry Field there!) but we can tell you right now that there is one, named \"Descriptive Metadata\", that is of <code>SBF</code> type.</p> Wait! Which are the fields in my Content Type? <p>How do we know that the field named \"Descriptive Metadata\" is a Strawberryfield? Well, we set-up the Digital Object Content Type for you that way, but also you can know what we know by pressing on \"Manage fields\" Tab on the top (don't forget to come back to \"Manage display\", afterwards!)</p> <p></p> <p>Also Surprise: You Content Entity has really really just 2 fields! And that, friends, is one of the secret ingredients of Archipelago. All goes into a Single Field. But wait: i see more fields in my Manage Display table. Why? Well. Some of them are base fields, part of what a Drupal Node is: base field means you can not remove them, they are part of the Definition itself. One obvious one is the <code>Title</code>.</p> <p>But there are also some fields very particular to Archipelago: You can see there are also ones named \"Formatter Object Metadata\", \"Media\" and one named \"Static Media\"!. Where does come from? Those are also Strawberryfields. It sounds confusing but it is really simple. They are really not \"fields\" in the sense of having different data than \"Descriptive Metadata\". Those are In Memory, realtime, copies of the \"Descriptive Metadata\" SBF field and are there to overcome one limitation of Drupal 8:</p> <p>Each Field can have a single \"Formatter\" setup per field.</p> <p>But we want to re-usue the JSON data to show a Viewer, Show Metadata as HTML directly on the ADO/NODE landing page, and we want also to, for example, format sometimes images as Thumbnails and not using a IIIF viewer only. This CopyFields (Legal term) have also a nice Performance advantage. Drupal needs to fetch only once the data from the real Field, \"Descriptive Metadata\", from the database. And then just makes the data available in real time to its copies. That makes all fast, very very fast! And of course flexible. As you dig more into Archipelago you will see the benefits of this approach. Finally, if you need to, you can make more CopyFields. But the reality is, there is a single, only one, SBF in each Digital Object and its named \"Descriptive Metadata\".</p> <p>You can also simply not care about the type and trust the UI. It will just show Formatters that are right for each type and expose Configuration options (and a little abstract of the current ones) under the Widget Column. Operations Columns allows you to setup each Widget. Widget term here is a bit confusing. These are not really Widget in terms of Data Input, but in terms of \"Configuration\" Input. But D8/9 is evolving and its getting better. Those settings apply always only to the current View Mode.</p> <p>You can play with this, experiment and change some settings to get more comfortable. We humbly propose you that you complete this info with the official Drupal 8 Documentation and also apply custom settings to your own, custom View Mode so you don't end changing base, expected functionality while you are still learning.</p>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#manage-form-display","title":"Manage Form Display","text":"<p>On the top you will see all your Form Modes Listed, with the <code>Default</code> one selected and expanded. The Table that follows has one row per Field attached/partof this Content Type. The list of fields here is shorter, the SBF CopyFields are not present because all data goes really only into real fields. Also some other, display only ones (means you can not modify them) will not appear here. Again, Some of the fields are part of the Content Type itself, in this case Digital Object (bundled) and some other ones are common to every Content Entity derived from a Node. \"Field\" column contains each field name and the Widget Column allows you to select what type of Input you are going to use to feed it on Ingest/edit. On the right you will see again a little gear, that allows you to configure the settings for a particular Widget. Those settings apply always only to the current Form Mode.</p> <p>So. The one we want to understand is the one attached to the \"Descriptive Metadata\" field. Currently one named \"Strawberryfield webform based input with inline rendering\". There are other two. But let's start with this. Press on the Gear to the right on the same row.</p> <p></p> <p>AS you can see there are not too many options. But, the main, first Text input is an Autocomplete field that will resolve against your existing Webforms. So, guess what. If you want to use your own Webform to feed a SBF, what do you do? You type the name, let the autocomplete work, select the right Webform, maybe your own custom one, and the you press \"Update\". Once that is done you need to \"Save\" your Form Mode (hint, button at the bottom of the page).</p> <p>We wish life was that easy (and it will once we are done with refining Drupal's UI) but for now there are some extra things you need to do to make sure the Webform, your custom one, can speak JSON. The default one you get named also \"Descriptive Metadata (descriptive_metadata)\", same as the field, is already setup to be used. Means if you create a new Webform by Copying that one, you can start using it inmediately. But if you created one from scratch (Different tutorial) you need to setup some settings.</p>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#setting-up-a-webform-to-talk-strawberryfield","title":"Setting up a Webform to talk Strawberryfield","text":"<p>Navigate to your Webform Managment form at <code>/admin/structure/webform</code></p> <p></p> <p>If you already created a Webform (different tutorial on how to do that) you will see your own named one in that list. I created for the purpose of this documentation one named \"Diego Test\" (Hi, i'm Diego..) and on the most right Column, \"Operations\" you will haven an Drop Down Menu. On your own Webform row, press on \"Settings\".</p> <p>First time, this can be a little bit intimitading. We recommend going baby steps since the Webform Module is a very powerful one but also exposes you to a lot (and sometimes too many) options. Even more, if you are new to Webforms, we recomment you to copy the \"Descriptive Metadata\" Webform we provided first, and make small changes to it (starting by naming it your own way!) so you can see how that affects your workflow and experience, and how that interacts with the created metadata. The Webform Module provides testing and building capabilities, so you have a Playground there before actually ingesting ADOs. Copying it will also make all the needed settings for SBF interaction to be moved over, so your work will be much easier.</p> <p>But we know you did not do that (where is the fun there right?). So lets setup one from scratch.</p>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#general-settings","title":"General Settings","text":"<p>Gist here is (look at the screenshot and copy the settings):</p> <ul> <li>GENERAL Settings: Check \"Disable saving of submissions\" option. You won't need this form to generate a Native Webform Submission entry.</li> <li>AJAX Settings: Check \"use ajax\" option. We want people to have the experience of staying in a single page while the create a new ADO via a Multi Step Webform Workflow.</li> </ul>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#confirmation-settings","title":"Confirmation Settings","text":"<p>Gist here is (again, look at the screenshot and copy the settings):</p> <ul> <li>Select \"Inline Confirmation\". You don't want Webform to send your user to another page while they are still ingesting their ADOs.</li> </ul>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#handler","title":"Handler","text":"<p>The glue, the piece of resistance. The handler is the one that knows how to talk to a SBF. In simple words, the handler (any handler) provides functionality that does something with a Webform Submission. The one that you want to select here, is the \"Strawberryfield harvester\" handler. Add it, name it whatever you like (or copy what you see in the screenshot) and make sure you select, if running using our deployment strategy, \"S3 File System\" as the option for \"Permanent Destination for uploaded files\". The wording is tricky there, its not really Permanent, since that is handled by Archipelago, but more to Temporary, while working in ingesting an Object, destination for the Webform. Its not really wrong neither. Its permanent for the Webform, but we have better plans for the files and metadata!</p> <p>Save your settings. And you are ready to roll. That webform can now be used as a Setting for any of the StrawberryField Widgets that use Webforms.</p> <p>Finally (the real finally). Archipelago encourages at least one Field/JSON key to be present always. One with \"type\" as key value. So make sure that your Custom Webform has that one.</p> <p>There are two ways of doing that:</p> <ul> <li> <p>You can copy how it is setup from the provided Webform's Elements, from the main Descriptive Metadata Webform and then add one \"select\" element to yours using the same \"type\" \"key\".Important in Archipelago is always the key value since that is what builds the JSON for your metadata. The Description can be any, but for UI consistency you could want to keep it the same across all your webforms.</p> <p></p> </li> <li> <p>Or, advanced, you can use the import/export capabilities (Webforms are just YAML files!) and export/copy your custom one as text, add the following element before or after some existing elements there</p> <pre><code> type:\n      '#type': select\n      '#title': 'Media Type'\n      '#options': schema_org_creative_works\n      '#required': true\n      '#label_attributes':\n        class:\n          - custom-form-input-heading\n</code></pre> <p>And then reimport.</p> <p>Having a \"type\" value will make your life easier. You don't need it, but everything works smoother that way.</p> <p>Since you have a single Content Type named Digital Object, having a Webform field that has as key \"type\", which leads to a \"type\" JSON key, allows you to discern the Nature of your Digital Object, book or Podcast, Image or 3D and do smart, nice things with them.</p> </li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"workingtwigs/","title":"Working with Twig in Archipelago","text":"<p>The following information can also be found in this Presentation from the \"Twig Templates and Archipelago\" Spring 2021 Workshop:</p> <ul> <li>Twig Templates and Archipelago</li> </ul>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#prerequisites-with-food-analogy","title":"Prerequisites (with food analogy)","text":"<ol> <li>Know your Data/Metadata. What do I have? <ul> <li>What do I have in my Fridge? Do I have Tofu? Do I have Peppermint ? One Bunch?</li> </ul> </li> <li>Know your final desired output Document: MODS, HTML, GEOJSON, etc. <ul> <li>What are you going to cook ? Do you have a picture of the Curry ? Have you ever had Curry ?</li> </ul> </li> <li>Know your Twig Basics<ul> <li>How to cut and dice , steam and saut\u00e9 </li> </ul> </li> <li>Do not be afraid<ul> <li>You can\u2019t get burned  here and Ingredients  do not expire!</li> </ul> </li> <li>Ask for help. Slack/Google Groups/Postcards  <ul> <li>Seeing others cook helps and also motivates. Others may share some spices. </li> </ul> </li> <li>Use and Share your findings!<ul> <li>Eat what you cook.  Share with friends and family. </li> </ul> </li> </ol> <p>Note</p> <p>All examples shown below are using the following JSON snipped from Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?].</p> Click to view image of the JSON snippet. <p></p> Click to view this snippet as JSON. <pre><code>{\n    \"type\": \"Photograph\",\n    \"label\": \"Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?]\",\n    \"owner\": \"New-York Historical Society, 170 Central Park West, New York, NY 10024, 212-873-3400.\",\n    \"rights\": \"This digital image may be used for educational or scholarly purposes without restriction. Commercial and other uses of the item are prohibited without prior written permission from the New-York Historical Society. For more information, please visit the New-York Historical Society's Rights and Reproductions Department web page at http:\\/\\/www.nyhistory.org\\/about\\/rights-reproductions\",\n    \"language\": [\n        \"English\"\n    ],\n    \"documents\": [],\n    \"publisher\": \"\",\n    \"ismemberof\": \"111\",\n    \"creator_lod\": [\n        {\n            \"name_uri\": \"\",\n            \"role_uri\": \"http:\\/\\/id.loc.gov\\/vocabulary\\/relators\\/pht\",\n            \"agent_type\": \"personal\",\n            \"name_label\": \"Stonebridge, George Ehler\",\n            \"role_label\": \"Photographer\"\n        }\n    ],\n    \"description\": \"George Ehler Stonebridge (d. 1941) was an amateur photographer who lived and worked in the Bronx, New York.\",\n    \"subject_loc\": [\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/subjects\\/sh85038796\",\n            \"label\": \"Dogs\"\n        }\n    ],\n    \"date_created\": \"1910-01-01\"\n}\n</code></pre>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#first-know-your-data","title":"First: Know Your Data","text":"<p>Understanding the basic structure of your JSON data. </p> <ol> <li> <p>Single JSON Value.</p> <p></p> <ul> <li>For <code>\"type\": \"Photograph\"</code><ul> <li>\"type\" = JSON Key or Property</li> <li>\"Photograph\" = Single JSON Value (string)</li> </ul> </li> </ul> </li> <li> <p>Multiple JSON Values (Array of Enumeration of Strings)</p> <p> - For <code>\"language\": [\"English\",\"Spanish\"]</code>     - \"language\" = JSON Key or Property     - \"[\"English\",\"Spanish\"]\" = Multiple JSON Values (Array of Enumeration of Strings)</p> </li> <li> <p>Multiple JSON Values (Array of Enumeration of Objects)</p> <p></p> <ul> <li>For <code>\"subject_loc\":[{\"uri\":\"http://..\",\"label\":\"Dogs\"},{\"uri\":\"http://..\",\"label\":\"Pets\"}]</code><ul> <li>\"subject_loc\" = JSON Key or Property</li> <li>[{\"uri\":\"http://..\",\"label\":\"Dogs\"},{\"uri\":\"http://..\",\"label\":\"Pets\"}] =<ul> <li>Object with two JSON Keys. Each one with a single Value</li> <li>Multiple JSON Values (Array of Enumeration of Objects)</li> </ul> </li> </ul> </li> </ul> </li> </ol>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#getting-started-with-the-twig-language-in-archipelago","title":"Getting Started with the Twig Language in Archipelago","text":"<ul> <li> <p>Data is known as Context in Twig Lingo.</p> </li> <li> <p>All your JSON Strawberryfield Metadata is accessible inside a Variable named data in your twig template. </p> </li> <li> <p>You can access the values by using <code>data DOT Property (attribute) Name</code>.</p> <ul> <li>In the Laddie the Dog example shown above (originally):<ul> <li><code>data.type</code> will contain \"Photograph\"</li> <li><code>data.language</code> will contain [ \"English\" ]</li> <li><code>data.language[0]</code> will contain \"English\" <ul> <li>0 means first entry in an Array or Enumeration</li> </ul> </li> <li><code>data.subject_loc</code> will contain [{ \"uri\":\"http://..\",\"label\": \"Dog\" }]</li> <li><code>data.subject_loc.uri</code> will contain <code>\"http://..\"</code></li> <li><code>data.subject_loc.label</code> will contain \"Dog\"</li> </ul> </li> </ul> </li> </ul> <p>Note</p> <p>You also have access to other info in your context <code>node</code>: such as<code>node.id</code> is the Drupal ID of your Current ADO; Also <code>is_front</code>, <code>language</code>, <code>is_admin</code>, <code>logged_in</code>; and more!</p>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#twig-statements-and-printing","title":"Twig Statements and Printing","text":"<p>Twig for Template Designers</p> <p>https://twig.symfony.com/doc/3.x/templates.html</p>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#simple-examples-using-printing-statements","title":"Simple examples using Printing Statements","text":"<p>Single JSON Value Example</p> Twig template<pre><code>Hello I am a {{ data.type }} and very happy to meet you\n</code></pre> Rendered output<pre><code>Hello I am a Photograph and very happy to meet you\n</code></pre> <p>Multiple JSON Values Example</p> Twig template<pre><code>Hello I was classified as \"{{ data.subject_loc[0].label }}\" and very happy to meet you\n</code></pre> Rendered output<pre><code>Hello I was classified as \"Dogs\" and very happy to meet you\n</code></pre>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#twig-statements-and-executing","title":"Twig Statements and Executing","text":"<p>If in Twig</p> <p>https://twig.symfony.com/doc/3.x/tags/if.html</p>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#rendered-output-based-upon-different-twig-conditionals-operators-tests-assignments-and-filters","title":"Rendered Output based upon different Twig <code>conditionals</code>, <code>operators</code>, <code>tests</code>, <code>assignments</code>, and <code>filters</code>","text":"<p>Conditionals, Operator, and Test Usage</p> Twig Template<pre><code>{% if data.subject_loc is defined %}\nHey I have a Subject Key\n{% else %}\nUps no Subject Key\n{% endif %}\n</code></pre> Rendered Output<pre><code>Hello I was classified as \"Dogs\" and very happy to meet you\n</code></pre> <ul> <li>if/else are conditionals</li> <li>is is an operator</li> <li>defined is a test </li> </ul> <p>Loop Usage</p> Twig Template<pre><code>{% for key, subject in data.subject_loc %}\n* Subject {{ subject.label }} found at position {{ key }}\n{% endfor %}\n</code></pre> Rendered Output<pre><code>* Subject Dogs found at position 0\n</code></pre> <ul> <li>for is a loop</li> <li>Inside the loop you have access to key, subject</li> </ul> <p>Assignment, Filter, and Loop Usage</p> Twig Template<pre><code>{% for subject in data.subject_loc %}\n{% set label_lowercase = subject.label|lower %}\nMy lower case Subject is {{ label_lowercase }}\n{% endfor %}\n</code></pre> Rendered Output<pre><code>`My lower case Subject is dogs`\n</code></pre> <ul> <li>set is an assignment </li> <li>| is a pipe, used after a value to apply a filter.</li> <li>lower is a filter</li> <li>Inside the loop you have have access to subject and label_lowercase</li> </ul> <p>Loop Scope</p> Twig Template<pre><code>{% for subject in data.subject_loc %}\n  {% set label_lowercase = subject.label|lower %}\nMy lower case Subject is {{ label_lowercase }}\n{% endfor %}\n{# \n The below won\u2019t display because it was assigned inside \n The For Loop\n#}\n{{ label_lowercase }}\n</code></pre> Rendered Output<pre><code>`My lower case Subject is dogs`\n</code></pre>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#full-examples-for-common-uses-cases","title":"Full Examples for Common Uses Cases:","text":"<p>Use Case #1</p> <p>I have multiple LoD Subjects and want to display them in my page as a clickable ordered list but I\u2019m a safe/careful person.</p> Twig Example for Use Case #1<pre><code>{% if data.subject_loc is iterable and data.subject_loc is not empty %}\n&lt;h2&gt;My Subjects&lt;/h2&gt;\n&lt;ul&gt;\n   {% for subject in data.subject_loc %}\n   &lt;li&gt;\n      &lt;a href=\"{{ subject.uri }}\" title=\"{{ subject.label|capitalize }}\" target=\"_blank\"&gt;\n      {{ subject.label }}\n      &lt;/a&gt;\n   &lt;/li&gt; \n   {% endfor %}\n&lt;/ul&gt;\n{% endif %}\n</code></pre> <p>Use Case #2</p> <p>I have sometimes a publication date. I want to show it in beautiful human readable language.</p> Twig Example for Use Case #2<pre><code>{% if data.date_published is not empty %}\n&lt;h2&gt;Date {{ data.label }} was published:&lt;/h2&gt;\n&lt;p&gt;\n{{ data.date_published|date(\"F jS \\\\o\\\\f Y \\\\a\\\\t g:ia\") }}\n&lt;/p&gt;\n{% endif %}\n</code></pre> <p>About <code>date</code></p> <ul> <li>date() is a function</li> <li>It uses a \u201cDate Format Pattern\u201d as argument.</li> </ul> <p>Use Case #3 (Full Curry)</p> <p>{# May 4th 2021 @dpino: I have sometimes a user provided creation date. I want to show it in beautiful human readable language but fallback to automatic date if absent. I also want in the last case to show it was either \u201ccreated\u201d or \u201cupdated\u201d. #}</p> <pre><code>    \"as:generator\": {\n        \"type\": \"Update\",\n        \"actor\": {\n            \"url\": \"https:\\/\\/archipelago.nyc\\/form\\/descriptive-metadata\",\n            \"name\": \"descriptive_metadata\",\n            \"type\": \"Service\"\n        },\n        \"endTime\": \"2021-03-17T13:24:01-04:00\",\n        \"summary\": \"Generator\",\n        \"@context\": \"https:\\/\\/www.w3.org\\/ns\\/activitystreams\"\n    }\n</code></pre> Twig Example for Use Case #3<pre><code>{% if data.date_created is not empty %}\n&lt;h2&gt;Date {{ data.label }} was created:&lt;/h2&gt;\n&lt;p&gt;\n  {{ data.date_created|date(\"F jS \\\\o\\\\f Y \\\\a\\\\t g:ia\") }}\n&lt;/p&gt;\n{% else %}\n&lt;h2&gt;Date {{ data.label }} was {{ attribute(data, 'as:generator').type|lower }}d  in this repository:&lt;/h2&gt;\n&lt;p&gt;\n  {{  attribute(data, 'as:generator').endTime|date(\"F jS \\\\o\\\\f Y \\\\a\\\\t g:ia\") }}\n&lt;/p&gt;\n{% endif %}\n</code></pre>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#a-recommended-workflow","title":"A Recommended Workflow","text":"<p>You want to create a New Metadata Display (HTML) or a new (XML) Schema based format?</p> <ol> <li>Get yourself an example document (Frame). If HTML copy the source. If XML copy the full XML. (Cmd+C or Ctrl+C)</li> <li>Create a new Metadata Display Entity. Copy the content (text) of your Frame into the Edit window. (Cmd+V or Ctrl+V)</li> <li>Select an existing (as complete as possible) ADO to use as preview, press Preview.</li> <li>Put your nice glasses  on. What do you see? What data in your Frame do you have in your ADO (data)?</li> <li>Start nimble. Select the <code>data.label</code> info and check where your Frame uses a Title or a Label. Remove that text (Cmd+X or Ctrl+X) and replace with a <code>{{ data.label }}</code>. Press Preview. Do you see your title?</li> <li>Keep doing 5, over and over. Leave complex values for the end. (e.g <code>data.subject_loc</code>)</li> <li>Document your changes. <code>{# I added this because .. #}</code></li> <li>Save.</li> </ol> <p>Once the Template is in place you can use it in a Formatter, as Endpoint, in your Search Results or just keep it around until you and the world are ready!</p>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#and-now-its-your-turn","title":"And now it's your turn!","text":"<p>We hope you found the information presented here to be helpful in getting started working with Twigs in Archipelago. Click here to return to the main Twigs in Archipelago documentation. Happy Twigging!</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"xdebug/","title":"Debugging PHP in Archipelago","text":"<p>This document describes how to enable Xdebug for local PHP development using the PHPStorm IDE and a docker container running the Archipelago <code>esmero-php:development</code> image. It involves interacting with the esmero/archipelago-docker-images repo and the esmero/archipelago-deployment repo.</p>"},{"location":"xdebug/#part-1-docker","title":"Part 1: Docker","text":"<ol> <li> <p>Run the following commands from your <code>/archipelago-deployment</code> directory:</p> <p><code>docker-compose down</code> \\ <code>docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d</code></p> <p>This version of <code>docker-compose up</code> uses an override file to modify our services. <code>docker-compose.dev.yml</code> we now have an extra PHP container called <code>esmero-php-debug</code>. </p> <p>To stop the containers in the future, run <code>docker-compose -f docker-compose.yml -f docker-compose.dev.yml down</code>.</p> <p>(To make these commands easier to remember, consider making bash aliases in your .bashrc file.) (If you are running your development on a Linux system, you may need to make a modification to your xdebug configuration file on the esmero-php-dev container. See appendix at the bottom of this page.)</p> <p>So we have reloaded the containers and now you are ready for Part 2.</p> </li> </ol>"},{"location":"xdebug/#part-2-phpstorm","title":"Part 2: PHPStorm","text":"<ol> <li> <p>In PHPStorm, open your <code>archipelago-deployment</code> project.</p> </li> <li> <p>Go to <code>Preferences &gt; Languages &amp; Frameworks &gt; PHP &gt; Debug</code> or <code>Settings &gt; PHP &gt; Servers</code>. In this window there is an Xdebug section. Use these settings:</p> <ul> <li>Debug port: <code>9003</code>. (do NOT use the default, 9000)</li> <li>Can accept external connections: yes, select checkbox</li> <li>(optional) Break at first line in PHP scripts: uncheck. If you leave this selected, you will have to manually step through a breakpoint from Drupal's main index.php file on every request, which is quite annoying. However, leaving this box checked can be useful for making sure the connection is working at first, before you have set any internal breakpoints.</li> </ul> <p>Your settings should look like this. Hit APPLY and OK.  </p> </li> <li> <p>Go to <code>Preferences &gt; Languages &amp; Frameworks &gt; PHP &gt; Servers</code>. We will create a new server here. Use these settings:</p> <ul> <li>Name: <code>docker-debug-server</code></li> <li>Host: <code>localhost</code></li> <li>Port: <code>8001</code></li> <li>Use path mappings: yes, select the checkbox</li> <li>Under project files, select the top-level <code>archipelago-deployment</code> directory in the <code>File/Directory</code> column.</li> <li>In the <code>Absolute path on the server</code> add <code>/var/www/html</code></li> </ul> <p>Hit APPLY and OK and close the window.</p> <p> </p> </li> <li> <p>Go to <code>Run &gt; Edit Configurations</code>. Hit the <code>+</code> Button to create a new PHP Remote Debug. Name whatever you want, I called mine <code>Archipelago</code>. Use these settings:</p> <ul> <li>Filter debug connection by IDE Key: yes, select the checkbox</li> <li>Server: select <code>docker-debug-server</code> from dropdown (we created this in step 3)</li> <li>IDE Key: <code>archipelago</code> (this matches the key set in our container)</li> </ul> <p> </p> </li> <li> <p>Note: If you try to validate your connection, it will fail. But that's ok.</p> </li> <li> <p>Validate your connection. With  <code>Run &gt; Edit Configurations</code> still open, you can hit the link that says \"Validate\". Use these settings in the following validation window:</p> <ul> <li>Path to create validation script <code>&lt;your local path&gt;/archipelago-deployment/web</code></li> <li>Url to validation script: <code>http://localhost:8001</code></li> </ul> <p>Hit VALIDATE. You should get a series of green check marks. If you get a warning about missing <code>php.ini</code> file, that is OK, our file has a different name in the container (<code>xdebug.ini</code>) and is still being read correctly.  </p> </li> </ol>"},{"location":"xdebug/#set-up-browser-integration","title":"Set up Browser Integration","text":"<ol> <li> <p>We have had success using the XDebug Helper extension in Chrome. Once you have the extension installed, right-click on the bug icon in the top right of your chrome browser window and select \"Options\" to configure the IDE key. Under \"IDE\", select \"Other\", and in the text box, enter \"archipelago\"</p> <p> </p> </li> </ol>"},{"location":"xdebug/#actually-debugging","title":"Actually Debugging!","text":"<ol> <li> <p>Hit the button (top right bar of PHPStorm) that looks like a telephone, for <code>Start Listening for PHP Debug Connections</code>.</p> <p></p> </li> <li> <p>Now, you can use <code>Run &gt; Debug</code> and select the <code>Archipelago</code> named configuration that we created in the previous steps. The debugging console will appear. It will say it is waiting for incoming connection from 'archipelago'  .</p> <p> </p> </li> <li> <p>Right now the debugging session is not enabled. Browse to  <code>localhost:8001</code>. Click on the gray XDebug Helper icon at the top right of your window and select the green \"Debug\" button. This will tell chrome to set the xdebug session key when you reload the page.</p> </li> <li> <p>Now set a breakpoint in your code, and refresh the page.  If you have breakpoints set, either manually, or from leaving \"Break at first line in PHP scripts\" checked, you should have output now in the debugger.</p> </li> <li> <p>If you are done actively debugging, it is best to click the green XDebug Helper icon and select \"Disable\". This will greatly improve speed and performance for your app in development. When you need to debug, just turn on debugging using the XDebug Helper button again.</p> </li> <li> <p>If you would like to see the output of your xdebug logs, run the following script:    <code>docker exec -ti esmero-php bash -c 'tail -f /tmp/xdebug.log &gt; /proc/1/fd/2'</code></p> </li> </ol> <p>Then, you can use the typical docker logs command on the <code>esmero-php</code> container, and you will see the xdebug output:    <code>docker logs esmero-php -f</code></p> <p>Xdebug makes accessing variables in Drupal kind of great. Many possibilities, including debugging for Twig templates. Happy debugging!</p>"},{"location":"xdebug/#appendix-xdebug-on-a-linux-host","title":"Appendix: XDebug on a linux host","text":"<p>If you are developing on a linux machine, you may need to make a change to the xdebug configuration file.</p> <ol> <li>Create a new file in the <code>/archipelago-deployment/xdebug</code> folder called <code>xdebug.ini</code> and enter the following text:     <pre><code>zend_extension=xdebug\n\n[xdebug]\nxdebug.mode=develop,debug\nxdebug.discover_client_host = 1\nxdebug.start_with_request=yes\n</code></pre></li> <li>Make a bind mount to this file in your docker-compose.dev.yml file:     <pre><code>  php-debug:\n  ...\n    volumes:\n  - ${PWD}:/var/www/html:cached\n    # Bind mount custom xdebug configuration file...\n  - ${PWD}/xdebug/xdebug.ini:/usr/local/etc/php/conf.d/docker-php-ext-xdebug.ini\n</code></pre></li> <li>Restart your docker containers using the method described at the top of this page.</li> </ol> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Archipelago Commons Intro","text":"<p>Archipelago Commons, or simply Archipelago, is an Open Source Digital Objects Repository / DAM Server Architecture based on the popular CMS <code>Drupal 9/10+</code> and released under <code>GLP V.3 License</code>. Archipelago is developed and supported at the Metropolitan New York Library Council (METRO).</p> <p>Archipelago is a mix of deeply integrated custom-coded Drupal modules (made with care by us, the Digital Services Team and METRO) and a curated and well-configured Drupal instance, running under a discrete and well-planned set of complementary additional service containers. You can learn more about the different Software Services used by Archipelago here, and Archipelago's unique approach to Metadata here.</p> <p>Archipelago's primary focus is to serve the greater <code>GLAM community</code> (libraries, archives, museums, universities and colleges, cultural heritage organizations) by providing a flexible, consistent, and unified way of describing, storing, linking, exposing metadata and media assets that make up rich repository collections all around our shared beautiful world. We respect identities and existing workflows, and we endeavor to design Archipelago in ways that empower communities of every size, shade, and shape.</p> <p>Finally, Archipelago tries to stay humble, slim, and nimble in nature with a small codebase full of inline comments and <code>@todos</code>. All of our work is driven by a clear and concise but thoughtful planned technical roadmap --updated in tandem with new releases.</p> <p>We recommend you start with the Core Documentation Guides listed here as you begin your Archipelago explorations. </p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p>"},{"location":"101_guides_list/","title":"Archipelago 101: Core Documentation Guides","text":"<p>Top 10 guides we recommend you review as you get started working with Archipelago:</p> <ol> <li> <p>Metadata in Archipelago: a long and worthwhile read that covers the fundamentals of Archipelago's architecture and approach to metadata and data</p> </li> <li> <p>Strawberryfield Formatters: overview of the general setup of an Archipelago Digital Object (ADO) page and the way your ADO JSON metadata and data are output</p> </li> <li> <p>Primer on Display Modes &amp; How to Create a Webform as an Input Method: deeper look at Display Modes and Form Modes, two ways you'll be interacting with your ADOs most frequently</p> </li> <li> <p>Twig Templates and Archipelago: a great place to dive into one of Archipelago's best loved feature areas</p> </li> <li> <p>Archipelago Multi Importer: all about Archipelago's batch ingest and update functionality</p> </li> <li> <p>Search and Solr Overview: for repositories, it's all about the search</p> <ul> <li>In-a-nutshell : JSON data to Strawberry Keyname Providers to Solr: essential overview of the pipeline from JSON data into and out of Solr</li> <li>Strawberry Key Name Providers, Solr Field, and Facet Configuration: fundamental information for site adminisrators</li> </ul> </li> <li> <p>Advanced Batch Find and Replace: targetted batch updates for your ADO metadata</p> </li> <li> <p>Strawberry Runners Post-Processing Configuration: background post-processing defaults and options for all your file transformation and data indexing needs</p> </li> <li> <p>Archipelago Local Deployment Guide: get your own local Archipelago up and running in about 15 minutes</p> </li> <li> <p>Archipelago Presentations, Events, and Additional Resources: features recordings and links to different Archipelago workshops, conference presentations, and other helpful references</p> </li> </ol> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p>","tags":["Archipelago 101","Documentation"]},{"location":"AMIviaSpreadsheets/","title":"Ingesting New Digital Objects and Collections using Spreadsheets or Google Sheets","text":"<p>Ingesting Only Digital Objects or Both Digital Objects and Collections uses similar processes, with a few key differences. Click here to jump to the Ingesting both Digital Objects and Collections and/or Creative Work Series (Compound) Objects section of this guide page.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#ingesting-only-new-digital-objects","title":"Ingesting Only New Digital Objects","text":"<p>From either the main Content page or the AMI Sets List page, select the 'Start an AMI set' button to begin.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-1-plugin-selection","title":"Step 1: Plugin Selection","text":"<p>Select the Plugin type you will be using from the dropdown menu.</p> <ul> <li>Google Sheets Importer</li> <li> <p>Spreadsheet Importer  (if using local CSV file)</p> <p></p> </li> </ul> <p>*The <code>Remote JSON API Importer</code> and additional remote import source options (for other repository systems) will be covered in separate tutorials following future releases.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-2-operation-and-spreadsheet-source-selection","title":"Step 2: Operation and Spreadsheet Source Selection","text":"<p>Select 'Create New ADOs' as the Operation you would like to perform.</p> <ul> <li> <p>If using Google Sheets Importer:</p> <ul> <li>Enter the ID of your Google Sheet</li> <li>Enter the Cell Range for your Google Sheet</li> </ul> <p></p> </li> <li> <p>If using Spreadsheet Importer:</p> <ul> <li>Under the 'Upload your file' section, select 'Choose File' to upload the CSV you will be using. After the file is finished uploading, you will have the option to 'Remove' and select a different file if needed.</li> </ul> <p> </p> </li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-3-data-transformation-selections","title":"Step 3: Data Transformation Selections","text":"<p>Select the data transformation approach--how your source data will be transformed into ADO (Archipelago Digital Object) Metadata.</p> <ul> <li> <p>You will have 3 options for your data transformation approach:</p> <ol> <li>Direct<ul> <li>Columns from your spreadsheet source will be cast directly to ADO metadata (JSON), without transformation/further processing (only intended for use with simple data strings or already JSON-encoded snippets/values).</li> </ul> </li> <li>Custom (Expert Mode)<ul> <li>Provides very granular custom data transformation and mapping options</li> <li>Needs to be used if importing Digital Objects and Digital Object Collections at the same time/from same spreadsheet source (see separate instructions below).</li> </ul> </li> <li>Template<ul> <li>Columns from your spreadsheet source will be cast to ADO metadata (JSON) using a Twig template setup for JSON output.</li> </ul> </li> </ol> </li> <li> <p>You will also need to Select which columns contain filenames, entities or URLS where files can be fetched from. Select what columns correspond to the Digital Object types found in your spreadsheet source.</p> </li> <li> <p>Lastly, for this step, you will need to select the destination Fields and Bundles for your New ADOs. If your spreadsheet source only contains Digital Objects, select <code>Strawberry (Descriptive Metadata source) for Digital Object</code></p> <p></p> </li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-4-global-ado-mappings","title":"Step 4: Global ADO Mappings","text":"<p>Select your global ADO mappings.</p> <ul> <li>Make sure to select the <code>ismemberof</code> collection membership relationship predicate column if applicable. For AMI source spreadsheets containing only non-Creative Work Series (Compound) Objects, only <code>ismemberof</code> can be mapped properly. To use <code>ispartof</code> relationship setup, please refer to the steps outlined in the separate section below.</li> </ul> Click to read more about Archipelago's default relationship mappings <ul> <li><code>ismemberof</code> and/or <code>ispartof</code> (and/or whatever predicate corresponds with the relationship you are mapping)</li> <li>these columns can be used to connect related objects using the object-to-object relationship that matches your needs</li> <li>in default Archipelago configurations, <code>ismemberof</code> is used for Collection Membership and <code>ispartof</code> is used for Parent-Child Object Relationships (so a Child ADO would reference the Parent ADO in <code>ispartof</code>)</li> <li>these columns can hold 3 types of values</li> <li>empty (no value)</li> <li>an integer to connect an object to another object's corresponding row in the same spreadsheet/CSV</li> <li>Ex: Row 2 corresponds to a Digital Object Collection; for a Digital Object corresponding to Row 3, the 'ismemberof' column contains a value of '2'. The Digital Object in Row 3 would be ingested as a member of the Digital Object Collection in Row 2.</li> <li>a UUID to connect with an already ingested object</li> </ul> <ul> <li>By default, the option to automatically assigns UUIDs is selected. Keep 'Automatically assign UUID' checked unless your source data already contains UUIDs in a <code>node_uuid</code> column.</li> <li> <p>Under the 'Base ADO mappings', select the <code>label</code> column for ADO Label. This selection is only used as a fail-safe (in case your AMI JSON Ingest Template does not have any mapping for a column to be mapped to the JSON <code>label</code> key, or your source data csv does not contain a <code>label</code> if going Direct for data transformation).</p> <p></p> </li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-5-zip-upload","title":"Step 5: ZIP upload","text":"<p>Provide an optional ZIP file containing your assets.</p> <ul> <li>You may choose to upload a ZIP file containing all or some of the corresponding files specified in your csv/spreadsheet.</li> <li> <p>The file upload size restrictions specified in your Archipelago instance will apply here (512MB maximum by default). </p> <p> </p> <ul> <li>Please note, when creating your ZIP file (in particular, within an OSX environment): only select the folders and files needed, not the top/enclosing folder they are in. </li> </ul> </li> </ul> Click to view screenshot of example ZIP file creation in OSX <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-6-ami-set-confirmation","title":"Step 6: AMI Set Confirmation","text":"<p>You will now see a message letting you know that 'Your source data was saved and is available as a CSV at <code>linktotheAMIgenerated.csv</code></p> <p>The message will also let you know that your New AMI Set was created and provide a link to the AMI Set page.</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-7-ami-set-processing","title":"Step 7: AMI Set Processing","text":"<p>Your newly created AMI Set will now need to be Processed.</p> <p>If you clicked on the 'see it here' link in Step 6, you will be brought to the AMI Set page for review. You may also select <code>Process</code> from the <code>Operations</code> menu for the AMI set from the main <code>AMI sets</code> page. From the <code>Process</code> page you can review the JSON configuration for your set (determined by your selections in the preceding steps).</p> Optional step to review the settings configured in your AMI Set <p>You may wish to double check the settings configured in your AMI Set in the Raw Metadata (JSON) on the AMI Set <code>View</code> tab before Processing.</p> <p></p> <p></p> <p>To Process this set, navigate to the <code>Process</code> tab. You will have multiple options related to the Processing outcome for your AMI Set.</p> <ul> <li>Skip ADO processing on missing File : If enabled a referenced missed file or one that can not be processed from the source, remote or local will make AMI skip the affected ROW. Enabled by default for better QA during processing. </li> <li>Desired ADOS Statuses After Process<ul> <li>The Statuses you have available will reflect the publication workflow/moderation states (such as Draft, Published, Archived/Unpublished) setup in your Archipelago instance, and the permissions associated your user account.</li> </ul> </li> <li>Please review the note about the 'remaining free space on your Drupal temporary filesystem. Please be aware of that before running a batch with large files'. If the amount of remaining free space you see does not seem sufficient for your AMI set processing needs, we recommend contacting your system administrator.  </li> <li> <p>Enqueuing and File Processing Options</p> <ul> <li>Enqueue but do not process Batch in realtime : Check this to enqueue but not trigger the interactive Batch processing. Cron or any other mechanism you have enabled will do the actual operation. This queue is shared by all AMI Sets in this repository and will be processed on a First-In First-Out basis.</li> <li>Force every File attached to an ADO to be processed in its own Queue item : Warning: This may make your ingest slower. Check this to force every file attached to an ADO to be downloaded and characterized as an independent process. This bypasses the Number of files Global setting that would otherwise trigger this behavior.</li> <li>Re download and reprocess every file : Check this to force every file attached to an ADO to be downloaded and characterized again, even if on a previous Batch run that data was already generated for reuse. IMPORTANT: Needed if e.g the URL of a file is the same but the remote source changed, if you have custom code that modifies the backend naming strategy of files.    </li> </ul> </li> <li> <p>Select <code>Confirm</code> to continue. </p> </li> </ul> <p>You will be returned to <code>AMI sets</code> page and see a brief confirmation message regarding the Enqueuing and Processing options you selected.</p> <p>If you chose to 'Confirm\" and Process your AMI Set immediately, proceed to Step 9: Processing and ADO Creation.</p> <p>If the chose to 'Enqueue' your AMI Set and the Queue operations for your Archipelago instance have been configured, you can simply leave your AMI Set in the Queue for Processing on the preconfigured schedule. Common timing for AMI Set Processes schedules are typically setup to run every three to six hours. Contact your Archipelago Administrators for details about your particular Archipelago's Processing schedule.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-8-queue-manager-push-may-be-restricted-to-administrator-users-only","title":"Step 8: Queue Manager Push (may be restricted to Administrator Users only)","text":"<p>If you chose to place your AMI set in the Queue to Process in step 7 and you wish to manually kickstart the Queue Processes, navigate to the Queue Manager found at <code>/admin/config/system/queue-ui</code>. (Be sure to select the <code>Queue Manager</code> under the System section, not the <code>Queue Manager for Hydroponic Service</code> under the Archipelago section).</p> <p></p> <p>To Process your AMI Set immediately from the Queue Manager page, select the checkbox next to the 'AMI Digital Object Ingester Queue Worker'. Keep the <code>Action</code> menu set to <code>Batch Process</code> and click the <code>Apply to selected items</code> button.</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-9-processing-and-ado-creation","title":"Step 9: Processing and ADO Creation","text":"<p>Your AMI set will now be Processed. You can follow the set's progress through the <code>Processing queues</code> loading screen.</p> <p></p> <p>After your AMI set is Processed, you will receive confirmation messages letting you know your Digital Objects were successfully created. </p> <p></p> <p>From this message, you can click on each ADO title to review the new created Digital Object (or Collection) if you wish. Or, you may proceed to step 10.  </p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-10-review-your-newly-created-digital-objects-directly-or-via-ami-set-report","title":"Step 10: Review your newly created Digital Objects directly or via AMI Set Report","text":"<ul> <li>Option 1: Return to the main Content page found at <code>/admin/content</code> and review your newly created Digital Objects. After ensuring that files and metadata elements were mapped correctly, you may choose to change the Status for your Digital Objects to 'Published'.</li> <li> <p>Option 2: Use the AMI Set Report</p> <ul> <li> <p>From the main <code>AMI sets</code> page, select <code>Report</code> from the <code>Operations</code> menu for the AMI set you wish to review. </p> </li> <li> <p>This Report will contain information related to the last Processing operation run against your AMI Set.</p> </li> <li>For each Digital Object or Collection row that was Processed, you will see:<ul> <li>a <code>datetime</code> stamp</li> <li>one of three <code>level</code> (INFO, WARNING, or ERRORS) applicability</li> <li>a <code>message</code> summarizing the Processing outcome--including a title/label link to the created ADO if successful</li> <li>a <code>details</code> summary containing system information related to the operations.</li> </ul> </li> </ul> <p></p> <ul> <li>You can use information found in the Reports tab to identify review your created ADOs one-by-one and identify any errors or issues that may have come up during the Process if needed.  </li> </ul> </li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#ingesting-both-new-digital-objects-and-collections-andor-creative-work-series-compound-objects-in-the-same-spreadsheet","title":"Ingesting Both New Digital Objects and Collections and/or Creative Work Series (Compound) Objects in the same spreadsheet","text":"<p>From either the main Content page or the AMI Sets List page, select the 'Start an AMI set' button to begin.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#steps-1-plugin-selection-step-2-operation-and-spreadsheet-source-selection","title":"Steps 1: Plugin Selection &amp; Step 2: Operation and Spreadsheet Source Selection","text":"<p>Follow the same instructions found above for Ingesting New Digital Objects.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-3-data-transformation-selections_1","title":"Step 3: Data Transformation Selections","text":"<p>To import Digital Objects and Digital Object Collections and/or Creative Work Series (Compound) Objects at the same time/from same spreadsheet source, you will need to select the <code>Custom (Expert Mode)</code> option for your data transformation approach.</p> <ul> <li>Custom (Expert Mode)<ul> <li>Provides very granular custom data transformation and mapping options</li> </ul> </li> </ul> <p>You will then need to 'Select your Custom Data Transformation and Mapping Options' for each of your Digital Object, Collection, and Creative Work Series (Compound) types.</p> <ul> <li> <p>For Collection and Creative Work Series (Compound) objects:</p> <ul> <li>Select either the Direct or Template (and corresponding JSON template) option for your data transformation approach.</li> <li>Select the destination Fields and Bundles for <code>Strawberry (Descriptive Metadata source) for Digital Object Collection</code><ul> <li>Use this same destination for your Creative Work Series (Compound) objects</li> </ul> </li> <li>You may also wish to Select which columns contain filenames, entities or URLS where files can be fetched from. For most Collection objects, you will likely leave unselected or choose <code>images</code> if you are uploading a thumbnail image for your Collection.</li> </ul> <p></p> </li> <li> <p>For each non-Creative Work Series (Compound) Digital Object type in your spreadsheet source:</p> <ul> <li>You will also need to select either the Direct or Template (and corresponding JSON template) option for your data transformation approach.</li> <li>Then Select the destination Fields and Bundles for <code>Strawberry (Descriptive Metadata source) for Digital Object</code></li> <li>Then select which columns contain filenames, entities or URLS where files can be fetched from. Select what columns correspond to the Digital Object types found in your spreadsheet source.</li> <li> <p>For example, for 'Map' type Digital Objects, you would select the following options (as depicted in this screenshot):</p> <p> </p> </li> </ul> </li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-4-global-ado-mappings_1","title":"Step 4: Global ADO Mappings","text":"<p>Select your global ADO mappings.</p> <ul> <li>Make sure to select the applicable relationship predicate columns (such as <code>ismemberof</code> and <code>ispartof</code>).</li> </ul> Click to read more about Archipelago's default relationship mappings <pre><code>- `ismemberof` and/or `ispartof` (and/or whatever predicate corresponds with the relationship you are mapping)\n- these columns can be used to connect related objects using the object-to-object relationship that matches your needs\n- in default Archipelago configurations, `ismemberof` is used for Collection Membership and `ispartof` is used for Parent-Child Object Relationships (so a Child ADO would reference the Parent ADO in `ispartof`)\n- these columns can hold 3 types of values\n    - empty (no value)\n    - an integer to connect an object to another object's corresponding row in the same spreadsheet/CSV\n      * Ex: Row 2 corresponds to a Digital Object Collection; for a Digital Object corresponding to Row 3, the 'ismemberof' column contains a value of '2'. The Digital Object in Row 3 would be ingested as a member of the Digital Object Collection in Row 2.\n    - a UUID to connect with an already ingested object\n</code></pre> <ul> <li>By default, the option to automatically assigns UUIDs is selected. Keep 'Automatically assign UUID' checked unless your source data already contains UUIDs in a <code>node_uuid</code> column.</li> <li>Under the 'Base ADO mappings', select the <code>label</code> column for ADO Label. This selection is only used as a fail-safe (in case your AMI JSON Ingest Template does not have any mapping for a column to be mapped to the JSON <code>label</code> key, or your source data csv does not contain a <code>label</code> if going Direct for data transformation).</li> <li>In order to make sure that Digital Objects containing the corresponding UUID or spreadsheet row number for any corresponding Collections, make sure <code>ismemberof</code> is also selected in the ADO Parent Columns. In order to make sure that Digital Objects containing the corresponding UUID or spreadsheet row number for any corresponding Parent ADOs (Creative Work Series/Compounds), make sure <code>ispartof</code> is also selected in the ADO Parent Columns.</li> <li> <p>Select the corresponding Columns for the Required ADO mappings.</p> <p></p> </li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"AMIviaSpreadsheets/#step-5-10","title":"Step 5-10:","text":"<p>Follow the same instructions found in Steps 5-10 above. As part of step 10, make sure your Digital Objects were ingested into the corresponding Collections you mapped them to in your spreadsheet source. Please note, you will need to Publish the Digital Objects before the Objects will appear in the Collection's View page (whether accessed as a logged-in Admin user or Anonymous/Public user). Celebrate your next AMI success with another fresh coffee, tea, or cookie!</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"CODE_OF_CONDUCT/","title":"Archipelago - code of conduct / anti-harassment policy","text":"<p>The Archipelago Commons community and the Metropolitan New York Library Council (METRO) are dedicated to providing a welcoming and positive experience for all participants, whether they are at a formal gathering, in a social setting, or taking part in activities online. This includes any forum, mailing list, wiki, web site, IRC channel, public meeting, conference, workshop/training or private correspondence. The Archipelago community welcomes participation from people all over the world, and these community members bring with them a wide variety of professional, personal and social backgrounds; whatever these may be, we treat colleagues with dignity and respect.</p> <p>This Code of Conduct governs how we behave in public or in private. We expect it to be honored by everyone who represents the project officially or informally, claims affiliation with the project, or participates directly.</p> <p>We ask that all community members adhere to the following expectations:</p> <ul> <li> <p>METRO and Archipelago have a zero-tolerance policy for verbal, physical, and sexual harassment. Anyone who is asked to stop a hostile or harassing behavior is expected to do so immediately. Here, for reference, are New York State\u2019s requirements.</p> <p>Harassment includes: Offensive verbal comments related to sex, gender, ethnicity, nationality, socioeconomic status, sexual orientation, disability, physical appearance, body size, age, race, religion; sexual or discriminatory images in public spaces; deliberate intimidation; stalking; harassing photography or recording; sustained disruption of talks or other events; inappropriate physical contact; and unwelcome sexual attention.</p> </li> <li> <p>Participation in discussions and activities should be respectful at all times. Please refrain from making inappropriate comments. Create opportunities for all people to speak, exercising tolerance of the perspectives and opinions of others. When we disagree, we do this in a polite and professional manner. We may not always agree. When frustrated, we back away and look for good intentions, not reasons to be more frustrated. When we see a flaw in a contribution, we offer guidance on how to fix it.</p> </li> <li>METRO and Archipelago honor the ability to be anonymous. If a person is going by their handle, a pseudonym, or doesn\u2019t wish to use their name, please respect their wishes and privacy. This also includes \u2018outing\u2019 someone\u2019s workplace, their age, their gender, their real name, etc, without their consent. We take people\u2019s privacy and their comfort very seriously.</li> </ul>"},{"location":"CODE_OF_CONDUCT/#protocol-for-conflict-resolution","title":"Protocol for Conflict Resolution","text":"<p>Participants in METRO and Archipelago communication channels violating this code of conduct may be sanctioned or expelled at the discretion of the organizers of the meeting (if the channel is an in-person event) or the Archipelago Advisory Board (if the channel is online).</p>"},{"location":"CODE_OF_CONDUCT/#initial-incident","title":"Initial Incident","text":"<p>If you are being harassed, notice that someone else is being harassed, or have any other concerns, and you feel comfortable speaking with the offender, please inform the offender that he/she/they has affected you negatively. Oftentimes, the offending behavior is unintentional, and the accidental offender and offended will resolve the incident by having that initial discussion. Participants asked to stop any harassing behavior are expected to comply immediately.</p>"},{"location":"CODE_OF_CONDUCT/#escalation","title":"Escalation","text":"<p>If the offender insists that they did not offend, if offender is actively harassing you, or if direct engagement is not a good option for you at this time, then you will need a third party to step in. To report any violation of the following code of conduct or if you have any questions or suggestions about this code of conduct, please contact archipelago-community@metro.org or fill out this form anonymously. This will be sent to leadership at METRO and the advisory board member currently acting as the Code of Conduct liaison. Our enforcement guidelines work in accordance with those published at the Contributor Covenant. </p> <p>Upon review, if METRO leadership and the Code of Conduct Liaison determine that the incident constitutes harassment they may take any action they deem appropriate, including warning the offender, expulsion from the meeting or other community channels, or contacting a higher authority such as a representative from the offender's institution.</p> <p>These policies draw from many other code of conduct documents, including but not limited to: code4lib, DLF, Islandora, ICG, Samvera, WikimediaNYC, and IDOCE</p>"},{"location":"I7solrImporter/","title":"Using the Islandora 7 Solr Importer","text":"<p>From either the main Content page or the AMI Sets List page, select the 'Start an AMI set' button to begin.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"I7solrImporter/#step-1-plugin-selection","title":"Step 1: Plugin Selection","text":"<p>Select the Islandora 7 Solr Importer from the dropdown menu.</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"I7solrImporter/#step-2-section-1-solr-server-configuration","title":"Step 2, section 1: Solr Server Configuration","text":"<p>You will only have the option to select 'Create New ADOs' as the Operation you would like to perform.</p> <p>For the Solr Server Configuration section, you will need to provide all of the following information:</p> <ul> <li>PID of the Islandora Collection Members you want to fetch (example: islandora:root)</li> <li>Host of your Solr Server (example: repositorydomain.org)</li> <li>Port (example: 8080)</li> <li>Path (example: /) <ul> <li>Note: if your Solr can be found at http://myrepo.com:8080/solr, then the path is always a single \"/\" (as in screenshot depiction below)</li> </ul> </li> <li>Type of Solr Deployment<ul> <li>Either Single Solr Server (most common) or Solr Cloud Ensemble</li> </ul> </li> <li>Core (example: islandora)</li> </ul> <p>You will also need to select the Starting Row you would like to begin fetching results from, and the Number of Rows to fetch. </p> <p>The Starting Row is an offset and defaults to 0, which is the most common (and recommended) approach. For the Total Number of Rows to Fetch, setting this to empty or null will automatically (refresh when selecting 'Next' button at bottom of page) prefill with the real Number Rows found by the Solr Query invoked. If you set this number higher than the actual results we will only fetch what can be fetched.</p> <p>For larger collections, you may wish to create multiple/split AMI ingest sets by selecting a specified number of rows. </p> <ul> <li>As an example, for a collection of 1500 objects that you wanted to split into three AMI ingests of 500 objects, you would specify the Starting Row as 0 for the first set and Number of Rows as 500. For the second set, your Starting Row would be Row 501; for the third set, 1001). In this example, the Number of Rows would always be 500.</li> </ul> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"I7solrImporter/#step-2-section-2-islandora-mappings","title":"Step 2, section 2: Islandora Mappings","text":"<p>In this step you will need to make determinations on how you would like to map your Islandora 7 digital objects to your Archipelago repository and whether or not you would like to fetch additional file datastreams, such as those for thumbnail images, transcripts, OCRs/HOCRs, etc.</p> <ul> <li> <p>Selecting \"Collapse Multi Children Objects\" will collapse Children Datastreams into a single ADO with many attached files (single row in the generated AMI set .csv file). Book Pages will be fetched but also the Top Level PDF (if one exists in your Islandora instance).</p> </li> <li> <p>In the Required ADO mappings, you will need to specify which Archipelago type you want to map each Islandora Content Model found in your source collection. </p> <ul> <li>For example, for info:fedora/islandora:sp_large_image_cmodel you may want to use Photograph.</li> </ul> </li> <li> <p>If you had left \"Collapse Multi Children Objects\" unselected, you will also need to specify the Islandora Content Model to ADO types mapping for possible Children.</p> </li> </ul> <p></p> <ul> <li> <p>You can also specify an ADO (Object or Collection) to be used as the Parent of Imported Objects. By selecting an existing ADO (Object or Collection) here using the autocomplete/search, the generated AMI set .csv file will contain an 'ismemberof' column containing the UUID of the selected ADO for every row.</p> </li> <li> <p>Under \"Additional Datastreams to Fetch\", you can select any number and/or combination of extra file datastreams to retrieve from your harvest. Please note that the I7 Importer will fetch every possible datastream that is present in your source I7 repository, but the additional file datastreams referenced may not be associated with actual files for every digital object.</p> </li> </ul> <p></p> <p>Language from form itself:</p> <p>Additional datastreams to fetch. OBJ datastream will always be fetched. Not all datastreams listed here might be present once your data is fetched.</p> <ul> <li>Selection of any additonal datastreams will generate a dropdown menu under \"Where extra datastreams should go.\" Here you will decide the organization and location of those datastreams by selecting one of the two options<ul> <li>Organize by mime type e.g TRANSCRIPT will go into the \"texts\" column</li> <li>Each in a separate column based on the datastream neame, TRANSCRIPT will go into the \"transcripts\" column</li> </ul> </li> </ul> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"I7solrImporter/#step-3-data-transformation-selections","title":"Step 3: Data Transformation Selections","text":"<p>Select the data transformation approach--how your source data will be transformed into ADO (Archipelago Digital Object) Metadata. As noted in the list below, 'Custom (Expert Mode)' is the recommended choice for AMI sets generated using the Islandora 7 Solr Importer plugin.</p> <ul> <li> <p>You will have 3 options for your data transformation approach:</p> <ol> <li>Direct<ul> <li>Columns from your spreadsheet source will be cast directly to ADO metadata (JSON), without transformation/further processing (only intended for use with simple data strings).</li> </ul> </li> <li>Custom (Expert Mode) Recommended choice for AMI sets generated using the Islandora 7 Solr Importer plugin<ul> <li>Provides very granular custom data transformation and mapping options</li> <li>Needs to be used if importing Digital Objects and Digital Object Collections at the same time/from same spreadsheet source (see separate instructions below).</li> </ul> </li> <li>Template<ul> <li>Columns from your spreadsheet source will be cast to ADO metadata (JSON) using a Twig template setup for JSON output.</li> </ul> </li> </ol> </li> <li> <p>You will also need to Select which columns contain filenames, entities or URLS where files can be fetched from. Select what columns correspond to the Digital Object types found in your spreadsheet source. If you fetched additional file datastreams during Step 2, you will see those columns listed here as well (see screenshot below for examples).</p> </li> <li> <p>Lastly, for this step, you will need to select the destination Fields and Bundles for your New ADOs. If your spreadsheet source only contains Digital Objects, select <code>Strawberry (Descriptive Metadata source) for Digital Object</code></p> <ul> <li>Select <code>Template</code> and use the AMI Ingest JSON template that corresponds with your metadata elements.</li> <li>Examples of potential file source columns include: <code>images</code>, <code>documents</code>, <code>audios</code>, <code>videos</code>, etc.</li> </ul> <p></p> </li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"I7solrImporter/#step-4-global-ado-mappings","title":"Step 4: Global ADO Mappings","text":"<p>Select your global ADO mappings.</p> <ul> <li>Even if empty (no values), select any relationship predicate columns (such as <code>ismemberof</code> and <code>ispartof</code>, if present) for ADO Parent columns.</li> <li>By default, the option to automatically assigns UUIDs is selected. Unchecking this option is not permitted when using the I7 Solr Importer.</li> <li> <p>Select the corresponding Columns for the Required ADO mappings.</p> <ul> <li>Select the <code>mods_titleinfo_title</code> column for ADO Label</li> </ul> <p></p> </li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"I7solrImporter/#step-5-zip-upload-and-ami-set-naming","title":"Step 5: ZIP upload and AMI Set naming","text":"<p>For standard Spreadsheet or Google Sheets AMI ingests, you would use this step to provide an optional ZIP file containing your assets. </p> <p>For your Islandora 7 Solr Importer process, the generated AMI set.csv file will contain the necessary URLs to the corresponding Islandora 7 file datastreams for each object as needed. Select next to skip this ZIP upload step and proceed. </p> <p>After you provide a title for your AMI set under \"Please Name your AMI set\", select \"Press to Create Set\"</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"I7solrImporter/#step-6-ami-set-confirmation","title":"Step 6: AMI Set Confirmation","text":"<p>You will now see a message letting your know your \"New AMI Set was created\". You will be able to review the generated .csv file directly from this page under Source Data File.</p> <p></p> <p>While you may immediately select \"Process\" from this AMI Set Confirmation page to use the Islandora 7 Importer generated .csv file as-is to ingest the ADOs in your AMI set, it is strongly recommended that you review the .csv file first. AMI is configured to trim unecessary (for Archipelago) and de-duplicate redundant Solr source data, but you may wish to pare down the sourced data even further and/or conduct general metadata review and cleanup before migrating your content. You will also likely want to make adjustments to your AMI Ingest JSON Template based on your review, depending on the variation of metadata columns/keys found in your source repostiory. </p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"I7solrImporter/#next-steps","title":"Next Steps","text":"<p>To proceed with Processing your AMI Set, click here to be directed to the main Ingesting Digital Objects via Spreadsheets.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"about/","title":"About this Documentation","text":"<p>This documentation was generated with Material for MkDocs. The repo/branch is at https://github.com/esmero/archipelago-documentation/tree/1.3.0, and the site is built using the following Github workflow: https://github.com/esmero/archipelago-documentation/blob/1.3.0/.github/workflows/ci.yml.</p>","tags":["Documentation","Contributing"]},{"location":"about/#contributing","title":"Contributing","text":"<ol> <li>First, please see this guide to set up the repo and branch locally and to create an issue and corresponding pull request.</li> <li>Install mkdocs-material and mike: <code>pip install mkdocs-material mike</code>.</li> <li>Make changes to local</li> <li>Run <code>mike delete --all &amp;&amp; mike deploy 1.0.0 default &amp;&amp; mike set-default 1.0.0 &amp;&amp; mike serve</code> to see and test changes.</li> <li>Follow the above guide to contribute the changes back.</li> </ol>","tags":["Documentation","Contributing"]},{"location":"acknowledgments/","title":"Caring &amp; Coding + Fixing","text":"<ul> <li>Diego Pino</li> <li>Giancarlo Birello</li> <li>Allison Sherrick</li> </ul>"},{"location":"acknowledgments/#acknowledgments","title":"Acknowledgments","text":"<p>Archipelago Commons is an Open Source repository software that is developed and supported at the Metropolitan New York Library Council (METRO).</p>"},{"location":"acknowledgments/#license","title":"License","text":"<p>GPLv3</p>"},{"location":"ami_index/","title":"Archipelago Multi-Importer (AMI)","text":"<p>Archipelago Multi-Importer (AMI) is a module for batch/bulk/mass ingests of Archipelago digital objects (ADOs) and collections. AMI also enables you to perform batch administrative actions, such as updating, patching/revising, or deleting digital objects and collections. AMI's Solr Importer plugin can be used to create AMI ingests and migrating content from existing Solr-sourcable digital repositories (such as Islandora 7).</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#ami-overview-and-under-the-hood-explanations","title":"AMI Overview and Under-the-Hood Explanations","text":"<p>From the desk of Diego Pino</p> <p>AMI provides Tabulated data ingest for ADOs with customizable input plugins. Each Spreadsheet (or Google Spreadsheet) goes through a Configuration Multi-step setup and generates at the end an AMI Set. AMI Sets then can be enqueued or directly ingested, its generated Objects purged and reingested again, its source data (generated and enriched with UUIDS) CSV replaced, improved and uploaded again and ingested.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#learn-more-about-metadata-in-archipelago-and-ami","title":"Learn More about Metadata in Archipelago and AMI","text":"<p>Please review the Metadata in Archipelago overview to learn about Archipelago's unique approach to metadata and how this applies in the context of AMI set adminstration.</p> Click to read the full AMI 0.4.0 (Archipelago - 1.0.0) Pre-Release Notes.","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#setup-steps","title":"Setup Steps","text":"<p>AMI has Ingest, Update and Patch capabilities. AMI has a plugin system to fetch data. The data can come from multiple sources and right now CSV/EXCEL or Google Spreadsheets are the ones enabled. It does parent/children validation, makes sure that parents are ingested first, cleans broken relationships, allows arbitrary multi relations to be generated in a single ROW (ismemberof, partOf, etc)  pointing to other rows or existing ADOs (via UUIDs) and can process rows directly as JSON or preprocessed via a Metadata Display entity (twig template) capable of producing JSON output. These templates can be configured by \u201ctype\u201d, Articles v/s 3DModel can have different ones. Even which columns contain Files can be configured at that level.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#ami-set-entity","title":"AMI Set Entity","text":"<p>Ami Sets are special custom entities that hold an Ingest Strategy generated via the previous Setup steps (as JSON with all it's settings), a CSV with data imported from the original source (with UUIDs prepopulated if they were not provided by the user). These AMI sets are simpler and faster than \u201cbatch sets\u201d because they do not have a single entry per Object to be ingested. All data lives in a CSV. This means the CSV of an AMI set can be corrected and reuploaded. Users can then Process a Set either putting the to be ingested ADOs in the queue and let Hydroponics Service do the rest or directly via Batch on the UI. ADOs generated by a set can also be purged from there. These sets can also be created manually if needed of any of the chosen settings modified anytime. Which AMI set generated the Ingest is also tracked in a newly created ADO\u2019s JSON and any other extra data (or fixed data e.g common Rights statements, or LoD) can be provided by a Twig Template. Ingest is amazingly fast. We monitored Ingest with Remote URL(islandora Datastreams) files of 15Mbytes average at a speed of 2 seconds per Object (including all post processing) continuously for a set of 100+.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#search-and-replace","title":"Search and Replace","text":"<p>This module also provides a simple search/replace text VBO action (handles JSON as text) and a full blown JSONPATCH VBO action to batch modify ADOs. The last one is extremely powerful permitting multiple operations at the same time with tests. E.g replace a certain value, add another value, remove another value only if a certain test (e.g \u201ctype\u201d:\u201dArticle\u201d and \u201cdate_of_digital\u201d: \u201c2020-09-09\u201d) matches. If any tests fail the whole operation will be canceled for that ADO. An incomplete \u201cWebform\u201d VBO action is present but not fully functional yet. This one allows you to choose a Webform, a certain element inside that Webform and then find and replace using the same Interface you would see while editing/adding a new ADO via the web form workflow.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#getting-started-with-ami","title":"Getting started with AMI","text":"<p>You can access AMI through the <code>AMI Sets</code> tab on the main Content page found at <code>/admin/content</code> or directly at <code>/amiset/list</code>.</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#google-sheets-api-configuration","title":"Google Sheets API Configuration","text":"<p>If you plan on using the Google Sheets Importer option, you will need to Configure the Google Sheets API.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#example-spreadsheetcsv","title":"Example Spreadsheet/CSV","text":"<p>Please refer to or use a fresh/new copy of the Demo Archipelago Digital Objects (ADOs) spreadsheet to import a small set of Digital Objects, using the same assets part of the One-Step Demo content ingest guide.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_index/#example-json-template","title":"Example JSON template","text":"<p>This JSON template can be used during the Data Transformation (step 3) of your AMI Import. This particular template corresponds with the metadata elements found in the Default Descriptive Metadata and Default Digital Object Collection webforms shipped with Archipelago 1.0.0.</p> Click to view the example 1.0.0 AMI JSON template <p>To use this template, copy and paste the JSON below directly into a new Metadata Display, found here for a local <code>http://localhost:8001/metadatadisplay/list</code> or <code>http://yoursite.org/metadatadisplay/list</code>. Select <code>JSON</code> as the 'Primary mime type this Twig Template entity will generate as output' for this new Metadata Display.</p> <pre><code>  {\n      \"type\": {{ data.type|json_encode|raw }},\n      \"label\": {{ data.label|json_encode|raw }},\n      \"issue_number\": {{ data.issue_number|json_encode|raw }},\n      \"interviewee\": {{ data.interviewee|json_encode|raw }},\n      \"interviewer\": {{ data.interviewer|json_encode|raw }},\n      \"duration\": {{ data.duration|json_encode|raw }},\n      \"website_url\": {{ data.website_url|json_encode|raw }},\n      \"description\": {{ data.description|json_encode|raw }},\n      \"date_created\": {{ data.date_created|json_encode|raw }},\n      \"date_created_edtf\": {{ data.date_created_edtf|json_encode|raw }},\n      \"date_created_free\": {{ data.date_created_free|json_encode|raw }},\n      \"creator\": {{ data.creator|json_encode|raw }},\n      \"creator_lod\": {{ data.creator_lod|json_encode|raw }},\n      \"publisher\": {{ data.publisher|json_encode|raw }},\n      \"language\": {{ data.language|json_encode|raw }},\n      \"ismemberof\": [],\n      \"ispartof\": [],\n      \"sequence_id\": {{ data.sequence_id|json_encode|raw }},  \n      \"owner\": {{ data.owner|json_encode|raw }},\n      \"local_identifier\": {{ data.local_identifier|json_encode|raw }},\n      \"related_item_host_title_info_title\": {{ data.related_item_host_title_info_title|json_encode|raw }},\n      \"related_item_host_display_label\": {{ data.related_item_host_display_label|json_encode|raw }},\n      \"related_item_host_type_of_resource\": {{ data.related_item_host_type_of_resource|json_encode|raw }},\n      \"related_item_host_local_identifier\": {{ data.related_item_host_local_identifier|json_encode|raw }},\n      \"related_item_note\": {{ data.related_item_note|json_encode|raw }},\n      \"related_item_host_location_url\": {{ data.related_item_host_location_url|json_encode|raw }},\n      \"note\": {{ data.note|json_encode|raw }},\n      \"physical_description_note_condition\": {{ data.physical_description_note_condition|json_encode|raw }},\n      \"note_publishinginfo\": {{ data.note_publishinginfo|json_encode|raw }},\n      \"physical_location\": {{ data.physical_location|json_encode|raw }},\n      \"physical_description_extent\": {{ data.physical_description_extent|json_encode|raw }},\n      \"date_published\": {{ data.date_published|json_encode|raw }},\n      \"date_embargo_lift\": {{ data.date_embargo_lift|json_encode|raw }},\n      \"rights_statements\": {{ data.rights_statements|json_encode|raw }},\n      \"rights\": {{ data.rights|json_encode|raw }},\n      \"subject_loc\": {{ data.subject_loc|json_encode|raw }},\n      \"subject_lcnaf_personal_names\": {{ data.subject_lcnaf_personal_names|json_encode|raw }},\n      \"subject_lcnaf_corporate_names\": {{ data.subject_lcnaf_corporate_names|json_encode|raw }},\n      \"subject_lcnaf_geographic_names\": {{ data.subject_lcnaf_geographic_names|json_encode|raw }},\n      \"subject_lcgft_terms\": {{ data.subject_lcgft_terms|json_encode|raw }},\n      \"subject_wikidata\": {{ data.subject_wikidata|json_encode|raw }},\n      \"edm_agent\": {{ data.edm_agent|json_encode|raw }},\n      \"term_aat_getty\": {{ data.term_aat_getty|json_encode|raw }},\n      \"viaf\": {{ data.viaf|json_encode|raw }},\n      \"pubmed_mesh\": {{ data.pubmed_mesh|json_encode|raw }},\n      \"europeana_concepts\": {{ data.europeana_concepts|json_encode|raw }},\n      \"europeana_agents\": {{ data.europeana_agents|json_encode|raw }},\n          \"europeana_places\": {{ data.europeana_places|json_encode|raw }},\n      \"geographic_location\": {{ data.geographic_location|json_encode|raw }},\n      \"subjects_local_personal_names\": {{ data.subjects_local_personal_names|json_encode|raw }},\n      \"subjects_local\": {{ data.subjects_locals|json_encode|raw }},\n      \"audios\": [],\n      \"images\": [],\n      \"models\": [],\n      \"videos\": [],\n      \"documents\": [],\n      \"as:generator\": {\n          \"type\": \"Create\",\n          \"actor\": {\n              \"url\": {{ setURL|json_encode|raw }},\n              \"name\": \"ami\",\n              \"type\": \"Service\"\n          },\n          \"endTime\": \"{{\"now\"|date(\"c\")}}\",\n          \"summary\": \"Generator\",\n          \"@context\": \"https:\\/\\/www.w3.org\\/ns\\/activitystreams\"\n      },\n      \"upload_associated_warcs\": []\n  }\n</code></pre> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/","title":"Using AMI's Linked Data Reconciliation","text":"<p>Archipelago Multi Importer (AMI)'s Linked Data Reconciliation tool can be used to enrich your metadata with Linked Data (LoD). Using this tool, you can map values from your topical/subject metadata elements to your preferred LoD vocabulary source. These mappings can then be transformed via a corresponding Metadata Display (Twig) template to process the values into JSON-formatted metadata for your specified AMI set.</p> <p>The aim of this tool is to automize as much of the reconciliation process as feasible within Archipelago. Please be aware that data reconciliation will still be in part a manual and potentially time intensive process.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/#important-note-preliminary-pre-requisite-ami-set-configuration","title":"Important Note: Preliminary / Pre-requisite AMI Set Configuration","text":"<p>In order to Reconciliate an AMI Set, you will need to have selected the 'Template' or 'Custom' data transformation approach (then also, via 'Template' for your Digital Object or Collection types) during Step 3 : Data Transformation of your AMI Set configuration.</p> <p>Your source spreadsheet will also need to contain at least one column containing terms/names (values) you want to reconcile against an LoD Authority Source. Multiple values should be separated by '|@|'.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/#step-1-select-the-ami-set-you-will-be-working-with","title":"Step 1: Select the AMI Set you will be working with.","text":"<p>From the main AMI Sets List page, click on your AMI Set's Name, or select the 'Edit' option from the Operations menu on the right-hand side of the Sets list.</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/#step-2-reconcile-lod-tab","title":"Step 2: Reconcile LoD Tab","text":"<p>Navigate to the Reconcile LoD tab.</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/#step-3-lod-reconciling-selections","title":"Step 3: LoD Reconciling Selections","text":"<p>From the list of columns from your spreadsheet source, select which columns you want to reconcile against LoD providers. </p> <p>Under the LoD Sources section, select how your chosen Columns will be LoD reconciled. - LoD reconcile options will be on the left, LoD Authority Sources will be on the right. - Example: 'local_subjects' will be mapped to 'LoC subjects (LCSH)'</p> <p></p> Full list of potential LoD Authority Sources <ul> <li>LoC subjects(LCSH)</li> <li>LoC Name Authority File (LCNAF)</li> <li>LoC Genre/Form Terms (LCGFT)</li> <li>LoC Thesaurus of Graphic Materials (TGN)</li> <li>LoC MARC List for Geographic Areas</li> <li>LoC Relators Vocabulary (Roles)</li> <li>LoC MADS RDF by type:</li> <li>Corporate Name</li> <li>Personal Name</li> <li>Family Name</li> <li>Topic</li> <li>Genre Form</li> <li>Geographic</li> <li>Temporal</li> <li>Extraterrestrial Area</li> <li>VIAF</li> <li>Getty aat Fuzzy / Terms / Exact Label Match</li> <li>Wikidata Q Items</li> </ul> <p>To preview the values contained in the column(s) you selected, click the 'Inspect cleaned/split up column values' button. </p> <p></p> <p>Tip: This preview step provides you with the opportunity to return to your AMI Set source CSV and make any necessary label/term corrections such as outliers and formatting errors before processing. This can be done multiple times until your source set is fully prepared. If using this workflow, you will tick the 'Re-process only adding new terms/LoD Authority Sources' processing option after replacing your updated source CSV (see screenshot below)</p> <p>When ready, there are multiple processing options to select from depending on your current need/workflow.  - To process immediately, select 'Process LoD from Source'  - To enqueue the batch process, select 'Enqueue but do not process Batch in real time.  - To add new data (i.e. terms, LoD Authority Sources) to existing reconciliation (e.g after replacing     source CSV data), select 'Re-process only adding new terms/LoD Authority Sources</p> <p></p> <p>Important note: if you have previously run LoD Reconciliation for your AMI set, this action will overwrite any manually corrected LoD on your Processed CSV. Please make sure you have a backup if unsure.</p> <p>Depending on the size of your AMI Set, the Reconciliation processing may take a few minutes. </p> <p></p> <p>When the process is finished, you will see a brief confirmation message.</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/#step-4-edit-reconciled-lod","title":"Step 4: Edit Reconciled LoD","text":"<p>Open the 'Edit Reconciled LoD' tab.</p> <p>You will see a table (form) containing: - Your Original term values (labels) - The CSV Column Header/Key from the source spreadsheet where the value is found - A Checked option you can use to denote that an LoD mapping has been reviewed/revisioned - The Linked Data Label and URL pairing selected during the LoD reconciliation process</p> <p></p> <p>The results table will show 10 original terms and mappings per page. You can advance through the pages using the page numbers and navigational arrows above and below the table.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/#step-5-review-and-edit-your-reconciled-lod-mappings","title":"Step 5: Review and Edit your Reconciled LoD Mappings","text":"<p>Review the LoD reconciliation mappings, to make sure the best terms were selected for your metadata.</p> <ul> <li>To revise and select a different term mapping, begin by typing in the 'Label' box in the corresponding LoD lookup element. (You can type directly over an incorrect term or within an empty cell if no value was mapped/identified.)</li> <li>Select your preferred term and URL pairing from the list.</li> <li>You can also add or remove multiple mappings using the +/- buttons beside the LoD lookup element.</li> <li>If desired, click the Checked option to mark that the term was reviewed/revisioned.</li> </ul> <p>As you advance through your review process, it is recommended that you use the 'Save Current LoD Page' at the bottom of each results page as you work. This will preserve the corrections you may have made and update the LoD Reconciled data for your AMI Set within the editing form.</p> <p></p> <p>When you have finished editing/reviewing your data, you must select 'Save all LoD back to CSV File' or else your LoD selections will not be preserved.</p> <p></p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/#step-6-ami-set-review-and-twig-metadata-display-preparation","title":"Step 6: AMI Set Review and Twig (Metadata Display) Preparation","text":"<p>You will now need to make sure that the Metadata Display (Twig) Template you selected to use during your initial AMI Set configuration is setup to Process your LoD mapped Label and URL selections into your Digital Objects and Collections JSON metadata.</p> <p>For every JSON key/element in your metadata that you need to process the LoD Reconciled data into, you need to specify in your Template that data for this element will be read from the 'Processed Data' LoD information. </p> <p>In the following example Twig snippet, the \"subject_loc\" JSON key will map corresponding values from the 'Processed Data' (data.lod) LoD information into a newly created Digital Object/Collection during the AMI Set Processing.</p> <pre><code>\"subject_loc\": {{ data_lod.mods_subject_topic.loc_subjects_thing|json_encode|raw }},\n</code></pre> <ul> <li>\"subject_loc\" = destination JSON Key or Property for your LoD values</li> <li>data.lod = directs the Twig template to source from the Processed Data LoD information (instead of the original AMI Source CSV accessed via 'data.xx_property_name')</li> <li>mods.subject.topic = the Column header in the original AMI Source CSV</li> <li>loc_subjects_thing = the Column containing the Label and URI/L pairs in the Processed Data LoD editable Table/Form (and reference CSV)</li> </ul> <p>The same general pattern can be adapted to apply to different mapping scenarios (original CSV source columns to Reconciled LoD Sources) as needed.</p> Full list of Column Options =&gt; Corresponding LoD Sources <ul> <li>'loc_subjects_thing' =&gt; LoC subjects(LCSH)</li> <li>'loc_names_thing' =&gt; LoC Name Authority File (LCNAF)</li> <li>'loc_genreForms_thing' =&gt; LoC Genre/Form Terms (LCGFT)</li> <li>'loc_graphicMaterials_thing' =&gt; LoC Thesaurus of Graphic Materials (TGN)</li> <li>'loc_geographicAreas_thing' =&gt; LoC MARC List for Geographic Areas</li> <li>'loc_relators_thing' =&gt; LoC Relators Vocabulary (Roles)</li> <li>'loc_rdftype_CorporateName' =&gt; LoC MADS RDF by type: Corporate Name</li> <li>'loc_rdftype_PersonalName' =&gt; LoC MADS RDF by type: Personal Name</li> <li>'loc_rdftype_FamilyName' =&gt; LoC MADS RDF by type: Family Name</li> <li>'loc_rdftype_Topic' =&gt; LoC MADS RDF by type: Topic</li> <li>'loc_rdftype_GenreForm' =&gt;  LoC MADS RDF by type: Genre Form</li> <li>'loc_rdftype_Geographic' =&gt; LoC MADS RDF by type: Geographic</li> <li>'loc_rdftype_Temporal' =&gt;  LoC MADS RDF by type: Temporal</li> <li>'loc_rdftype_ExtraterrestrialArea' =&gt; LoC MADS RDF by type: Extraterrestrial Area </li> <li>'viaf_subjects_thing' =&gt; VIAF</li> <li>'getty_aat_fuzzy' =&gt; Getty aat Fuzzy</li> <li>'getty_aat_terms' =&gt; Getty aat Terms</li> <li>'getty_aat_exact' =&gt; Getty aat Exact Label Match</li> <li>'wikidata_subjects_thing' =&gt; Wikidata Q Items</li> </ul>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_lod_rec/#next-steps","title":"Next Steps","text":"<p>To proceed with Processing your AMI Set, click here to be directed to the main Ingesting Digital Objects via Spreadsheets.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_spreadsheet_overview/","title":"Spreadsheet Formatting Overview","text":"","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_spreadsheet_overview/#spreadsheet-formatting-overview","title":"Spreadsheet Formatting Overview","text":"<p>There are multiple ways a spreadsheet/CSV file can be structured to work with AMI, depending on the data transformation and mapping you will be using.</p> <ul> <li>For most standard AMI ingests, each Row of your spreadsheet/CSV will correspond to a single Digital Object, Creative Work Series (Compound Object Parent) or Collection.</li> <li> <p>Columns in your spreadsheet/CSV can be mapped to different data (files) and metadata elements (label, description, subjects, etc.).</p> </li> <li> <p>It is recommended that different types of files are placed into separate columns--\"images\", \"documents\", \"models\", \"videos\", \"audios\", \"texts\".</p> <ul> <li>Filepaths can point to remote files, to existing files within your docker container, s3 (or other storage type/location that is accessible to Archipelago), and to paths within zip files.<ul> <li>Example path for existing file within docker container:     <code>/var/www/html/d8content/myAMIimage.jpg</code></li> <li>Example s3 path:     <code>s3://myAMIuploads/myAMIdocument.pdf</code></li> <li>Example remote filepath:     <code>https://dogsaregreat.edu/dogs.tiff</code></li> </ul> </li> <li>Multiple files (of the same type) can be placed in a single cell, separated by a semicolon ( ; ).</li> <li>For Digital Objects comprised of multiple types of files, such as an Oral History Interview with an audio file and a PDF transcript file, you can place different file types within different corresponding columns for the same Row.</li> <li>It is recommended that filepaths are copied/stored as plain (non-hyperlinked) formatted text.</li> </ul> </li> </ul> <p>Caution with using Quotation Marks</p> <p>Archipelago is very JSON-forward (friendly!) and makes assumptions about JSON encoding coming from CSV documents. If you are planning to start and end values with quotation marks in your CSV file, such as for <code>label</code> values, you need to enter the values encapsulated between the UTF-8 (unicode) character for straight quotations--in the CSV cell (corresponding row + column).  For example, in order for the ingested object to have the label (title) value processed as \"Strawberry-Bed\" for the ingested Archipelago Digital Object you need to enter the data as \"\\u0022Strawberry-Bed\\u0022\" in the corresponding <code>label</code> cell in your AMI Set CSV.</p> <ul> <li> <p>Every spreadsheet/CSV file should contain the following Columns:</p> <ul> <li><code>type</code><ul> <li>the Digital Object or Digital Object Collection Type, such as 'Photograph' or 'Collection'</li> </ul> </li> <li><code>label</code><ul> <li>the title of the Digital Object or Collection</li> </ul> </li> <li>Soft-requirement <code>node_uuid</code><ul> <li>this can be empty</li> <li>if empty, Archipelago will automatically generate UUIDs</li> <li>can be used with existing UUIDs during migrations</li> </ul> </li> <li>Soft-requirement <code>sequence_id</code> for Creative Work Series (compound) children objects<ul> <li>this is used to determine the sequence order for children objects within a Creative Work Series (compound) object</li> <li>should be an integer only (ie, '1' and not 'Page 1')</li> <li>if not present, the objects will present in the original ingest order</li> <li>we strongly recommend mapping the correct sequence using 'sequence_id'</li> </ul> </li> </ul> </li> <li> <p>Recommended Columns:</p> <ul> <li>Files as defined above<ul> <li>\"images\", \"documents\", \"models\", \"videos\", \"audios\", \"text\"</li> <li>.warc/.wacz files should be placed in a column \"upload_associated_warcs\"</li> </ul> </li> <li><code>ismemberof</code> and/or <code>ispartof</code> (and/or whatever predicate corresponds with the relationship you are mapping)<ul> <li>these columns can be used to connect related objects using the object-to-object relationship that matches your needs</li> <li>in default Archipelago configurations, <code>ismemberof</code> is used for Collection Membership and <code>ispartof</code> is used for Parent-Child Object Relationships (so a Child ADO would reference the Parent ADO in <code>ispartof</code>)</li> <li>these columns can hold 3 types of values<ul> <li>empty (no value)</li> <li>an integer to connect an object to another object's corresponding row in the same spreadsheet/CSV</li> <li>Ex: Row 2 corresponds to a Digital Object Collection; for a Digital Object corresponding to Row 3, the 'ismemberof' column contains a value of '2'. The Digital Object in Row 3 would be ingested as a member of the Digital Object Collection in Row 2.</li> <li>a UUID to connect with an already ingested object</li> </ul> </li> </ul> </li> <li>Metadata - for all the rich, detailed information associated with your Digital Objects and Collections<ul> <li>Every Column header will become a JSON Key and each cell a JSON value for that Key</li> <li> <p>You can use direct JSON snippets such as:</p> <p><pre><code>[{\"uri\": \"http://id.loc.gov/authorities/subjects/sh95008857\",\"label\": \"Digital libraries\"}]\n</code></pre>         - If you have an advanced twig template with the necessary logic, you can place data in cells that can be parsed and structured in various ways (such as multiple values separated by semicolons split accordingly, capitalization of values based on defined patterns, etc.)</p> </li> </ul> </li> </ul> </li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["AMI","Archipelago Multi Importer"]},{"location":"ami_update/","title":"Using AMI's Update Operations","text":"<p>Archipelago Multi Importer (AMI)'s Update Operations can be used to Update, Replace, or Append metadata values or files for existing Digital Objects and Collections found in your Archipelago. You can prepare and use AMI Update Sets in different ways using one of three functional operations, depending on your update needs. This guide will provide a general overview of the three main functions and how each operation may be useful.</p>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"ami_update/#important-notes-preliminary-pre-requisites","title":"Important Notes: Preliminary / Pre-requisites","text":"<p>You need to have existing Digital Objects or Collections (ADOs) in your Archipelago to work with. You should have a prepared AMI Update Set CSV that contains at least the following columns/headers:</p> <ul> <li>node_uuid<ul> <li>required</li> <li>should contain the values for any existing ADOs you wish to update</li> <li>needs to be a 1:1 match; AMI Update functionality cannot be used to update/change node_uuid values</li> </ul> </li> <li>label<ul> <li>required</li> <li>contains the corresponding label (title) for your existing ADOs</li> <li>can contain modifications for the label (title) values</li> </ul> </li> <li>type<ul> <li>optional, only required if using the Custom (Expert Mode) approach for your AMI set configuration or targeting changes for 'type' mappings for your ADOs</li> <li>contains the corresponding type values for your existing ADOs</li> <li>can contain modifications for the type values  </li> <li>related to this, AMI Update functionality cannot be used to update/change the underlying Drupal bundle mappings for already-ingested objects. In other words, you cannot use an AMI Update Set to change a Child Object with a 'Page' type originally ingested as a Digital Object -&gt; to a Parent Object with a 'Book' type mapped to the Digital Object Collection/Compound bundle. For these changes, you will need to delete and re-ingest your ADOs, or use an external (not provided in default Archipelagos) Drupal module for bundle/Drupal Content Type changes.</li> </ul> </li> <li>additional metadata elements you wish to Update, Replace, or Append values for, such as \"subjects\"<ul> <li>optional (but likely pertains to the reason you are Updating your ADOs)</li> <li>follow recommendations and practices described in more below detail</li> </ul> </li> <li>files<ul> <li>optional</li> <li>on the final 'Process' tab of your AMI Update Set Configuration, if the \"Do not touch existing files\" option is checked, then existing files will be left untouched. It is recommended to always keep this option checked if you are targeting only Metadata updates.</li> </ul> </li> </ul> <p>You should be familiar with the basic mechanics of AMI Set Configuration noted in Steps 1-6.</p> <p>Best Practices</p> <p>For all AMI Update operations, it is strongly recommended to both:</p> <ol> <li> <p>Before Updating, use the 'Export Archipelago Digital Objects to CSV content item' Action available on the main <code>Content</code> page and the <code>Find and Replace</code> page menus to generate a CSV of your non-modified objects. If something unintended occurs with your Update execution, you could use this CSV of your non-modified objects to restore your objects (or just a field or two) as needed.</p> </li> <li> <p>Create a small test batch CSV referencing one to two/three ADOs to test the execution of your desired Update actions on before running your larger Update Sets. There is no 'Undo' or 'Revert Changes' button that can be used for an AMI Update Set. You do have the option to 'Revert Revisions' on an object-by-object basis, but that is not ideal for reverting changes that were executing across large batches of ADOs. See the 'Checking Your Changes' documentation section for more information about reviewing and potentially reverting Revisions.</p> </li> </ol>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"ami_update/#data-transformation-options-for-ami-update-sets","title":"Data Transformation Options for AMI Update Sets","text":"<p>As with regular/Create New AMI Sets, you will have to select your preferred Data Tranformation configuration during Step 3 : of your AMI Update Set Configuration.</p> <ul> <li>Direct<ul> <li>Columns from your spreadsheet source will be cast directly to ADO metadata (JSON), without transformation/further processing (only intended for use with simple data strings or already JSON-encoded snippets/values).</li> <li>This is likely the most common Data Transformation configuration you will use for simple AMI Update Replace and Append Sets.</li> </ul> </li> <li>Custom (Expert Mode)<ul> <li>Provides very granular custom data transformation and mapping options</li> <li>You will likely only use this Data Transformation configuration for more complex AMI Update Sets when you want to differentiate between Data Transformation setups for Digital Objects and Digital Object Collections/Compound Objects/Creative Work Series (such as passing Digital Objects through 'Template' tranformation, and Collections/Compounds through 'Direct' transformation).</li> </ul> </li> <li>Template<ul> <li>Columns from your spreadsheet source will be cast to ADO metadata (JSON) using a Twig template setup for JSON output.</li> <li>This is likely the most common Data Transformation configuration you will use for more complex AMI Update Sets. This is the setup you would need to use if you want to use an AMI Update Set with AMI's LoD Reconciliation to update existing ADOs subject metadata with enriched LoD.</li> </ul> </li> </ul> <p>Caution with using Templates for Data Transformation</p> <p>If you are planning to use the 'Template' or 'Custom (Export Mode)' data transformation approach for your AMI Update Set configuration, you will need to have prepared your corresponding AMI Ingest Template to account for the specific Update actions you have planned. </p> <p>It is important to keep in mind that all of the metadata elements for your existing ADOs metadata may not necessarily be present in your AMI Update Set Source CSV. For example, you may have only prepared your AMI Update Set Source CSV to contain a limited number of headers/columns, such as only those required (node_uuid, label) and one or two metadata elements you wish to update (such as \"subjects\"). If you choose to pass your AMI Update Set through a Twig template, the output after Processing your AMI Update Set may overwrite your existing data if you do not have all of the necessary logic/checks in place to preserve the existing metadata if desired.</p> <p>In other words, imagine your twig template contains this statement: <pre><code>  \"subjects\": {{ data.subjects|json_encode|raw }}, \n</code></pre></p> <p>Independently of IF your CSV contains \"subjects\" as a header/column, the Twig template will still output an empty \"subjects\", which, when using the \"Replace\" mode will wipe out any existing \"subjects\" in your ADO.</p> <p>During any update operation (independently of the functional operation chosen) and IF you are using/passing your CSV through a template, AMI will provide an extra Context key for you to reference in your Twig Template. You can always reference 'dataOriginal.subjects' for example -- all dataOriginal.xx keys will contain the values of the existing metadata for your ADOs. This allows you to make \"smart\" templates that check IF a certain key/values exists, compare the unmodified (and to be modified) ADO(s) with the new data passed, then generate the desired output. </p>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"ami_update/#update-set-processing-options","title":"Update Set Processing Options","text":"<p>Beginning from Step 7, Processing of your AMI Set Configuration, select the Update operation that best corresponds to your targeted Update scenario.</p> <p></p> <p></p>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"ami_update/#1-replace-update-operation","title":"1. Replace Update Operation","text":"<p>The Replace Update Operation Replace 'will replace JSON keys found in an ADO's configured target field(s) with new JSON values. Not provided JSON keys will be kept.'</p> <ul> <li>If the processed data contains a JSON key that is already in the ADO's metadata to be updated, the values in the AMI Update Set CSV will be used, replacing completely the values found in that key in the existing ADO.</li> <li>The Replace update operation paired with the 'Direct' data transformation is likely the update operation you will use.</li> </ul>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"ami_update/#common-use-case-scenario-for-replace-updates","title":"Common use case scenario for Replace updates:","text":"<ul> <li>You notice a missing or incorrectly processed field in your original AMI Set/ADOs.</li> <li>You create an AMI set that contains only the 'node_uuid', 'label', and one column for the missing field and values.<ul> <li>If the values are singular, you do not need to JSON-encode the values in the CSV</li> <li>If the values are multiple, you need to JSON-encode them, formatted as: [\"value1\",\"value2\"]</li> </ul> </li> <li>You select the Direct data transformation approach on the AMI configuration.</li> <li>You select the Replace update operation and keep 'Do not touch existing files' checked.</li> <li>With this setup, the new field and values are added to the existing JSON for the impacted ADOs.</li> </ul>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"ami_update/#2-complete-all-json-keys-update-operation","title":"2. Complete (All JSON keys) Update Operation","text":"<p>The Complete (All JSON keys) Update Operation (formerly labeled 'Normal' in previous Archipelago releases) 'will update a complete existing ADO's JSON data with all new JSON data.' This will replace all the existing JSON, everything in an ADO with new processed data. </p> <ul> <li>The Complete (All JSON keys) update operation is powerful and can overwrite your whole JSON object record if not paired with a template that has all the extra checks/logic needed to preserve existing data if desired (see note of 'Caution with Templates for Data Transformation' above). </li> <li>It is also recommended to only use the Complete (All JSON keys) approach if you need to re-process the majority of the metadata fields for ADOs.</li> </ul>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"ami_update/#3-append-update-operation","title":"3. Append Update Operation","text":"<p>The Append update operation 'will append values to existing JSON key(s) in an ADO's configured target field(s). New JSON keys will be added too.' </p> <ul> <li>If the processed data contains a key that is already in the ADO\u2019s metadata to be updated, attempts will be made to match the \"source\" type (array, complex object) to add to it. If you have 2 values in a key, and your original/existing data contains a single value, the result will have 3 values (and then it will try to deduplicate too). If the Source data did not contain a key present in the processed data, then it will be added.</li> <li>The Append operation can be very useful, but it should be used with caution if targeting single values versus arrays. AMI will not permit malformed JSON data to be generated. But you need to consider if your Append update tranforms a previously single-value key into a multiple-value array, how this change may impact any references made in you display or other templates, Views throughout your Archipelago. <ul> <li>For example, if your Object Description Display template is not setup to check for iterable (multiple value/array) values for a given element, then the multiple values for an updated ADO may not output as expected.</li> </ul> </li> </ul>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"ami_update/#other-process-setup-options","title":"Other Process Setup Options","text":"<p>For the other AMI Set Process options and steps, please refer to the information found from Steps 7-10 in this complementary documentation for Create New ADOs AMI Sets. See the 'Checking Your Changes' documentation section for more information about reviewing and potentially reverting Revisions.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["AMI","Archipelago Multi Importer","AMI Update Sets","Update","Replace","Append"]},{"location":"annotations/","title":"Annotations in Archipelago","text":"<p>Archipelago extends Annotorius to provide W3C-compliant Web Annotations for Digital Objects. These annotations can be added per image (when multiple), edited for text and shape adjustments, and saved/discarded using the regular Edit mode (bonus track 1: temp storage that persists when you log out and come back in to your session). Archipelago also exposes a full API for WebAnnotations, that keeps track of which Images (referenced in the Strawberryfield @ <code>as:image</code>) were annotated and creates the W3C valid entries inside your Digital Object's JSON @ <code>ap:annotationCollection</code> (bonus track 2: multiple users can annotate the same resource, enabling digital scholarship collaboration opportunities). </p> <p>Important Note: For any image-based Digital Objects you would like to apply annotations to, the Digital Object <code>type</code> must be setup to display the image file(s) using the Open SeaDragon viewer. More information about about Managing Display Modes in Archipelago can be found here. Please stay tuned for updates announcing web annotation integration for Mirador 3.</p>"},{"location":"annotations/#enabling-annotations","title":"Enabling Annotations","text":"<ol> <li>Navigate to Admin --&gt; Structure --&gt; Content types --&gt; Digital Object --&gt; Manage Display and select the \"Digital Object Full view\" mode. <code>https://yoursite.org/admin/structure/types/manage/digital_object/display/digital_object_viewmode_fullitem</code> </li> <li>On the \u201cFragola\u201d row, click on the small gear icon on the far right, which will be open the configurations for this display type. </li> <li>Select the \u201cEnable loading/editing of W3C webAnnotations\u201d option.   <ul> <li>Learn more about the JSON format of WebAnnotations here: https://www.w3.org/TR/annotation-model/#index-of-json-keys.</li> </ul> </li> <li>Under \"What tool to enable\", select either the Rectangular or Polygon (freehand drawing) tool for your annotation style.</li> <li>Select the 'Update' button.</li> <li>Also Save your settings using the button at the bottom of the page.</li> </ol> <p> You are now ready to get started adding annotations! </p>"},{"location":"annotations/#adding-and-saving-annotations","title":"Adding and Saving Annotations","text":"<ol> <li>Navigate to the image-based Digital Object you would like to apply annotations to.</li> <li>To add a new annotation, select and hold the <code>Shift</code> key. Click and then drag to apply either a Rectangular box or multi-point Polygon shape.</li> <li>Double click to exit the annotation drawing mode.</li> <li>Enter the text for your annotation in the pop-up window.     </li> <li>Click the \"Ok\" button when you are ready.</li> <li>To save your annotation (or annotations if you created multiple), navigate to the main Digital Object \"Edit\" tab, where you will see a message about Unsaved Web Annotation Changes.     </li> <li>Select \"Save\" to preserve your Annotation(s). They will now become part of your Digital Object's JSON, found under the <code>ap:annotationCollection</code> key.<ul> <li>Pressing the \"Discard\" button will discard only the unsaved Annotations, and will reload the page.</li> </ul> </li> </ol>"},{"location":"annotations/#editing-and-deleting-annotations","title":"Editing and Deleting Annotations","text":"<ol> <li>Navigate to the image-based Digital Object you would whose annotation(s) you want to edit or delete.</li> <li>Click within the Annotation and select the downwards arrow in the upper right-hand corner of the pop-up window.</li> <li>Select either the \"Edit\" option and Edit the Annotation as desired; Or select the \"Delete\" option.     </li> <li>To preserve your editing or deleting actions, navigate to the main Digital Object \"Edit\" tab, where you will see a message about Unsaved Web Annotation Changes. (See screenshot in Step 6 of Adding and saving Annotations above.)</li> <li>Select \"Save\" to preserve your Annotation(s) edits or deletions. Pressing the \"Discard\" button will discard only the unsaved Annotations changes, and will reload the page.</li> </ol> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"archifilepersistencestrategy/","title":"Archipelago's File Persistence Strategy","text":""},{"location":"archifilepersistencestrategy/#how-are-files-for-archipelago-digital-objects-ados-persisted-what-happens-with-those-fishtanks","title":"How are files for Archipelago Digital Objects (ADOs) persisted? (What happens with those fishtanks?)","text":"<p>A few Event Subscribers/Data describing logics happen in a certain order:</p> <ol> <li> <p>User Uploads via a webform Element a new File or via Drush/Batch ingest that attaches (via JSONAPI) a file.</p> </li> <li> <p>If the webform is involved, Archipelago acts quickly and calls directly (before the Node even exists) the file classifier, that will:</p> <ol> <li>Add/complement a <code>as:somefiletype</code> JSON structure into the main <code>ADO</code> SBF <code>JSON</code>, with info about the file, checksums, size, Drupal fids, uuid, etc. This is a heavy function part of the <code>StrawberryfieldFilePersisterService</code>. It does a lot, making use of optimized logic, but may do more in the future to handle too-many/too-big files needs (FYI: solution will be simple, add to a queue and process later).</li> <li>The most (yes) important info added here is the desired future storage location of the file.</li> </ol> </li> <li> <p>The user finishes the form, saves and and confirms the ADO creation, and finally all the Node events fire.</p> </li> <li> <p>On presave <code>StrawberryfieldEventPresaveSubscriberAsFileStructureGenerator</code> runs and checks if 2.1 already was processed. This is needed since the user could have triggered an ingest via drush/JSONAPI/Webhooks etc. If all is well (this is a less expensive check) Archipelago continues.</p> </li> <li> <p>On presave (next) <code>StrawberryfieldEventPresaveSubscriberFilePersister</code> runs, checking all TEMPORARY files described in <code>as:somefiletype</code> and actually copying them to the right \"desired\" location.</p> </li> <li> <p>And on Save <code>StrawberryfieldEventInsertFileUsageUpdater</code> also marking the file as \"being\" used by a Strawberry driven Node (different Event).</p> </li> </ol> <p>Note: Anytime we remove directly from the raw JSON a full <code>as:somefiletype</code> structure of a sub-element from an <code>as:structure</code> we force Archipelago to do all the above again, and Archipelago can regenerate technical metadata. This has been used when updating EXIF binaries or even when something went wrong (while testing, but this stuff is safe no worries). Eventually, there will be a BIG red button that does that if you do not like JSON editing.</p> <p>Discussions related to Archipelago's file persistence strategy and planned potential strategies can be be found here: Strawberryfield Issues: 107, and here: Strawberryfield Issues: 76. This page will be updated with additional information following future developments.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"archipelag-logo-usage/","title":"Archipelago Commons Logo Usage Guidelines","text":"","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#our-logo","title":"Our Logo","text":"<p>The Archipelago Commons Logo features an illustrated dumbo octopus, named Dumpling, swimming beside the Archipelago Commons name. This logo is used for branding identity and visual representation for Archipelago Commons, an open-source repository software that is developed and supported at the Metropolitan New York Library Council (METRO).</p> <p>This logo was developed with care by METRO's Digital Services Team, and was drawn by Diego Pino Navarro.</p>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#logo-license","title":"Logo License","text":"<p>The Archipelago Commons Logo is protected by a Mozilla Public License 2.0. Please refer to the full License Description and Terms found at https://opensource.org/license/mpl-2-0.</p>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#logo-versions","title":"Logo Versions","text":"<p>The Archipelago Commons Logo is available in a few different iterations. You should use our Primary version for the vast majority of use cases.</p>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#primary","title":"Primary","text":"<ul> <li>Link to Zip File Containing 720px, 480px, 320px, and 240px versions</li> </ul>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#alternate-colors","title":"Alternate Colors","text":"<ul> <li>Link to Zip File Containing 480px, 320px, and 240px versions</li> </ul> <ul> <li>Link to Zip File Containing 480px, 320px, and 240px versions</li> </ul>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#solid-black-outline","title":"Solid Black Outline","text":"<ul> <li>Link to Zip File Containing 480px, 320px, and 240px versions</li> </ul>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#solid-white-outline","title":"Solid White Outline","text":"<ul> <li>Link to Zip File Containing 480px, 320px, and 240px versions</li> </ul>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#stamp-versions-black-or-white-outline","title":"Stamp Versions: Black or White Outline","text":"<p>Please only use the Stamp Versions of the Archipelago Commons logo if you do not have space to use the Primary or Alternate Color versions of the Logo.</p> <p></p> <ul> <li>Link to Zip File Containing Multiple Versions</li> </ul>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#favicon","title":"Favicon","text":"<ul> <li>Link to Zip File Containing Multiple Versions</li> </ul>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#placement-clear-space","title":"Placement &amp; Clear Space","text":"<p>To ensure our Archipelago Commons Logo maintains legibility and integrity when placing within your own website or documents, always preserve a minimum clear space between the logo and other elements. Please see further notes relating to proper Logo placement in the Logo Improper Uses section below.</p>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#scaling-proportionally","title":"Scaling proportionally","text":"<p>The proportions of the Archipelago Commons Logo should never be altered. When resizing any version of the Archipelago Commons logo, use the original scaling to maintain the correct proportions.</p>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#background-colors","title":"Background colors","text":"<p>Our primary Archipelago Commons Logo (black type with a pink dumbo octopus) should be used over most backgrounds. </p> <p>Do not use the Archipelago Commons Logo over patterned backgrounds.</p>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#logo-improper-uses","title":"Logo Improper Uses","text":"<p>We are very excited to see the Archipelago Commons Logo presented on your institutional websites. We are proud to have you and your colleagues as part of the wider Archipelago Community. We kindly ask that you abide by the following notes to help ensure that the Archipelago Commons Logo is well presented, and thank you in advance for your respectful usage of the Archipelago Commons Logo.</p> <ol> <li>Do not use the Archipelago Commons Logo in a way that incorrectly implies affiliation with, or sponsorship, endorsement, or approval by Archipelago Commons or METRO of your products or services.</li> <li>Do not display the Archipelago Commons Logo more prominently than your own product, service, or company name.</li> <li>Do not use the Archipelago Commons Logo on merchandise for sale (e.g., selling t-shirts, tote bags, mugs, stickers, etc.).</li> <li>Do not use the Archipelago Commons Logo on merchandise produced for free distribution (e.g., t-shirts, tote bags, mugs, stickers, etc.) without prior written consent of the METRO Digital Services Team.</li> <li>Do not use the Archipelago Commons Logo for any other form of commercial use (e.g. offering technical support services), unless such use is limited to a truthful and descriptive reference (e.g. \u201cIndependent technical support for Archipelago Commons\u201d or \"This repository runs on Archipelago Commons\").</li> <li>Do not modify any Archipelago Commons Logo versions, abbreviate them, or combine them with any other symbols, words, or images, or incorporate them into a tagline or slogan.</li> <li>Do not separate the illustration of Dumpling (the dumbo octopus) and the Archipelago Commons Wordmark.</li> <li>Do not rearrange the elements, change the scale of elements, or flip or rotate the elements in the Archipelago Commons Logo.</li> <li>Do not stretch, distort, or add decorative effects such as emboss or drop shadow, or otherwise modify the Archipelago Commons Logo.</li> <li>Do not alter the colors of the Archipelago Commons Logo or make it transparent.</li> <li>Do not place or embed the Archipelago Commons Logo within a box or carrier.</li> <li>Do not use the Archipelago Commons Favicon images outside actual Favicon application.</li> </ol>","tags":["Archipelago Commons","Logo"]},{"location":"archipelag-logo-usage/#questions-about-proper-usage","title":"Questions About Proper Usage","text":"<p>If you have any questions or concerns about proper usage of the Archipelago Commons Logo not specified in this guide, please reach out to METRO's Digital Services Team, Diego Pino Navarro and Allison Sherrick.</p>","tags":["Archipelago Commons","Logo"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/","title":"Archipelago-deployment: upgrading Drupal 9 to Drupal 10 (1.1.0 to 1.3.0)","text":"","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#what-is-this-documentation-for","title":"What is this documentation for?","text":"<p>If you already have a well-set-up and well-loved Archipelago (1.1.0) running Drupal 9 (D9), this documentation will allow you to update to 1.3.0 on Drupal 10 (D10) without any major issues.</p> <p>D9 is no longer supported as of the end of November 2023. D10 has been around for a little while, and even if every module is not supported yet, what you need and want for Archipelago has long been ready for D10. However, Archipelago is still D9 compatible if it's necessary for you to stay back a little longer.</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#requirements","title":"Requirements","text":"<ul> <li>An archipelago-deployment local instance 1.1.0 (working, tested) deployed using provided instructions via Docker and running Drupal 9.</li> <li>Basic knowledge and instincts (+ courage) on how to run Terminal Commands, <code>composer</code> and <code>drush</code>.</li> <li>Patience. You can't skip steps here.</li> <li>For shell Commands documented here please copy line by line--not the whole block.</li> </ul>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#backing-up-and-preparing-for-the-upgrade","title":"Backing up and preparing for the upgrade","text":"<p>Backups are always going to be your best friends. Archipelago's code, database, and settings are mostly self-contained in your current <code>archipelago-deployment</code> repo folder, and backing up is simple because of that.</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-1","title":"Step 1:","text":"<p>On a terminal, <code>cd</code> into your running <code>archipelago-deployment</code> folder and shut down your <code>docker-compose</code> ensemble by running the following:</p> <pre><code>docker-compose down\n</code></pre>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-2","title":"Step 2:","text":"<p>Verify that all containers are actually down. The following command should return an empty listing:</p> <pre><code>docker ps\n</code></pre> <p>If anything is still running, wait a little longer and run the command again.</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-3","title":"Step 3:","text":"<p>Now let's <code>tar.gz</code> the whole ensemble with data and configs. As an example we will save this into your <code>$HOME</code> folder. As a good practice we append the current date (YEAR-MONTH-DAY) to the filename. Here we assume today is October 31st of 2023.</p> <pre><code>cd ..\nsudo tar -czvpf $HOME/archipelago-deployment-D9-20231031.tar.gz archipelago-deployment\ncd archipelago-deployment\n</code></pre> <p>The process may take a few minutes. Now let's verify that all is there and that the <code>tar.gz</code> is not corrupt.</p> <pre><code>tar -tvvf $HOME/archipelago-deployment-D9-20231031.tar.gz\n</code></pre> <p>You will see a listing of files, and at the end you will see something like this: <code>Archive Format: POSIX pax interchange format, Compression: gzip</code>. If corrupt (Do you have enough space? Did your ssh connection drop?) you will see the following:</p> <pre><code>tar: Unrecognized archive format\n</code></pre>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-4","title":"Step 4:","text":"<p>Restart your <code>docker-compose</code> ensemble, and wait a little while for all to start.</p> <pre><code>docker-compose up -d\n</code></pre>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-5","title":"Step 5:","text":"<p>Export/backup all of your live Archipelago configurations (this allows you to compare/come back in case you lose something custom during the upgrade).</p> <pre><code>docker exec esmero-php mkdir config/backup\ndocker exec esmero-php drush cex --destination=/var/www/html/config/backup\n</code></pre> <p>Good. Now it's safe to begin the upgrade process.</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#upgrading-to-130","title":"Upgrading to 1.3.0","text":"","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-1-edit-docker-composeryml","title":"Step 1: Edit docker-composer.yml","text":"<p>First we are going to edit your docker-compose.yml file to reference the latest PHP container as needed. Starting in your Archipelago deployment directory location, run the following commands:</p> <p>If you have not already, run:</p> <pre><code>docker-compose down\n</code></pre> <p>Then open your docker-compose.yml file:</p> <pre><code>nano docker-compose.yml\n</code></pre> <p>Inside your <code>docker-compose.yml</code> file, page down to the <code>php</code> section and change the <code>image</code> section to match exactly as follows:</p> <pre><code>image: \"esmero/php-8.1-fpm:1.2.0-multiarch\"\n</code></pre> <p>Next page down to the <code>iiif</code> section and change the <code>image</code> section to match exactly as follows:</p> <pre><code>image: \"esmero/cantaloupe-s3:6.0.1-multiarch\"\n</code></pre> <p>Save your changes.</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-2-docker-pull-and-check","title":"Step 2: docker pull and check","text":"<p>Time to fetch the latest branch and update our <code>docker compose</code> and <code>composer</code> dependencies. To pull the images and bring up the ensemble, run:</p> <pre><code>docker compose pull\ndocker compose up -d\n</code></pre> <p>Give all a little time to start. Please be patient. To ensure all is well, run (more than once if necessary) the following:</p> <pre><code>docker ps\n</code></pre> <p>You should see something like this:</p> <pre><code>CONTAINER ID   IMAGE                                      COMMAND                  CREATED          STATUS          PORTS                              NAMES\n355e13878b7e   nginx                                      \"/docker-entrypoint.\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8001-&gt;80/tcp               esmero-web\n86b685008158  solr:8.11.2                                 \"docker-entrypoint.s\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8983-&gt;8983/tcp             esmero-solr\na8f0d9c6d4a9   esmero/cantaloupe-s3:6.0.1-multiarch       \"sh -c 'java -Dcanta\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8183-&gt;8182/tcp             esmero-cantaloupe\n6642340b2496   mariadb:10.6.12-focal                      \"docker-entrypoint.s\u2026\"   10 minutes ago   Up 10 minutes   3306/tcp                           esmero-db\n0aef7df34037   minio/minio:RELEASE.2022-06-11T19-55-32Z   \"/usr/bin/docker-ent\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:9000-9001-&gt;9000-9001/tcp   esmero-minio\n28ee3fb4e7a7   esmero/php-8.1-fpm:1.2.0-multiarch         \"docker-php-entrypoi\u2026\"   10 minutes ago   Up 10 minutes   9000/tcp                           esmero-php\na81c36d51a81   esmero/esmero-nlp:fasttext-multiarch       \"/usr/local/bin/entr\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:6400-&gt;6400/tcp             esmero-nlp\n</code></pre> <p>Important here is the <code>STATUS</code> column. It needs to be a number that goes up in time every time you run <code>docker ps</code> again (and again).</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-3-updates-for-key-drupal-and-archipelago-modules","title":"Step 3. Updates for key Drupal and Archipelago modules","text":"<p>Now we are going to tell <code>composer</code> to update the key Drupal and Archipelago modules.</p> <p>First we are going to disable and remove a few minor Drupal modules. Run the following commands (in order):</p> <pre><code>docker exec -ti esmero-php bash -c \"drush pm-uninstall fancy_file_delete\"\ndocker exec -ti esmero-php bash -c \"drush pm:uninstall quickedit\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/fancy_file_delete\"\n</code></pre> <p>Now update the versions used for these Drupal modules:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require drupal/jquery_ui_slider:^2 drupal/jquery_ui_effects:^2 drupal/jquery_ui:1.6 drupal/jquery_ui_datepicker:^2 drupal/jquery_ui_touch_punch:^1 drupal/better_exposed_filters:6.0.3 --no-update --with-all-dependencies\"\n</code></pre> <p>And now update one other Drupal module and the main Archipelago modules:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/views_bulk_operations:^4.2' 'strawberryfield/strawberryfield:1.3.0.x-dev' 'strawberryfield/webform_strawberryfield:1.3.0.x-dev' 'strawberryfield/format_strawberryfield:1.3.0.x-dev' 'strawberryfield/strawberry_runners:0.7.0.x-dev' 'archipelago/ami:0.7.0.x-dev' --no-update --with-all-dependencies\"\n</code></pre> <p>From inside your <code>archipelago-deployment</code> repo folder we are now going to open up file <code>permissions</code> for some of your most protected Drupal files.</p> <pre><code>sudo chmod 777 web/sites/default\nsudo chmod 666 web/sites/default/*settings.php\nsudo chmod 666 web/sites/default/*services.yml\n</code></pre>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-4-disableremove-additional-drupal-modules-and-loosen-up-dependencies","title":"Step 4: Disable/Remove additional Drupal modules and loosen up dependencies","text":"<p>We are going to remove additional select Drupal modules that are not 1.3.0 or D10 compliant.</p> <p>Please run each of the following commands separately, in order, and do not skip any commands.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer remove symfony/http-kernel symfony/yaml --no-update\"\ndocker exec -ti esmero-php bash -c 'composer remove egulias/email-validator --no-update'\ndocker exec -ti esmero-php bash -c \"composer require drupal/config_inspector:^2 --no-update\"\ndocker exec -ti esmero-php bash -c \"drush pm:uninstall jsonapi_earlyrendering_workaround\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/jsonapi_earlyrendering_workaround --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/core:^10' 'drupal/core-recommended:^10' 'drupal/core-composer-scaffold:^10' 'drupal/core-project-message:^10' --update-with-dependencies --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/core-dev:^10' --dev --update-with-dependencies --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drush/drush:^12' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/twig_tweak:^2 --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/config_inspector:^2 --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/config_update:2.0.x-dev --no-update\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/config_update_ui --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/context:^5.0@RC' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/devel:^5.1' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/sophron --no-update\"\ndocker exec -ti esmero-php bash -c \"composer remove fileeye/mimemap --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/imagemagick:^3 --no-update\"  \ndocker exec -ti esmero-php bash -c \"composer remove fileeye/mimemap --no-update\"    \ndocker exec -ti esmero-php bash -c \"composer require 'drupal/imce:^3.0' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/search_api_attachments:^9.0' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/search_api_solr:^4.3 solarium/solarium ^6.3 --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/twig_tweak:^3.2' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/webform_entity_handler:^2.0' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/webformnavigation:^2.0' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/form_mode_manager --no-update\"\n</code></pre> <p>Well done! If you see no issues and all ends in Green colored messages, all is good!  Jump to Step 5</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#what-if-all-is-not-ok-and-i-see-red-and-a-lot-of-dependency-explanations","title":"What if all is not OK, and I see red and a lot of dependency explanations?","text":"<p>If you have manually installed packages via composer in the past that are NO longer Drupal 10 compatible you may see errors. In that case you need to check each package website's (normally https://www.drupal.org/project/the_module_name) and check if there is a Drupal 10 compatible version.</p> <p>If so run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require drupal/the_module_name:^VERSION_NUMBER_THAT_WORKS_ON_DRUPAL10_ --update-with-dependencies --no-update\" and run **Step 3** again (and again until all is cleared)\n</code></pre> <p>If not, try to find a replacement module that does something similar, but in any case you may end up having to remove before proceeding. Give us a ping/slack/google group/open a github ISSUE if you find yourself uncertain about this.</p> <pre><code>docker exec -ti esmero-php bash -c \"drush pm-uninstall the_module_name\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/the_module_name --no-update\"\n</code></pre>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-5-update-composerjson","title":"Step 5: Update composer.json","text":"<p>Now you need to update your <code>composer.json</code> file to include an important patch. Starting in your Archipelago deployment directory location, run the following commands:</p> <pre><code>nano composer.json\n</code></pre> <p>Inside your <code>composer.json</code> file, page down to the <code>\"patches\"</code> section and change the section to match exactly as follows:</p> <pre><code>\"patches\": {\n      \"drupal/form_mode_manager\": {\n       \"D10 compatibility\": \"https://www.drupal.org/files/issues/2023-10-11/3297262-20.patch\"\n      },\n      \"drupal/ds\": {\n       \"https://www.drupal.org/project/ds/issues/3338860\": \"https://www.drupal.org/files/issues/2023-04-04/3338860-5-d10-compatible_0.patch\"\n      }\n    }\n</code></pre> <p>Save your changes and then run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer update -W\"\n</code></pre>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-6-one-final-round-of-drupal-module-updates","title":"Step 6: One final round of Drupal module updates","text":"<p>We will now run a few more updates for additional Drupal modules.</p> <p>Please run each of the following commands separately, in order, and do not skip any commands.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require mglaman/composer-drupal-lenient\"\ndocker exec -ti esmero-php bash -c \"composer config --merge --json extra.drupal-lenient.allowed-list '[\\\"drupal/form_mode_manager\\\"]'\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/form_mode_manager:2.x-dev@dev'\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/color:^1.0'\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/hal\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/aggregator\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/ckeditor\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/seven\"\ndocker exec -ti esmero-php bash -c \"composer require archipelago/archipelago_subtheme:1.3.0.x-dev\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/quickedit drupal/classy drupal/stable\"\ndocker exec -ti esmero-php bash -c \"drush pm:uninstall quickedit\"\ndocker exec -ti esmero-php bash -c \"drush pm:uninstall hal\"\ndocker exec -ti esmero-php bash -c \"drush pm:install jquery_ui\"\n</code></pre> <p>Whew, that's a lot of module updates! Now run one final database update command:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-7-optional-syncs","title":"Step 7: Optional Syncs","text":"<p>Optionally, you can sync your new Archipelago 1.3.0 and bring in all the latest configs and settings. For this you have two options (no worries, remember you made a backup!):</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#a-partial-sync-which-will-bring-new-configs-and-update-existing-ones-but-will-not-remove-ones-that-only-exist-in-your-custom-setup-eg-new-webforms-or-view-modes","title":"A Partial Sync, which will bring new configs and update existing ones but will not remove ones that only exist in your custom setup, e.g. new Webforms or View Modes.","text":"<pre><code>docker exec esmero-php drush cim -y --partial   \n</code></pre>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#a-complete-sync-which-will-bring-new-things-and-update-existing-but-will-also-remove-all-the-ones-that-are-not-part-of-130-its-a-like-clean-factory-reset","title":"A Complete Sync, which will bring new things and update existing but will also remove all the ones that are not part of 1.3.0. It's a like clean factory reset.","text":"<pre><code>docker exec esmero-php drush cim -y\n</code></pre> <p>If all goes well here and you see no errors it's time to reindex <code>Solr</code> because there are new Fields. Run the following:</p> <pre><code>docker exec esmero-php drush search-api-reindex\ndocker exec esmero-php drush search-api-index\n</code></pre> <p>You might see some warnings related to modules dealing with previously non-existent data\u2014no worries, just ignore those.</p> <p>If you made it this far you are done with code/devops (are we ever ready?), and that means you should be able to (hopefully) stay in the Drupal 10 realm for a few years!</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-8-update-or-not-your-metadata-display-entities-and-menu-items","title":"Step 8: Update (or not) your Metadata Display Entities and menu items","text":"<p>Recommended: If you want to add new templates and menu items 1.3.0 provides, run this:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/deploy.sh'\n</code></pre> <p>Once that is done, you can choose to update all Metadata Displays (twig templates) we ship with new 1.3.0 versions (heavily adjusted IIIF manifests, better Object description template using Macros). Before you do this, we strongly recommend that you first make sure to manually (copy/paste) backup any Twig templates you have modified. If unsure, do not run the command that comes after this warning! You can always manually copy the new templates from the <code>d8content/metadatadisplays</code> folder which contains text versions (again, copy/paste) of each shipped template you can pick and use when you feel ready.</p> <p>If you are sure (like really?) you want to overwrite the ones you modified (sure, just checking?), then you can run this:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/update_deployed.sh'\n</code></pre> <p>Done! (For realz now)</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#step-9-or-should-we-say-10","title":"Step 9 (or should we say 10)","text":"<p>Please login to your Archipelago and test/check all is working! Enjoy 1.3.0 and Drupal 10. Thanks!</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#need-help-blue-screen-missed-a-step-need-a-hug-and-such","title":"Need help? Blue Screen? Missed a step? Need a hug and such?","text":"<p>If you see any issues or errors or need help with a step, please let us know (ASAP!). You can either open an <code>issue</code> in this repository or use the Google Group. We are here to help.</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Sherrick</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-UpgradeDrupalD9toD10/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-democontent/","title":"Adding Demo Archipelago Digital Objects (ADOs) to your Repository","text":"<p>We make this optional since we feel  not everyone wants to have Digital Objects from other people using space in their system.  Still, if you are new to Archipelago we encourage you to do this. Its a simply way to get started without thinking too much.  You can learn and test. Then delete and move over. </p>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#prerequisites","title":"Prerequisites","text":"","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#the-new-way-archipelago-100-rc2-or-higher","title":"The new way Archipelago 1.0.0-RC2 or higher.","text":"<ul> <li>You installed it either via the Step by Step deployment on OSX, the one for Ubuntu or using your secret powers directly on a VM/Metal/Cloud/EC2 or even a raspberryPI.</li> <li>You followed the guides without being too creative which means you have an <code>jsonapi</code> drupal user and an <code>admin</code> one and you can login and out of your server.</li> <li>You remember your <code>admin</code> user. (If you followed one of the deployment guides, password will be <code>archipelago</code>)</li> </ul>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#step-1-only-step","title":"Step 1: (only step)","text":"<ul> <li>Log into your Archipelago using the <code>admin</code> user. </li> <li>Navigate to <code>Content</code> -&gt; <code>Ami Sets</code>. You will see a single <code>AMI Set</code> already in place. </li> <li>On the Drop Down Menu to the right (The <code>edit</code> Button), press on the little <code>down arrow</code> and choose <code>Process</code>. </li> <li>A new Form will appear. Under <code>DESIRED ADOS STATUSES AFTER PROCESS</code>, change all from Draft to Published, leave <code>Enqueue but do not process Batch in realtime</code> unchecked and press \"Confirm\". The Ingest will start and a progress bar will advance. Once ready a list of Ingest Objects should appear.</li> <li>You are done!</li> </ul>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#old-way-a-running-archipelago-10-beta3-or-higher","title":"Old way, A running Archipelago 1.0-Beta3 or higher.","text":"<ul> <li>You installed it either via the Step by Step deployment on OSX, the one for Ubuntu or using your secret powers directly on a VM/Metal/Cloud/EC2 or even a raspberryPI.</li> <li>You followed the guides without being too creative which means you have a <code>jsonapi</code> drupal user and you can login and out of your server.</li> </ul>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#step-1-get-the-content","title":"Step 1: Get the content","text":"<p>Go into your <code>archipelago-deployment</code> folder and into the <code>d8content</code> folder that is inside it, e.g.</p> <pre><code>cd archipelago-deployment/d8content\ngit clone https://github.com/esmero/archipelago-recyclables\n</code></pre>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#step-2-ingest-the-objects","title":"Step 2: Ingest the Objects","text":"<ul> <li>If running Docker execute:</li> </ul> <pre><code>docker exec -ti esmero-php bash -c 'd8content/archipelago-recyclables/deploy_ados.sh'\n</code></pre> <p>You will see multiple outputs similar to this:</p> <pre><code>Files in provided location:\n - anne_001.jpg\n - anne_002.jpg\n - anne_003.jpg\n - anne_004.jpg\n - anne_005.jpg\n - anne_006.jpg\n - anne_007.jpg\n - anne_008.jpg\n - anne_009.jpg\n - anne_010.jpg\nFile anne_001.jpg sucessfully uploaded with Internal Drupal file ID 5\nFile anne_002.jpg sucessfully uploaded with Internal Drupal file ID 6 \nFile anne_003.jpg sucessfully uploaded with Internal Drupal file ID 7\nFile anne_004.jpg sucessfully uploaded with Internal Drupal file ID 8\nFile anne_005.jpg sucessfully uploaded with Internal Drupal file ID 9 \nFile anne_006.jpg sucessfully uploaded with Internal Drupal file ID 10 \nFile anne_007.jpg sucessfully uploaded with Internal Drupal file ID 11 \nFile anne_008.jpg sucessfully uploaded with Internal Drupal file ID 12\nFile anne_009.jpg sucessfully uploaded with Internal Drupal file ID 13 \nFile anne_010.jpg sucessfully uploaded with Internal Drupal file ID 14\nNew Object 'Anne of Green Gables : Chapters 1 and 2' with UUID 9eb28775-d73a-4904-bc79-f0e925075bc5 successfully ingested. Thanks!\n</code></pre> <p>The gist here is that if the script says <code>Thanks</code> you are good.</p> <ul> <li>If you are not running Docker (You are a unicorn or at least a hacker) you will need to tune/copy/modify the following script: <code>archipelago-deployment/d8content/archipelago-recyclables/deploy_ados.sh</code></li> </ul> <p>Inside you will find lines like this one: </p> <pre><code>drush archipelago:jsonapi-ingest /var/www/html/d8content/archipelago-recyclables/ado/0c2dc01a-7dc2-48a9-b4fd-3f82331ec803.json --uuid=0c2dc01a-7dc2-48a9-b4fd-3f82331ec803 --bundle=digital_object --uri=http://esmero-web --files=/var/www/html/d8content/archipelago-recyclables/ado/0c2dc01a-7dc2-48a9-b4fd-3f82331ec803 --user=jsonapi --password=jsonapi --moderation_state=published;\n</code></pre> <p>What you want here is to modify/replace the absolute paths that point your demo objects (.json) and their assets (folders with the same name). Basically replace every entry of <code>/var/www/html/d8content/archipelago-recyclables/</code> with the path to <code>archipelago-recyclables</code>.</p>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#need-help-blue-screen-missed-a-step-need-a-hug-another-hug","title":"Need help? Blue Screen? Missed a step? Need a hug? Another Hug?","text":"<p>If you have trouble running this or see errors or need help with a step (its only two steps), please let us know (ASAP!). You can either open an <code>issue</code> in this repository or use the Google Group. We are here to help.</p>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#caring-coding-fixing","title":"Caring &amp; Coding + Fixing","text":"<ul> <li>Diego Pino</li> <li>Giancarlo Birello</li> <li>Allison Lund</li> </ul>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-democontent/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago Digital Objects","Demo Content"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/","title":"Archipelago-deployment-live: upgrading Drupal 9 to Drupal 10 (1.1.0 to 1.3.0)","text":"","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#what-is-this-documentation-for","title":"What is this documentation for?","text":"<p>If you already have a well-set-up and well-loved Archipelago (1.1.0) running Drupal 9 (D9), this documentation will allow you to update to 1.3.0 on Drupal 10 (D10) without any major issues.</p> <p>D9 is no longer supported as of the end of November 2023. D10 has been around for a little while, and even if every module is not supported yet, what you need and want for Archipelago has long been ready for D10. However, Archipelago is still D9 (1.2.0) compatible if it's necessary for you to stay back a little longer.</p> <p>This documentation is meant to be a guide/helper, not a single-click-magic solution. Why? Because your Drupal environment and your local github branches for this whole project might have diverged really a lot from a vanilla Archipelago base deployment. You might have new templates, custom composer.json, new modules, changed webforms and Views. That is not an Archipelago issue, more like a <code>How do i keep any Drupal instance, YAML files, custom configurations and modules</code> in version control and in sync as i upgrade other parts, issue.</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#requirements","title":"Requirements","text":"<ul> <li>An archipelago-deployment-live instance 1.1.0 (working, tested) deployed using provided instructions via Docker and running Drupal 9.</li> <li>Basic knowledge, patience and instincts (+ courage) on how to run Terminal Commands, <code>composer</code> and <code>drush</code>.</li> <li>Patience(again). You can't skip steps here.</li> <li>For shell Commands documented here please copy line by line--not the whole block.</li> <li>You are running already version control and know how to git pull/push/merge.</li> </ul>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#backing-up-and-preparing-for-the-upgrade","title":"Backing up and preparing for the upgrade","text":"<p>Backups are always going to be your best friends. Archipelago's code, database, and settings are mostly self-contained in your current <code>archipelago-deployment-live</code> repo folder, and backing up is simple because of that.</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-1","title":"Step 1:","text":"<p>On a terminal, <code>cd</code> into your running <code>archipelago-deployment-live</code> folder, then <code>cd</code> inside the <code>deploy/ec2-docker</code> subfolders and shut down your <code>docker-compose</code> ensemble by running the following:</p> <pre><code>docker-compose down\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-2","title":"Step 2:","text":"<p>Verify that all containers are actually down. The following command should return an empty listing:</p> <pre><code>docker ps\n</code></pre> <p>If anything is still running, wait a little longer and run the command again.</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-3","title":"Step 3:","text":"<p>Now let's <code>tar.gz</code> the whole ensemble with data and configs. We will exclude here the local source caches generated by Cantaloupe. If these or not exist will depend on how custom your deployment is. As an example we will save this into your <code>$HOME</code> folder. As a good practice we append the current date (YEAR-MONTH-DAY) to the filename. Here we assume today is December 12th of 2023.</p> <p>We will cd back to the parent folder of your running <code>archipelago-deployment-live</code> folder, so three levels down, assuming you are right now inside <code>archipelago-deployment-live/deploy/ec2-docker</code></p> <pre><code>cd ../../..\nsudo tar --exclude=archipelago-deployment-live/data_storage/iiifcache --exclude=archipelago-deployment-live/data_storage/iiiftmp -czvpf $HOME/archipelago-deployment-D9-20231212.tar.gz archipelago-deployment-live\n</code></pre> <p>The process may take a few minutes. Now let's verify that all is there and that the <code>tar.gz</code> is not corrupt.</p> <pre><code>tar -tvvf $HOME/archipelago-deployment-D9-20231212.tar.gz\n</code></pre> <p>You will see a listing of files, and at the end you will see something like this: <code>Archive Format: POSIX pax interchange format, Compression: gzip</code>. If corrupt (Do you have enough space? Did your ssh connection drop?) you will see the following:</p> <pre><code>tar: Unrecognized archive format\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-4","title":"Step 4:","text":"<p><code>cd</code> again into your running <code>archipelago-deployment-live</code> folder, then <code>cd</code> inside the <code>deploy/ec2-docker</code> Restart your <code>docker-compose</code> ensemble, and wait a little while for all to start.</p> <pre><code>docker-compose up -d\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-5","title":"Step 5:","text":"<p>Export/backup all of your live Archipelago 1.1.0, Drupal 9 configurations (this allows you to compare/come back in case you lose something custom during the upgrade).</p> <pre><code>docker exec esmero-php mkdir config/backup\ndocker exec esmero-php drush cex --destination=/var/www/html/config/backup\n</code></pre> <p>Good. Now it's safe to begin the upgrade process.</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#upgrading-to-130","title":"Upgrading to 1.3.0","text":"","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-1-edit-docker-composeryml","title":"Step 1: Edit docker-composer.yml","text":"<p>First we are going to edit your docker-compose.yml file to reference the latest PHP container as needed. Starting in your Archipelago deployment directory location, run the following commands:</p> <p>If you have not already, run:</p> <pre><code>docker-compose down\n</code></pre> <p>Then open your docker-compose.yml file:</p> <pre><code>nano docker-compose.yml\n</code></pre> <p>Inside your <code>docker-compose.yml</code> file, page down to the <code>php</code> section and change the <code>image</code> section to match exactly as follows:</p> <pre><code>image: \"esmero/php-8.1-fpm:1.2.0-multiarch\"\n</code></pre> <p>Next page down to the <code>iiif</code> section and change the <code>image</code> section to match exactly as follows:</p> <pre><code>image: \"esmero/cantaloupe-s3:6.0.1-multiarch\"\n</code></pre> <p>Save your changes.</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-2-docker-pull-and-check","title":"Step 2: docker pull and check","text":"<p>Time to fetch the latest branch and update our <code>docker compose</code> and <code>composer</code> dependencies. To pull the images and bring up the ensemble, run:</p> <pre><code>docker compose pull\ndocker compose up -d\n</code></pre> <p>Give all a little time to start. Please be patient. To ensure all is well, run (more than once if necessary) the following:</p> <pre><code>docker ps\n</code></pre> <p>You should see something like this (your versions might vary):</p> <pre><code>CONTAINER ID   IMAGE                                      COMMAND                  CREATED          STATUS          PORTS                              NAMES\n355e13878b7e   nginx                                      \"/docker-entrypoint.\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8001-&gt;80/tcp               esmero-web\n86b685008158   solr:8.11.2                                \"docker-entrypoint.s\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8983-&gt;8983/tcp             esmero-solr\na8f0d9c6d4a9   esmero/cantaloupe-s3:6.0.1-multiarch       \"sh -c 'java -Dcanta\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8183-&gt;8182/tcp             esmero-cantaloupe\n6642340b2496   mariadb:10.6.12-focal                      \"docker-entrypoint.s\u2026\"   10 minutes ago   Up 10 minutes   3306/tcp                           esmero-db\n0aef7df34037   minio/minio:RELEASE.2022-06-11T19-55-32Z   \"/usr/bin/docker-ent\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:9000-9001-&gt;9000-9001/tcp   esmero-minio\n28ee3fb4e7a7   esmero/php-8.1-fpm:1.2.0-multiarch         \"docker-php-entrypoi\u2026\"   10 minutes ago   Up 10 minutes   9000/tcp                           esmero-php\na81c36d51a81   esmero/esmero-nlp:fasttext-multiarch       \"/usr/local/bin/entr\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:6400-&gt;6400/tcp             esmero-nlp\n</code></pre> <p>Important here is the <code>STATUS</code> column. It needs to be a number that goes up in time every time you run <code>docker ps</code> again (and again). <code>Note</code>: We will update from Solr 8.11.2 to Solr 9.1.1 only after we are sure all is running correctly.</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-3-updates-for-key-drupal-and-archipelago-modules","title":"Step 3. Updates for key Drupal and Archipelago modules","text":"<p>Now we are going to tell <code>composer</code> to update the key Drupal and Archipelago modules.</p> <p>First we are going to disable and remove a few minor Drupal modules. Run the following commands (in order):</p> <pre><code>docker exec -ti esmero-php bash -c \"drush pm-uninstall fancy_file_delete\"\ndocker exec -ti esmero-php bash -c \"drush pm:uninstall quickedit\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/fancy_file_delete\"\n</code></pre> <p>Now update the versions used for these Drupal modules:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require drupal/jquery_ui_slider:^2 drupal/jquery_ui_effects:^2 drupal/jquery_ui:1.6 drupal/jquery_ui_datepicker:^2 drupal/jquery_ui_touch_punch:^1 drupal/better_exposed_filters:6.0.3 --no-update --with-all-dependencies\"\n</code></pre> <p>And now update one other Drupal module and the main Archipelago modules:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/views_bulk_operations:^4.2' 'strawberryfield/strawberryfield:1.3.0.x-dev' 'strawberryfield/webform_strawberryfield:1.3.0.x-dev' 'strawberryfield/format_strawberryfield:1.3.0.x-dev' 'strawberryfield/strawberry_runners:0.7.0.x-dev' 'archipelago/ami:0.7.0.x-dev' --no-update --with-all-dependencies\"\n</code></pre> <p>From inside your <code>archipelago-deployment-live</code> repo folder, and inside the <code>drupal</code> subfolder, we are now going to open up file <code>permissions</code> for some of your most protected Drupal files.</p> <pre><code>sudo chmod 777 web/sites/default\nsudo chmod 666 web/sites/default/*settings.php\nsudo chmod 666 web/sites/default/*services.yml\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-4-disableremove-for-additional-select-drupal-modules","title":"Step 4: Disable/Remove for additional select Drupal modules","text":"<p>We are going to remove additional select Drupal modules that are not 1.3.0 or D10 compliant.</p> <p>Please run each of the following commands separately, in order, and do not skip any commands.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer remove symfony/http-kernel symfony/yaml --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/config_inspector:^2 --no-update\"\ndocker exec -ti esmero-php bash -c \"drush pm:uninstall jsonapi_earlyrendering_workaround\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/jsonapi_earlyrendering_workaround --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/core:^10' 'drupal/core-recommended:^10' 'drupal/core-composer-scaffold:^10' 'drupal/core-project-message:^10' --update-with-dependencies --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/core-dev:^10' --dev --update-with-dependencies --no-update\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/google_api_client --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drush/drush:^12' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/twig_tweak:^2 --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/config_inspector:^2 --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/config_update:2.0.x-dev --no-update\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/config_update_ui --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/context:^5.0@RC' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/devel:^5.1' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/sophron --no-update\"\ndocker exec -ti esmero-php bash -c \"composer remove fileeye/mimemap --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/imagemagick:^3 --no-update\"  \ndocker exec -ti esmero-php bash -c \"composer remove fileeye/mimemap --no-update\"    \ndocker exec -ti esmero-php bash -c \"composer remove drupal/jsonapi_earlyrendering_workaround --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/imce:^3.0' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/search_api_attachments:^9.0' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/twig_tweak:^3.2' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/webform_entity_handler:^2.0' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/webformnavigation:^2.0' --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/webform_jqueryui_datepicker:^6 --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/rdf --no-update \"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/form_mode_manager --no-update\"\n</code></pre> <p>Well done! If you see no issues and all ends in Green colored messages, all is good!  Jump to Step 5</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#what-if-all-is-not-ok-and-i-see-red-and-a-lot-of-dependency-explanations","title":"What if all is not OK, and I see red and a lot of dependency explanations?","text":"<p>If you have manually installed packages via composer in the past that are NO longer Drupal 10 compatible you may see errors. In that case you need to check each package website's (normally https://www.drupal.org/project/the_module_name) and check if there is a Drupal 10 compatible version.</p> <p>If so run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/the_module_name:^VERSION_NUMBER_THAT_WORKS_ON_DRUPAL10_' --update-with-dependencies --no-update\" and run **Step 3 ** again (and again until all is cleared)\n</code></pre> <p>If not, try to find a replacement module that does something similar, but in any case you may end up having to remove before proceeding. Give us a ping/slack/google group/open a github ISSUE if you find yourself uncertain about this.</p> <pre><code>docker exec -ti esmero-php bash -c \"drush pm-uninstall the_module_name\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/the_module_name --no-update\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-5-update-composerjson","title":"Step 5: Update composer.json","text":"<p>Now you need to update your <code>composer.json</code> file to include an important patch. Starting in your Archipelago deployment directory location, run the following commands:</p> <pre><code>nano composer.json\n</code></pre> <p>Inside your <code>composer.json</code> file, page down to the <code>\"patches\"</code> section and change the section to match exactly as follows:</p> <pre><code>\"patches\": {\n      \"drupal/form_mode_manager\": {\n       \"D10 compatibility\": \"https://www.drupal.org/files/issues/2023-11-07/3297262-62-form-mode-manager-D10.patch\"\n      },\n      \"drupal/ds\": {\n       \"https://www.drupal.org/project/ds/issues/3338860\": \"https://www.drupal.org/files/issues/2023-04-04/3338860-5-d10-compatible_0.patch\"\n      }\n    }\n</code></pre> <p>Save your changes and then run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer update -W\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-6-one-final-round-of-drupal-module-updates","title":"Step 6: One final round of Drupal module updates","text":"<p>We will now run a few more updates for additional Drupal modules.</p> <p>Please run each of the following commands separately, in order, and do not skip any commands.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require mglaman/composer-drupal-lenient\"\ndocker exec -ti esmero-php bash -c \"composer config --merge --json extra.drupal-lenient.allowed-list '[\\\"drupal/form_mode_manager\\\"]'\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/form_mode_manager:2.x-dev@dev'\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/color:^1.0'\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/hal\"  \ndocker exec -ti esmero-php bash -c \"composer require drupal/aggregator\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/ckeditor\"  \ndocker exec -ti esmero-php bash -c \"composer require drupal/seven\"\ndocker exec -ti esmero-php bash -c \"composer require archipelago/archipelago_subtheme:1.3.0.x-dev\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/classy drupal/stable\"\ndocker exec -ti esmero-php bash -c \"drush pm:uninstall hal\"\ndocker exec -ti esmero-php bash -c \"drush pm:uninstall rdf\"\ndocker exec -ti esmero-php bash -c \"drush pm:install jquery_ui\"\ndocker exec -ti esmero-php bash -c \"drush pm:install jquery_ui_effects\"\n</code></pre> <p>Whew, that's a lot of module updates! Now run one final database update command:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-7-optional-syncs","title":"Step 7: Optional Syncs","text":"<p>Optionally, you can sync your new Archipelago 1.3.0 and bring in all the latest configs and settings. This requires you do fetch, either via <code>git</code> or manually via <code>wget</code> or <code>curl</code> newer configs from https://github.com/esmero/archipelago-deployment-live/tree/1.3.0/drupal/config/sync into the same folder structure/location of your deployment.</p> <p>For this you have two options (no worries, remember you made a backup!):</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#a-partial-sync-which-will-bring-new-configs-and-update-existing-ones-but-will-not-remove-ones-that-only-exist-in-your-custom-setup-eg-new-webforms-or-view-modes","title":"A Partial Sync, which will bring new configs and update existing ones but will not remove ones that only exist in your custom setup, e.g. new Webforms or View Modes.","text":"<pre><code>docker exec esmero-php drush cim -y --partial\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#a-complete-sync-which-will-bring-new-things-and-update-existing-but-will-also-remove-all-the-ones-that-are-not-part-of-130-its-a-like-clean-factory-reset","title":"A Complete Sync, which will bring new things and update existing but will also remove all the ones that are not part of 1.3.0. It's a like clean factory reset.","text":"<pre><code>docker exec esmero-php drush cim -y\n</code></pre> <p>If all goes well here and you see no errors it's time to reindex <code>Solr</code> because there are new Fields. Run the following:</p> <pre><code>docker exec esmero-php drush search-api-reindex\ndocker exec esmero-php drush search-api-index\n</code></pre> <p>You might see some warnings related to modules dealing with previously non-existent data\u2014no worries, just ignore those.</p> <p>If you made it this far you are done with code/devops (are we ever ready?), and that means you should be able to (hopefully) stay in the Drupal 10 realm for a few years!</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-8-update-or-not-your-metadata-display-entities-and-menu-items","title":"Step 8: Update (or not) your Metadata Display Entities and menu items","text":"<p>Recommended: If you want to add new templates and menu items 1.3.0 provides, you need to fetch everything from this remote https://github.com/esmero/archipelago-deployment-live/tree/1.3.0/drupal/d8content to your local installation into the same folder structure/location and then run:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/deploy.sh'\n</code></pre> <p><code>Important</code>: If you don't download/sync/git merge (or your prefered method) then the command will add nothing, since you will be running this command against 1.1.0 content.</p> <p>Once that is done, you can choose to update all Metadata Displays (twig templates) we ship with new 1.3.0 versions (heavily adjusted IIIF manifests, better Object description template using Macros). Before you do this, we strongly recommend that you first make sure to manually (copy/paste) backup any Twig templates you have modified. If unsure, do not run the command that comes after this warning! You can always manually copy the new templates from the <code>d8content/metadatadisplays</code> folder which contains text versions (again, copy/paste) of each shipped template you can pick and use when you feel ready.</p> <p>If you are sure (like really?) you really want to overwrite the ones you modified (sure, just checking?), then you can run this (make sure you edit that file):</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/update_deployed.sh'\n</code></pre> <p>Done! (For realz now)</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-9","title":"Step 9","text":"<p>Please login to your Archipelago and test/check all is working! Enjoy 1.3.0 and Drupal 10. Thanks! Also please keep all your new changes under version control. Check what has changed. <code>git add</code> and <code>git commit -m \"What i changed\"</code> as needed.</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#step-10","title":"Step 10","text":"<p>If all looks good and you are not missing any functionality (please be thorough about testing), it is time to Jump to Upgrading to Solr 9.x</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#need-help-blue-screen-missed-a-step-need-a-hug-or-someone-that-listens-to-you-in-silence","title":"Need help? Blue Screen? Missed a step? Need a hug or someone that listens to you in silence?","text":"<p>If you see any issues or errors or need help with a step, please let us know (ASAP!). You can either open an <code>issue</code> in this repository or use the Google Group. We are here to help.</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Sherrick</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeDrupalD9toD10/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment-live","Drupal 9","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/","title":"Archipelago-deployment-live: upgrading Archipelago 1.3.0 to 1.4.0 (Drupal 10.1 to 10.2.x)","text":"","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#what-is-this-documentation-for","title":"What is this documentation for?","text":"<p>If you already have a well-set-up and well-loved Archipelago (1.3.0) running Drupal 10.1 or 10.2 (D10), this documentation will allow you to update to 1.4.0 on latest Drupal 10.2 without any major issues. Even if Archipelago is 10.3.1 (as the time of writing this guide) compatible we do not recommend upgrading still until some of the Drupal core and contributed module issues are solved and that minor release is more stable.</p> <p>This documentation is meant to be a guide/helper, not a single-click-magic solution. Why? Because your Drupal environment and your local github branches for this whole project might have diverged really a lot from a vanilla Archipelago base deployment. You might have new templates, custom composer.json, new modules, changed webforms and Views. That is not an Archipelago issue, more like a 'How do I keep any Drupal instance, YAML files, custom configurations and modules in version control and in sync as I upgrade other parts' issue.</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#requirements","title":"Requirements","text":"<ul> <li>An archipelago-deployment-live instance 1.3.0 (working, tested) deployed using provided instructions via Docker and running Drupal 10.</li> <li>Basic knowledge, patience and instincts (+ courage) on how to run Terminal Commands, <code>composer</code> and <code>drush</code>.</li> <li>Patience(again). You can't skip steps here.</li> <li>For shell Commands documented here please copy line by line--not the whole block.</li> <li>You are running already version control and know how to git pull/push/merge.</li> </ul>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#backing-up-and-preparing-for-the-upgrade","title":"Backing up and preparing for the upgrade","text":"<p>Backups are always going to be your best friends. Archipelago's code, database, and settings are mostly self-contained in your current <code>archipelago-deployment-live</code> repo folder, and backing up is simple because of that.</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-1","title":"Step 1:","text":"<p>On a terminal, <code>cd</code> into your running <code>archipelago-deployment-live</code> folder, then <code>cd</code> inside the <code>deploy/ec2-docker</code> subfolders and shut down your <code>docker-compose</code> ensemble by running the following:</p> <pre><code>docker-compose down\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-2","title":"Step 2:","text":"<p>Verify that all containers are actually down. The following command should return an empty listing:</p> <pre><code>docker ps\n</code></pre> <p>If anything is still running, wait a little longer and run the command again.</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-3","title":"Step 3:","text":"<p>Now let's <code>tar.gz</code> the whole ensemble with data and configs. We will exclude here the local source caches generated by Cantaloupe. If these or not exist will depend on how custom your deployment is. As an example we will save this into your <code>$HOME</code> folder. As a good practice we append the current date (YEAR-MONTH-DAY) to the filename. Here we assume today is July 7th of 2024.</p> <p>We will <code>cd</code> back to the parent folder of your running <code>archipelago-deployment-live</code> folder, so three levels down, assuming you are right now inside <code>archipelago-deployment-live/deploy/ec2-docker</code></p> <pre><code>cd ../../..\nsudo tar --exclude=archipelago-deployment-live/data_storage/iiifcache --exclude=archipelago-deployment-live/data_storage/iiiftmp -czvpf $HOME/archipelago-deployment-D10-20240707.tar.gz archipelago-deployment-live\n</code></pre> <p>The process may take a few minutes. Now let's verify that all is there and that the <code>tar.gz</code> is not corrupt.</p> <pre><code>tar -tvvf $HOME/archipelago-deployment-D10-20240707.tar.gz\n</code></pre> <p>You will see a listing of files, and at the end you will see something like this: <code>Archive Format: POSIX pax interchange format, Compression: gzip</code>. If corrupt (Do you have enough space? Did your ssh connection drop?) you will see the following:</p> <pre><code>tar: Unrecognized archive format\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-4","title":"Step 4:","text":"<p><code>cd</code> again into your running <code>archipelago-deployment-live</code> folder, then <code>cd</code> inside the <code>deploy/ec2-docker</code> Restart your <code>docker-compose</code> ensemble, and wait a little while for all to start.</p> <pre><code>docker-compose up -d\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-5","title":"Step 5:","text":"<p>Export/backup all of your live Archipelago 1.3.0, Drupal 10 configurations (this allows you to compare/come back in case you lose something custom during the upgrade).</p> <pre><code>docker exec esmero-php mkdir config/backup\ndocker exec esmero-php drush cex --destination=/var/www/html/config/backup\n</code></pre> <p>Good. Now it's safe to begin the upgrade process.</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#upgrading-to-140","title":"Upgrading to 1.4.0","text":"","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-0-get-familiar-with-what-changed","title":"Step 0: Get familiar with what changed.","text":"<p>We mean this. This is a new step. Running a Production Server requires some informed decision making and thus, we believe, a good pre-step is reviewing what changed between releases. </p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-1-edit-docker-composeryml-optional","title":"Step 1: Edit docker-composer.yml (optional)","text":"<p>For this release there are no larger updates on the Docker backend level (good!). Solr 9 was updated from 9.1 to 9.2 and Database (MYSQL and MariaDB) also got a version bump. Esmero NLP with Machine Learning Model Endpoints is not meant for production/public access and right now is for community evaluation only. Reasons are many: some technical (CPU/GPU and memory consumption not suited for a medium server) but mostly about usefulness, use cases, real needs driving the use of the tools we developed and the very much needed Community Discussion and Consensus on our Field's (GLAM) ethical concerns about this.</p> <p>If you decide to upgrade your services please review one of these (based on your running Platform): - https://github.com/esmero/archipelago-deployment-live/blob/1.4.0/deploy/ec2-docker/docker-compose-aws-s3-arm64.yml - https://github.com/esmero/archipelago-deployment-live/blob/1.4.0/deploy/ec2-docker/docker-compose-aws-s3.yml</p> <p>And selectively copy images/enviromentals into your production <code>docker-compose.yml</code> file.</p> <p>For this, if you have not already, run:</p> <pre><code>docker-compose down\n</code></pre> <p>Then open your docker-compose.yml file and with one of the previous links open, edit the corresponding service definitions:</p> <pre><code>nano docker-compose.yml\n</code></pre> <p>Save your changes.</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-2-docker-pull-and-check","title":"Step 2: docker pull and check","text":"<p>Time to fetch the latest branch and update our <code>docker compose</code> and <code>composer</code> dependencies. To pull the images and bring up the ensemble, run:</p> <pre><code>docker compose pull\ndocker compose up -d\n</code></pre> <p>Give all a little time to start. Please be patient. To ensure all is well, run (more than once if necessary) the following:</p> <pre><code>docker ps\n</code></pre> <p>You should see something like this if you synced all containers to the latest (your versions and databse might vary depending on your server's platform):</p> <pre><code>CONTAINER ID   IMAGE                                      COMMAND                  CREATED          STATUS          PORTS                              NAMES\n5b06ee366f58   jonasal/nginx-certbot                      \"/docker-entrypoint.\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8001-&gt;80/tcp               esmero-web\n86b685008158   solr:9.2.1                                 \"docker-entrypoint.s\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8983-&gt;8983/tcp             esmero-solr\na4872b237e17   esmero/cantaloupe-s3:6.0.1-multiarch       \"sh -c 'java -Dcanta\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8183-&gt;8182/tcp             esmero-cantaloupe\nbec0b31f3421   mariadb:10.6.18-focal                      \"docker-entrypoint.s\u2026\"   10 minutes ago   Up 10 minutes   3306/tcp                           esmero-db\n85bedadf9732   redis:6.2-alpine                           \"docker-entrypoint.s\u2026\"   10 minutes ago   10 minutes ago                                     esmero-redis\n6a9e9d8647a9   minio/minio:RELEASE.2022-06-11T19-55-32Z   \"/usr/bin/docker-ent\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:9000-9001-&gt;9000-9001/tcp   esmero-minio\nbc5327680ca7   esmero/php-8.1-fpm:1.2.0-multiarch         \"docker-php-entrypoi\u2026\"   10 minutes ago   Up 10 minutes   9000/tcp                           esmero-php\nd53729be1211   esmero/esmero-nlp:fasttext-multiarch       \"/usr/local/bin/entr\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:6400-&gt;6400/tcp             esmero-nlp\n</code></pre> <p>Important here is the <code>STATUS</code> column. It needs to be a number that goes up in time every time you run <code>docker ps</code> again (and again).</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-3-updates-for-key-drupal-and-archipelago-modules","title":"Step 3. Updates for key Drupal and Archipelago modules","text":"<p>Now we are going to tell <code>composer</code> to update the key Drupal and Archipelago modules.</p> <p>We don't need to disable any modules this time (great) but we will need to lose up some dependencies to target a stable Drupal 10.2.7.</p> <p>Notice how we will run composer commands using the <code>--no-update</code> argument which means composer won't have to resolve (nor fetch) library dependencies on every execution.</p> <p>First, update the versions used for these Drupal modules:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/color:1.x-dev@dev' 'drupal/config_inspector:dev-2.1.x' 'drupal/search_api_solr:4.3.4' 'drupal/search_api:^1.3' 'solarium/solarium:^6' 'egulias/email-validator:^4' 'drupal/upgrade_status:^4.3'  --no-update --with-all-dependencies\"\n</code></pre> <p>And now update the main Archipelago modules:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require cebe/php-openapi:^1 'league/openapi-psr7-validator:0.22' 'devizzent/cebe-php-openapi:1.0.3' 'strawberryfield/strawberryfield:1.4.0.x-dev' 'strawberryfield/webform_strawberryfield:1.4.0.x-dev' 'strawberryfield/format_strawberryfield:1.4.0.x-dev' 'strawberryfield/strawberry_runners:0.8.0.x-dev' 'archipelago/ami:0.8.0.x-dev'  'archipelago/archipelago_subtheme:1.4.0.x-dev' --no-update --with-all-dependencies\"\n</code></pre> <p>From inside your <code>archipelago-deployment-live</code> repo folder, and inside the <code>drupal</code> subfolder, we are now going to open up file <code>permissions</code> for some of your most protected Drupal files.</p> <pre><code>sudo chmod 777 web/sites/default\nsudo chmod 666 web/sites/default/*settings.php\nsudo chmod 666 web/sites/default/*services.yml\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-4-update-drupal-core-to-stable-1027","title":"Step 4: Update Drupal Core to stable 10.2.7","text":"<p>We are going to remove additional select Drupal modules that are not 1.3.0 or D10 compliant.</p> <p>Please run each of the following commands separately, in order, and do not skip any commands.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/core:10.2.7' 'drupal/core-recommended:10.2.7' 'drupal/core-composer-scaffold:10.2.7' 'drupal/core-project-message:10.2.7' --update-with-dependencies --no-update\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/core-dev:^10' --dev --update-with-dependencies --no-update\"\n</code></pre> <p>Well done! If you see no issues and all ends in Green colored messages, all is good!  Jump to Step 5</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#what-if-all-is-not-ok-and-i-see-red-and-a-lot-of-dependency-explanations","title":"What if all is not OK, and I see red and a lot of dependency explanations?","text":"<p>If you have manually installed packages via composer in the past that are NO longer Drupal 10.2 compatible you may see errors. In that case you need to check each package website's (normally https://www.drupal.org/project/the_module_name) and check if there is a Drupal 10 compatible version.</p> <p>If so run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/the_module_name:^VERSION_NUMBER_THAT_WORKS_ON_DRUPAL10_' --update-with-dependencies --no-update\" and run **Step 3 ** again (and again until all is cleared)\n</code></pre> <p>If not, try to find a replacement module that does something similar, but in any case you may end up having to remove before proceeding. Give us a ping/slack/google group/open a github ISSUE if you find yourself uncertain about this.</p> <pre><code>docker exec -ti esmero-php bash -c \"drush pm-uninstall the_module_name\"\ndocker exec -ti esmero-php bash -c \"composer remove drupal/the_module_name --no-update\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-5-update-composerjson","title":"Step 5: Update composer.json","text":"<p>Now you need to update your <code>composer.json</code> file to include an important patch. Starting in your Archipelago deployment directory location, run the following commands:</p> <pre><code>nano composer.json\n</code></pre> <p>Inside your <code>composer.json</code> file, page down to the <code>\"patches\"</code> section and change the section to match exactly as follows:</p> <pre><code>\"patches\": {\n           \"drupal/core\": {\n              \"https://www.drupal.org/project/drupal/issues/3364109\": \"https://git.drupalcode.org/project/drupal/-/merge_requests/4092.diff\"\n           }\n    }\n</code></pre> <p>Save your changes and then run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer update -W\"\n</code></pre> <p>Note: again. If you see error messages of missing dependencies, invalid (stuck/old versions) libraries or conflicts, review each message and check what modules/libraries are not allowing you to upgrade. One way of understanding the chained dependency errors is to run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer why some_namespace/some_library\" \n</code></pre> <p>Example given:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer why solarium/solarium\"\n</code></pre> <p>Will output:</p> <pre><code>drupal-composer/drupal-project 1.4.0.x-dev requires solarium/solarium (^6)\ndrupal/search_api_solr         4.3.4       requires solarium/solarium (^6.3.5)\n</code></pre> <p>If a given library/module/dependency is only required by <code>drupal-composer/drupal-project</code> you have space to loosen it up to a higher/more permissive version. If it dependends on other library/modules, those might be the culprits of not being able to upgrade it and normally the solution is to upgrade that library first (always using the <code>--no-update</code> argument) and then run <code>docker exec -ti esmero-php bash -c \"composer update -W\"</code> again.</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-6-update-your-database-and-run-hook-updates","title":"Step 6: Update your Database (and run hook updates)","text":"<p>Now run one final database update command:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-7-optional-syncs","title":"Step 7: Optional Syncs","text":"<p>Optionally, you can sync your new Archipelago 1.4.0 and bring in all the latest configs and settings. This requires you do fetch, either via <code>git</code> or manually via <code>wget</code> or <code>curl</code> newer configs from https://github.com/esmero/archipelago-deployment-live/tree/1.4.0/drupal/config/sync into the same folder structure/location of your deployment.</p> <p>For this you have two options (no worries, actually do worry, just remember you made a backup! Did you? Double check!):</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#option-1-a-partial-sync-which-will-bring-new-configs-and-update-existing-ones-but-will-not-remove-ones-that-only-exist-in-your-custom-setup-eg-new-webforms-or-view-modes","title":"Option 1. A Partial Sync, which will bring new configs and update existing ones but will not remove ones that only exist in your custom setup, e.g. new Webforms or View Modes.","text":"<pre><code>docker exec esmero-php drush cim -y --partial\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#option-2-a-complete-sync-which-will-bring-new-things-and-update-existing-but-will-also-remove-all-the-ones-that-are-not-part-of-130-its-a-like-clean-factory-reset","title":"Option 2. A Complete Sync, which will bring new things and update existing but will also remove all the ones that are not part of 1.3.0. It's a like clean factory reset.","text":"<pre><code>docker exec esmero-php drush cim -y\n</code></pre> <p>If all goes well here and you see no errors it's time to reindex <code>Solr</code> because there are new Fields. Run the following:</p> <pre><code>docker exec esmero-php drush search-api-reindex\ndocker exec esmero-php drush search-api-index\n</code></pre> <p>You might see some warnings related to modules dealing with previously non-existent data\u2014-no worries, just ignore those.</p> <p>If you made it this far you are done with code/devops (are we ever really?), and that means you should be able to (hopefully) stay in the Drupal 10 realm for a few years!</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-8-update-or-not-your-metadata-display-entities-and-menu-items","title":"Step 8: Update (or not) your Metadata Display Entities and menu items","text":"<p>Recommended: If you want to add new templates and menu items 1.4.0 provides, you need to fetch everything from this remote https://github.com/esmero/archipelago-deployment-live/tree/1.4.0/drupal/d8content to your local installation into the same folder structure/location and then run:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/deploy.sh'\n</code></pre> <p><code>Important</code>: If you don't download/sync/git merge (or your prefered method) then the command will add nothing, since you will be running this command against 1.3.0 content.</p> <p>Once that is done, you can choose to update all Metadata Displays (twig templates) we ship with new 1.4.0 versions (e.g heavily adjusted IIIF manifests with Content Search API service definitions). Before you do this, we strongly recommend that you first make sure to manually (copy/paste) backup any Twig templates you have modified. If unsure, do not run the command that comes after this warning! You can always manually copy the new templates from the <code>d8content/metadatadisplays</code> folder which contains text versions (again, copy/paste) of each shipped template you can pick and use when you feel ready.</p> <p>If you are sure (like really?) you really want to overwrite the ones you modified (sure, just checking?), then you can run this (make sure you edit that file):</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/update_deployed.sh'\n</code></pre> <p>Done! (For realz now)</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-9","title":"Step 9","text":"<p>Please login to your Archipelago and test/check all is working! Enjoy 1.4.0 under Drupal 10. Thanks! Also please keep all your new changes under version control. Check what has changed. <code>git add</code> and <code>git commit -m \"What i changed\"</code> as needed.</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#step-10","title":"Step 10","text":"<p>If all looks good and you are not missing any functionality (please be thorough about testing), it is time to Jump to Upgrading to Solr 9.x</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#need-help-blue-screen-missed-a-step-need-a-hug-or-someone-that-listens-to-you-in-silence","title":"Need help? Blue Screen? Missed a step? Need a hug or someone that listens to you in silence?","text":"<p>If you see any issues or errors or need help with a step, please let us know (ASAP!). You can either open an <code>issue</code> in this repository or use the Google Group. We are here to help.</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Sherrick</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-UpgradeFrom1dot3/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment-live","Drupal 10"]},{"location":"archipelago-deployment-live-gitworkflow/","title":"Managing, sheltering, pruning and nurturing your own custom Archipelago","text":"<p>Now that you have your base Archipelago Live Deployment running (Do you? If not, go back!) you may be wondering about things like:</p> <ol> <li>What happens when I need to update to the next release?</li> <li>How do I keep my Drupal and Modules updated in between releases?</li> <li>Can I add Drupal Modules?</li> <li>Will a new release overwrite all my customizations?</li> <li>What things are safe to customize?</li> <li>How do I keep my very own things in Version Control and safe from others?</li> <li>And many (many) other similar questions.</li> </ol>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#1-keep-your-archipelago-under-version-control-via-github","title":"1. Keep your Archipelago under Version Control via Github","text":"<p><code>Archipelagos</code> are living beings. They evolve and become beautiful, closer and closer to your needs. Because of that <code>resetting</code> your particularities on every <code>Archipelago</code> code release is not a good idea, nor even recommended. What you want is to keep your own <code>Drupal Settings</code>\u2014your facets, your themes, your Solr fields, your own modules, and all their configurations\u2014safe and be able to restore all in case something goes wrong.</p> <p>The ones we ship with every Release will <code>reset</code> your Archipelago's settings to Factory defaults if applied <code>wildly</code>.</p> <p>This is where <code>Github</code> comes in place.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#basic-steps","title":"Basic steps","text":"<p>Prerequisites:</p> <ul> <li>Have an Archipelago Deployment Live instance running</li> <li>Have Terminal access to your Live Instance</li> <li>Have a Github account</li> <li>Have a personal Github Access Token Created</li> <li>Run <code>git config --global --edit</code> on your Live Instance and Set your user name/email/etc.</li> <li>Note: Opens in <code>Vi</code>! In case of emergency/panic press <code>ESC</code> and type <code>:x</code> to escape and/or run away in terror. To edit Press <code>i</code> and uncomment the lines. Once Done press <code>ESC</code> and type <code>:x</code> to save.</li> </ul>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#11-start-by-creating-a-git-fork","title":"1.1 Start by creating a Git Fork","text":"<p>Let's fork https://github.com/esmero/archipelago-deployment-live under your own Account via the web. Happy Note: Since 2021 also keeping forked branches in sync with the origin can be done via the UI directly.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#12-connect-your-live-instance-terminal","title":"1.2 Connect your Live instance terminal.","text":"<p>Move to your repository's base folder, and let's start by adding your New Fork as a secondary Git <code>Origin</code>. Replace in this command <code>yourOwnAccount</code> with (guess what?) your own account:</p> <pre><code>git remote add upstream https://github.com/yourOwnAccount/archipelago-deployment-live\n</code></pre> <p>Now check if you have two remotes (<code>origin</code> =&gt; This repository, <code>upstream</code> =&gt; your own fork):</p> <pre><code>git remote -v\n</code></pre> <p>You will see this:</p> <pre><code>origin  https://github.com/esmero/archipelago-deployment-live (fetch)\norigin  https://github.com/esmero/archipelago-deployment-live (push)\nupstream    https://github.com/yourOwnAccount/archipelago-deployment-live (fetch)\nupstream    https://github.com/yourOwnAccount/archipelago-deployment-live (push)\n</code></pre> <p>Good!</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#13-now-lets-create-from-your-current-live-instance-a-new-branch","title":"1.3 Now let's create from your current Live Instance a new Branch.","text":"<p>We will push this branch into your Fork and it will be all yours to maintain. Please replace <code>yourOwnOrg</code> with any Name you want for this. We like to keep the current Branch name in place after your personal prefix:</p> <pre><code>git checkout -b yourOwnOrg-1.0.0-RC3\n</code></pre> <p>Good, you now have a new <code>local</code> branch named <code>yourOwnOrg-1.0.0-RC3</code>, and it's time to decide what we are going to push into Github.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#14-push-the-basics","title":"1.4 Push the Basics.","text":"<p>By default our deployment strategy (this repository) ignores a few files you want to have in Github.  Also, there are things like the Installed Drupal Modules and PHP Libraries (the Source Code), the Database, Caches, your Secrets (<code>.env</code> file), and your Drupal <code>settings.php</code> file. You FOR SURE do not want to have these in Github and are better suited for a private Backup Storage.</p> <p>Let's start by <code>push</code>ing what you have (no commits, your new <code>yourOwnOrg-1.0.0-RC3</code> as it is) to your new Fork. From there on we can add new Commits and files:</p> <pre><code>git push upstream yourOwnOrg-1.0.0-RC3\n</code></pre> <p>And Git will respond with the following (use your <code>yourOwnAccount</code> personal Github Access Token as password):</p> <pre><code>Username for 'https://github.com': yourOwnAccount\nPassword for 'https://yourOwnAccount@github.com': \nTotal 0 (delta 0), reused 0 (delta 0)\nremote: \nremote: Create a pull request for 'yourOwnOrg-1.0.0-RC3' on GitHub by visiting:\nremote:      https://github.com/yourOwnAccount/archipelago-deployment-live/pull/new/yourOwnOrg-1.0.0-RC3\nremote: \nTo https://github.com/yourOwnAccount/archipelago-deployment-live\n * [new branch]      yourOwnOrg-1.0.0-RC3 -&gt; yourOwnOrg-1.0.0-RC3\n</code></pre>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#15-first-commit","title":"1.5 First Commit","text":"<p>Right now this new Branch (go and check it out at https://github.com/yourOwnAccount/archipelago-deployment-live/tree/yourOwnOrg-1.0.0-RC3) will not differ at all from 1.0.0-RC3. That is OK. To make your Branch unique, what we want is to \"commit\" our changes. How do we do this?</p> <p>Let's add our <code>composer.json</code> and <code>composer.lock</code> to our change list. Both of these files are quite personal, and as you add more Drupal Modules, dependencies, or Upgrade your Archipelgo and/or Drupal Core and Modules, all of these corresponding files will change. See the <code>-f</code>? Because our base deployment ignores that file and you want it, we \"Force\" add it. Note: At this stage <code>composer.lock</code> won't be added at all because it's still the same as before. So you can only \"add\" files that have changes.</p> <pre><code>git add drupal/composer.json \ngit add -f drupal/composer.lock\n</code></pre> <p>Now we can see what is new and will be committed by executing:</p> <pre><code>git status\n</code></pre> <p>You may see something like this:</p> <pre><code>On branch yourOwnOrg-1.0.0-RC3 \nChanges to be committed:\n  (use \"git restore --staged &lt;file&gt;...\" to unstage)\n    new file:   drupal/composer.json\n\nChanges not staged for commit:\n  (use \"git add &lt;file&gt;...\" to update what will be committed)\n  (use \"git restore &lt;file&gt;...\" to discard changes in working directory)\n    modified:   drupal/scripts/archipelago/deploy.sh\n    modified:   drupal/scripts/archipelago/update_deployed.sh\n\nUntracked files:\n  (use \"git add &lt;file&gt;...\" to include in what will be committed)\n    deploy/ec2-docker/docker-compose.yml\n    drupal/.editorconfig\n    drupal/.gitattributes\n</code></pre> <p>If you do not want to add each <code>Changes not staged for commit</code> individually (WE recommend you only commit what you need. Be warned and take caution.), you can also issue a <code>git add .</code>, which means add all.</p> <pre><code>git add drupal/scripts/archipelago/deploy.sh\ngit add drupal/scripts/archipelago/update_deployed.sh\ngit add deploy/ec2-docker/docker-compose.yml\n</code></pre> <p>In this case we are also committing <code>docker-compose.yml</code>, which you may have customized and modified to your domain (See Install Guide Step 3), <code>deploy.sh</code> and <code>update_deployed.sh</code> scripts. If you ever need to avoid tracking certain files at all, you can edit the <code>.gitignore</code> file and add more patterns to it (look at it, it's fun!).</p> <pre><code>git commit -m \"Fresh Install of Archipelago for yourOwnOrg\"\n</code></pre> <p>If you had your email/user account setup correctly (see Prerequisites) you will see:</p> <pre><code>Fresh Install of Archipelago yourOwnOrg\n 4 files changed, 360 insertions(+), 46 deletions(-)\n create mode 100644 deploy/ec2-docker/docker-compose.yml\n create mode 100644 drupal/composer.json\n</code></pre> <p>And now finally you can push this back to your Fork:</p> <pre><code>git push upstream yourOwnOrg-1.0.0-RC3\n</code></pre> <p>And Git will respond with the following (use your <code>yourOwnAccount</code> personal Github Access Token as password):</p> <pre><code>Username for 'https://github.com': yourOwnAccount\nPassword for 'https://yourOwnAccount@github.com': \nEnumerating objects: 18, done.\nCounting objects: 100% (18/18), done.\nCompressing objects: 100% (10/10), done.\nWriting objects: 100% (10/10), 2.26 KiB | 2.26 MiB/s, done.\nTotal 10 (delta 5), reused 0 (delta 0)\nremote: Resolving deltas: 100% (5/5), completed with 5 local objects.\nTo https://github.com/yourOwnAccount/archipelago-deployment-live\n   d9fa835..3427ce5  yourOwnOrg-1.0.0-RC3 -&gt; yourOwnOrg-1.0.0-RC3\n</code></pre> <p>And done.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#2-keeping-your-archipelago-modules-updated-during-releases","title":"2. Keeping your Archipelago Modules Updated during releases","text":"<p>Releases in Archipelago are a bit different to other OSS projects. When a Release is done (let's say 1.0.0-RC2) we freeze the current release branches in every module we provide, package the release, and inmediatelly start with a new Release Cycle (6 months long normally) by creating in each repository a new Set of Branches (for this example 1.0.0-RC3). All new commits, fixes, improvements, features now will ALWAYS go into the Open/on-going new cycle branches (for this example 1.0.0-RC3), and once we are done we do it all over again. We freeze (for this example 1.0.0-RC3), and a new release cycle starts with fresh new \"WIP\" branches (for this example 1.1.0).</p> <p>Some Modules like AMI or Strawberry Runners have their independent Version but are released together anyway, e.g. for 1.0.0-RC3 both AMI and Strawberry Runners are 0.2.0. Why? Because work started later than the core Archipelago and also because they are not really CORE. So what happens with <code>main</code> branches? In our project <code>main</code> branches are never experimental. They are always a 1:1 with the latest stable release. So <code>main</code> will contain a full commit of 1.0.0-RC2 until we freeze 1.0.0-RC3 when <code>main</code> gets that code. Over and over. Nice, right?</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#which-modules-belong-to-archipelago-and-follow-a-release-cycle","title":"Which modules belong to Archipelago and follow a release cycle?","text":"<p>The following modules are the ones we update on every release:</p> <ul> <li>https://github.com/esmero/strawberryfield -&gt; Composer name: <code>strawberryfield/strawberryfield</code></li> <li>https://github.com/esmero/format_strawberryfield -&gt; Composer name: <code>strawberryfield/format_strawberryfield</code></li> <li>https://github.com/esmero/webform_strawberryfield -&gt; Composer name: <code>strawberryfield/webform_strawberryfield</code></li> <li>https://github.com/esmero/ami -&gt; Composer name: <code>archipelago/ami</code></li> <li>https://github.com/esmero/strawberry_runners    -&gt; Composer name: <code>strawberryfield/strawberry_runners</code></li> </ul> <p>We also update macro modules that are meant for deployment like this Repository and https://github.com/esmero/archipelago-deployment.</p> <p>To keep your Archipelago up to date, especially once you \"go custom\" as described in this Documentation, the process is quite simple, e.g. to fetch latest <code>1.0.0-RC3</code> updates during the <code>1.0.0-RC3</code> release cycle run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require strawberryfield/strawberryfield:dev-1.0.0-RC3 strawberryfield/format_strawberryfield:dev-1.0.0-RC3 strawberryfield/webform_strawberryfield:dev-1.0.0-RC3 archipelago/ami:0.2.0.x-dev strawberryfield/strawberry_runners:0.2.0.x-dev strawberryfield/strawberry_runners:0.2.0.x-dev archipelago/archipelago_subtheme:dev-1.0.0-RC3 -W\"\n</code></pre> <p>And then run any Database updates that may be needed:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre> <p>This will bring all the new code and all (except if there are BUGS!) should work as expected.</p> <p>Note: Archipelago really tries hard to be as backwards compatible as possible and rarely will you see a non-documented or non-dealt-with deprecation. </p> <p>Note 2: We of course recommend always running the Stable (frozen) release, but since code is plastic and fixes will go into a WIP open branch, you should be safe enough to move all modules together.</p> <p>You can run these commands any time you need, and while the release is open you will always get the latest code (even if it's always the same branch). Please follow/subscribe to each Module's Github to be aware of changes/issues and improvements.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#3-keeping-your-archipelagos-drupal-contributed-modules-and-core-updated","title":"3. Keeping your Archipelago's Drupal Contributed Modules and Core updated","text":"","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#31-contributed-modules","title":"3.1 Contributed Modules.","text":"<p>To keep your Archipelago's Drupal up to date check your Drupal at https://yoursite.org/admin/modules/update. Make sure you check mostly (yes mostly, no need to overreact) for Security Updates. Not every Drupal contributed module (project) keeps backwards compatibility, and we try to test every version we ship (as in this repository's <code>composer.lock</code> files) before releasing. Once you detect a major change/requirement, visit the Project's Changelog Website, and take some time reading it. If you feel confident it's not going to break all, copy the suggested Composer command, e.g. if you visit https://www.drupal.org/project/google_api_client/releases/8.x-3.2 you will see that the update is suggested as:</p> <pre><code>Install with Composer: $ composer require 'drupal/google_api_client:^3.2'\n</code></pre> <p>Using the same module as an example, before doing any final updates, check your current running version (take note in case you need to revert):</p> <pre><code>docker exec -ti esmero-php bash -c \"composer info 'drupal/google_api_client\"\n</code></pre> <p>Keep the version around.</p> <p>Now let's update, which means using the suggested command translated to our own Docker world like this (notice the <code>-W</code>):</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/google_api_client:^3.2 -W\"\n</code></pre> <p>And then run any Database updates that may be needed:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre> <p>This will update that module. Test your website. Depending on what you update, you want to focus first on the functionality it provides, and then create/edit/delete a fictitious Digital Object to ensure it did not add any errors to your most beloved Digital Objects workflows.</p> <p>If you see errors or you feel it's not acting as it should, you can revert by doing:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/google_api_client:^VERSION_YOU_KEPT_AROUND -W\"\n</code></pre> <p>And then run any Database updates that may be needed:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre> <p>If this happens we encourage you to please \ud83d\udc4f share your findings with our community/slack/Github ISSUE here.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#31-drupal-core-inside-the-same-major-version","title":"3.1 Drupal Core inside the same major version:","text":"<p>This is quite similar to a contributed module but normally involves at least 3 dependencies and of course larger changes.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#exact-version","title":"Exact Version","text":"<p>Inside the same major version, e.g. inside Drupal 9, if you are currently running Drupal <code>9.0.1</code> and you want to update to an exact latest (as I write <code>9.2.4</code>):</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require drupal/core:9.2.4 drupal/core-dev:9.2.4 drupal/core-composer-scaffold:9.2.4 drupal/core-project-message:9.2.4 --update-with-dependencies\"\n</code></pre> <p>Or under Drupal 8, if you are currently running Drupal <code>8.9.14</code> and you want to update to an exact latest (as I write <code>8.9.18</code>):</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require drupal/core-dev:8.9.18 drupal/core:8.9.18 drupal/core-composer-scaffold:8.9.18 --update-with-dependencies\"\n</code></pre> <p>And then for both cases run any Database updates that may be needed:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#alternative-major-version","title":"Alternative Major Version","text":"<p>If you want to only remember a <code>single command</code> and want to be sure to also get all extra packages for Drupal 9, run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require drupal/core-dev:^9 drupal/core:^9 drupal/core-composer-scaffold:^9 drupal/core-project-message:^9 -W\"\n</code></pre> <p>Or for Drupal 8:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require drupal/core-dev:^8 drupal/core:^8 drupal/core-composer-scaffold:^8 drupal/core-project-message:^8 -W\"\n</code></pre> <p>And then for both cases run any Database updates that may be needed:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre> <p>This will always get you the latest <code>Drupal</code> and <code>dependencies</code> allowed by your <code>composer.json</code>.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-gitworkflow/#32-drupal-core-between-major-versions","title":"3.2 Drupal Core between major versions:","text":"<p>Since major versions may bring larger deprecations, contributed modules will stay behind, and the world (and your database may collapse), we really recommend that you do some tests first (locally) or follow one of our guides. We at Archipelago will always document a larger version update. Currently, the Drupal 8 to Drupal 9 Update is documented in detail here.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback, or open an ISSUE in this Archipelago Deployment Live Repository.</p> <p>Return to Archipelago Live Deployment.</p>","tags":["Github"]},{"location":"archipelago-deployment-live-moveToLive/","title":"Moving from <code>archipelago-deployment</code> to <code>archipelago-deployment-live</code>","text":"","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#what-is-this-documentation-for","title":"What is this documentation for?","text":"<p>If you have been using/running/populating an instance with Archipelago Digital Objects that was set up using our simpler-to-deploy but harder-to-customize archipelago-deployment strategy and can't wait to move to this one\u2014meant for a larger (and somehow easier to maintain and upgrade on the long run) instance\u2014but (wait!) you do not want to ingest again, set up again, configure users, etc. (You already did that!), this is your documentation.</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#what-is-this-documentation-not-for","title":"What is this documentation not for?","text":"<p>To install an <code>archipelago-deployment-live</code> from scratch or to keep (forever) syncing between the two deployment options in a quantum phase shifting eternum like a time crystal.</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#requirements","title":"Requirements","text":"<ul> <li>An instance of Archipelago (working, tested) installed using <code>archipelago-deployment</code> as a basis.</li> <li>Basic knowledge and instincts on how to run Terminal Commands, copy files, run <code>composer</code>, <code>drush</code>, Linux Permissions, and <code>git</code> of course.</li> <li>Patience. You can't skip steps here. Also, Patience again since you may have stretched (good) your current instance to do way more than we thought it could.</li> <li>Time to read the main Documentation of this Repo to have a basic knowledge of how this is deployed. Recommended even if you are not going to deploy one from scratch here.</li> <li>For Shell Commands documented here please copy line by line\u2014not the whole block.</li> </ul>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#differences-between-both-deployments-strategies","title":"Differences between both deployments strategies.","text":"<p>In a nutshell: <code>archipelago-deployment-live</code> uses a different folder structure moving configuration storage, data storage outside of your webroot, and allows a much finer control of your settings (safer) and Docker containers. In a nutshell inside the first nutshell: <code>archipelago-deployment-live</code> also ignores more files so keeping customized versions, your own packages, your own settings around, and version controlled is much easier. Lastly: <code>archipelago-deployment-live</code> makes more use of Cloud Services, e.g. so if you have been running <code>min.io</code> as local mounted storage you may now consider moving storage (files) to a cloud service like AWS S3.</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#commonalities-between-both-deployment-strategies","title":"Commonalities between both deployment strategies.","text":"<p>In a nutshell: Since both run the same code and use the same Docker Containers, the data is actually the same. Everything is just persisted in different places.</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#getting-the-new-repo-in-place","title":"Getting the new repo in place","text":"<p>First you need to clone this repository and (hopefully) store in the same parent folder to your current <code>archipelago-deployment</code> one. For the purpose of this tutorial we will assume you have <code>archipelago-deployment</code> cloned in this location: <code>$HOME/archipelago-deployment</code>.</p> <p>Locate your <code>archipelago-deployment</code> folder in your terminal. Do an <code>ls</code> to make sure you can see the folder (not the content) and run:</p> <pre><code>git clone https://github.com/esmero/archipelago-deployment-live\ncd archipelago-deployment-live\ngit checkout 1.0.0-RC3\ncd ..\ncd archipelago-deployment\n</code></pre> <p>Now you have side by side <code>$HOME/archipelago_deployment</code> and <code>$HOME/archipelago-deployment-live</code>.</p> <p>This will give you the base structure.</p> <p>Before touching anything let's start by generating a backup of your current deployment (safety first).</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#backing-up","title":"Backing up","text":"","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#step-1","title":"Step 1:","text":"<p>Shut down your <code>docker-compose</code> ensemble. Inside your original <code>archipelago-deployment</code> folder run this:</p> <pre><code>docker-compose down\n</code></pre>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#step-2","title":"Step 2:","text":"<p>Verify all containers are actually down:</p> <pre><code>docker ps\n</code></pre> <p>The following command should return an empty listing. If anything is still running, wait a little longer and run the previous command again.</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#step-3","title":"Step 3:","text":"<p>Now let's tar.gz the whole ensemble with data and configs. As an example we will save this into your <code>$HOME</code> folder. As a good practice we append the current date (YEAR-MONTH-DAY) to the filename. Here we assume today is December 1st of 2021:</p> <pre><code>sudo tar -czvpf $HOME/archipelago-deployment-backup-20211201.tar.gz ../archipelago-deployment\n</code></pre> <p>The process may take a few minutes. Now let's verify that all is there and that the <code>tar.gz</code> is not corrupt:</p> <pre><code>tar -tvvf $HOME/archipelago-deployment-backup-20211201.tar.gz \n</code></pre> <p>You will see a listing of files. If corrupt (do you have enough space? did your ssh connection drop?) you will see:</p> <pre><code>tar: Unrecognized archive format\n</code></pre> <p>Done! If you are running a public instance we can allow ourselves to start Docker again to avoid downtime:</p> <pre><code>docker-compose up -d\n</code></pre>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#the-directory-structures","title":"The directory structures","text":"<p>Now that you backed all up we can spend some minutes looking at both directory structures.</p> <p>If you observe both deployment strategies side by side you will inmediately notice the most important similarities and also differences:</p> archipelago-deployment Live archipelago-deployment <pre>.\n\u251c\u2500\u2500 config_storage\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiifconfig\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 nginxconfig\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 nginxconfig_selfcert\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 php-fpm\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 solrconfig\n\u251c\u2500\u2500 data_storage\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 db\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiifcache\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiiftmp\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 letsencrypt\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 minio-data\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 ngnixcache\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 selfcert\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 solrcore\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 solrlib\n\u251c\u2500\u2500 deploy\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 azure-kubernetes\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 ec2-docker\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 kubernetes\n\u251c\u2500\u2500 docs\n\u2514\u2500\u2500 drupal\n\u2502   \u251c\u2500\u2500 config\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 d8content\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 docs\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 drush\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 patches\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 persistent\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 private\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 scripts\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 vendor\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 web\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 xdebug\n</pre> <pre>.\n\u251c\u2500\u2500 config\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 sync\n\u251c\u2500\u2500 d8content\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 metadatadisplays\n\u251c\u2500\u2500 docs\n\u251c\u2500\u2500 drush\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 Commands\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 sites\n\u251c\u2500\u2500 nginxconfigford8\n\u251c\u2500\u2500 patches\n\u251c\u2500\u2500 persistent\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 db\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiifcache\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiifconfig\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 miniodata\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 solrconfig\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 solrcore\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 solrlib\n\u251c\u2500\u2500 private\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 webform\n\u251c\u2500\u2500 scripts\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 archipelago\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 composer\n\u251c\u2500\u2500 vendor\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 archipelago\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 asm89\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 aws\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 behat\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 bin\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 brick\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 chi-teck\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 composer\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 consolidation\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 container-interop\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 cweagans\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 data-values\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 dflydev\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 doctrine\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 drupal\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 drush\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 easyrdf\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 egulias\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 enlightn\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 erusev\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 evenement\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 ezyang\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 fabpot\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 fileeye\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 firebase\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 frictionlessdata\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 google\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 graham-campbell\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 grasmash\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 guzzlehttp\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 instaclick\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 jcalderonzumba\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 jean85\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 jmikola\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 justinrainbow\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 laminas\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 league\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 lsolesen\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 maennchen\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 markbaker\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 masterminds\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 mglaman\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 mhor\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 mikey179\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 mixnode\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 ml\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 monolog\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 mtdowling\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 myclabs\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 nesbot\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 nette\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 nikic\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 paragonie\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 pear\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phar-io\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phenx\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phpdocumentor\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phplang\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phpmailer\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phpoffice\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phpoption\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phpseclib\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phpspec\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phpstan\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 phpunit\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 professional-wiki\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 psr\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 psy\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 ralouphie\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 ramsey\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 react\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 sebastian\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 seld\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 sirbrillig\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 solarium\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 squizlabs\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 stack\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 strawberryfield\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 swaggest\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 symfony\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 symfony-cmf\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 theseer\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 twbs\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 twig\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 typo3\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 vlucas\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 web64\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 webflo\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 webmozart\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 wikibase\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 wikimedia\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 zaporylie\n\u251c\u2500\u2500 web\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 core\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 libraries\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 modules\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 profiles\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 sites\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 themes</pre>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#the-data","title":"The Data","text":"<p>Let's start by focusing on the <code>data</code>, in our case the Database, Solr, and File (S3 + Private) storage. Collapsing here a few folders will make this easier to read. Marked with a <code>*</code> are matching folders that contain DB, Solr Core, the S3 min.io data (if you are using local storage) and also Drupal's very own <code>private</code> folder:</p> archipelago-deployment Live archipelago-deployment <pre>.\n\u251c\u2500\u2500 config_storage\n\u251c\u2500\u2500 data_storage\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 * db *\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiifcache\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiiftmp\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 letsencrypt\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 * minio-data *\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 ngnixcache\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 selfcert\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 solrcore\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 solrlib\n\u251c\u2500\u2500 deploy\n\u251c\u2500\u2500 docs\n\u2514\u2500\u2500 drupal\n\u2502   \u251c\u2500\u2500 config\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 d8content\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 docs\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 drush\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 patches\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 persistent\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 * private *\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 scripts\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 vendor\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 web\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 xdebug\n</pre> <pre>.\n\u251c\u2500\u2500 config\n\u251c\u2500\u2500 d8content\n\u251c\u2500\u2500 docs\n\u251c\u2500\u2500 drush\n\u251c\u2500\u2500 nginxconfigford8\n\u251c\u2500\u2500 patches\n\u251c\u2500\u2500 persistent\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 * db *\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiifcache\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 iiifconfig\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 * miniodata *\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 solrconfig\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 * solrcore *\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 solrlib\n\u251c\u2500\u2500 * private *\n\u251c\u2500\u2500 scripts\n\u251c\u2500\u2500 vendor\n\u251c\u2500\u2500 web\n</pre>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#copying-the-data-into-the-new-structure","title":"Copying the Data into the new Structure","text":"<p>To do so we need to stop Docker again. This is needed because Databases sometimes keep an open Change Log and Locks in place, and if there is any interaction or cron running, your data may end up corrupted.</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#step-1_1","title":"Step 1:","text":"<p>Shut down your <code>docker-compose</code> ensemble. Inside your original <code>archipelago-deployment</code> folder run this:</p> <pre><code>docker-compose down\n</code></pre>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#step-2_1","title":"Step 2:","text":"<p>Verify all containers are actually down:</p> <pre><code>docker ps\n</code></pre>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#step-3_1","title":"Step 3:","text":"<p>We will copy DB, min.io (File and ADO storage as files) and Drupal's private (temporary files, caches) folders to its new place:</p> <pre><code>sudo cp -rpv persistent/db ../archipelago-deployment-live/data_storage/db\nsudo cp -rpv persistent/solrcore ../archipelago-deployment-live/data_storage/solrcore\nsudo cp -rpv persistent/miniodata ../archipelago-deployment-live/data_storage/minio-data\nsudo cp -rpv private ../archipelago-deployment-live/drupal/private\n</code></pre> <p>Running <code>-rpv</code> will copy verbosely and recursively while preserving original permissions.</p> <p>Done!</p> <p>You can now start <code>docker-compose</code> again:</p> <pre><code>docker-compose up -d\n</code></pre>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#the-web","title":"The Web","text":"<p>Collapsing again a few folders to aid in readability, we can now focus on your actual Drupal/Archipelago Code/Web and settings. To be honest (we are), you can easily reinstall and restore all this via <code>composer</code>, but we can also move folders as a learning experience/time and bandwidth experience. Marked with a <code>*</code> are matching folders you want to copy over:</p> archipelago-deployment Live archipelago-deployment <pre>.\n\u251c\u2500\u2500 config_storage\n\u251c\u2500\u2500 data_storage\n\u251c\u2500\u2500 deploy\n\u251c\u2500\u2500 docs\n\u2514\u2500\u2500 drupal\n\u2502   \u251c\u2500\u2500 * config *\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 d8content\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 docs\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 drush\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 patches\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 persistent\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 private\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 scripts\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 * vendor *\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 * web *\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 xdebug\n</pre> <pre>.\n\u251c\u2500\u2500 * config *\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 sync\n\u251c\u2500\u2500 d8content\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 metadatadisplays\n\u251c\u2500\u2500 docs\n\u251c\u2500\u2500 drush\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 Commands\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 sites\n\u251c\u2500\u2500 nginxconfigford8\n\u251c\u2500\u2500 patches\n\u251c\u2500\u2500 persistent\n\u251c\u2500\u2500 private\n\u251c\u2500\u2500 scripts\n\u251c\u2500\u2500 * vendor *\n\u251c\u2500\u2500 * web *\n</pre>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#copying-the-web-into-the-new-structure","title":"Copying the Web into the new Structure","text":"<p>No need to stop Docker again. We can do this while your Archipelago is still running.</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#step-1_2","title":"Step 1:","text":"<p>We will copy all important folders over. From your <code>archipelago-deployment</code> folder run:</p> <pre><code>sudo cp -rpv vendor ../archipelago-deployment-live/drupal/vendor\nsudo cp -rpv web ../archipelago-deployment-live/drupal/web\nsudo cp -rpv config ../archipelago-deployment-live/drupal/config\n</code></pre> <p>And also, selectively, a few files we know you are very fond of!</p> <pre><code>sudo cp -rpv composer.json ../archipelago-deployment-live/drupal/composer.json\nsudo cp -rpv composer.lock ../archipelago-deployment-live/drupal/composer.lock\n</code></pre> <p>Done!</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#ssl-enviromentals-configurations-settings-and-docker","title":"SSL, Enviromentals, Configurations, Settings and Docker","text":"<p>We are almost done, but <code>archipelago-deployment-live</code> has a different, safer way of defining SSL Certs, credentials, and global settings for your Archipelago.  We will start first by copying settings as they are (most likely not very safe), and then we can update passwords/etc. to make your system better-prepared for the world.</p> <p>To learn more about these general settings please read this section of the parent Documentation (who likes duplicated documentation? Nobody.). The gist here is (after reading, please do not skip) that we need to add our service definitions into a <code>.env</code> file.</p> <p>Coming from <code>archipelago-deployment</code> means and assumes that you are running AWS Linux 2 using the suggested locations in this document, that you have a vanilla deployment, and that you followed these instructions) so your values for <code>$HOME/archipelago-deployment-live/deploy/ec2-docker/.env</code> will be the following:</p> <pre><code>ARCHIPELAGO_ROOT=/home/ec2-user/archipelago-deployment-live\nARCHIPELAGO_EMAIL=your@validemail.org\nARCHIPELAGO_DOMAIN=your.domain.org\nMINIO_ACCESS_KEY=minio\nMINIO_SECRET_KEY=minio123\nMYSQL_ROOT_PASSWORD=esmero-db\nMINIO_BUCKET_MEDIA=archipelago\nMINIO_FOLDER_PREFIX_MEDIA=/\nMINIO_BUCKET_CACHE=archipelago\nMINIO_FOLDER_PREFIX_CACHE=/\n</code></pre> <p>If you plan on staying on local storage driven <code>min.io</code>, <code>MINIO_BUCKET_CACHE</code> and <code>MINIO_FOLDER_PREFIX_CACHE</code> are not going to be used. If you are planning on moving your Storage from local to cloud driven please replace with the right values, e.g. AWS IAM keys and Secrets + bucket names and prefixes (folders). Again, refer to the parent Documentation for setting this up.</p> <p>Once you have that in place (Double-check. If something goes wrong here we can always fine-tune and fix again.), we need to decide on a new <code>docker-compose</code> file, and you may need to customize it depending on your choices and current and future needs.</p> <p>If you already have an SSL certificate, and it's provided by <code>CertBot</code> you can either copy the certs from your current system (will totally depend on your setup since <code>archipelago-deployment</code> does not provide out-of-the-box SSL Certs) to <code>$HOME/archipelago-deployment-live/data_storage/letsencrypt</code>.</p> <p>A normal folder structure for that is:</p> <pre><code>.\n\u251c\u2500\u2500 accounts\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 acme-v02.api.letsencrypt.org\n\u2502\u00a0\u00a0     \u2514\u2500\u2500 directory\n\u2502\u00a0\u00a0         \u2514\u2500\u2500 cac9f8218ef18e4f11ec053785bbf648\n\u2502\u00a0\u00a0             \u251c\u2500\u2500 meta.json\n\u2502\u00a0\u00a0             \u251c\u2500\u2500 private_key.json\n\u2502\u00a0\u00a0             \u2514\u2500\u2500 regr.json\n\u251c\u2500\u2500 archive\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 your.domain.org\n\u2502\u00a0\u00a0  \u00a0\u00a0 \u251c\u2500\u2500 cert1.pem\n\u2502\u00a0\u00a0  \u00a0\u00a0 \u251c\u2500\u2500 chain1.pem\n\u2502\u00a0\u00a0  \u00a0\u00a0 \u251c\u2500\u2500 fullchain1.pem\n\u2502\u00a0\u00a0  \u00a0\u00a0 \u2514\u2500\u2500 privkey1.pem\n\u2502\u00a0\n\u251c\u2500\u2500 csr\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 0000_csr-certbot.pem\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 0001_csr-certbot.pem\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 0002_csr-certbot.pem\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 0003_csr-certbot.pem\n\u251c\u2500\u2500 keys\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 0000_key-certbot.pem\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 0001_key-certbot.pem\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 0002_key-certbot.pem\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 0003_key-certbot.pem\n\u251c\u2500\u2500 live\n\u2502\u00a0\u00a0 \u251c\u2500\u2500 README\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 your.domain.org\n\u2502\u00a0\u00a0 \u00a0\u00a0 \u251c\u2500\u2500 cert.pem -&gt; ../../archive/your.domain.org/cert1.pem\n\u2502\u00a0\u00a0 \u00a0\u00a0 \u251c\u2500\u2500 chain.pem -&gt; ../../archive/your.domain.org/chain1.pem\n\u2502\u00a0\u00a0 \u00a0\u00a0 \u251c\u2500\u2500 fullchain.pem -&gt; ../../archive/your.domain.org/fullchain1.pem\n\u2502\u00a0\u00a0 \u00a0\u00a0 \u251c\u2500\u2500 privkey.pem -&gt; ../../archive/your.domain.org/privkey1.pem\n\u2502\u00a0\u00a0 \u00a0\u00a0 \u2514\u2500\u2500 README\n\u251c\u2500\u2500 renewal\n\u2502\u00a0\u00a0 \u2514\u2500\u2500 your.domain.org.conf\n\u2502\u00a0\u00a0 \n\u2514\u2500\u2500 renewal-hooks\n    \u251c\u2500\u2500 deploy\n    \u251c\u2500\u2500 post\n    \u2514\u2500\u2500 pre\n</code></pre> <p>Or if your SSL cert is up for renewal, you can just let Archipelago request it for you. Renewal will happen auto-magically, and you may never ever need to worry about that in the future.</p> <p>Finally, let's adapt the <code>docker-compose</code> file we need to our previous (but still current!) <code>archipelago-deployment</code> reality.</p> <p>For x86/AMD, run (for ARM64/Apple M1 please check the parent Documentation):</p> <pre><code>cp $home/archipelago-deployment-live/deploy/ec2-docker/docker-compose-aws-s3.yml $home/archipelago-deployment-live/deploy/ec2-docker/docker-compose.yml\nnano $home/archipelago-deployment-live/deploy/ec2-docker/docker-compose.yml\n</code></pre> <p>And replace the content with this slightly modified version. Note: we really only changed the lines after this comment: <code># THIS DIFFERS FROM THE NORMAL ONE...</code>.</p> <pre><code># Run docker-compose up -d\n\nversion: '3.5'\nservices:\n  web:\n    container_name: esmero-web\n    image: staticfloat/nginx-certbot\n    restart: always\n    environment:\n      CERTBOT_EMAIL: ${ARCHIPELAGO_EMAIL}\n      ENVSUBST_VARS: FQDN\n      FQDN: ${ARCHIPELAGO_DOMAIN}\n    ports:\n      - \"80:80\"\n      - \"443:443\"\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/config_storage/nginxconfig/conf.d:/etc/nginx/user.conf.d\n      - ${ARCHIPELAGO_ROOT}/config_storage/nginxconfig/certbot_extra_domains:/etc/nginx/certbot/extra_domains:ro\n      - ${ARCHIPELAGO_ROOT}/drupal:/var/www/html:cached\n      - ${ARCHIPELAGO_ROOT}/data_storage/ngnixcache:/var/cache/nginx\n      - ${ARCHIPELAGO_ROOT}/data_storage/letsencrypt:/etc/letsencrypt\n    depends_on:\n      - solr\n      - php\n      - db\n    tty: true\n    networks:\n      - host-net\n      - esmero-net\n  php:\n    container_name: esmero-php\n    restart: always\n    image: \"esmero/php-7.4-fpm:1.0.0-RC2-multiarch\"\n    tty: true\n    networks:\n      - host-net\n      - esmero-net\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/config_storage/php-fpm/www.conf:/usr/local/etc/php-fpm.d/www.conf\n      - ${ARCHIPELAGO_ROOT}/drupal:/var/www/html:cached\n    environment:\n      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}\n      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}\n      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD}\n      MINIO_BUCKET_MEDIA: ${MINIO_BUCKET_MEDIA}\n      MINIO_FOLDER_PREFIX_MEDIA: ${MINIO_FOLDER_PREFIX_MEDIA}\n  solr:\n    container_name: esmero-solr\n    restart: always\n    image: \"solr:8.8.2\"\n    tty: true\n    ports:\n      - \"8983:8983\"\n    networks:\n      - host-net\n      - esmero-net\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/data_storage/solrcore:/var/solr/data\n      - ${ARCHIPELAGO_ROOT}/config_storage/solrconfig:/drupalconfig\n      - ${ARCHIPELAGO_ROOT}/data_storage/solrlib:/opt/solr/contrib/archipelago/lib\n    entrypoint:\n      - docker-entrypoint.sh\n      - solr-precreate\n      - drupal\n      - /drupalconfig\n  db:\n    image: mysql:8.0.22\n    command: mysqld --default-authentication-plugin=mysql_native_password  --max_allowed_packet=256M\n    container_name: esmero-db\n    restart: always\n    environment:\n      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD}\n    networks:\n      - host-net\n      - esmero-net\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/data_storage/db:/var/lib/mysql\n  nlp:\n    container_name: esmero-nlp\n    restart: always\n    image: \"esmero/esmero-nlp:1.0\"\n    ports:\n      - \"6400:6400\"\n    networks:\n      - host-net\n      - esmero-net\n  iiif:\n    container_name: esmero-cantaloupe\n    image: \"esmero/cantaloupe-s3:4.1.9RC\"\n    restart: always\n    ports:\n      - \"8183:8182\"\n    networks:\n      - host-net\n      - esmero-net\n    environment:\n      AWS_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY}\n      AWS_SECRET_ACCESS_KEY: ${MINIO_SECRET_KEY}\n      # THIS DIFFERS FROM THE STANDARD ONE AND ENABLES LOCAL FILESYSTEM CACHE INSTEAD OF AWS S3 one\n      CACHE_SERVER_DERIVATIVE: FilesystemCache\n      S3SOURCE_BASICLOOKUPSTRATEGY_BUCKET_NAME: ${MINIO_BUCKET_MEDIA}\n      S3SOURCE_BASICLOOKUPSTRATEGY_PATH_PREFIX: ${MINIO_FOLDER_PREFIX_MEDIA}\n      S3CACHE_BUCKET_NAME: ${MINIO_BUCKET_CACHE} \n      S3CACHE_OBJECT_KEY_PREFIX: ${MINIO_FOLDER_PREFIX_CACHE} \n      XMS: 2g\n      XMX: 4g\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/config_storage/iiifconfig:/etc/cantaloupe\n      - ${ARCHIPELAGO_ROOT}/data_storage/iiifcache:/var/cache/cantaloupe\n      - ${ARCHIPELAGO_ROOT}/data_storage/iiiftmp:/var/cache/cantaloupe_tmp\n  minio:\n    container_name: esmero-minio\n    restart: always\n    image: minio/minio:latest\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/data_storage/minio-data:/data:cached\n    ports:\n      - \"9000:9000\"\n      - \"9001:9001\"\n    networks:\n      - host-net\n      - esmero-net\n    environment:\n      MINIO_HTTP_TRACE: /tmp/minio-log.txt\n      MINIO_ROOT_USER: ${MINIO_ACCESS_KEY}\n      MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY}\n      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}\n      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}\n      # THIS DIFFERS FROM THE STANDARD ONE AND ENABLES LOCAL MINIO INSTEAD OF AWS S3 one \n    command: server /data --console-address \":9001\"\nnetworks:\n  host-net:\n    driver: bridge\n  esmero-net:\n    driver: bridge\n    internal: true\n</code></pre> <p>Press CNTRL-X, and you are done. Now the final test!!</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-moveToLive/#shutdown-the-old-one-start-the-new-one","title":"Shutdown the old one, start the new one","text":"<p>So we are ready. Testing may be a hit-or-miss thing here. Did we cover all the steps? Did a command fail? The good thing is that we can start the new ensemble, and all our old ones will survive. And we can come back over and over until we are ready. Let's try!</p> <p>We will start by shutting down the running Docker ensemble:</p> <pre><code>cd $HOME/archipelago-deployment\ndocker-compose down\n</code></pre> <p>Now let's go to our new deployment. Docker starts here in a different folder:</p> <pre><code>cd $HOME/archipelago-deployment-live/deploy/ec2-docker\ndocker-compose up\n</code></pre> <p>You may notice that we removed the <code>-d</code>. Why? We want to see all the messages and notice/mark/copy any errors, e.g. did the SSL CERT load correctly? Did the MYSQL import work out? To avoid shutting it down while all starts, please open another Terminal and type:</p> <pre><code>docker ps\n</code></pre> <p>And look at the up-times. Do you see any Containers restarting (where Created and the Status differ for a lot and Status keeps resetting to 0?)? A healthy deployment will look similar to this:</p> <pre><code>CONTAINER ID   IMAGE                                    COMMAND                  CREATED          STATUS         PORTS                                      NAMES\nf794c25db64c   esmero/cantaloupe-s3:4.1.9RC2-arm64      \"sh -c 'java -Dcanta\u2026\"   6 seconds ago    Up 3 seconds   0.0.0.0:8183-&gt;8182/tcp                     esmero-cantaloupe\n5b791445720f   jonasal/nginx-certbot                    \"/docker-entrypoint.\u2026\"   6 seconds ago    Up 3 seconds   0.0.0.0:80-&gt;80/tcp, 0.0.0.0:443-&gt;443/tcp   esmero-web\ne38fbbd86edf   esmero/esmero-nlp:1.0.1-RC2-arm64        \"/usr/local/bin/entr\u2026\"   11 seconds ago   Up 6 seconds   0.0.0.0:6400-&gt;6400/tcp                     esmero-nlp\nc84a0a4d43e9   minio/minio:latest                       \"/usr/bin/docker-ent\u2026\"   11 seconds ago   Up 6 seconds   0.0.0.0:9000-9001-&gt;9000-9001/tcp           esmero-minio\n3ec176a960c3   esmero/php-7.4-fpm:1.0.0-RC2-multiarch   \"docker-php-entrypoi\u2026\"   11 seconds ago   Up 6 seconds   9000/tcp                                   esmero-php\ne762ad7ea5e2   solr:8.8.2                               \"docker-entrypoint.s\u2026\"   11 seconds ago   Up 6 seconds   0.0.0.0:8983-&gt;8983/tcp                     esmero-solr\n381166d61f8c   mariadb:10.5.10-focal                    \"docker-entrypoint.s\u2026\"   11 seconds ago   Up 6 seconds   3306/tcp      \n</code></pre> <p>If you feel that all seems to be fine, open a browser window and visit your website. See if you can log in and see ADOs. If not you can momentarily shut down this new Docker ensemble and restart the older one. Nothing is lost! Then with time and tea/coffee and fresh eyes come back and re-trace your steps. 95% of the issues are incorrect values in the <code>.env</code> file. The other 5% may be on us. If you run into any trouble please get in touch!</p> <p>Happy deploying!</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback, or open an ISSUE in this Archipelago Deployment Live Repository.</p> <p>Return to Archipelago Live Deployment.</p>","tags":["Archipelago-deployment","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/","title":"Archipelago Deployment Live","text":"<p>A Cloud / Local production ready Archipelago Deployment using Docker and soon Kubernetes.</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#what-is-this-repo-for","title":"What is this repo for?","text":"<p>Running Archipelago Commons on a live public instance using SSL with Blob/Object Storage backend </p> <ul> <li>Cloud-based deployment, e.g. AWS/Azure under Linux</li> <li>Self-managed servers running Linux</li> <li>x86/AMD86 or ARM64/v8 CPU architectures</li> </ul>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#what-is-this-repo-not-for","title":"What is this repo not for?","text":"<ul> <li>Running your own local/development Archipelago. For that we suggest using https://github.com/esmero/archipelago-deployment</li> </ul>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#requirements","title":"Requirements","text":"","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#minimal-not-recommended-for-production","title":"Minimal (not recommended for production)","text":"<ul> <li>4 Gbytes of RAM (e.g AWS EC2 t3.medium) 2 CPUs, Single SSD Drive of 100 Gbytes</li> </ul>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#base-line-for-production","title":"Base line for production","text":"<ul> <li>8 Gbytes of RAM (AWS EC2 t3.medium)  2 CPUs, Single SSD Drive of 100 Gbytes, optional: one magnetic Drive of 500 Gbytes for Caches/Temp files/Backups.</li> </ul>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#recommended-for-production","title":"Recommended for production","text":"<ul> <li>16 Gbytes of RAM (AWS EC2 m6g.xlarge - Graviton) 4 CPUs, Single SSD Drive of 200 Gbytes, optional: one magnetic Drive of 1TB for Caches/Temp files/Backups.</li> </ul>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#os","title":"OS:","text":"<ul> <li>Ubuntu 22.04 LTS /Ubuntu 24.04 LTS/Amazon Linux 2023/Debian 10 \"Buster\" matching your CPU architecture (of course)</li> <li>Most recent <code>Docker</code> running as a service and <code>docker-compose</code> or <code>docker compose</code></li> </ul>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#skills-and-the-important-human-aspect","title":"Skills and the important human aspect","text":"<ul> <li>Basic Unix/Linux terminal skills and a root/sudo account</li> <li><code>Docker</code> basics knowledge and how to manage packages in your System</li> <li>Knowledge of how to manage AWS/Azure or any other cloud-based provider for Computing Instances and S3 compatible Object Storage system and the associated permissions credentials to do so.</li> <li>To be patient. Please read everything. Do not skip steps. Do not only copy and paste commands. Explanations give context and might help you troubleshoot issues.</li> </ul> <p>Basically this guide is meant for humans with basic to medium <code>DevOps</code> background or humans with patience that are willing to troubleshoot, ask, and try again when that background is not (yet) enough. And we are here to help.</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#deployment-on-linuxx86amd-system","title":"Deployment on Linux/X86/AMD system","text":"","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#step-1","title":"Step 1:","text":"<p>Deploy your base system</p> <p>Make sure your Firewall/AWS Security group has these ports open for everyone to access</p> <ul> <li>443 (NGINX SSL)</li> <li>80 (NGINX HTTP) And protected/modally open for your own development/testing/administration</li> <li>8183 (Cantaloupe)</li> <li>8983 (Solr)</li> <li>6400 (NLP64)</li> <li>9001 (Minio)</li> <li>22 (so you can ssh into your machine)</li> </ul> <p>Setup your system using your favorite package manager with </p> <ul> <li>Docker</li> <li>git</li> <li>htop</li> <li>tree</li> <li>docker-compose or <code>docker compose</code> (V2) if running the most recent docker service. See https://docs.docker.com/compose/migrate/</li> </ul> <p>e.g. for Amazon Linux 2023 (x86/amd64) these steps are tested:</p> <pre><code>sudo yum update -y\nsudo yum install -y docker\nsudo service docker start\nsudo usermod -a -G docker ec2-user\nsudo chkconfig docker on\nsudo systemctl enable docker\nsudo yum install -y git htop tree jq perl-JSON-PP\nsudo curl -L \"https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)\" -o /usr/local/bin/docker-compose\nsudo chmod +x /usr/local/bin/docker-compose\nsudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose\nsudo reboot\n</code></pre> <p>Reboot is needed to allow Docker to take full control over your OS resources.</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#step-2","title":"Step 2:","text":"<p>In your location of choice clone this repo</p> <pre><code>git clone https://github.com/esmero/archipelago-deployment-live\ncd archipelago-deployment-live\ngit checkout 1.4.0\n</code></pre>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#step-3-setup-your-enviromental-variables-for-dockerservices","title":"Step 3. Setup your enviromental variables for Docker/Services","text":"","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#setup-enviromentals","title":"Setup Enviromentals","text":"<p>Setup your deployment enviromental variables by copying the template</p> <p><pre><code>cp deploy/ec2-docker/.env.template deploy/ec2-docker/.env\n</code></pre> and editing it</p> <pre><code>nano deploy/ec2-docker/.env\n</code></pre> <p>The content of that file would be similar to this.  <code>Note</code>: There are a few extra commented lines at the end only used for: https://docs.archipelago.nyc/1.4.0/security_bots/ if you decide to go that way.</p> <pre><code>ARCHIPELAGO_ROOT=/home/ec2-user/archipelago-deployment-live\nARCHIPELAGO_EMAIL=your@validemail.org\nARCHIPELAGO_DOMAIN=your.domain.org\nMINIO_ACCESS_KEY=THE_S3_AZURE_OR_LOCAL_MINIO_KEY\nMINIO_SECRET_KEY=THE_S3_AZURE_OR_LOCAL_MINIO_SECRET\nMYSQL_ROOT_PASSWORD=YOUR_MYSQL_PASSWORD_FOR_ARCHIPELAGO\nMINIO_BUCKET_MEDIA=THE_NAME_OF_YOUR_S3_BUCKET_FOR_PERSISTEN_STORAGE\nMINIO_FOLDER_PREFIX_MEDIA=media/\nMINIO_BUCKET_CACHE=THE_NAME_OF_YOUR_S3_BUCKET_FOR_IIIF_STORAGE\nMINIO_FOLDER_PREFIX_CACHE=iiifcache/\nREDIS_PASSWORD=YOUR_REDIS_PASSWORD\n</code></pre> <p>What does each key mean?</p> <ul> <li><code>ARCHIPELAGO_ROOT</code>: the absolute path to your <code>archipelago-deployment-live</code> git repo in your host machine.</li> <li><code>ARCHIPELAGO_EMAIL</code>: a valid email, will be used to register your SSL Certificate via Certbot.</li> <li><code>ARCHIPELAGO_DOMAIN</code>: a valid domain name for your repository. This domain will be also used to request your SSL Certificate via Certbot.</li> <li><code>MINIO_ACCESS_KEY</code>: If you are running a Cloud Service backed S3/Azure Storage this needs to be generated there. The user/IAM owner of this ACCESS KEY needs to have access to read/write the bucket you will configure in this same <code>.env</code>. If running local <code>min.io</code> whatever you set will be used.</li> <li><code>MINIO_SECRET_KEY</code>: If you are running a Cloud Service backed S3/Azure Storage this needs to generated there. The user/IAM owner of the matching SECRET_KEY needs to have access to read/write the bucket you will configure in this same <code>.env</code> file. If running local <code>min.io</code> whatever you set will be used.</li> <li><code>MYSQL_ROOT_PASSWORD</code>: The MYSQL 8 or Mariadb 15 password. This password will be used later also during Drupal deployment via <code>drush</code></li> <li><code>MINIO_BUCKET_MEDIA</code>: The name of your Persistant Storage Bucket. If using mini.io local we recommend keeping it simple, e.g. <code>archipelago</code>.</li> <li><code>MINIO_FOLDER_PREFIX_MEDIA</code>: The <code>folder</code> (a prefix really) where your DO Storage and File storage will go inside the <code>MINIO_BUCKET_MEDIA</code> Bucket. <code>media/</code> is a fine name for this one and common in archipelago deployments. IMPORTANT: Always terminate these with a <code>/</code>. </li> <li><code>MINIO_BUCKET_CACHE</code>: The name of your IIIF Cache storage Bucket. May be the same as <code>MINIO_BUCKET_MEDIA</code>. If different make sure your your <code>MINIO_ACCESS_KEY</code> and/or IAM role ACL have permission to read write to this one too.</li> <li><code>MINIO_FOLDER_PREFIX_CACHE</code>:  The <code>folder</code> (a prefix really) where Cantaloupe will/can write its <code>iiif</code> caches. <code>iiifcache/</code> is a lovely name we use a lot. IMPORTANT: Always terminate these with a <code>/</code>.</li> <li><code>REDIS_PASSWORD</code>: Password for your REDIS (Drupal Cache/Queue storage) if you decide to enable the Drupal REDIS module.</li> </ul> <p><code>IMPORTANT NOTE</code>: For AWS EC2. If you selected an <code>IAM role</code> for your server when setting it up/deploying it, <code>min.io</code> will use the AWS EC2-backed internal API to request access to your S3. This means the ROLE itself needs to have read/write access (ACL) to the given Bucket(s) and your key/secrets won't be able to override that. Please do not ignore this note. It will save you a LOT of frustration and coffee. You can also run an EC2 instace without a given IAM and in that case just the ACCESS_KEY/SECRET will matter.</p> <p>Now that you know, you also know that these values should not be shared and this <code>.env</code> file should not be committed/kept in version control. Please be careful.</p> <p><code>docker-compose</code> will read this <code>.env</code> and start all services for you based on its content.</p> <p>Once you have modified this you are ready for your first big decision. </p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#running-a-fully-qualified-domain-you-wish-a-validsigned-certificate-for-amdintel-architecture","title":"Running a fully qualified domain you wish a valid/signed certificate for AMD/INTEL Architecture?","text":"<p>This means you will use the <code>docker-compose-aws-s3.yml</code>. Do the following:</p> <pre><code>cp deploy/ec2-docker/docker-compose-aws-s3.yml deploy/ec2-docker/docker-compose.yml\n</code></pre>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#running-a-fully-qualified-domain-you-wish-a-validsigned-certificate-for-arm64apple-m1-architecture","title":"Running a fully qualified domain you wish a valid/signed certificate for ARM64/Apple M1 Architecture?","text":"<p>This means you will use the <code>docker-compose-aws-s3-arm64.yml</code>. Do the following:</p> <pre><code>cp deploy/ec2-docker/docker-compose-aws-s3-arm64.yml deploy/ec2-docker/docker-compose.yml\n</code></pre>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#or-running-self-signed-optional-and-does-not-apply-to-arm64apple-m1-architecture","title":"OR Running self-signed? (optional and does not apply to ARM64/Apple M1 Architecture):","text":"<p>Only if you are not running a fully qualified domain you wish a valid/signed. We really DO not recommend this route. IF you plan on using this deployment for local testing or running on non SSL please go for https://github.com/esmero/archipelago-deployment which delivers the same experience in less than 20 minutes deployment time.</p> <p>Generate a self signed Cert</p> <pre><code>sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout data_storage/selfcert/private/nginx.key -out data_storage/selfcert/certs/nginx.crt \nsudo openssl dhparam -out data_storage/selfcert/dhparam.pem 4096\ncp deploy/ec2-docker/docker-compose-selfsigned.yml deploy/ec2-docker/docker-compose.yml\n</code></pre> <p>Note: Self signed docker-compose.yml file is setup to use min.io with local storage</p> <pre><code>    volumes:\n      - ${ARCHIPELAGO_ROOT}/data_storage/minio-data:/data:cached\n</code></pre> <p>This folder will be created by min.io. If you are using a secondary Drive (e.g. magnetic) you can modify your <code>deploy/ec2-docker/docker-compose.yml</code> to use a folder there, e.g.</p> <pre><code>    volumes:\n      - /persistentinotherdrive/data_storage/minio-data:/data:cached\n</code></pre> <p>Make sure your logged in user can read/write to it.</p> <p>NOTE: If you want to use AWS S3 storage for the self signed version replace the minio Service yaml block with this Service Block in your new <code>deploy/ec2-docker/docker-compose.yml</code>. You can mix and match services and even remove all <code>:cached</code> statements for improved R/W volumen performance.</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#step-4-first-run","title":"Step 4. First Run","text":"","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#first-permissions","title":"First Permissions","text":"<pre><code>sudo chown 8183:8183 config_storage/iiifconfig/cantaloupe.properties\nsudo chown -R 8183:8183 data_storage/iiifcache\nsudo chown -R 8183:8183 data_storage/iiiftmp\nsudo chown -R 8983:8983 data_storage/solrcore\n</code></pre>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#actual-first-run","title":"Actual first run","text":"<p>Time to spin our docker containers for the first time. We will start all without going into background so log/error checking is easier, especially if you have selected a Valid/Signed Cert choice and also want to be sure S3 keys/access are working.</p> <pre><code>cd deploy/ec2-docker\ndocker-compose up\n</code></pre> <p>You will see a lot of things happening. Check for errors/problems/clear alerts and give all a minute or so to start.  Ok, let's assume your setup managed to request a valid signed SSL cert, you will see a nice message!</p> <pre><code>- Congratulations! Your certificate and chain have been saved at:XXXXX\n   Your certificate will expire on 20XX-XX-XX. To obtain a new or\n   tweaked version of this certificate in the future, simply run\n   certbot again. To non-interactively renew *all* of your\n   certificates, run \"certbot renew\"\n</code></pre> <p>Archipelago will do that for you whenever it's about to expire so no need to deal with this manually, even when <code>docker-compose</code> restarts.</p> <p>Now press CTRL+C. <code>docker-compose</code> will shutdown gracefully. Good!</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#step-5-deploy-drupal-10","title":"Step 5. Deploy Drupal 10","text":"","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#composer-and-drupal","title":"Composer and Drupal","text":"<p>Copy the shipped default composer.default.json to composer.json and composer.default.lock to composer.lock (ONLY if you are installing from scratch):</p> <pre><code>cp ../../drupal/composer.default.json ../../drupal/composer.json\ncp ../../drupal/composer.default.lock ../../drupal/composer.lock\n</code></pre> <p>Start Docker again</p> <pre><code>docker-compose up -d\n</code></pre> <p>Wait a few seconds and run:</p> <pre><code>docker exec -ti esmero-php bash -c \"chown -R www-data:www-data private\"\ndocker exec -ti esmero-php bash -c \"chown -R www-data:www-data web/sites\"\ndocker exec -ti esmero-php bash -c \"composer install\"\n</code></pre> <p>Composer install will take a little while and bring all your PHP libraries.</p> <p>Once done, execute our setup script that will prepare your Drupal <code>settings.php</code> and bring some of the <code>.env</code> enviromental variables to the Drupal environment. </p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/setup.sh'\n</code></pre> <p>And now you can deploy Drupal! </p> <p>IMPORTANT: Make sure you replace in the following command inside <code>root:MYSQL_ROOT_PASSWORD</code> the <code>MYSQL_ROOT_PASSWORD</code> string with the value you used/assigned in your <code>.env</code> file for <code>MYSQL_ROOT_PASSWORD</code>. And replace <code>ADMIN_PASSWORD</code> with a password that is safe and you won't forget! That passwords is for your Drupal super user (uid:0).</p> <pre><code>docker exec -ti -u www-data esmero-php bash -c \"cd web;../vendor/bin/drush -y si --verbose --existing-config --db-url=mysql://root:MYSQL_ROOT_PASSWORD@esmero-db/drupal --account-name=admin --account-pass=ADMIN_PASSWORD -r=/var/www/html/web --sites-subdir=default --notify=false;drush cr;chown -R www-data:www-data sites;\"\n</code></pre>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#step-6-users-and-initial-content","title":"Step 6. Users and initial Content.","text":"<p>After installation is done (may take a few) you can install initial users and assign them roles. Copy each line separately. A missing permission will not let you ingest the initial Metadata Displays and AMI set.</p> <pre><code>docker exec -ti esmero-php bash -c 'drush ucrt demo --password=\"demo\"; drush urol metadata_pro \"demo\"'\n</code></pre> <pre><code>docker exec -ti esmero-php bash -c 'drush ucrt jsonapi --password=\"jsonapi\"; drush urol metadata_api \"jsonapi\"'\n</code></pre> <pre><code>docker exec -ti esmero-php bash -c 'drush urol administrator \"admin\"'\n</code></pre> <p>Before ingesting the base content we need to make sure we can access your <code>JSON-API</code> on for your new domain. That means we need to change internal urls (<code>https://esmero-web</code>) to the new valid SSL driven ones. This is easy:</p> <p>On your host machine (no need to <code>docker exec</code> these ones), replace first in the following command <code>your.domain.org</code> with the domain you setup in your <code>.env</code> file. Go to (cd into) your base git clone folder (Important: YOUR BASE CLONE FOLDER) and then run</p> <pre><code> sed -i 's/http:\\/\\/esmero-web/https:\\/\\/your.domain.org/g' drupal/scripts/archipelago/deploy.sh\n sed -i 's/http:\\/\\/esmero-web/https:\\/\\/your.domain.org/g' drupal/scripts/archipelago/update_deployed.sh\n</code></pre> <p>Now your <code>deploy.sh</code> and <code>update_deployed.sh</code> are update and ready. Let's ingest some Twig Templates, an AMI Set, menus and a Blocks.</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/deploy.sh'\n</code></pre> <p>IMPORTANT: <code>update_deployed.sh</code> is not needed when deploying for the first time and totally discouraged on a customized Archipelago.  If you make modifications to your <code>Twig templates</code>, that command will replace the ones shipped by us with fresh copies overwriting all your modifications. Only run to restore larger errors or when needing to update everything ones with newer versions and you don't care for your own customization. Please read https://docs.archipelago.nyc/1.4.0/utility_scripts/ for more ways of managing exporting/importing Metadata Display Entities (Twig templates).</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#step-7-set-your-public-iiif-server-url-to-your-actual-domain","title":"Step 7. Set your public IIIF server URL to your actual domain","text":"<p>By default archipelago ships with a public facing and an internal facing IIIF Server URLs configured. These urls are used by a number of IIIF enabled viewers and need to be changed to reflect your new reality (a real Domain name and a proxied path!). These settings belong to the <code>strawberryfield/format_strawberryfield</code> module. </p> <p>First check your current settings:</p> <pre><code>docker exec -ti esmero-php bash -c \"drush config-get format_strawberryfield.iiif_settings\"\n</code></pre> <p>You will see the following:</p> <pre><code>pub_server_url: 'http://localhost:8183/iiif/2'\nint_server_url: 'http://esmero-cantaloupe:8182/iiif/2'\n</code></pre> <p>Let's modify <code>pub_server_url</code>. Replace in the following command <code>your.domain.org</code> with the domain you defined in your <code>.env</code> file.  NOTE: We are passing the <code>-y</code> flag to <code>drush</code> avoid that way having to answer \"yes\".</p> <pre><code>docker exec -ti esmero-php bash -c \"drush config-set -y format_strawberryfield.iiif_settings pub_server_url https://your.domain.org/cantaloupe/iiif/2\"\n</code></pre> <p>Finally Done! Now you can log into your new Archipelago using <code>https</code> and start exploring. Thank you for following this guide!</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#deployment-on-arm64v8graviton-apple-m1-system","title":"Deployment on ARM64/v8(Graviton, Apple M1) system:","text":"<p>This applies to AWS <code>m6g</code> and <code>t3g</code> Instances and is documented inline in this guide. Please open an ISSUE in this repository if you run into any problems. Please review https://github.com/esmero/archipelago-deployment-live/blob/1.4.0/deploy/ec2-docker/docker-compose-aws-s3-arm64.yml for more info.</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#how-do-i-know-my-architecture","title":"How do I know my Architecture?","text":"<p>Run </p> <pre><code>uname -m \n</code></pre> <ul> <li>For an <code>x86(64 bit)</code> processor system output will be <code>x86_64</code></li> <li>For an <code>ARM(64 bit)</code> processor system output will be <code>aarch64</code></li> </ul>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Lund</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#acknowledgments","title":"Acknowledgments","text":"<p>This software is a Metropolitan New York Library Council Open-Source initiative and part of the Archipelago Commons project.</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-readme/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-search_solr_index/","title":"Archipelago-deployment-live: Upgrading Solr","text":"","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#what-is-this-documentation-for","title":"What is this documentation for?","text":"<p>This documentation will help you ugprade your Solr Index from 8.x to 9.x or inside 9.x releases. And is meant to be a guide/helper. There is no simple way of saying this, but because the way a Solr index (sort of a Binary tree) is build, any larger change in the schema, field type definitions requires either a complete reindex but really, most of the time, a wipe, and start fresh situation. There is no perfect way around. There are very complex ways of keeping an old Server running and serving searches while you re-index a new one, but honestly, the how and approach will depend on your existing knowledge of Solr, your skills (even memory!) to execute so, and documenting those hacks are beyond the scope of this documentation. What is proven is what we explain in this document</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#requirements","title":"Requirements","text":"<ul> <li>An archipelago-deployment-live instance (working, tested) deployed using provided instructions via Docker running either Solr 8.x or 9.x</li> <li>Good knowledge, patience and instincts (+ courage and time) on how to run Terminal Commands.</li> <li>Patience(again but also patience from your users since search will be unavailable until you reindex). You can't skip steps here.</li> <li>For shell Commands documented here please copy line by line--not the whole block.</li> <li>You are running already version control and know how to git pull/push/merge.</li> </ul>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#backing-up-and-preparing-for-the-upgrade","title":"Backing up and preparing for the upgrade","text":"<p>Backups are always going to be your best friends. Archipelago's code, database, and settings are mostly self-contained in your current <code>archipelago-deployment-live</code> repo folder, and backing up is simple because of that.</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-1","title":"Step 1:","text":"<p>To make upgrading simpler we will clone archipelago-deployment-live (empty one) into a different folder. That way we can copy complete folders of configs and files instead of fetching them from github one by one.</p> <p>Go to your home folder (for the sake of this documentation it will be <code>/home/ec2-user</code> but you can also use the <code>$HOME</code> environmental variable instead )</p> <pre><code>cd /home/ec2-user\ngit clone https://github.com/esmero/archipelago-deployment-live archipelago-deployment-live-1.4.0\ncd archipelago-deployment-live-1.4.0\ngit switch 1.4.0\n</code></pre> <p>Now, on a terminal, <code>cd</code> into your running <code>archipelago-deployment-live</code> folder, then <code>cd</code> inside the <code>deploy/ec2-docker</code> subfolders and shut down your <code>docker-compose</code> ensemble by running the following:</p> <pre><code>docker-compose down\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-2","title":"Step 2:","text":"<p>Verify that all containers are actually down. The following command should return an empty listing:</p> <pre><code>docker ps\n</code></pre> <p>If anything is still running, wait a little longer and run the command again.</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-3","title":"Step 3:","text":"<p>Now let's <code>tar.gz</code> the whole ensemble with data and configs. We will exclude here the local source caches generated by Cantaloupe. If these or not exist will depend on how custom your deployment is.</p> <p>As an example we will save this into your <code>$HOME</code> folder. As a good practice we append the current date (YEAR-MONTH-DAY) to the filename. Here we assume today is July 7th of 2024.</p> <p>We will <code>cd</code> back to the parent folder of your running <code>archipelago-deployment-live</code> folder, so three levels down, assuming you are right now inside <code>archipelago-deployment-live/deploy/ec2-docker</code></p> <pre><code>cd ../../..\nsudo tar --exclude=archipelago-deployment-live/data_storage/iiifcache --exclude=archipelago-deployment-live/data_storage/iiiftmp -czvpf $HOME/archipelago-deployment-D10-20240707.tar.gz archipelago-deployment-live\n</code></pre> <p>The process may take a few minutes. Now let's verify that all is there and that the <code>tar.gz</code> is not corrupt.</p> <pre><code>tar -tvvf $HOME/archipelago-deployment-D10-20240707.tar.gz\n</code></pre> <p>You will see a listing of files, and at the end you will see something like this: <code>Archive Format: POSIX pax interchange format, Compression: gzip</code>. If corrupt (Do you have enough space? Did your ssh connection drop?) you will see the following:</p> <pre><code>tar: Unrecognized archive format\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-4","title":"Step 4:","text":"<p><code>cd</code> again into your running <code>archipelago-deployment-live</code> folder, then <code>cd</code> inside the <code>deploy/ec2-docker</code> Restart your <code>docker-compose</code> ensemble, and wait a little while for all to start.</p> <pre><code>docker-compose up -d\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-5","title":"Step 5:","text":"<p>Export/backup all of your live Archipelago 1.3.0, Drupal 10 configurations (this allows you to compare/come back in case you lose something custom during the upgrade).</p> <pre><code>docker exec esmero-php mkdir config/backup\ndocker exec esmero-php drush cex --destination=/var/www/html/config/backup\n</code></pre> <p>Good. Now it's safe to begin the upgrade process.</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#upgrading-to-solr-921","title":"Upgrading to Solr 9.2.1","text":"","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-0-get-familiar-with-what-changed","title":"Step 0: Get familiar with what changed.","text":"<p>Running a Production Server requires some informed decision making and thus, we believe, a good pre-step is reviewing what changed between releases.  In specific focus on this folder.</p> <p>https://github.com/esmero/archipelago-deployment-live/tree/1.4.0/config_storage/solrconfig/conf</p> <p>and</p> <p>https://github.com/esmero/archipelago-deployment-live/tree/1.4.0/data_storage/solrlib</p> <p>Also (please) read the official documentation here https://solr.apache.org/guide/8_9/solr-upgrade-notes.html</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-1-edit-docker-composeryml","title":"Step 1: Edit docker-composer.yml","text":"<p>You want to replace your current Solr Service (in its enterity. Please make sure indendation is 1:1). </p> <pre><code> solr:\n    container_name: esmero-solr\n    restart: always\n    image: \"solr:9.2.1\"\n    # If running Docker &lt; 20.10.10 please uncomment the following lines\n    # See https://solr.apache.org/guide/solr/latest/upgrade-notes/major-changes-in-solr-9.html#solr-9-2 \n    #security_opt:\n    #  - seccomp:unconfined\n    tty: true\n    environment:\n      SOLR_HEAP: 1024m\n      SOLR_OPTS: -Dsolr.jetty.request.header.size=65535 -Dsolr.modules=scripting\n    ports:\n      - \"8983:8983\"\n    networks:\n      - host-net\n      - esmero-net\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/data_storage/solrcore:/var/solr/data\n      - ${ARCHIPELAGO_ROOT}/config_storage/solrconfig:/drupalconfig\n      - ${ARCHIPELAGO_ROOT}/data_storage/solrlib:/opt/solr/contrib/archipelago/lib\n    entrypoint:\n      - docker-entrypoint.sh\n      - solr-precreate\n      - drupal\n      - /drupalconfig\n</code></pre> <p>You can also use any of these as reference:</p> <ul> <li>https://github.com/esmero/archipelago-deployment-live/blob/1.4.0/deploy/ec2-docker/docker-compose-aws-s3-arm64.yml</li> <li>https://github.com/esmero/archipelago-deployment-live/blob/1.4.0/deploy/ec2-docker/docker-compose-aws-s3.yml</li> </ul> <p>For this, if you have not already, run:</p> <pre><code>docker-compose down\n</code></pre> <p>Then open your docker-compose.yml file, find the <code>solr:</code> key and replace with this new settings. If for some unknown reason your voluments do not match our defaults, please adapt to your custom edits so they match where the Solr Core and Libraries are saved:</p> <pre><code>nano docker-compose.yml\n</code></pre> <p>Save your changes.</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-2-wipe-clean-get-the-new-configs-get-the-new-ocr-highlight-library","title":"Step 2: Wipe clean. Get the new configs. Get the new OCR Highlight library","text":"<p>Wait! (breath.)</p> <p>This step is only required if you are moving from Solr 8.x to 9.x or inside 9.x you have <code>solr field type</code> definition changes. If you had a stock 1.3.0 with solr 9.1 and want to move to solr 9.2 you can skip deleting everything and can jump to Step 3!</p> <p>This step requires some nerve. Be sure you know where you are inside your terminal (always)</p> <p>Inside your archipelago-deployment-live folder run:</p> <pre><code>cd data_storage/solrcore\npwd\n</code></pre> <p>You should see something like  <pre><code>/home/ec2-user/archipelago-deployment-live/data_storage/solrcore\n</code></pre></p> <p>Which means you are in the correct folder. Now time to clean your index (really think twice here ok? You have a backup. Never run any of these without a backup)</p> <pre><code>sudo rm -rf *\n</code></pre> <p>Now we need the new configurations for your Solr (so then docker container can re-create the index from scratch). Remember we downloaded a reference/empty Archipelago Deployment Live 1.4.0 at <code>/home/ec2-user/archipelago-deployment-live-1.4.0</code>. We are going to use the files there to replace your own configs. <code>cd</code> back to your live deployment assuming here it is (still) <code>/home/ec2-user/archipelago-deployment-live</code></p> <pre><code>cd /home/ec2-user/archipelago-deployment-live\ncp -rpv /home/ec2-user/archipelago-deployment-live-1.4.0/config_storage/solrconfig/conf/* /home/ec2-user/archipelago-deployment-live/config_storage/solrconfig/conf/.\n</code></pre> <p>Now we need to remove the old OCR library and replace with the new one</p> <pre><code>rm /home/ec2-user/archipelago-deployment-live/data_storage/solrlib/solr-ocrhighlighting-0.7.1.jar\ncp -rpv /home/ec2-user/archipelago-deployment-live-1.4.0/data_storage/solrlib/solr-ocrhighlighting-0.8.4-SNAPSHOT.jar /home/ec2-user/archipelago-deployment-live//data_storage/solrlib/.\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-3-docker-pull-and-check","title":"Step 3: docker pull and check","text":"<p>Optional: You might want to review/compare (now) your current Drupal Search API Index against our most current one here:</p> <p>In specific:</p> <ul> <li>https://github.com/esmero/archipelago-deployment-live/blob/1.4.0/drupal/config/sync/search_api.server.esmero_solr.yml</li> <li>https://github.com/esmero/archipelago-deployment-live/blob/1.4.0/drupal/config/sync/search_api.index.default_solr_index.yml</li> </ul> <p>and anything that starts with <code>search_api.</code> too.</p> <p>Time to fetch the latest Solr that will re-create the index:</p> <pre><code>docker compose pull\ndocker compose up -d\n</code></pre> <p>Give all a little time to start. Please be patient. To ensure all is well, run (more than once if necessary) the following:</p> <pre><code>docker ps\n</code></pre> <p>You should see something like this if you synced all containers to the latest (your versions and databse might vary depending on your server's platform):</p> <pre><code>CONTAINER ID   IMAGE                                      COMMAND                  CREATED          STATUS          PORTS                              NAMES\n5b06ee366f58   jonasal/nginx-certbot                      \"/docker-entrypoint.\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8001-&gt;80/tcp               esmero-web\n86b685008158   solr:9.2.1                                 \"docker-entrypoint.s\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8983-&gt;8983/tcp             esmero-solr\na4872b237e17   esmero/cantaloupe-s3:6.0.1-multiarch       \"sh -c 'java -Dcanta\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:8183-&gt;8182/tcp             esmero-cantaloupe\nbec0b31f3421   mariadb:10.6.18-focal                      \"docker-entrypoint.s\u2026\"   10 minutes ago   Up 10 minutes   3306/tcp                           esmero-db\n85bedadf9732   redis:6.2-alpine                           \"docker-entrypoint.s\u2026\"   10 minutes ago   10 minutes ago                                     esmero-redis\n6a9e9d8647a9   minio/minio:RELEASE.2022-06-11T19-55-32Z   \"/usr/bin/docker-ent\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:9000-9001-&gt;9000-9001/tcp   esmero-minio\nbc5327680ca7   esmero/php-8.1-fpm:1.2.0-multiarch         \"docker-php-entrypoi\u2026\"   10 minutes ago   Up 10 minutes   9000/tcp                           esmero-php\nd53729be1211   esmero/esmero-nlp:fasttext-multiarch       \"/usr/local/bin/entr\u2026\"   10 minutes ago   Up 10 minutes   0.0.0.0:6400-&gt;6400/tcp             esmero-nlp\n</code></pre> <p>Important here is the <code>STATUS</code> column. It needs to be a number that goes up in time every time you run <code>docker ps</code> again (and again).</p> <p>Check your Solr logs</p> <pre><code>docker logs -f esmero-solr -n 100 for failure messages.\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#step-4-re-index-drupal-search-api","title":"Step 4: Re-index Drupal Search API","text":"<p>If you decide/not decide to syncronize Drupal's Search API Server, Index and fields is personal decision (optional). We do recommend it but your Solr Index might have many customizations already, so a better/pro approach would be do <code>diff</code> the .yml files and decide selectively. Once you decided/did that/skipped. Time to reindex</p> <p>Run the following:</p> <p><pre><code>docker exec esmero-php drush search-api-reindex\ndocker exec esmero-php drush search-api-index\n</code></pre> Check your Drupal logs, try some searches.</p> <p>If you made it this far you are done with code/devops (are we ever ready?), and that means you should be able to (hopefully) stay in the Drupal 10.x realm for a few years!</p> <p>Done!</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#need-help-blue-screen-missed-a-step-need-a-hug-or-someone-that-listens-to-you-in-silence","title":"Need help? Blue Screen? Missed a step? Need a hug or someone that listens to you in silence?","text":"<p>If you see any issues or errors or need help with a step, please let us know (ASAP!). You can either open an <code>issue</code> in this repository or use the Google Group. We are here to help.</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Sherrick</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-search_solr_index/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment-live","Drupal 10","Solr 8","Solr 9"]},{"location":"archipelago-deployment-live-updatingContainers/","title":"How to update your Docker containers","text":"<p>From time to time you may have a need to update the containers themselves. Primarily this is done for security releases.</p>","tags":["Github","Docker","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-updatingContainers/#1-update-docker-composeyml","title":"1. Update docker-compose.yml","text":"<p>The first thing you need to do is to edit your <code>docker-compose.yml</code> file and replace the version of the container with the new one you wish to use.</p> <p>Navigate to your <code>docker-compose.yml</code> file and open it to edit. On Debian installs it would look like this:</p> <pre><code>  cd /usr/local/archipelago/deploy/archipelago-deployment-live/deploy/ec2-docker\n  vi docker-compose.yml\n</code></pre> <p>You want to change the image line to reflect the name of the new image you wish to use:</p> <pre><code>image: esmero/php-7.4-fpm:1.0.0-RC3-multiarch\n</code></pre> <p>might become:</p> <pre><code>image: esmero/php-8.0-fpm:1.1.0-multiarch\n</code></pre> <p>Save your change. If use vi like in the above it would look like this:</p> <pre><code>  :wq\n</code></pre>","tags":["Github","Docker","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-updatingContainers/#pull-the-new-images","title":"Pull the new image(s)","text":"<p>Docker Compose will now allow us to grab the new image(s) while your current system is running:</p> <pre><code>  docker-compose pull\n</code></pre>","tags":["Github","Docker","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-updatingContainers/#stop-and-restart-the-container","title":"Stop and restart the container","text":"<p>It is necesary to stop and start the container or the current image will continue to be used:</p> <pre><code>  docker-compose stop container-name\n</code></pre> <p>Wait for it to stop. Then bring it back up:</p> <pre><code>docker-compose up -d \n</code></pre> <p>It is important to use the -d flag or you will have your live instance stuck in your terminal. You want it to run in the background. The <code>-d</code> flag stands for detached.</p>","tags":["Github","Docker","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-updatingContainers/#down-and-up","title":"Down and up","text":"<p>If you are more comfortable having the all the containers go down and up you can do that with the following:</p> <pre><code>docker-compose down\ndocker-compose up\n</code></pre> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback, or open an ISSUE in this Archipelago Deployment Live Repository.</p> <p>Return to Archipelago Live Deployment.</p>","tags":["Github","Docker","Archipelago-deployment-live"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/","title":"Archipelago-deployment-live: upgrading Drupal 8 to Drupal 9 (1.0.0-RC2 to 1.0.0-RC3)","text":"","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#what-is-this-documentation-for","title":"What is this documentation for?","text":"<p>If you already have a well-set-up and well-loved Archipelago (RC2 or your own custom version) running Drupal 8 (D8), this documentation will allow you to update to Drupal 9 (D9) without major issues.</p> <p>D8 is no longer supported as of the end of November 2021. D9 has been around for a little while and even if every module is not supported yet, what you need and want for Archipelago has long been D9-ready.</p>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#requirements","title":"Requirements","text":"<ul> <li>An archipelago-deployment-live instance 1.0.0-RC2 (working, tested) deployed using provided instructions via Docker and running Drupal 8.</li> <li>Basic knowledge and instincts (+ courage) on how to run Terminal Commands, <code>composer</code> and <code>drush</code>.</li> <li>Patience. You can't skip steps here.</li> <li>For Shell Commands documented here please copy line by line\u2014not the whole block.</li> </ul>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#backing-up-and-preparing-for-the-upgrade","title":"Backing up and preparing for the upgrade","text":"<p>Backups are always going to be your best friends. Archipelago's code, database and settings are mostly self-contained in your current <code>archipelago-deployment-live</code> repo folder, and backing up is simple because of that.</p>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-1","title":"Step 1:","text":"<p>Shut down your <code>docker-compose</code> ensemble. Move to your <code>archipelago-deployment-live</code> folder and run this:</p> <pre><code>cd deploy/ec2-docker\ndocker-compose down\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-2","title":"Step 2:","text":"<p>Verify that all containers are actually down. The following command should return an empty listing. If anything is still running, wait a little longer, and run the following comman again.</p> <pre><code>docker ps\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-3","title":"Step 3:","text":"<p>Now let's <code>tar.gz</code> the whole ensemble with data and configs. As an example we will save this into your <code>$HOME</code> folder. As a good practice we append the current date (YEAR-MONTH-DAY) to the filename. Here we assume today is December 1st of 2021.</p> <pre><code>sudo tar -czvpf $HOME/archipelago-deployment-live-backup-20211201.tar.gz ../../../archipelago-deployment-live\n</code></pre> <p>The process may take a few minutes. Now let's verify that all is there and that the <code>tar.gz</code> is not corrupt.</p> <pre><code>tar -tvvf $HOME/archipelago-deployment-live-backup-20211201.tar.gz\n</code></pre> <p>You will see a listing of files. If corrupt (Do you have enough space? Did your ssh connection drop?) you will see the following:</p> <pre><code>tar: Unrecognized archive format\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-4","title":"Step 4:","text":"<p>Restart your <code>docker-compose</code> ensemble, and wait a little while for all to start.</p> <pre><code>docker-compose up -d\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-5","title":"Step 5:","text":"<p>Export/backup all of your live Archipelago configurations (this allows you to compare/come back in case you lose something custom during the upgrade).</p> <pre><code>docker exec esmero-php mkdir config/backup\ndocker exec esmero-php drush cex --destination=/var/www/html/config/backup\n</code></pre> <p>Good. Now it's safe to begin the upgrade.</p>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#upgrading-to-100-rc3","title":"Upgrading to 1.0.0-RC3","text":"","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-1_1","title":"Step 1:","text":"<p>First we are going to disable modules that are not part of 1.0.0-RC3 or are not yet compatible with D9. Run the following command:</p> <pre><code>docker exec esmero-php drush pm-uninstall module_missing_message_fixer markdown webprofiler key_value webform_views\n</code></pre> <p>From inside your <code>archipelago-deployment-live</code> repo folder we are going to open up the file <code>permissions</code> for some of your most protected Drupal files.</p> <pre><code>cd ../../\nsudo chmod 777 drupal/web/sites/default\nsudo chmod 666 drupal/web/sites/default/*settings.php\nsudo chmod 666 drupal/web/sites/default/*services.yml\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-2_1","title":"Step 2:","text":"<p>Time to fetch the <code>1.0.0-RC3</code> branch and update our <code>docker-compose</code> and <code>composer</code> dependencies. We are also going to stop the current <code>docker</code> ensemble to update all containers to newer versions:</p> <pre><code>cd deploy/ec2-docker\ndocker-compose down\ngit checkout 1.0.0-RC3\n</code></pre> <p>Then copy the appropriate <code>docker-compose</code> file for your architecture:</p> OSX (macOS)/x86-64 <pre><code>cp docker-compose-osx.yml docker-compose.yml\n</code></pre> Linux/x86-64/AMD64 <pre><code>cp docker-compose-linux.yml docker-compose.yml\n</code></pre> OSX (macOS)/Linux/ARM64 <pre><code>cp docker-compose-arm64.yml docker-compose.yml\n</code></pre> <p>Finally, pull the images, and bring up the ensemble:</p> <pre><code>docker compose pull\ndocker compose up -d\n</code></pre> <p>Give all a little time to start. The latest <code>min.io</code> adds a new console, and your <code>Solr</code> core and <code>Database</code> need to be upgraded. Please be patient. To ensure all is well, run (more than once if necessary) the following:</p> <pre><code>docker ps\n</code></pre> <p>You should see something like this:</p> <pre><code>CONTAINER ID   IMAGE                                    COMMAND                  CREATED          STATUS          PORTS                                                           NAMES\n867fd2a42134   nginx                                    \"/docker-entrypoint.\u2026\"   32 seconds ago   Up 27 seconds   0.0.0.0:8001-&gt;80/tcp, :::8001-&gt;80/tcp                           esmero-web\n8663e84a9b48   solr:8.8.2                               \"docker-entrypoint.s\u2026\"   33 seconds ago   Up 30 seconds   0.0.0.0:8983-&gt;8983/tcp, :::8983-&gt;8983/tcp                       esmero-solr\n9b580fa0088f   minio/minio:latest                       \"/usr/bin/docker-ent\u2026\"   33 seconds ago   Up 28 seconds   0.0.0.0:9000-9001-&gt;9000-9001/tcp, :::9000-9001-&gt;9000-9001/tcp   esmero-minio\n50e2f41c7b60   esmero/esmero-nlp:1.0                    \"/usr/local/bin/entr\u2026\"   33 seconds ago   Up 30 seconds   0.0.0.0:6400-&gt;6400/tcp, :::6400-&gt;6400/tcp                       esmero-nlp\n300810fd6f03   esmero/cantaloupe-s3:4.1.9RC             \"sh -c 'java -Dcanta\u2026\"   33 seconds ago   Up 30 seconds   0.0.0.0:8183-&gt;8182/tcp, :::8183-&gt;8182/tcp                       esmero-cantaloupe\n248e4638ba2a   mysql:8.0.22                             \"docker-entrypoint.s\u2026\"   33 seconds ago   Up 28 seconds   3306/tcp, 33060/tcp                                             esmero-db\n141ace919344   esmero/php-7.4-fpm:1.0.0-RC2-multiarch   \"docker-php-entrypoi\u2026\"   33 seconds ago   Up 28 seconds   9000/tcp                                                        esmero-php\n</code></pre> <p>Important here is the <code>STATUS</code> column. It needs to be a number that goes up in time every time you run <code>docker ps</code> again (and again).</p>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-3_1","title":"Step 3:","text":"<p>Now we are going to tell <code>composer</code> to actually fetch the new code and dependencies using the 1.0.0-RC3 provided <code>composer.lock</code> and update the whole Drupal/PHP/JS environment.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer install\"\n</code></pre> <p>This will fail (sorry!) for a few packages but no worries, they need to be patched and composer is not that smart. So simply run it again:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer install\"\n</code></pre> <p>Well done! If you see no issues and all ends in a Green colored message all is good! Jump to Step 4</p>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#what-if-not-all-is-ok-and-i-see-red-and-a-lot-of-dependency-explanations","title":"What if not all is OK and I see red and a lot of dependency explanations?","text":"<p>If you have manually installed packages via composer in the past that are NO longer Drupal 9 compatible you may see errors. In that case you need to check each package website's (normally https://www.drupal.org/project/the_module_name) and check if there is a Drupal 9 compatible version.</p> <p>If so run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/the_module_name:^VERSION_NUMBER_THAT_WORKS_ON_DRUPAL9_' --update-with-dependencies --no-update\" and run **Step 3 ** again (and again until all is cleared)\n</code></pre> <p>If not: try to find a replacement module that does something simular, but in any case you may end having to remove before proceding. Give us a ping/slack/google group/open a github ISSUE if you find yourself uncertain about this.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer remove drupal/the_module_name --no-update\"\ndocker exec -ti esmero-php bash -c \" drush pm-uninstall the_module_name\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-4_1","title":"Step 4:","text":"<p>We will now ask Drupal to update some internal configs and databases. They will bring you up to date with RC3 settings and D9 particularities.</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/setup.sh'\ndocker exec -ti esmero-php bash -c 'drush urol administrator \"admin\"'\ndocker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-5_1","title":"Step 5:","text":"<p>Previously D8 installations had a \"module/profile\" driven installation. Those are no longer used or even exist as part of core, but a profile can't be changed once installed so you have to do the following to avoid Drupal complaining about our new and simpler way of doing things (a small roll back):</p> <pre><code>docker exec -ti esmero-php bash -c \"sed -i 's/minimal: 1000/standard: 1000/g' config/sync/core.extension.yml\"\ndocker exec -ti esmero-php bash -c \"sed -i 's/profile: minimal/profile: standard/g' config/sync/core.extension.yml\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-6","title":"Step 6:","text":"<p>Now you can Sync your new Archipelago 1.0.0-RC3 and bring all the new configs and settings in. For this you have two options (no worries, remember you made a backup!):</p>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#a-partial-sync-which-will-bring-new-configs-and-update-existing-ones-but-will-not-remove-ones-that-only-exist-in-your-custom-setup-eg-new-webforms-or-view-modes","title":"A Partial Sync, which will bring new configs and update existing ones but will not remove ones that only exist in your custom setup, e.g. new Webforms or View Modes.","text":"<pre><code>docker exec esmero-php drush cim -y --partial\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#a-complete-sync-which-will-bring-new-configs-and-update-existing-ones-but-will-also-remove-all-the-ones-that-are-not-part-of-rc3-its-a-like-clean-factory-reset","title":"A Complete Sync, which will bring new configs and update existing ones but will also remove all the ones that are not part of RC3. It's a like clean factory reset.","text":"<pre><code>docker exec esmero-php drush cim -y\n</code></pre> <p>If all goes well here and you see no errors it's time to reindex <code>Solr</code> because there are new Fields. Run the following:</p> <pre><code>docker exec esmero-php drush search-api-reindex\ndocker exec esmero-php drush search-api-index\n</code></pre> <p>You might see some warnings related to modules dealing with previously non-existent data\u2014no worries, just ignore those.</p> <p>If you made it this far you are done with code/devops (are we ever ready?), and that means you should be able to (hopefully) stay in the Drupal 9 realm for a few years!</p>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromD8ToD9/#step-7-update-or-not-your-metadata-display-entities-and-menu-items","title":"Step 7: Update (or not) your Metadata Display Entities and Menu items.","text":"<p>Recommended: If you want to add new templates and menu items 1.0.0-RC3 provides, run this:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/deploy.sh'\n</code></pre> <p>Once that is done, you can choose to update all Metadata Displays (Twig templates) we ship with new 1.0.0-RC3 versions (heavily fixed IIIF manifest, Markdown to HTML for Metadata, better Object descriptions). But before you do this, we really recommend that you first make sure to manually (copy/paste) backup any Twig templates you have modified. If unusure, do not run the command that comes after this warning! You can always manually copy the new templates from the <code>d8content/metadatadisplays</code> folder which contains text versions (again, copy/paste) of each shipped template you can pick and use when you feel ready.</p> <p>If you are sure (like really?) you want to overwrite the ones you modified (sure, just checking?), then you can run this:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/update_deployed.sh'\n</code></pre> <p>Done! (For realz now)</p> <p>Please log into your Archipelago and test/check all is working! Enjoy 1.0.0-RC3 and Drupal 9. Thanks!</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback, or open an ISSUE in this Archipelago Deployment Live Repository.</p> <p>Return to Archipelago Live Deployment.</p>","tags":["Archipelago-deployment-live","Drupal 8","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/","title":"Archipelago-deployment-live: upgrading from 1.0.0-RC3 to 1.0.0","text":"","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#what-is-this-documentation-for","title":"What is this documentation for?","text":"<p>If you already have a well-set-up and well-loved Archipelago (RC3 or your own custom version) running Drupal 9, this documentation will allow you to update to 1.0.0 without major issues.</p>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#requirements","title":"Requirements","text":"<ul> <li>An archipelago-deployment-live instance 1.0.0-RC3 (working, tested) deployed using provided instructions via Docker and running Drupal 9.</li> <li>Basic knowledge and instincts (+ courage) on how to run Terminal Commands, <code>composer</code> and <code>drush</code>.</li> <li>Patience. You can't skip steps here.</li> <li>For Shell Commands documented here please copy line by line\u2014not the whole block.</li> </ul>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#backing-up-and-preparing-for-the-upgrade","title":"Backing up and preparing for the upgrade","text":"<p>Backups are always going to be your best friends. Archipelago's code, database and settings are mostly self-contained in your current <code>archipelago-deployment-live</code> repo folder, and backing up is simple because of that.</p>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-1","title":"Step 1:","text":"<p>Shut down your <code>docker-compose</code> ensemble. Move to your <code>archipelago-deployment-live</code> folder and run this:</p> <pre><code>cd deploy/ec2-docker\ndocker-compose down\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-2","title":"Step 2:","text":"<p>Verify that all containers are actually down. The following command should return an empty listing. If anything is still running, wait a little longer, and run the following comman again.</p> <pre><code>docker ps\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-3","title":"Step 3:","text":"<p>Now let's <code>tar.gz</code> the whole ensemble with data and configs. As an example we will save this into your <code>$HOME</code> folder. As a good practice we append the current date (YEAR-MONTH-DAY) to the filename. Here we assume today is December 1st of 2021.</p> <pre><code>sudo tar -czvpf $HOME/archipelago-deployment-live-backup-20220803.tar.gz ../../../archipelago-deployment-live\n</code></pre> <p>The process may take a few minutes. Now let's verify that all is there and that the <code>tar.gz</code> is not corrupt.</p> <pre><code>tar -tvvf $HOME/archipelago-deployment-live-backup-20220803.tar.gz \n</code></pre> <p>You will see a listing of files. If corrupt (Do you have enough space? Did your ssh connection drop?) you will see the following:</p> <pre><code>tar: Unrecognized archive format\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-4","title":"Step 4:","text":"<p>Restart your <code>docker-compose</code> ensemble, and wait a little while for all to start.</p> <pre><code>docker-compose up -d\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-5","title":"Step 5:","text":"<p>Export/backup all of your live Archipelago configurations (this allows you to compare/come back in case you lose something custom during the upgrade).</p> <pre><code>docker exec esmero-php mkdir config/backup\ndocker exec esmero-php drush cex --destination=/var/www/html/config/backup\n</code></pre> <p>Good. Now it's safe to begin the upgrade.</p>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#upgrading-to-100","title":"Upgrading to 1.0.0","text":"","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-1_1","title":"Step 1:","text":"<p>First we are going to disable modules that are not part of 1.0.0 or are not yet compatible with Drupal 9.4.x or higher . Run the following command:</p> <pre><code>docker exec esmero-php drush pm-uninstall search_api_solr_defaults entity_reference\n</code></pre> <p>From inside your <code>archipelago-deployment-live</code> repo folder we are going to open up the file <code>permissions</code> for some of your most protected Drupal files.</p> <pre><code>cd ../../\nsudo chmod 777 drupal/web/sites/default\nsudo chmod 666 drupal/web/sites/default/*settings.php\nsudo chmod 666 drupal/web/sites/default/*services.yml\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-2_1","title":"Step 2:","text":"<p>First let's back up our current composer.lock:</p> <pre><code>cp drupal/composer.lock drupal/composer.original.lock\n</code></pre> <p>Time to fetch the <code>1.0.0</code> branch and update our <code>docker-compose</code> and <code>composer</code> dependencies. We are also going to stop the current <code>docker</code> ensemble to update all containers to newer versions:</p> <pre><code>cd deploy/ec2-docker\ndocker-compose down\ngit fetch\ngit checkout 1.0.0 \n</code></pre> <p>If you decide to enable the Drupal REDIS module, make sure to add the <code>REDIS_PASSWORD</code> variable to your <code>.env</code> file.</p> <p><code>IMPORTANT NOTE</code>: For AWS EC2. If you selected an <code>IAM role</code> for your server when setting it up/deploying it, <code>min.io</code> will use the AWS EC2-backed internal API to request access to your S3. This means the ROLE itself needs to have read/write access (ACL) to the given Bucket(s) and your key/secrets won't be able to override that. Please do not ignore this note. It will save you a LOT of frustration and coffee. You can also run an EC2 instance without a given IAM and in that case just the ACCESS_KEY/SECRET will matter.</p> <p>Now that you know, you also know that these values should not be shared and this <code>.env</code> file should not be committed/kept in version control. Please be careful.</p> <p>Now let's back up the existing <code>docker-compose</code> file:</p> <pre><code>cp docker-compose.yml docker-compose-original.yml\n</code></pre> <p>Then copy the appropriate <code>docker-compose</code> file for your architecture:</p> Linux/x86-64/AMD64 <pre><code>cp docker-compose-aws-s3.yml docker-compose.yml\n</code></pre> Linux/ARM64/Apple Silicon (M1 and M2) <pre><code>cp docker-compose-aws-s3-arm64.yml docker-compose.yml\n</code></pre> <p>Next, let's review what's changed in case any customizations need to be brought into the new <code>docker-compose</code> configurations:</p> <pre><code>git diff --no-index docker-compose-original.yml docker-compose.yml\n</code></pre> <p>You should encounter something like the following:</p> <pre><code>diff --git a/docker-compose-original.yml b/docker-compose.yml\nindex 6f5b17e..282417f 100644\n--- a/docker-compose-original.yml\n+++ b/docker-compose.yml\n@@ -1,5 +1,5 @@\n # Run docker-compose up -d\n-\n+# Docker file for AMD64/X86 machines\n version: '3.5'\n services:\n   web:\n@@ -23,6 +23,7 @@ services:\n       - solr\n       - php\n       - db\n+      - redis\n     tty: true\n     networks:\n       - host-net\n@@ -30,7 +31,7 @@ services:\n   php:\n     container_name: esmero-php\n     restart: always\n-    image: \"esmero/php-7.4-fpm:1.0.0-RC2-multiarch\"\n+    image: \"esmero/php-8.0-fpm:1.1.0-multiarch\"\n     tty: true\n     networks:\n       - host-net\n@@ -44,10 +45,11 @@ services:\n       MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD}\n       MINIO_BUCKET_MEDIA: ${MINIO_BUCKET_MEDIA}\n       MINIO_FOLDER_PREFIX_MEDIA: ${MINIO_FOLDER_PREFIX_MEDIA}\n+      REDIS_PASSWORD: ${REDIS_PASSWORD}\n   solr:\n     container_name: esmero-solr\n     restart: always\n-    image: \"solr:8.8.2\"\n+    image: \"solr:8.11.2\"\n</code></pre> <p>As you can see, most of the changes in this example are for new images and a new service/container/environment variable (REDIS), but you may have custom settings for your containers. Review any differences carefully and make adjustments as needed.</p> <p>Finally, pull the images:</p> <pre><code>docker compose pull \n</code></pre> <p>1.0.0 provides a new Cantaloupe that uses different permissions so we need to adapt those. From your current folder (../ec2-deploy) run:</p> <pre><code>sudo chown 8183:8183 ../../config_storage/iiifconfig/cantaloupe.properties\nsudo chown -R 8183:8183 ../../data_storage/iiifcache\nsudo chown -R 8183:8183 ../../data_storage/iiiftmp\n</code></pre> <p>Time to start the ensemble again</p> <pre><code>docker compose up -d\n</code></pre> <p>Give all a little time to start. <code>Solr</code> core and <code>Database</code> need to be upgraded, Cantaloupe is new and this brings also Redis for caching. Please be patient. To ensure all is well, run (more than once if necessary) the following:</p> <pre><code>docker ps\n</code></pre> <p>You should see something like this:  e.g if running on ARM64 You should see something like this: </p> <pre><code>CONTAINER ID   IMAGE                                    COMMAND                  CREATED          STATUS          PORTS                                                           NAMES\n4ed2f62e866e   jonasal/nginx-certbot                      \"/docker-entrypoint.\u2026\" 32 seconds ago   Up 27 seconds   0.0.0.0:80-&gt;80/tcp, :::80-&gt;80/tcp, 0.0.0.0:443-&gt;443/tcp, :::443-&gt;443/tcp   esmero-web\ne6b4383039c3   minio/minio:RELEASE.2022-06-11T19-55-32Z   \"/usr/bin/docker-ent\u2026\" 33 seconds ago   Up 30 seconds   0.0.0.0:9000-9001-&gt;9000-9001/tcp, :::9000-9001-&gt;9000-9001/tcp              esmero-minio\nf2b6b173b7e2   solr:8.11.2                                \"docker-entrypoint.s\u2026\" 33 seconds ago   Up 28 seconds   0.0.0.0:8983-&gt;8983/tcp, :::8983-&gt;8983/tcp                                  esmero-solr\na553bf484343   esmero/php-8.0-fpm:1.0.0-multiarch         \"docker-php-entrypoi\u2026\" 33 seconds ago   Up 30 seconds   9000/tcp                                                                   esmero-php\necb47349ae94   esmero/esmero-nlp:fasttext-multiarch       \"/usr/local/bin/entr\u2026\" 33 seconds ago   Up 30 second    0.0.0.0:6400-&gt;6400/tcp, :::6400-&gt;6400/tcp                                  esmero-nlp\n61272dce034a   redis:6.2-alpine                           \"docker-entrypoint.s\u2026\" 33 seconds ago   Up 28 seconds                                                                              esmero-redis\n0ee9869f809b   esmero/cantaloupe-s3:6.0.0-multiarch       \"sh -c 'java -Dcanta\u2026\" 33 seconds ago   Up 28 seconds   0.0.0.0:8183-&gt;8182/tcp, :::8183-&gt;8182/tcp                                  esmero-cantaloupe\n131d072567ce   mariadb:10.6.8-focal                       \"docker-entrypoint.s\u2026\" 33 seconds ago   Up 28 seconds   3306/tcp                                                                   esmero-db                                            esmero-php\n</code></pre> <p>Important here is the <code>STATUS</code> column. It needs to be a number that goes up in time every time you run <code>docker ps</code> again (and again).</p>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-3_1","title":"Step 3:","text":"<p>Instead of using the provided <code>composer.default.lock</code> out of the box we are going to loosen certain dependencies and bring manually Archipelago modules, all this to make update easier and future upgrades less of a pain.</p> <p>First, as a sanity check let's make sure nothing happened to our original <code>composer.lock</code> fileby doing a diff against our backed up file:</p> <pre><code>git diff --no-index ../../drupal/composer.original.lock ../../drupal/composer.lock\n</code></pre> <p>If all is ok, there should be no output. If there's any output, copy your backed up file back to default:</p> <pre><code>cp ../../drupal/composer.original.lock ../../drupal/composer.lock\n</code></pre> <p>Finally, we bring over the modules:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require drupal/core:^9 drupal/core-composer-scaffold:^9 drupal/core-project-message:^9 drupal/core-recommended:^9\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/core-dev:^9 --dev\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/tokenuuid:^2\"\ndocker exec -ti esmero-php bash -c \"composer require 'drupal/facets:^2.0'\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/moderated_content_bulk_publish:^2\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/queue_ui:^3.1\"\ndocker exec -ti esmero-php bash -c \"composer require drupal/jquery_ui_touch_punch:^1.1\"\ndocker exec -ti esmero-php bash -c \"composer require archipelago/ami:0.4.0.x-dev strawberryfield/format_strawberryfield:1.0.0.x-dev strawberryfield/strawberryfield:1.0.0.x-dev strawberryfield/strawberry_runners:0.4.0.x-dev strawberryfield/webform_strawberryfield:1.0.0.x-dev drupal/views_bulk_operations:^4.1\"\n</code></pre> <p>Now we are going to tell <code>composer</code> to actually fetch the new code and dependencies using <code>composer.lock</code> and update the whole Drupal/PHP/JS environment.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer update -W\"\ndocker exec -ti esmero-php bash -c \"drush cr\"\ndocker exec -ti esmero-php bash -c \"drush en jquery_ui_touch_punch\"\ndocker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre> <p>Well done! If you see no issues and all ends in a Green colored message all is good! Jump to Step 4</p>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#what-if-not-all-is-ok-and-i-see-red-and-a-lot-of-dependency-explanations","title":"What if not all is OK and I see red and a lot of dependency explanations?","text":"<p>If you have manually installed packages via composer in the past that are NO longer Drupal 9 compatible you may see errors.  In that case you need to check each package website's (normally https://www.drupal.org/project/the_module_name) and check if there is a Drupal 9 compatible version. </p> <p>If so run:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer require 'drupal/the_module_name:^VERSION_NUMBER_THAT_WORKS_ON_DRUPAL9_' --update-with-dependencies --no-update\" and run **Step 3 ** again (and again until all is cleared)\n</code></pre> <p>If not: try to find a replacement module that does something simular, but in any case you may end having to remove before proceding. Give us a ping/slack/google group/open a github ISSUE if you find yourself uncertain about this. </p> <pre><code>docker exec -ti esmero-php bash -c \"composer remove drupal/the_module_name --no-update\"\ndocker exec -ti esmero-php bash -c \" drush pm-uninstall the_module_name\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-4_1","title":"Step 4:","text":"<p>We will now ask Drupal to update some internal configs and databases. They will bring you up to date with 1.0.0 settings and D9 particularities.</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/setup.sh'\ndocker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-5_1","title":"Step 5:","text":"<p>Now you can Sync your new Archipelago 1.0.0 and bring all the new configs and settings in. For this you have two options (no worries, remember you made a backup!):</p>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#a-partial-sync-which-will-bring-new-configs-and-update-existing-ones-but-will-not-remove-ones-that-only-exist-in-your-custom-setup-eg-new-webforms-or-view-modes","title":"A Partial Sync, which will bring new configs and update existing ones but will not remove ones that only exist in your custom setup, e.g. new Webforms or View Modes.","text":"<pre><code>docker exec esmero-php drush cim -y --partial\n</code></pre>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#a-complete-sync-which-will-bring-new-configs-and-update-existing-ones-but-will-also-remove-all-the-ones-that-are-not-part-of-rc3-its-a-like-clean-factory-reset","title":"A Complete Sync, which will bring new configs and update existing ones but will also remove all the ones that are not part of RC3. It's a like clean factory reset.","text":"<pre><code>docker exec esmero-php drush cim -y\n</code></pre> <p>If all goes well here and you see no errors it's time to reindex <code>Solr</code> because there are new Fields. Run the following:</p> <pre><code>docker exec esmero-php drush search-api-reindex\ndocker exec esmero-php drush search-api-index\n</code></pre> <p>You might see some warnings related to modules dealing with previously non-existent data\u2014no worries, just ignore those.</p> <p>If you made it this far you are done with code/devops (are we ever ready?), and that means you should be able to (hopefully) stay in the Drupal 9 realm for a few years!</p>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-live-upgradeFromRC3/#step-7-update-or-not-your-metadata-display-entities-and-menu-items","title":"Step 7: Update (or not) your Metadata Display Entities and Menu items.","text":"<p>Recommended: If you want to add new templates and menu items 1.0.0 provides, go to your base Github repo folder, replace in the following commands <code>your.domain.org</code> with the actual domain of your Server and run those individually:</p> <pre><code>sed -i 's/http:\\/\\/esmero-web/https:\\/\\/your.domain.org/g' drupal/scripts/archipelago/deploy.sh\n</code></pre> <pre><code>sed -i 's/http:\\/\\/esmero-web/https:\\/\\/your.domain.org/g' drupal/scripts/archipelago/update_deployed.sh\n</code></pre> <p>Now update your Metadata Display Templates and Blocks</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/deploy.sh'\n</code></pre> <p>Once that is done, you can choose to update all Metadata Displays (Twig templates) we ship with new 1.0.0 versions (heavily fixed IIIF manifest, Markdown to HTML for Metadata, better Object descriptions). But before you do this, we really recommend that you first make sure to manually (copy/paste) backup any Twig templates you have modified. If unusure, do not run the command that comes after this warning! You can always manually copy the new templates from the <code>d8content/metadatadisplays</code> folder which contains text versions (again, copy/paste) of each shipped template you can pick and use when you feel ready.</p> <p>If you are sure (like really?) you want to overwrite the ones you modified (sure, just checking?), then you can run this:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/update_deployed.sh'\n</code></pre> <p>Done! (For realz now)</p> <p>Please log into your Archipelago and test/check all is working! Enjoy 1.0.0. Thanks!</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback, or open an ISSUE in this Archipelago Deployment Live Repository.</p> <p>Return to Archipelago Live Deployment.</p>","tags":["Archipelago-deployment-live","Drupal 9"]},{"location":"archipelago-deployment-osx/","title":"Installing Archipelago Drupal 10 on OSX (macOS)","text":"","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#about-running-terminal-commands","title":"About running terminal commands","text":"<p>This guide assumes you are comfortable enough running terminal (bash) commands on an OSX Computer.</p> <p>We made sure that you can <code>copy</code> and <code>paste</code> each of these commands from this guide directly into your terminal.</p> <p>You will notice sometimes commands span more than a single line of text. If that is the case, always make sure you copy and paste a single line at a time and press the <code>Enter</code> key afterwards. We suggest also you look at the output.</p> <p>If something fails (and we hope it does not) troubleshooting will be much easier if you can share that output when asking for help.</p> <p>Happy deploying!</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#osx-macos","title":"OSX (macOS):","text":"<ul> <li>Install Docker for Mac</li> <li>For OSX (macOS) <code>Ventura</code> or Higher on Intel (i5/i7) and Apple Silicon Chips (M1/M2/M3) the tested version is: <code>4.31.0(153195)</code> with Docker engine v26.1.4. You may go newer of course.</li> <li>In <code>Preferences</code> -&gt; <code>General</code>: check under <code>Choose file sharing implementation for your containers</code> either the new <code>VirtioFS</code>(faster) or <code>gRPC FUSE</code> and restart. Specially if you are using your <code>$HOME</code> folder for deploying, e.g. <code>/Users/username</code>.</li> <li>In <code>Preferences</code> -&gt; <code>Resources</code>: 4 Gbytes of RAM is the recommended minimun and works; 8 Gbytes is faster and snappier.</li> <li>Install Github Desktop.</li> <li>At least 10 Gbytes of free space (to get started).</li> <li>Being able to open a terminal.</li> </ul> <p>Note: Most Recent Docker Desktop for macOS removed <code>docker-compose</code> and replace it with <code>docker compose</code>. We updated this documentation to reflect the newer version. If you are running an older one, either upgrade or please replace every mention of <code>docker composer</code> with the legacy <code>docker-compose</code> when going through the steps.</p> <p>Note: Recent OSX (macOS) and newer Macs ship with 2 annoying things: Apple Cloud Syncing User Folders and (wait for it) Case insensitive File Systems. If you are happy with your shiny new Mac (like i was) we are aware that it's better to deploy anything mounted outside of the <code>/User</code> folder or even better, in an external drive formatted using a Case Sensitive Unix Filesystem (Mac OS Extended (Case-sensitive, Journaled)).</p> <p>Note 2: \"gRPC FUSE\" experience may vary, recent Docker for Mac does it well. In older RC1 ones it was evil. Changing/Disabling it after having installed Archipelago may affect your S3/Minio storage accessibility. Please let us know what your experience on this is.</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#wait-question-do-you-have-a-previous-version-of-archipelago-running","title":"Wait! Question: Do you have a previous version of Archipelago running?","text":"<p>If so, let's give that hard working repository a break first. If not, skip to Step 1:</p> <ul> <li>Open a terminal (you have that already right?) and go to your previous download/git clone folder and run:</li> </ul> <pre><code>docker compose down\ndocker compose rm\n</code></pre> <ul> <li>Can't remember where you downloaded it? Ok. We can deal with that!</li> </ul> <p>Let's stop the containers gracefully first, run:</p> <pre><code>docker stop esmero-web\ndocker stop esmero-solr\ndocker stop esmero-db\ndocker stop esmero-cantaloupe\ndocker stop esmero-php\ndocker stop esmero-minio\ndocker stop esmero-nlp\n</code></pre> <p>Now we need to remove them, run:</p> <pre><code>docker rm esmero-web\ndocker rm esmero-solr\ndocker rm esmero-db\ndocker rm esmero-cantaloupe\ndocker rm esmero-php\ndocker rm esmero-minio\ndocker rm esmero-nlp\n</code></pre> <p>Ok, now we are ready to start. Depending on what type of Chip your Apple uses you have two options:</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#step-1-intel-docker-deployment-on-intel-chips-apple-machines","title":"Step 1 (Intel): Docker Deployment on Intel Chips Apple Machines","text":"<pre><code>git clone https://github.com/esmero/archipelago-deployment.git archipelago-deployment\ncd archipelago-deployment\ngit checkout 1.4.0\ncp docker-compose-osx.yml docker-compose.yml\ndocker compose pull\ndocker compose up -d\n</code></pre>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#step-1-m1m2m3-docker-deployment-on-apple-silicon-chips-m1m2m3","title":"Step 1 (M1/M2/M3): Docker Deployment on Apple Silicon Chips (M1/M2/M3)","text":"<pre><code>git clone https://github.com/esmero/archipelago-deployment.git archipelago-deployment\ncd archipelago-deployment\ngit checkout 1.4.0\ncp docker-compose-arm64.yml docker-compose.yml\ndocker compose pull\ndocker compose up -d\n</code></pre> <p>Note: If you are running on an Intel Apple Machine from an external Drive or a partition/filesystem that is <code>Case Sensitive</code> and is not syncing automatically to <code>Apple Cloud</code> you can also use <code>docker-compose-linux.yml</code>. Note2: <code>docker-compose.yml</code> is git ignored in case you make local adjustments or changes to it.</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#step-2-set-up-your-minio-s3-bucket","title":"Step 2: Set up your Minio S3 bucket","text":"<p>Once all containers are up and running (you can do a <code>docker ps</code> to check), access the minio console at <code>http://localhost:9001</code> using your most loved Web Browser with the following credentials:</p> <pre><code>user:minio\npass:minio123\n</code></pre> <p>and once logged in, press on \"Buckets\" (left tools column) and then on \"Create Bucket\"  (top right) and under \"Bucket Name\" type <code>archipelago</code>. Leave all other options unchecked for now (you can experiment with those later), and make sure you write <code>archipelago</code> (no spaces, lowercase) and press \"Save\". Done! That is where we will persist all your Files and also your File copies of each Digital Object. You can always go there and explore what Archipelago (well really Strawberryfield does the hard work) has persisted so you can get comfortable with our architecture.</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#step-3-deploy-drupal-10-and-the-awesome-archipelago-modules","title":"Step 3: Deploy Drupal 10 and the awesome Archipelago Modules","text":"<p>The following will run composer inside the esmero-php container to download all dependencies and Drupal Core too:</p> <pre><code>docker exec -ti esmero-php bash -c \"composer install\"\n</code></pre> <p>Once that command finishes run our setup script:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/setup.sh'\n</code></pre> <p>Explanation: That script will append some important configurations to your local <code>web/sites/default/settings.php</code>.</p> <p>Note: We say <code>local</code> because your whole Drupal web root (the one you cloned) is also mounted inside the esmero-php and esmero-web containers. So edits to PHP files, for example, can be done without accessing the container directly from your local folder.</p> <p>If this is the first time you deploy Drupal using the provided Configurations run:</p> <pre><code>docker exec -ti -u www-data esmero-php bash -c \"cd web;../vendor/bin/drush -y si --verbose --existing-config --db-url=mysql://root:esmerodb@esmero-db/drupal --account-name=admin --account-pass=archipelago -r=/var/www/html/web --sites-subdir=default --notify=false;drush cr;chown -R www-data:www-data sites;\"\n</code></pre> <p>Note: You will see these warnings:</p> <p><code>[warning] The \"block_content:9aa72fb1-2817-44a7-8fb5-a3eb51166e83\" was not found</code> <code>[warning] The \"block_content:1cdf7155-eb60-4f27-9e5e-64fffe93127a\" was not found</code> <code>[warning] The \"facets_summary_block:advance\" was not found</code> <code>[warning] The \"facets_summary_block:search_page_facet_summary\" was not found</code> <code>[notice] Missing required data for configuration: role_theme_switcher.settings</code></p> <p>Nothing to worry about. We will provide the missing part in Step 5.</p> <p>Note 2: Please be patient. This step takes since composer 2.0 25-30% longer because of how the most recent Drupal Installation code fetches translations and other resources (see <code>Performed install task</code>). This means progress might look like getting \"stuck\", go and get a coffee/tea and let it run to the end.</p> <p>Once finished, this will give you an <code>admin</code> Drupal user with <code>archipelago</code> as password (Change this if running on a public instance!).</p> <p>Final Note about Steps 2-3: You don't need to, nor should you do this more than once. You can destroy/stop/update, recreate your Docker containers, and start again (<code>git pull</code>), and your Drupal and Data will persist once you're past the <code>Installation complete</code> message. I repeat, all other containers' data is persisted inside the <code>persistent/</code> folder contained in this cloned git repository. Drupal and all its code is visible, editable, and stable inside your <code>web/</code> folder.</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#step-4-create-a-demo-and-a-jsonapi-user-using-drush-and-assign-your-admin-user-the-administrator-role","title":"Step 4: Create a \"demo \"and a \"jsonapi\" user using drush and assign your \"admin\" user the Administrator Role.","text":"<p><pre><code>docker exec -ti esmero-php bash -c 'drush ucrt demo --password=\"demo\"; drush urol metadata_pro \"demo\"'\n</code></pre> <pre><code>docker exec -ti esmero-php bash -c 'drush ucrt jsonapi --password=\"jsonapi\"; drush urol metadata_api \"jsonapi\"'\n</code></pre> <pre><code>docker exec -ti esmero-php bash -c 'drush urol administrator \"admin\"'\n</code></pre></p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#step-5-ingest-some-metadata-displays-to-make-playing-much-more-interactive","title":"Step 5: Ingest some Metadata Displays to make playing much more interactive","text":"<p>Archipelago is more fun without having to start writing Metadata Displays (in Twig) before you know what they actually are. Since you should now have a <code>jsonapi</code> user and jsonapi should be enabled, you can use that awesome functionality of D8 to get that done. We have 4 demo Metadata Display Entities that go well with the demo Webform we provided. To do that execute in your shell (copy and paste):</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/deploy.sh'\n</code></pre> <p>Open your most loved Web Browser and point it to <code>http://localhost:8001</code>.</p> <p>Note: It can take some time to start the first time (Drupal needs some warming up).</p> <p>Also, to make this docker-compose easier to use we are doing something named <code>bind mounting</code> (or similar...) your folders. The good thing is that you can edit files in your machine, and they get updated instantly to docker. The bad thing is that the OSX (macOS) driver runs slower than on Linux. Speed is a huge factor here, but you get the flexibility of changing, backing up, and persisting files without needing a Docker University Degree.</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#step-6-optional-but-more-fun-if-you-add-content","title":"Step 6: Optional but more fun if you add content","text":"<p>One-Step Demo content ingest</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#need-help-blue-screen-missed-a-step-need-a-hug-and-such","title":"Need help? Blue Screen? Missed a step? Need a hug and such?","text":"<p>If you see any issues or errors or need help with a step, please let us know (ASAP!). You can either open an <code>issue</code> in this repository or use the Google Group. We are here to help.</p> <p>If you like this, let us know!</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Sherrick</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-osx/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment","Drupal 10","macOS","OSX"]},{"location":"archipelago-deployment-readme/","title":"Archipelago Docker Deployment","text":"<p>Updated: July 1st 2024</p> <p>Previously Updated: January 30th 2024</p> <p>This repository serves as bootstrap for a Archipelago 1.4.0 deployment on a localhost for development/testing/customizing via Docker and provides a more unified experience this time:</p> <ul> <li>minio.io (latest)for local S3 with Console.</li> <li>Apache Solr 9.2.1 with the wizardly Solr OCR Highlight library v0.8.4 built by the Developement Team at the Bavarian State Library. Thanks Johannes Baiter and team.</li> <li>MySQL 8.x (amd64/x86)/MariaDB 10.6.x(Arm64/M1/M2/M3)</li> <li>NGINX 11</li> <li>Custom PHP-FPM 8.1 multi architecture, fine-tuned for Drupal 10 , WARC to WACZ processing, Tesseract 5 with JP2 support, PDFAlto and Composer 2.x, Drush 12, etc</li> <li>Natural Language Processing via NLPWEB64 multi architecture with FastText Language detection (Thanks Mike Bennet!) or alternatively new ML (Image similarity: YOLO,MobileNet,Insightface and Text transformer: SBERT)</li> <li>Cantaloupe 6.0.1 Snapshot multi architecture as IIIF2/3 Server with Video Frame extraction and PDF support (with custom fix for tiled PDF)</li> <li>A Skeleton Project setup to run latest Version of Drupal (10.2.x), our new Bootstrap 5 theme and Strawberry Field modules on 1.4.0 &amp; friends on 0.8.0</li> <li>Complete support for Apple Silicon M1 Machines and in general <code>arm64</code> architecture Chips like Raspberry Pi 4, with specially built arm64 docker containers. The only differences now between deployment strategies is the DB. Blazing fast OCR.</li> </ul> <p>The skeleton project contains all the pieces needed to run a local deployment of a vanilla Archipelago including (YES!) content provided as an optional feature from archipelago-recyclables</p>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-readme/#starting-from-zero","title":"Starting from ZERO","text":"<p>This is the recommended, simplest way for this release. There are a too many, tons of fun new features, Metadata Displays, Webforms, New formatters and Twig extensions, improved viewers, new and improved JS libraries, OpenCV/Face Detection, smarter NLP, File composting, better HUGE import/update capabilities, bug fixes (yes so many) so please try them out. The team has also updated the DEMO AMI set (Content) to showcase metadata/display improvements.</p>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-readme/#macos-intel-or-apple-silicon-m1m2m3","title":"macOS Intel or Apple Silicon M1/M2/M3:","text":"<p>Step by Step deployment on macOS</p>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-readme/#ubuntu-1804-or-2004","title":"Ubuntu 18.04 or 20.04:","text":"<p>Step by Step deployment on Ubuntu</p>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-readme/#windows-10-or-11","title":"Windows 10 or 11:","text":"<p>Step by Step deployment on Windows</p>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-readme/#more-fun-if-you-add-content","title":"More fun if you add content:","text":"<p>One-Step Demo content ingest</p> <p>If you like it (or not), want new features, or want to be part of making this better (documenting, coding and planning) let us know. Make your voice and opinion be heard, this is a community effort.</p>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-readme/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Sherrick</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-readme/#acknowledgments","title":"Acknowledgments","text":"<p>This software is a Metropolitan New York Library Council Open-Source initiative and part of the Archipelago Commons project.</p>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-readme/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment","Drupal 10","Docker"]},{"location":"archipelago-deployment-ubuntu/","title":"Installing Archipelago Drupal 10 on Ubuntu 18.04 or 20.04","text":"","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#about-running-terminal-commands","title":"About running terminal commands","text":"<p>This guide assumes you are comfortable enough running terminal (bash) commands on a Linux Computer.</p> <p>We made sure that you can <code>copy</code> and <code>paste</code> each of these commands from this guide directly into your terminal.</p> <p>You will notice sometimes commands span more than a single line of text. If that is the case, always make sure you copy and paste a single line at a time and press the <code>Enter</code> key afterwards. We suggest you also look at the output.</p> <p>If something fails (and we hope it does not) troubleshooting will be much easier if you can share that output when asking for help.</p> <p>Happy deploying!</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#prerequisites","title":"Prerequisites","text":"<ul> <li>At least 10 Gbytes of free space (to get started)</li> <li>Some basic Unix/Terminal Skills</li> <li>2-4 Gbytes of RAM (4 Recommended)</li> <li>Install Docker if you don't have it already by running:</li> </ul> <pre><code>sudo apt install apt-transport-https ca-certificates curl software-properties-common\ncurl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -\nsudo add-apt-repository \"deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable\"\nsudo apt update\nsudo apt-cache policy docker-ce\nsudo apt install docker-ce\nsudo systemctl status docker\n\nsudo usermod -aG docker ${USER}\n</code></pre> <p>Log out, and log in again!</p> <pre><code>sudo apt install docker-compose\n</code></pre> <p>Git tools are included by default in Ubuntu.</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#wait-question-do-you-have-a-previous-version-of-archipelago-running","title":"Wait! Question: Do you have a previous version of Archipelago running?","text":"<p>If so, let's give that hard working repository a break first. If not, Step 1:</p> <ul> <li>Open a terminal (you have that already right?) and go to your previous download/git clone folder and run:</li> </ul> <pre><code>docker-compose down\ndocker-compose rm\n</code></pre> <ul> <li>Can't remember where you downloaded it? Ok. We can deal with that!</li> </ul> <p>Let's stop the containers gracefully first, run:</p> <pre><code>docker stop esmero-web\ndocker stop esmero-solr\ndocker stop esmero-db\ndocker stop esmero-cantaloupe\ndocker stop esmero-php\ndocker stop esmero-minio\ndocker stop esmero-nlp\n</code></pre> <p>Now we need to remove them so we run the following:</p> <pre><code>docker rm esmero-web\ndocker rm esmero-solr\ndocker rm esmero-db\ndocker rm esmero-cantaloupe\ndocker rm esmero-php\ndocker rm esmero-minio\ndocker rm esmero-nlp\n</code></pre> <p>Ok, now we are ready to start.</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#step-1-deployment","title":"Step 1: Deployment","text":"","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#prefer-to-watch-a-video-to-see-what-its-like-to-install-go-to-our-user-contributed-documentation1","title":"Prefer to watch a video to see what it's like to install? Go to our <code>user contributed documentation</code>[^1]!","text":"","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#important","title":"IMPORTANT","text":"<p>If you run <code>docker-compose</code> as root user (using <code>sudo</code>) some enviromental variables, like the current folder used inside the <code>docker-compose.yml</code> to mount the Volumes, will not work and you will see a bunch of errors.</p> <p>There are two possible solutions.</p> <ul> <li>The best is to add your user to the docker group (so no <code>sudo</code> needed).</li> <li>The second option is to replace every <code>{$PWD}</code> inside your <code>docker-compose.yml</code> with either the full path to your current folder, or with a <code>.</code> and wrap that whole line in double quotes, basically making the paths for volumes relatives.</li> </ul> <p>Instead of: <code>- ${PWD}:/var/www/html:cached</code> use: <code>- \".:/var/www/html:cached\"</code></p> <p>Now that you got it, let's deploy:</p> <pre><code>git clone https://github.com/esmero/archipelago-deployment.git archipelago-deployment\ncd archipelago-deployment\ngit checkout 1.4.0\n</code></pre> <pre><code>cp docker-compose-linux.yml docker-compose.yml\ndocker-compose pull\ndocker-compose up -d\n</code></pre> <p>Note: <code>docker-compose.yml</code> is git ignored in case you make local adjustments or changes to it.</p> <p>You need to make sure Docker can read/write to your local Drive, a.k.a mounted volumes (especially if you decided not to run it as <code>root</code> because we told you so!).</p> <p>This means in practice running:</p> <pre><code>sudo chown -R 8183:8183 persistent/iiifcache\nsudo chown -R 8983:8983 persistent/solrcore\n</code></pre> <p>And then:</p> <pre><code>docker exec -ti esmero-php bash -c \"chown -R www-data:www-data private\"\n</code></pre> <p>Question: Why is this last command different? Answer: Just a variation. The long answer is that the internal <code>www-data</code> user in that container (Alpine Linux) has uid:82, but on Ubuntu the <code>www-data</code> user has a different one so we let Docker assign the uid from inside instead. In practice you could also run directly <code>sudo chown -R 82:82 private</code> which would only apply to an Alpine use case, which can differ in the future! Does this make sense? No worries if not.</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#step-2-set-up-your-minio-s3-bucket","title":"Step 2: Set up your Minio S3 bucket","text":"<p>Once all containers are up and running (you can do a <code>docker ps</code> to check), access <code>http://localhost:9001</code> using your most loved Web Browser with the following credentials:</p> <pre><code>user:minio\npass:minio123\n</code></pre> <p>and create a bucket named \"archipelago\". To do so go to the <code>Buckets</code> section in the navigation pane, and click <code>Create Bucket +</code>. Type <code>archipelago</code> under <code>Bucket Name</code> and submit, done! That is where we will persist all your Files and also your File copies of each Digital Object. You can always go there and explore what Archipelago (well really Strawberryfield does the hard work) has persisted so you can get comfortable with our architecture.</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#step-3-deploy-drupal-10-and-the-awesome-archipelago-modules","title":"Step 3: Deploy Drupal 10 and the awesome Archipelago Modules","text":"<p>The following will run composer inside the esmero-php container to download all dependencies and Drupal Core too.</p> <pre><code>docker exec -ti esmero-php bash -c \"composer install\"\n</code></pre> <p>You might see a warning: <code>Do not run Composer as root/super user! See https://getcomposer.org/root for details</code> and the a long list of PHP packages. Don't worry. All is good here. Keep following the instructions! Once that command finishes run our setup script:</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/setup.sh'\n</code></pre> <p>Explanation: That script will append some important configurations to your local <code>web/sites/default/settings.php</code>.</p> <p>Note: We say <code>local</code> because your whole Drupal web root (the one you cloned) is also mounted inside the esmero-php and esmero-web containers. So edits to PHP files, for example, can be done without accessing the container directly from your local folder.</p> <p>If this is the first time you're deploying Drupal using the provided Configurations run:</p> <pre><code>docker exec -ti -u www-data esmero-php bash -c \"cd web;../vendor/bin/drush -y si --verbose --existing-config --db-url=mysql://root:esmerodb@esmero-db/drupal --account-name=admin --account-pass=archipelago -r=/var/www/html/web --sites-subdir=default --notify=false;drush cr;chown -R www-data:www-data sites;\"\n</code></pre> <p>Note: You will see these warnings:</p> <p><code>[warning] The \"block_content:9aa72fb1-2817-44a7-8fb5-a3eb51166e83\" was not found</code> <code>[warning] The \"block_content:1cdf7155-eb60-4f27-9e5e-64fffe93127a\" was not found</code> <code>[warning] The \"facets_summary_block:advance\" was not found</code> <code>[warning] The \"facets_summary_block:search_page_facet_summary\" was not found</code></p> <p>Nothing to worry about. We will provide the missing part in Step 5.</p> <p>Note 2: Please be patient. This step takes now 25-30% longer because of how the most recent Drupal Installation code fetches translations and other resources (see <code>Performed install task</code>). This means progress might look like getting \"stuck\", go and get a coffee/tea and let it run to the end.</p> <p>Once finished, this will give you an <code>admin</code> Drupal user with <code>archipelago</code> as password (change this if running on a public instance!) and also set the right Docker Container owner for your Drupal installation files.</p> <p>Final note about Steps 2-3: You don't need to, nor should you do this more than once. You can destroy/stop/update, recreate your Docker containers, and start again (<code>git pull</code>), and your Drupal and Data will persist once you've passed the <code>Installation complete</code> message. I repeat, all other containers' data is persisted inside the <code>persistent/</code> folder contained in this cloned git repository. Drupal and all its code is visible, editable, and stable inside your <code>web/</code> folder.</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#step-4-create-a-demo-and-a-jsonapi-user-using-drush-and-assign-your-admin-user-the-administrator-role","title":"Step 4: Create a \"demo \"and a \"jsonapi\" user using drush and assign your \"admin\" user the Administrator Role.","text":"<p><pre><code>docker exec -ti esmero-php bash -c 'drush ucrt demo --password=\"demo\"; drush urol metadata_pro \"demo\"'\n</code></pre> <pre><code>docker exec -ti esmero-php bash -c 'drush ucrt jsonapi --password=\"jsonapi\"; drush urol metadata_api \"jsonapi\"'\n</code></pre> <pre><code>docker exec -ti esmero-php bash -c 'drush urol administrator \"admin\"'\n</code></pre></p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#step-5-ingest-some-metadata-displays-to-make-playing-much-more-interactive","title":"Step 5: Ingest some Metadata Displays to make playing much more interactive","text":"<p>Archipelago is more fun without having to start writing Metadata Displays (in Twig) before you know what they actually are. Since you should now have a <code>jsonapi</code> user and jsonapi should be enabled, you can use that awesome functionality of D8 to get that done. We have 4 demo Metadata Display Entities that go well with the demo Webform we provided. To do that execute in your shell (copy and paste):</p> <pre><code>docker exec -ti esmero-php bash -c 'scripts/archipelago/deploy.sh'\n</code></pre> <p>You are done! Open your most loved Web Browser and point it to <code>http://localhost:8001</code></p> <p>Note: It can take some time to start the first time (Drupal needs some warming up). The Ubuntu deployment is WAY faster than the OSX deployment because of the way the bind mount volumes are handled by the driver. Our experience is that Archipelago basically reacts instantly!</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#step-6-optional-but-more-fun-if-you-add-content","title":"Step 6: Optional but more fun if you add content","text":"<p>One-Step Demo content ingest</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#need-help-blue-screen-missed-a-step-need-a-hug","title":"Need help? Blue Screen? Missed a step? Need a hug?","text":"<p>If you see any issues or errors or need help with a step, please let us know (ASAP!). You can either open an <code>issue</code> in this repository or use the Google Group. We are here to help.</p> <p>If you like this, let us know!</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#user-contributed-documentation-a-video1","title":"User contributed documentation (A Video!)[^1]:","text":"<p>Installing Archipelago on AWS Ubuntu by Zach Spalding: https://youtu.be/RBy7UMxSmyQ</p> <p>[^1]: You may find this user contributed tutorial video, which was created for an earlier Archipelago release, to be helpful. Please note that there are significant differences between the executed steps and that you need to follow the current release instructions in order to have a successful deployment.</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Sherrick</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-ubuntu/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment","Drupal 10","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-windows/","title":"Installing Archipelago Drupal 10 on Windows 10/11","text":"","tags":["Archipelago-deployment","Drupal 10","Windows","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-windows/#prerequisites","title":"Prerequisites","text":"<ul> <li>Windows 11 64-bit: Home or Pro version 21H2 or higher, or Enterprise or Education version 21H2 or higher (see Docker for Windows link below).</li> <li>Windows 10 64-bit: Home or Pro 21H1 (build 19043) or higher, or Enterprise or Education 20H2 (build 19042) or higher (see Docker for Windows link below).</li> <li>Install Ubuntu on WSL 2</li> <li>Install Docker for Windows</li> <li>Install Github for Desktop</li> <li>At least 10 Gbytes of free space (to get started)</li> <li>Some basic Unix/Terminal Skills</li> <li>4 Gbytes of RAM (8 Recommended)</li> </ul>","tags":["Archipelago-deployment","Drupal 10","Windows","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-windows/#optional","title":"Optional","text":"<ul> <li>Install Github Desktop. Ubuntu comes with Git by default, but for a more full-featured way to work with Github, you can install this application.</li> </ul> <p>Open the Docker Desktop app. The Docker service should start up automatically with a status showing when the service is up and running.</p> <p>Open an Ubuntu Terminal session (type <code>Ubuntu</code> in the Windows Start menu).</p> <p>Bring everything up to date: <code>sudo apt update &amp;&amp; sudo apt upgrade -y</code></p>","tags":["Archipelago-deployment","Drupal 10","Windows","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-windows/#deployment","title":"Deployment","text":"<p>Follow the steps for deployment in Ubuntu.</p>","tags":["Archipelago-deployment","Drupal 10","Windows","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-windows/#acknowledgment","title":"Acknowledgment","text":"<p>Thanks to Corinne Chatnik for documenting these steps!</p>","tags":["Archipelago-deployment","Drupal 10","Windows","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-windows/#need-help-blue-screen-missed-a-step-need-a-hug","title":"Need help? Blue Screen? Missed a step? Need a hug?","text":"<p>If you see any issues or errors or need help with a step, please let us know (ASAP!). You can either open an <code>issue</code> in this repository or use the Google Group. We are here to help.</p> <p>If you like this, let us know!</p>","tags":["Archipelago-deployment","Drupal 10","Windows","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-windows/#caring-coding-fixing-testing","title":"Caring &amp; Coding + Fixing + Testing","text":"<ul> <li>Diego Pino</li> <li>Allison Lund</li> <li>Giancarlo Birello</li> </ul>","tags":["Archipelago-deployment","Drupal 10","Windows","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-deployment-windows/#license","title":"License","text":"<p>GPLv3</p>","tags":["Archipelago-deployment","Drupal 10","Windows","Ubuntu 18.04","Ubuntu 20.04"]},{"location":"archipelago-glossary/","title":"Archipelago Glossary of Terms","text":"<p>This list provides quick reference points for some of the most common Archipelago terminology you are likely to encounter.</p> <p>If you are not seeing a term listed here, try using the 'Search' found above to look for your term across all documentation pages.</p>"},{"location":"archipelago-glossary/#ados-archipelago-digital-objects-and-collections","title":"ADOs : Archipelago Digital Objects and Collections","text":"<p>Abbreviation of Archipelago Digital Object. Can be used to refer to ADO Collections/Compounds/Creative Work Series Parents as well.</p> <p>Essentially, any Content type that uses a 'Strawberryfield' (see reference below) is an ADO.</p>"},{"location":"archipelago-glossary/#ado-to-view-mode-mapping","title":"ADO to View Mode Mapping","text":"<p>This refers to the 'ADO to View Mode Mapping' form found at <code>/admin/config/archipelago/viewmode_mapping</code>. This form is used to determine the 'Display Mode' used for an ADO based on the object's <code>type</code>.</p> <p>See also the 'Display Mode' and <code>type</code> references below.</p>"},{"location":"archipelago-glossary/#ami-archipelago-multi-importer","title":"AMI: Archipelago Multi Importer","text":"<p>Archipelago's Module for module for batch/bulk/mass ingests of Archipelago digital objects (ADOs) and collections. AMI also enables you to perform batch administrative actions, such as updating, patching/revising, or deleting digital objects and collections.</p> <p>Related documentation: Archipelago Multi-Importer (AMI).</p>"},{"location":"archipelago-glossary/#ami-sets","title":"AMI Sets","text":"<p>This refers to the special AMI custom entities that hold an Ingest or Update Strategy, a source data CSV with data/metadata, and possibly a CSV for Processed Data coming from Archipelago's Linked Data Reconciliation.</p> <p>Related documentation: Archipelago Multi-Importer (AMI) - AMI Set Entity.</p>"},{"location":"archipelago-glossary/#ami-reports-reports-tab","title":"AMI Reports / Reports Tab","text":"<p>The AMI Reports tab contains information related to the last Processing operation performed for an AMI Set.</p> <p>Related documentation: Step 10: Review your newly created Digital Objects directly or via AMI Set Report.</p>"},{"location":"archipelago-glossary/#ami-update","title":"AMI Update","text":"<p>AMI's Update Operations can be used to Update, Replace, or Append metadata values or files for existing Digital Objects and Collections found in your Archipelago. You can prepare and use AMI Update Sets in different ways using one of three functional operations, depending on your update needs.</p> <p>Related documentation: AMI Update Sets.</p>"},{"location":"archipelago-glossary/#apentitymapping-json-key","title":"<code>ap:entitymapping</code> (JSON key)","text":"<p>This key is used to provide structural mapping hints for Archipelago, such as whether an ADO uses <code>images</code> or <code>ispartof</code>. </p> <p>Related documentation: Metadata in Archipelago - The ap:entitymapping key</p>"},{"location":"archipelago-glossary/#asas_file_type-json-key","title":"<code>as:{as_file_type}</code> (JSON key)","text":"<p>This key is used to provide filetype mapping information for Archipelago, and will be automatically generated depending on the type of file(s) associated with an ADO. </p> <p>Related documentation: Metadata in Archipelago - The as:{as_file_type} key</p>"},{"location":"archipelago-glossary/#compound-object","title":"Compound Object","text":"<p>See 'Digital Object Collection / Compound Object / Creative Work Series' below.</p>"},{"location":"archipelago-glossary/#creative-work-series","title":"Creative Work Series","text":"<p>See 'Digital Object Collection / Compound Object / Creative Work Series' below.</p>"},{"location":"archipelago-glossary/#data-transformation-selections","title":"Data Transformation Selections","text":"<p>The data transformation approach used in an AMI Set that determines how your source data will be transformed into ADOs upon AMI Set Processing.</p> <p>Related documentation: Step 3. Data Transformation Selections.</p>"},{"location":"archipelago-glossary/#digital-object","title":"Digital Object","text":"<p>This is the Drupal Content Type that refers to Digital Objects in Archipelago. </p> <p>This is the Content Type used for standalone/non-hierarchically structured or Child Objects in Parent-Child Relationships.</p> <p>For most Archipelago repositories, this is the most common Drupal Content Type used.</p> <p>More context can be found here: Metadata in Archipelago - Drupal and JSON</p>"},{"location":"archipelago-glossary/#digital-object-collection-compound-object-creative-work-series","title":"Digital Object Collection / Compound Object / Creative Work Series","text":"<p>This is the Drupal Content Type that refers to Digital Object Collections or Parent Compound Objects / Creative Work Series in Archipelago. </p> <p>This is Content Type used for Parent Objects in Parent-Child Relationships (such as Book to Page, where Book = Parent and Page = Child). </p> <p>More context can be found here: Metadata in Archipelago - Drupal and JSON</p>"},{"location":"archipelago-glossary/#display-mode-view-mode","title":"Display Mode / View Mode","text":"<p>A Display Mode is a configureable page layout, using Formatters and other options, for the general display of an ADO in Archipelago. You can define Display Modes for different <code>types</code> of Digital Objects and Collections. </p> <p>Display Mode and View Mode can be used interchangeably.</p> <p>Related documentation: Primer on Display Modes. </p>"},{"location":"archipelago-glossary/#find-and-replace","title":"Find and Replace","text":"<p>Archipelago's Advanced Batch Find and Replace functionality provides different ways for you to efficiently Find/Search and Replace metadata values found in the raw JSON of your Digital Objects and Collections.</p> <p>Related documentation: Advanced Batch Find and Replace</p>"},{"location":"archipelago-glossary/#formatters-strawberryfield-formatters","title":"Formatters / Strawberryfield Formatters","text":"<p>See 'Strawberryfield Formatters' below.</p>"},{"location":"archipelago-glossary/#fragaria-redirects","title":"Fragaria Redirects","text":"<p>Archipelago's Fragaria Redirect Module is a special module that provides dynamic redirect routes matched against existing Search API field values.</p> <p>Related documentation: Fragaria Redirects Module. </p>"},{"location":"archipelago-glossary/#ismemberof-json-key","title":"<code>ismemberof</code> (JSON key)","text":"<p>This key, in default Archipelago configurations, is used for Collection Membership.</p> <p>This key can contain multiple values if appropriate for your repository's Collections structure.</p> <p>For ingested ADOs, this key will contain 'Drupal Node id' integers for corresponding Collection(s).</p> <p>In AMI sets, this key can contain integers to connect to an object's corresponding row in the same spreadsheet/CSV, or a UUID for an already ingested Collection object.</p> <p>This key should never contain string values for Collection labels/titles.</p> <p>See also <code>ap:entitymapping</code> key above.</p>"},{"location":"archipelago-glossary/#ispartof-json-key","title":"<code>ispartof</code> (JSON key)","text":"<p>This key, in default Archipelago configurations, is used for Parent-Child Object Relationships (so a Child ADO would reference the Parent ADO in <code>ispartof</code>).</p> <p>This key should contain single values only, due to the nature of the Parent-Child Object Relationship.</p> <p>For ingested ADOs, this key will contain 'Drupal Node id' integers for a corresponding Parent Object.</p> <p>In AMI sets, this key can contain integers to connect to an object's corresponding row in the same spreadsheet/CSV, or a UUID for an already ingested Parent object.</p> <p>This key should never contain string values for Parent Object labels/titles.</p> <p>See also <code>ap:entitymapping</code> key above.</p>"},{"location":"archipelago-glossary/#label-json-key","title":"<code>label</code> (JSON key)","text":"<p>This key is used for the title of the Digital Object or Collection. This key is required for all ADOs. </p> <p>Related documentation: Metadata in Archipelago - The label key</p>"},{"location":"archipelago-glossary/#linked-data-reconciliation-lod-reconciliation","title":"Linked Data Reconciliation / LoD Reconciliation","text":"<p>AMI's (Archipelago Multi Importer's) Linked Data Reconciliation tool can be used to enrich your metadata with Linked Data (LoD). Using this tool, you can map values from your topical/subject metadata elements to your preferred LoD vocabulary source. These mappings can then be transformed via a corresponding Metadata Display (Twig) template to process the values into JSON-formatted metadata for your specified AMI set.</p> <p>Related documentation: Linked Data Reconciliation.</p>"},{"location":"archipelago-glossary/#metadata-api-module","title":"Metadata API Module","text":"<p>Archipelago's Metadata API Module is a special Strawberryfield Formatter Module that implements Metadata API Endpoints and uses Views and Metadata Display Entities as a configuration option to provide dynamic APIs based on Metadata present in each Archipelago Digital Object (or Node) that contains a Strawberryfield type of field (JSON).</p> <p>Related documentation: Metadata API Module</p>"},{"location":"archipelago-glossary/#metadata-display-entities-twig-templates","title":"Metadata Display Entities / Twig Templates","text":"<p>Metadata Display Entities are particular Twig Templates used in Archipelago to transform or 'cast' JSON metadata/data into different displays and outputs.</p> <p>Related documentation: Twig Templates and Archipelago</p>"},{"location":"archipelago-glossary/#metadata-display-preview","title":"Metadata Display Preview","text":"<p>Archipelago's Metadata Display Preview is a very handy tool for your repository toolkit that enables you to preview the output of your Metadata Display (Twig) Templates (found at <code>/metadatadisplay/list</code>).</p> <p>Related documentation: Metadata Display Preview</p>"},{"location":"archipelago-glossary/#metadata-display-usage","title":"Metadata Display Usage","text":"<p>Archipelago's Metadata Display Usage tab is an extension of Metadata Display Preview that provides a convenient way for you to check where a Metadata Display Template is currently being used across your Archipelago. </p> <p>Related documentation: Metadata Display Usage</p>"},{"location":"archipelago-glossary/#process-tab-ami-set-processing","title":"Process Tab / AMI Set Processing","text":"<p>The Process Tab contains configuration options for selecting outcomes related to the Processing of your AMI Set.</p> <p>Related documentation: Step 7: AMI Set Processing.</p>"},{"location":"archipelago-glossary/#search-and-replace","title":"Search and Replace","text":"<p>See 'Find and Replace' above.</p>"},{"location":"archipelago-glossary/#search-and-solr","title":"Search and Solr","text":"<p>Refers to the integrations of related (Archipelago, Drupal) Search and Solr Index components in Archipelago.</p> <p>Related documentation: Search &amp; Solr Overview</p>"},{"location":"archipelago-glossary/#strawberryfield-or-strawberry-field","title":"Strawberryfield (or Strawberry Field)","text":"<p>The flexible, extensible JSON blob Field attached to every Archipelago Digital Object / Collection / Compound Object / Creative Work Series.</p> <p>The Strawberryfield contains all of the data and metadata associated with ADOs in a single Field (per ADO).</p> <p>In default Archipelagos, can be viewed in the 'Raw JSON' display beneath every ADO when logged in as an authenticated user.</p> <p>Related documentation: Metadata in Archipelago.</p>"},{"location":"archipelago-glossary/#strawberry-flavour","title":"Strawberry Flavour","text":"<p>A 'Strawberry flavour' refers to a post-processing related document, such as a generated HOCR solr document or new filetype created by Archipelago such as a WACZ file generated from an original WARC file.</p>"},{"location":"archipelago-glossary/#strawberryfield-formatter","title":"Strawberryfield Formatter","text":"<p>Strawberryfield Formatters are Archipelago specific Field Formatters that can transform Strawberryfield JSON data into different formats for display, including in IIIF Manifests and/or different Viewers.</p> <p>More information found at: Strawberryfield Formatters</p>"},{"location":"archipelago-glossary/#strawberry-keyname-provider","title":"Strawberry Keyname Provider","text":"<p>Strawberry Keyname Providers feed the values from your desired JSON keys to Solr fields. They are a crucial part of the Search and Solr setup in Archipelago.</p> <p>Related Documentation: Search &amp; Solr Overview - Strawberry Keyname Providers</p>"},{"location":"archipelago-glossary/#strawberryfield-modules","title":"Strawberryfield Modules","text":"<p>Strawberryfield Modules are the Archipelago specific modules at the heart of our system.</p> <ul> <li>Strawberryfield<ul> <li>Github Repository</li> <li>Related documentation</li> </ul> </li> <li>Format Strawberryfield<ul> <li>Github Repository</li> <li>Related documentation</li> </ul> </li> <li>Webform Strawberryfield<ul> <li>Github Repository</li> <li>Related documentation</li> </ul> </li> <li>Strawberry Runners<ul> <li>Github Repository</li> <li>Related documentation</li> </ul> </li> <li>Archipelago Multi-Importer (AMI)<ul> <li>Github Repository</li> <li>Related documentation</li> </ul> </li> <li>Fragaria (Redirects)<ul> <li>Github Repository</li> <li>Related Documentation</li> </ul> </li> </ul> <p>Related documentation: Strawberryfields Forever.</p>"},{"location":"archipelago-glossary/#strawberry-runners-sbr","title":"Strawberry Runners / SBR","text":"<p>Archipelago's Strawberry Runners (SBR) module provides provides a set of post-processing capabilities for the JSON based metadata, files and entities that comprise your Archipelago Digital Objects (ADOs). </p> <p>Related documentation: Strawberry Runners Post-Processing Configuration.</p>"},{"location":"archipelago-glossary/#twig-templates-metadata-display-entities","title":"Twig Templates / Metadata Display Entities","text":"<p>See 'Metadata Display Entities' above.</p>"},{"location":"archipelago-glossary/#type-json-key","title":"<code>type</code> (JSON key)","text":"<p>This key is used for the Digital Object or Digital Object Collection/semantic Type, such as 'Photograph' or 'Collection'. This key is required for all ADOs.</p> <p>Related documentation: Metadata in Archipelago - The type key</p>"},{"location":"archipelago-glossary/#view-mode-display-mode","title":"View Mode / Display Mode","text":"<p>See 'Display Mode' entry above.</p>"},{"location":"archipelago-glossary/#webforms","title":"Webforms","text":"<p>The 'Webform Strawberryfield' module provides Drupal Webform ( == awesome piece of code) integrations for StrawberryField, so you can really have control over your Metadata ingests entered and edited via webforms.</p> <p>Related documentation: Webforms in Archipelago</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"createdisplaymodes/","title":"Creating Display Modes for Archipelago Digital Objects","text":"<p>We recommend checking out our primer on Display Modes for a broader overview on Form Modes and View Modes for Archipelago Digital Objects (ADOs).</p> <p>But how do you create and enable these Display Modes in the first place? Let's find out.</p>"},{"location":"createdisplaymodes/#adding-a-new-form-mode","title":"Adding a new Form Mode","text":"<p>Why would you want to create a new form mode? One common reason is to create different data entry experiences for users with different roles. Let's create an example form mode called \"Student Webform\" --  we can imagine a deployment where Students need a simplified form for ADO creation. We are going to create a form mode, enable it for Digital Objects, and give it some custom settings that differentiate it from existing form modes.</p> <ol> <li> <p>Navigate to <code>yoursite/admin/structure/display-modes</code></p> <p></p> </li> <li> <p>Click on Form modes. This image shows the basic Form Modes shipped with Archipelago</p> <p></p> </li> <li> <p>Click the \"Add Form mode\" button at the top of the page. Then select the \"Content\" entity type from the list. In this example, we ultimately want the form mode to be applied to Archipelago Digital Objects, which is a Content entity type.</p> <p></p> </li> <li> <p>Enter the name of your Form Mode and hit save. Here we are entering \"Student Webform\".</p> <p></p> </li> <li> <p>Great. Now you will see your new Form mode in the list! Let's put it to use.</p> <p></p> </li> <li> <p>Head to <code>yoursite/admin/structure/types/manage/digital_object</code> and click the \"Manage Form Display\" tab. As mentioned above, in this example we want to add a new Form Mode for ADOs, so we are dealing with the Digital Object content type. Scroll to the bottom of this page and look for the \"Custom Display Settings\" area, which is collapsed by default. Expand it, and you should see this.</p> <p></p> </li> <li> <p>Enable \"Student Webform\" and hit save! Now scroll back up the page. You'll see it enabled like so.</p> <p></p> </li> <li> <p>Now select our new \"Student Webform\" tab. From here, you have many options and can configure input fields as you see fit! To finish out our specific example though, let's finally add our Student Webform to the display. Click on the settings gear icon next to the Descriptive Metadata field.</p> <p></p> <p>You'll see that the default webform named \"Descriptive Metadata\" is entered. To add custom content to this Field Widget, start typing in the autocomplete. This example assumes you've created a webform called <code>Student Webform</code> in <code>yoursite/admin/structure/webform</code>. For info on how to create a new Webform with proper settings, see our Webforms as input guide.</p> </li> <li> <p>After you've selected your \"Student Webform\" in the Field Widget setting, hit Update, and then Save at the bottom of the page.</p> </li> </ol> <p>All done! So let's recap. We created a new form mode. We added this form mode to the Manage Form Display &gt; Custom Display Settings options for Digital Objects. And finally we configured the Field Widget for Descriptive Metadata in our new Form Mode to use a new Webform. This last step is arbitrary to this example. We could have enabled or disabled fields, or changed other field widget settings depending on our needs. But configuring different Webforms as Field Widgets for Descriptive Metadata is a common use case in Archipelago.</p> <p>Thanks for reading this far! But there is more. We might want to display, in addition to ingest, our ADOs in custom ways. The process for creating new View Modes (the other type of Display Mode) is quite similar to creating new Form Modes, but let's walk through it with another example case.</p>"},{"location":"createdisplaymodes/#adding-a-new-view-mode","title":"Adding a new View Mode","text":"<p>Why would you want to create a new View Mode? Maybe there is a new type of media you are attaching to ADOs that you want to display using the proper player or tool. Or maybe you want to simplify the ADO display, removing fields from the display page. In this example let's create a new View Mode for ADOs that adds some fields to the display to show the Author and Published date of the object.</p> <ol> <li> <p>Navigate to <code>yoursite/admin/structure/display-modes</code></p> <p></p> </li> <li> <p>Select View modes, and click the \"Add View mode\" at the top of the page.</p> <p></p> </li> <li> <p>Select Content as your entity type.</p> <p></p> </li> <li> <p>Enter the name of your new View Mode and save. Ours is \"Digital Object with Publishing Information\"</p> <p></p> </li> <li> <p>Now let's enable this View mode. Go to <code>yoursite/admin/structure/types/manage/digital_object</code> and click the \"Manage Display\" tab.</p> </li> <li> <p>Scroll to the bottom of the page and expand the \"Custom Display Settings\" area. You will see our newly created View Mode. Enable it and hit save.</p> <p></p> </li> <li> <p>Now scroll back to the page top. You will see \"Digital Object with Publishing Information\" in the list of View Modes, so go ahead and select it.</p> <p></p> </li> <li> <p>Scroll down until you see the \"Disabled\" section. This section contains fields that are available to the ADO content type, but are not enabled in this display mode. Let's enable Author and Post date by changing the \"Region\" column dropdown from \"Disabled\" to \"Content\". (To learn more about Regions in Drupal, see here). Basically, this ensures that this field has a home in the page layout. Hit save.</p> <p></p> <p></p> </li> <li> <p>Now, if you want ADOs to use this View Mode for display, there is one last step. You need to select \"Digital Object with Publishing Info\" as the view mode Display Settings when adding new content. This area is located on the right side of the page. See below:</p> <p></p> </li> <li> <p>Now, when we view the individual ADO, these new fields have been added to the display.</p> <p></p> </li> </ol> <p>All done! This was quite a simple example, but now you are aware of how to customize your own ADO display. It can only get more complex and exciting from here.</p> <p>Let's recap. We created a new View Mode. We enabled this View Mode in Manage Display &gt; Custom Display Settings for Digital Objects. We enabled new fields (in this case, just for instruction, the Author and Post date fields) to make our new View Mode unique, and learned about Disabled fields in the process. We selected our new View Mode in the Display Settings area (slightly confusing wording because yes, this is a View Mode, subset of Display Mode) during ADO creation (for more on creating new objects, see this guide).</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"customwebformelements/","title":"Archipelago Custom Webform Elements","text":"<p>In addition to the core elements provided by the Drupal Webform module, Archipelago also deploys a robust set of custom webform elements specific to digital repositories metadata needs and use cases.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"customwebformelements/#linked-data","title":"Linked Data:","text":"<p>(*found under Composite Elements in \"Add Element\" menu)</p> <ul> <li> <p>Library of Congress (LoC) Linked Open data</p> <ul> <li>Provides a form element to reconciliate against LoC Headings and similar LoD Sources</li> <li>Able to configure which LoC Autocomplete Source Provider is used:<ul> <li>Subjects (LCSH)</li> <li>LC Name Authority File (LCNAF)</li> <li>LC Genre/Form Terms (LCGFT</li> <li>Thesaurus of Graphic Materials (TGN)</li> <li>MARC list for Geographic areas</li> <li>Filter Suggest results by RDF Type<ul> <li>Any of the Classes listed here: https://id.loc.gov/ontologies/madsrdf/v1.html</li> </ul> </li> </ul> </li> </ul> </li> <li> <p>Multi LoD Source Agent Items</p> <ul> <li>Provides a form element to reconciliate Agents against Multiple sources of Agents</li> </ul> </li> <li> <p>Wikidata</p> <ul> <li>Provides a form element to reconciliate against Wikidata Items and Agents<ul> <li>via Wikidata Action API</li> </ul> </li> </ul> </li> <li> <p>Getty Vocabulary Term</p> <ul> <li>Provides a form element to reconciliate against the Getty Vocabularies</li> </ul> </li> <li> <p>VIAF</p> <ul> <li>Provides a form element to reconciliate against VIAF (OCLC) Items</li> </ul> </li> <li> <p>Location GEOJSON (Nominatim--Open Street Maps)</p> <ul> <li>Provides a form element to collect valid location information (address, longitude, latitude, geolocation) using Nominatim/Openstreetmap open API</li> </ul> </li> <li> <p>PubMed MeSH Suggest</p> <ul> <li>Provides a form element to reconciliate against PubMed MeSH</li> </ul> </li> <li> <p>SNAC Constellation Linked Open Data</p> <ul> <li>Provides a form element to reconciliate against Social Networks and Archival Context (SNAC)</li> </ul> </li> <li> <p>Europeana Entity Suggest</p> <ul> <li>Provides a form element to reconciliate against Europeana Entity</li> </ul> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"customwebformelements/#customlocal-webform-lod-elements","title":"Custom/Local Webform LoD Elements","text":"<p>(*also found under Composite Elements in \"Add Element\" menu)</p> <ul> <li> <p>Webform Options LoD suggest</p> <ul> <li>Provides a form element autocomplete labels/urls(values) from Webform Options.</li> </ul> </li> <li> <p>Webform LoD from CSV attached to an ADO suggest</p> <ul> <li>Provides a form element autocomplete labels/urls(values) from a CSV attached to a Digital Object.</li> <li>Documentation Guide found here: Using Archipelago's 'Webform LoD from CSV attached to an ADO suggest'</li> </ul> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"customwebformelements/#file-upload-elements","title":"File Upload Elements:","text":"<ul> <li> <p>Enhancements for Audio, Document, Image, Video file uploads</p> <ul> <li>Backend processing for technical metadata (such as pronom, exif extraction)</li> </ul> </li> <li> <p>Import Metadata from a File (such as XML)</p> <ul> <li>Provides a form element for uploading, saving a file and parsing the content as metadata/webform submission data.</li> </ul> </li> <li> <p>Import Metadata in CSV format from a File</p> <ul> <li>Provides a form element for uploading, saving a file and parsing the content as metadata/webform submission data.</li> </ul> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"customwebformelements/#datetime-elements","title":"Date/Time Elements:","text":"<ul> <li>Multi Format Date and Date Range<ul> <li>Provides a form element for setting/reading Dates indifferent formats suitable for metadata.</li> <li>Optional EDTF Validation. </li> </ul> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"customwebformelements/#composite-elements","title":"Composite Elements:","text":"<ul> <li>Panorama Tour Builder<ul> <li>Provides a form element to build multi-scene Panorama Tour Builder with hotspots</li> </ul> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"customwebformelements/#computed-elements","title":"Computed Elements:","text":"<ul> <li> <p>Computed Metadata Transplant</p> <ul> <li>Provides an item that takes source values (not only elements) and distributes them into other places/elements via a twig template</li> </ul> </li> <li> <p>Computed Token</p> <ul> <li>Provides an item to display computed webform submission values using tokens.</li> </ul> </li> <li> <p>Computed Twig  </p> <ul> <li>Provides an item to display computed webform submission values using Twig</li> </ul> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"customwebformelements/#but-wait-theres-more","title":"But wait there's more!","text":"<p>You can review the coding behind these custom elements here: https://github.com/esmero/webform_strawberryfield/tree/1.3.0/src/Element</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"devops/","title":"Archipelago Software Services","text":"<p>At the core of the Archipelago philosophy is our commitment to both simplicity and flexibility.</p>"},{"location":"devops/#under-the-hood-archipelagos-architecture-is","title":"Under the hood, Archipelago's architecture is:","text":"<ul> <li>Drupal (CMS)</li> <li>Solr (Search)</li> <li>Cantaloupe (Image Server)</li> <li>S3 Storage (Mini.io or any other S3 flavor)</li> <li>NLP + ML Server</li> </ul> <p>Installation is entirely Dockerized and scripted with easy-to-follow directions.</p> <ul> <li>Docker containers are as follows:</li> </ul> Container Purpose Description esmero-web NGNIX Routes calls to esmero-php esmero-php PHP-FPM Has all binaries for postprocessing/exif/ocr/etc. Runs PHP code. esmero-db Database AMD and INTEL processors: MYSQL 8ARM processors: MariaDB esmero-minio Storage S3 API compatible Backend file and ADO as file storage. In a local it will do all the S3 stuff, on a live instance it can server as file routing to AWS S3, Azure Blob Storage, etc. esmero-solr Solr Currently version 9.1.1 esmero-nlp Natural Language Processing NLP64 server for entity extraction, language detection, ML, etc. esmero-cantaloupe IIIF Media Server Customized Server used to provide IIIF Image API/Media access <ul> <li>Information related to non-Dockerized installation and configruation can be found here: Traditional Installation Notes</li> </ul>"},{"location":"devops/#strawberryfield-modules-at-the-heart-of-every-archipelago","title":"Strawberryfield Modules at the heart of every Archipelago:","text":"<ul> <li>Strawberryfield</li> <li>Format Strawberryfield</li> <li>Webform Strawberryfield</li> <li>Strawberry Runners</li> <li>Archipelago Multi-Importer (AMI)</li> <li>Fragaria Redirects</li> </ul> <p>Documentation related to the Strawberryfield modules can be found here: Strawberryfields Forever</p>"},{"location":"devops/#archipelago-also-extends-these-powerful-tools","title":"Archipelago also extends these powerful tools:","text":"<ul> <li>Annotorius</li> <li>Drupal Webform Module</li> <li>International Image Interoperability Framework (IIIF)</li> <li>Internet Archive BookReader</li> <li>Mirador</li> <li>OpenSeadragon</li> <li>Pannellum</li> <li>Replayweb.page Webarchive Player</li> <li>Solr OCR Highlighting</li> <li>Twig Templating In Symfony / In Drupal</li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"documentation_about/","title":"About This Documentation","text":"<p>Documentation is vital to our community, and contributions are welcome, greatly appreciated, and encouraged. </p>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_about/#how-to-contribute","title":"How to Contribute","text":"<p>Difficulty Level: Moderate\u2013Difficult</p> <ol> <li>Follow the GitHub Workflow.</li> <li>Check out the examples of additional features.</li> </ol>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_features/","title":"Additional Documentation Features","text":"<p>Below are some examples of features that are currently in use on the site. To explore more visit the Material for MkDocs documentation.</p>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_features/#examples","title":"Examples","text":"","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_features/#images","title":"Images","text":"<p>Images are located in the <code>docs/images</code> folder. You can add new ones there and link to them by relative path. For example, if you added <code>strawberries_color.png</code>, you would embed it like so:</p> <p>Image</p> MarkupResult <pre><code>![New Documentation Image](images/strawberries_color.png)\n</code></pre> <p></p>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_features/#admonitions","title":"Admonitions","text":"<p>Question Admonition</p> MarkupResult <pre><code>??? question \"What is a collapsible admonition?\"\n\n    This is a collapsible admonition. It can have a title, and it collapses so as not to interrupt the flow the of the document, but it provides useful information as needed.\n</code></pre> What is a collapsible admonition? <p>This is a collapsible admonition. It can have a title, and it collapses so as not to interrupt the flow the of the document, but it provides useful information as needed.</p> <p>You can read more about admonitions with further examples in the Material for MkDocs documentation.</p>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_features/#code-blocks","title":"Code blocks","text":"<p>Code block with title and highlighted lines</p> MarkupResult <pre><code>```html+twig title=\"HTML in a TWIG template\" hl_lines=\"8 9 10\"\n{% for image in attribute(data, 'as:image')|slice(0,1) %}\n {% if image[\"flv:exif\"] %}\n   {% set height = image[\"flv:exif\"].ImageHeight%}\n {% else  %}\n   {% set width = 1200 %}\n {% endif %}\n   {% set imageurl =  IIIFserverurl ~ image['url']|replace({'s3://':''})|replace({'private://':''})|url_encode %}\n&lt;a href=\"{{ nodeurl }}\" title=\"{{ data.label }}\" alt=\"{{ data.label }}\"&gt;\n&lt;img src=\"{{ imageurl }}/pct:5,5,90,30/,400/0/default.jpg\" class=\"d-block.w-auto\" alt=\"{{ image.name }}\" height=\"400px\" style=\"min-width:1200px\"&gt;\n&lt;/a&gt; \n{% endfor %}\n```\n</code></pre> HTML in a TWIG template<pre><code>{% for image in attribute(data, 'as:image')|slice(0,1) %}\n {% if image[\"flv:exif\"] %}\n   {% set height = image[\"flv:exif\"].ImageHeight%}\n {% else  %}\n   {% set width = 1200 %}\n {% endif %}\n   {% set imageurl =  IIIFserverurl ~ image['url']|replace({'s3://':''})|replace({'private://':''})|url_encode %}\n&lt;a href=\"{{ nodeurl }}\" title=\"{{ data.label }}\" alt=\"{{ data.label }}\"&gt;\n&lt;img src=\"{{ imageurl }}/pct:5,5,90,30/,400/0/default.jpg\" class=\"d-block.w-auto\" alt=\"{{ image.name }}\" height=\"400px\" style=\"min-width:1200px\"&gt;\n&lt;/a&gt; \n{% endfor %}\n</code></pre>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_features/#quirks","title":"Quirks","text":"<p>Because of the use of front matter (the block of YAML at the top that contains settings and data for the file) the markup for a horizontal rule is restricted. To create one you have to use the following:</p> <p>Horizontal Rule</p> MarkupResult <pre><code>___\n</code></pre> <p>Info</p> <p>The above are underscore (<code>_</code>) characters, as opposed to hyphens (<code>-</code>).</p> <p>Some of the documentation that is automatically deployed from the repos have special comments that are converted to theme-specific elements via script.</p> <p>Front Matter</p> Deployment Repo with Front MatterDocumentation Repo with Front Matter <pre><code>&lt;!--documentation\n---\ntitle: \"Adding Demo Archipelago Digital Objects (ADOs) to your Repository\"\ntags:\n  - Archipelago Digital Objects\n  - Demo Content\n---\ndocumentation--&gt;\n</code></pre> <pre><code>---\ntitle: \"Adding Demo Archipelago Digital Objects (ADOs) to your Repository\"\ntags:\n  - Archipelago Digital Objects\n  - Demo Content\n---\n</code></pre> <p>Switching Elements</p> Deployment Repo with Theme-specific MarkupDocumentation Repo with Theme-specific Markup <pre><code>&lt;!--switch_below\n\n??? info \"OSX (macOS)/x86-64\"\n\n    ```shell\n    cp docker-compose-osx.yml docker-compose.yml\n    ```\n\n??? info \"Linux/x86-64/AMD64\"\n\n    ```shell\n    cp docker-compose-linux.yml docker-compose.yml\n    ```\n\n??? info \"OSX (macOS)/Linux/ARM64\"\n\n    ```shell\n    cp docker-compose-arm64.yml docker-compose.yml\n    ```\n\nswitch_below--&gt;\n\n___\n\nOSX (macOS)/x86-64:\n\n```shell\ncp docker-compose-osx.yml docker-compose.yml\n```\n\n___\n\nLinux/x86-64/AMD64:\n\n```shell\ncp docker-compose-linux.yml docker-compose.yml\n```\n\n___\n\nOSX (macOS)/Linux/ARM64:\n\n```shell\ncp docker-compose-arm64.yml docker-compose.yml\n```\n\n___\n\n&lt;!--switch_above\nswitch_above--&gt;\n</code></pre> <pre><code>??? info \"OSX (macOS)/x86-64\"\n\n    ```shell\n    cp docker-compose-osx.yml docker-compose.yml\n    ```\n\n??? info \"Linux/x86-64/AMD64\"\n\n    ```shell\n    cp docker-compose-linux.yml docker-compose.yml\n    ```\n\n??? info \"OSX (macOS)/Linux/ARM64\"\n\n    ```shell\n    cp docker-compose-arm64.yml docker-compose.yml\n    ```\n\n&lt;!--repo_docs\n\n___\n\nOSX (macOS)/x86-64:\n\n```shell\ncp docker-compose-osx.yml docker-compose.yml\n```\n\n___\n\nLinux/x86-64/AMD64:\n\n```shell\ncp docker-compose-linux.yml docker-compose.yml\n```\n\n___\n\nOSX (macOS)/Linux/ARM64:\n\n```shell\ncp docker-compose-arm64.yml docker-compose.yml\n```\n\n___\n\nrepo_docs--&gt;\n</code></pre>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_technical/","title":"Documentation Technical Details","text":"<p>Archipelago documentation is generated using the following open source projects:</p> <ul> <li>MkDocs generates the static site</li> <li>Material for MkDocs provides the site's theme</li> <li>mike generates the routing for versions</li> <li>Github Actions builds and deploys the files to Github Pages</li> </ul> <p>To use any advanced features not mentioned in these pages, you can look through the documentation for each of the above projects.</p> <p>In addition to the pages added directly via this repository, there are some pages automatically deployed here with GitHub Actions from the following repositories:</p> <ul> <li>https://github.com/esmero/archipelago-deployment</li> <li>https://github.com/esmero/archipelago-deployment-live</li> </ul> <p>Both the main READMEs and documentation in the <code>docs</code> folders for those repositories are prepended with <code>archipelago-deployment</code> and <code>archipelago-deployment-live</code> respectively and copied to the <code>docs</code> folder here with the rest of the documentation. In practice that means those pieces of documentation need to be edited in those repositories directly.</p>","tags":["Documentation"]},{"location":"documentation_template/","title":"Example Documentation Page Title","text":"<p>A brief overview of the specific functionality or workflow area that will be covered in your documentation.</p>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_template/#stepsguides","title":"Steps/Guides","text":"<ol> <li>Step one example.</li> <li> <p>Step two example with images:</p> <p></p> </li> <li> <p>Step three example with Details section:</p> Click to open this Details Section <p>More Details in a List</p> <ul> <li>Apples</li> <li>Oranges</li> <li>Peaches</li> <li>Pumpkins</li> </ul> </li> <li> <p>Step four example with a simple code block:</p> <pre><code>{% for image in attribute(data, 'as:image')|slice(0,1) %}\n {% if image[\"flv:exif\"] %}\n   {% set height = image[\"flv:exif\"].ImageHeight%}\n {% else  %}\n   {% set width = 1200 %}\n {% endif %}\n   {% set imageurl =  IIIFserverurl ~ image['url']|replace({'s3://':''})|replace({'private://':''})|url_encode %}\n&lt;a href=\"{{ nodeurl }}\" title=\"{{ data.label }}\" alt=\"{{ data.label }}\"&gt;\n&lt;img src=\"{{ imageurl }}/pct:5,5,90,30/,400/0/default.jpg\" class=\"d-block.w-auto\" alt=\"{{ image.name }}\" height=\"400px\" style=\"min-width:1200px\"&gt;\n&lt;/a&gt; \n{% endfor %}\n</code></pre> </li> <li> <p>Step five example with a code block that has a title and highlighted lines:</p> HTML in a TWIG template<pre><code>{% for image in attribute(data, 'as:image')|slice(0,1) %}\n {% if image[\"flv:exif\"] %}\n   {% set height = image[\"flv:exif\"].ImageHeight%}\n {% else  %}\n   {% set width = 1200 %}\n {% endif %}\n   {% set imageurl =  IIIFserverurl ~ image['url']|replace({'s3://':''})|replace({'private://':''})|url_encode %}\n&lt;a href=\"{{ nodeurl }}\" title=\"{{ data.label }}\" alt=\"{{ data.label }}\"&gt;\n&lt;img src=\"{{ imageurl }}/pct:5,5,90,30/,400/0/default.jpg\" class=\"d-block.w-auto\" alt=\"{{ image.name }}\" height=\"400px\" style=\"min-width:1200px\"&gt;\n&lt;/a&gt; \n{% endfor %}\n</code></pre> </li> <li> <p>Last Step Example. </p> </li> </ol> <p>Congratulations! \ud83c\udf89</p>","tags":["Documentation","Contributing","Examples"]},{"location":"documentation_workflow/","title":"GitHub Workflow","text":"<ol> <li>Find an issue to resolve or create a new issue here.</li> <li>Fork the documentation repo. If you're not familiar with forking see this guide.</li> <li>Create an issue branch in your forked repo. For example, if the issue you're resolving is ISSUE-100:    <pre><code>git checkout -b ISSUE-100\n</code></pre></li> <li>Copy this template to create a new piece of documentation:    <pre><code>cp docs/documentation_template.md docs/new_documentation.md\n</code></pre></li> <li>Make your changes to the copied markdown file.</li> <li>If this is new documentation add it to the <code>nav</code> section of the <code>mkdocs.yml</code> configuration file at the root of the repo. For example:    <pre><code>nav:\n  - Home: index.md\n  - About Archipelago:\n    - Archipelago's Philosophy &amp; Guiding Principles: ourtake.md\n    - Strawberryfields Forever: strawberryfields.md\n    - Software Services: devops.md\n    - New Documentation: new_documentation.md\n  - Code of Conduct: CODE_OF_CONDUCT.md\n  - Instructions and Guides:\n    - Archipelago-Deployment:\n      - Start: archipelago-deployment-readme.md\n      - Installing Archipelago Drupal 9 on OSX (macOS): archipelago-deployment-osx.md\n      - Installing Archipelago Drupal 9 on Ubuntu 18.04 or 20.04: archipelago-deployment-ubuntu.md\n      - Installing Archipelago Drupal 9 on Windows 10/11: archipelago-deployment-windows.md\n      - Adding Demo Archipelago Digital Objects (ADOs) to your Repository: archipelago-deployment-democontent.md\n...\n</code></pre></li> <li> <p>To view the changes locally, first install the Python libraries using the Python package manager pip:    <pre><code>pip install mkdocs-material mike git+https://github.com/jldiaz/mkdocs-plugin-tags.git mkdocs-git-revision-date-localized-plugin mkdocs-glightbox\n</code></pre>    You may need to install Python on your machine. Download Python or use your favorite operating system package manager such as Homebrew. </p> </li> <li> <p>Now you can build the site locally, e.g. for the documentation using the 1.4.0 branch:    <pre><code>mike deploy 1.4.0\nmike set-default 1.4.0\n</code></pre>    If you create a new branch to match the issue number as in step 3, you would use your branch instead of 1.4.0. For example, a branch of ISSUE-129.    <pre><code>mike deploy ISSUE-129\nmike set-default ISSUE-129\n</code></pre></p> </li> <li>Start the web server:    <pre><code>mike serve\n</code></pre></li> <li>Check the results in your browser by going to: http://localhost:8000</li> <li>If everything looks good, you can push to your forked repo issue branch:     <pre><code>git add .\ngit commit -m \"Create new docs with useful information.\"\ngit push origin ISSUE-100\n</code></pre></li> <li>Create a pull request and link to the issue by tagging it, e.g. <code>Resolves #100</code>.</li> </ol>","tags":["Documentation","Contributing","Examples"]},{"location":"drupal_core_update/","title":"Updating Drupal Core in Archipelago","text":"<p>Drupal, the project, puts out new core releases on a regular schedule. Your Archipelago site needs to apply the security updates and possibly minor releases between major core updates. Major core updates will typically coincide with an updated Archipelago stable release. </p> <p>Updating core is done via Composer:</p>","tags":["Archipelago-deployment","Archipelago-deployment-live","DevOps","Drupal","Drupal Core"]},{"location":"drupal_core_update/#stepsguides","title":"Steps/Guides","text":"<ol> <li>First you will want to verify Drupal Core will update itself and the dependencies. To do so, you run the following command:     <pre><code>docker exec -ti esmero-php bash -c \"composer update \"drupal/core-*:^9\" --with-all-dependencies --dry-run\n</code></pre>     The <code>--dry-run</code> flag will allow you to see what will be updated. Once you review the updates and are ready to go with the full update, you will run the same command without the <code>dry-run</code> flag.</li> <li>Update Core and the dependencies:     <pre><code>docker exec -ti esmero-php bash -c \"composer update \"drupal/core-*:^9\" --with-all-dependencies\"\n</code></pre></li> <li>Sometimes the database needs to be updated after the update of the files. The following command will prompt you to type yes if there are updates to be done, or it will not return anything if no updates are needed:     <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre></li> <li>Clear the Drupal cache:     <pre><code>docker exec -ti esmero-php bash -c \"drush cache:rebuild\"\n</code></pre> Occasionally there will be other Drupal modules that Archipelago uses, and they need to be updated at the same time you run a Core update. This is an example of updating Drupal Webform, which was required for moving to Drupal 9.5.x:</li> </ol> <p>Updating a Drupal module</p> <ol> <li>Use Composer to update a module to a specific release:     <pre><code>docker exec -ti esmero-php bash -c \"composer update \"drupal/webform:6.1.4\n</code></pre></li> <li>Check for database updates and apply them if necessary:     <pre><code>docker exec -ti esmero-php bash -c \"drush updatedb\"\n</code></pre>     or     <pre><code>docker exec -ti esmero-php bash -c \"drush updb\"\n</code></pre></li> <li>Clear the cache again:     <pre><code>docker exec -ti esmero-php bash -c \"drush cache:rebuild\"\n</code></pre>     or     <pre><code>docker exec -ti esmero-php bash -c \"drush cr\"\n</code></pre></li> </ol>","tags":["Archipelago-deployment","Archipelago-deployment-live","DevOps","Drupal","Drupal Core"]},{"location":"embargo/","title":"Embargo &amp; Access Restrictions in Archipelago","text":"<p>Archipelago features two primary different Metadata Based Embargo / Access Restriction options for limiting access to materials when needed.</p> <p>You can find the main Metadata Based Embargo settings configuration form at:</p> <ul> <li><code>admin/config/archipelago/metadatabased_Embargo</code></li> <li>Through the <code>Configuration</code> menu &gt; <code>Archipelago</code> &gt; <code>Metadata based Embargo settings</code></li> </ul> <p></p> <p>This form allows you to enable/disable Embargo functionality enforced at the Formatter level and configure on which JSON Key/values those will act.</p> <p>For either (or both) Embargo options, you will to select the checkbox for 'Is Embargo checking and enforcing globally active?'</p> <p>Important Note: Your Metadata Keys and Values Matter</p> <p>Please also keep in mind that the Embargo Options noted below need to draw the necessary related metadata values as defined in specific keys. You cannot specify to use either Embargo by Date or Embargo Bypass by IP without also defining the values in your Digital Object/Collection metadata.</p>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#option-1-embargo-by-date","title":"Option 1: Embargo by Date","text":"<p>The first Embargo option you have is to define a \"JSON key present in your metadata that contains an Embargo lift date that will be used to Embargo Metadata and Media.\"</p> <ul> <li>Archipelago recommended default key is 'date_embargo_lift'</li> </ul> <p>Functional mechanism for your specified key: </p> <ul> <li>value for future date in the 'date_embargo_lift' key (or what key you define)</li> <li>set at the individual digital object level</li> </ul> Example of JSON Metadata key and values for 'date_embargo_lift <pre><code>{\n    \"date_embargo_lift\": \"2044-08-23\"\n}\n</code></pre> <p>Functional results for objects with future dates in the 'date_embargo_lift' key: </p> <ul> <li>File media viewer does not display until the date set in the 'date_embargo_lift' key in the Digital Object's JSON metadata has passed.</li> <li>Metadata record displays as usual.</li> </ul>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#option-2-embargo-bypass-by-ip","title":"Option 2: Embargo Bypass by IP","text":"<p>The second Embargo option you have is to define a \"JSON key present in your metadata that contains an allowed to bypass Embargo through a visitor IP or IP range that will be used to Embargo Metadata and Media\".</p> <p>Functional mechanism for your specified key: </p> <ul> <li>value for individual IP address or IP ranges set in the 'embargo_ip_bypass' key (or what key you define)</li> <li>set at the individual digital object level</li> </ul> Example of JSON Metadata key and values for 'embargo_ip_bypass <pre><code>{\n    \"embargo_ip_bypass\": \"[\n    \"123.123.123.123\",\n    \"456.789.0.0\\/11\",\n    \"222.33.444.0\\/56\"\n    ],\n}\n</code></pre> <p>Functional results for objects with IP range/s specified in the 'embargo_ip_bypass' key: </p> <ul> <li>File media viewer does not display for individuals accessing the Digital Object from IP addresses not listed in the 'embargo_ip_bypass' key in </li> <li>Metadata record displays as usual.</li> <li>No caching for any objects with values for this 'embargo_ip_bypass' key</li> </ul>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#additional-configurations-needed-display-mode-formatters-and-twig-templates","title":"Additional Configurations Needed: Display Mode Formatters and Twig Templates","text":"<p>In order to enforce the Embargo options noted above, you have multiple options for configuration. You can use a singular or combination of the options described below, depending on your use cases and desired Embargo application. It is recommended that you at least apply the necessary Display Mode Formatter Configuration Options. You may also optionally wish to apply Twig Template changes noted further below.</p>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#display-mode-formatter-configuration-options","title":"Display Mode Formatter Configuration Options","text":"<p>First, to better understand this functionality area, please first familiarize yourself with Archipelago's Primer on Display Modes and Strawberryfield Formatters.</p> <p>For every Display Mode you have configured for your Digital Objects and Digital Object Collection (Compounds/Creative Work Series), you can configure Embargo settings within the corresponding Strawberryfield Formatter.</p>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#step-1-navigate-to-display-suites-manage-display-tab","title":"Step 1. Navigate to Display Suite's Manage Display Tab","text":"<ul> <li>For Digital Objects, found at <code>/admin/structure/types/manage/digital_object/display</code></li> <li>For Digital Object Collections (Compounds/Creative Work Series), found at <code>/admin/structure/types/manage/digital_object_collection/display</code></li> </ul>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#step-2-edit-the-strawberryfield-formatter-settings","title":"Step 2. Edit the StrawberryField Formatter Settings","text":"<ul> <li>For every Display Mode potentially impacted by Embargo restrictions, select the corresponding StrawberryField Formatter used to render/display the Media Files and Viewer.</li> </ul>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#step-3-setup-your-embargo-configurations","title":"Step 3. Setup your Embargo Configurations","text":"<p>Within the selected StrawberryField Formatter, navigate to the bottom section of the form where you can setup your desired Embargo options.</p> <p></p> <ul> <li> <p>Option to 'Hide the Viewer in the presence of an Embargo'. This is the simplest option for configuring an Embargo restriction. If you leave this option unchecked, acting on an Embargo will be delegated to the IIIF Manifest driving the viewer.</p> </li> <li> <p>Option will be to specify 'When Embargo is used or applied, alternate JSON Key(s) where the files to be used by this formatter where uploaded/store in your JSON.' </p> <ul> <li>Please be careful about providing the same keys used for users that can by pass an Embargo. You can add multiple ones (keys) separated by comma. Some viewers use multiple file types, such as audio files and text-based subtitle files, and in that case please add all of the corresponding file type keys. In case of multiple keys for the same type (e.g. \"audios1\", \"audios2\"), the key names will be also used for grouping. Leave empty to not provide any alternate Embargo option at all.</li> <li>This means you can add conditional logic to your IIIF Manifest (Twig Template) to do things such as \"return an empty manifest and skip the images\", or provide alternative images in case of an Embargo.</li> </ul> </li> <li> <p>Option to 'Use Global IIIF Urls'. This is enabled by default.</p> </li> </ul> <p>Depending on which Display Mode and particular StrawberryField Formatter you are editing, you may see additional options related to the IIIF Manifest.</p> <p></p> <ul> <li>If the particular viewer you are editing is setup to use an Exposed Metadata Endpoint, you can also simplify the response and use the option at the end--'Return 401 in case of an Embargo', removing your need of delegating the decision to twig logic. Archipelago will resolve the Embargo and if the user can't see it, will return an access denied (401 code).</li> <li>You can find the 'Exposed Metadata using Twig Configuration entities' at <code>/admin/config/archipelago/metadataexpose</code>.</li> </ul> <p></p> <ul> <li>If you decide not to code the your response to an Embargo on the IIIF manifest, you should at least enable the Formatter setting noted above to Hide the Viewer in the presence of an Embargo'.</li> </ul>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#twig-template-configuration-option","title":"Twig Template Configuration Option","text":"<p>Please first familiarize yourself with Twig Templates and Archipelago and Working With Twig in Archipelago.</p> <p>To apply either (or both) of the examples noted below, begin by navigating to the Metadata Display List in your Archipelago.</p> <ul> <li>Found at <code>/metadatadisplay/list</code> or <code>Content &gt; Metadata Displays</code></li> </ul> <p>You will most likely only need to apply Embargo-related conditional logic to the primary 'Object Description' template, depending on the particular Metadata Display Usage setup in your Archipelago.</p>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#object-description-template-example-1","title":"Object Description Template Example 1","text":"<p>In your selected Metadata Display Twig Template, such as the primary Object Description, navigate to the section of the template where the main metadata field display output begins. Add the following conditional statement before the output of other elements:</p> <pre><code>  {% if data_embargo.embargoed == true %}\n    &lt;h3&gt;Access to this Object has been restricted.&lt;/h3&gt;\n    &lt;p&gt;Please contact the Repository Administrator for any access questions or concerns.&lt;p&gt;\n  {% endif %}\n</code></pre> <p>Example output when applied within the default Archipelago Object Description Output --and paired with the Formatter Embargo Configuration noted above:</p> <p></p>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#object-description-template-example-2","title":"Object Description Template Example 2","text":"<p>The following example conditional logic is already provided in the default Archipelago Object Description template. To apply within your particular Object Description (or other) template, navigate to the section of the template where the file download section begins. Add the following conditional statement before the output of file download section:</p> <pre><code>{# -- Embargo option -- #} \n{% set file_download_restricted = false %} \n\n{% if data_embargo.embargoed == true %} \n   {# also restrict if Embargoed #} \n   {% set file_download_restricted = true %} \n{% endif %} \n</code></pre> <p>Example output when applied within the default Archipelago Object Description Output --and paired with the Formatter Embargo Configuration noted above (the Download menu tab does not show the media file downloand link):</p> <p></p>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"embargo/#final-considerations","title":"Final Considerations","text":"<p>There is a specialized Permission found at <code>admin/people/permissions</code> under the Strawberry Metadata and Media Field Formatters section, which allows you to specify that certain User Roles can 'See Embargoed object metadata and assets'. Please note, this Permission is not ACL. Enforced Embargo configured JSON keys will not act on Formatters if a user has this enabled. If an Object is Embargoed this permission will allow any role with this assigned to bypass it.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Embargo","Access Restrictions","Restricted Access","Metadata Based Embargo"]},{"location":"experimental_tools/","title":"Experimental Machine Learning (ML) Tools","text":"<p>Archipelago 1.4.0 Local Deployment Release was shipped with a set of experimental Machine Learning (ML) Tools for testing and assessing the applicable use of such tools/workflows within a general repository environment. These tools should not be considered 'final product' integrations, and should not be exposed to end users--there are intentional configurations in place that limit exposure of these endpoints to authenticated users only.</p> <p>Please review Allison &amp; Diego's recent 2024 Conference Presentations related to this important topic in our shared field for a fuller understanding of our team's ethical perspectives and the considerations we keep related to ML tools and applications.</p>","tags":["Experimental ML Tools","Machine Learning","ML"]},{"location":"experimental_tools/#whats-under-the-hood-powering-these-tools","title":"What's under the hood powering these tools?","text":"<ul> <li>New Archipelago ML supporting code (lots of maths!)</li> <li>New Strawberry Runners Post-Processing pipelines for Image and Text Vectorization</li> <li>New Solr Fields for storing the vectorized image or text fragments</li> <li>New Views and contextual and exposed filters for enabling search and results from Solr</li> <li>New UI Interfaces for interacting with these tools</li> <li>New <code>nlp</code> Docker Container (<code>esmero/esmero-nlp:1.4.1-arm64</code> or <code>esmero/esmero-nlp:1.4.1-amd64</code>)</li> </ul> <p>All of the above items, except for the new <code>nlp</code> Docker Container, are available out-of-the-box in default local deployment configurations. However, to use the tools you will first need to switch the <code>nlp</code> Docker Container being used from the default <code>esmero/esmero-nlp:fasttext-multiarch</code> to <code>esmero/esmero-nlp:1.4.1-arm64</code> or <code>esmero/esmero-nlp:1.4.1-amd64</code> in your <code>docker-compose.yml</code> file.</p>","tags":["Experimental ML Tools","Machine Learning","ML"]},{"location":"experimental_tools/#enabling-the-new-nlp-container-triggering-related-post-processing","title":"Enabling the new <code>nlp</code> container &amp; triggering related post-processing","text":"<ol> <li>In your <code>archipelago-deployment</code> folder, navigate to and open the <code>docker-compose.yml</code> file.</li> <li>Comment out (add # at the beggining of) line 76 for <code>image: \"esmero/esmero-nlp:fasttext-multiarch\"</code></li> <li>Remove the comment (# at the begnning) for line 78 for <code>image: \"esmero/esmero-nlp:1.4.1-arm64\"</code> OR <code>image: \"esmero/esmero-nlp:1.4.1-amd64\"</code> (depending on your particular computer platform CPU type).</li> <li>Save your changes.</li> <li> <p>In your terminal, while still in your <code>archipelago-deployment</code> folder, run: <pre><code>docker-compose pull\n</code></pre></p> </li> <li> <p>After the new <code>nlp</code> container finishes pulling and downloading, run: <pre><code>docker-compose up -d\ndocker ps\n</code></pre> Then check that the new <code>nlp</code>container now shows one of <code>esmero/esmero-nlp:1.4.1-arm64</code> or <code>esmero/esmero-nlp:1.4.1-amd64</code> depending on your Platform/CPU under IMAGE.</p> </li> <li> <p>While logged in as an adminsitrator, use Archipelago's Advanced Batch Find and Replace to select all of the objects ingested into your repository and run the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code> Action. If you have not yet ingested any objects into your Archipelago, you can skip this and proceed to the next step. </p> </li> <li>Wait for the Background Post-processing Queues and your Solr Index to finish updating after you have either run the AMI Demo Set and/or ingested objects through the webforms/your own AMI set(s). Feel free to give the Cron a push to start ahead of the default 3 hour schedule. Please be patient with this post-processing and indexing process, as the ML parts of this process (image and text extraction, vector generation) can take some time to complete.</li> </ol> <p>When all of the above steps are completed, you will be ready to start using the following experimental tools.</p>","tags":["Experimental ML Tools","Machine Learning","ML"]},{"location":"experimental_tools/#experimental-image-ml-comparison-integrated","title":"Experimental Image ML comparison (integrated)","text":"<p>As a logged in user, you can find the 'Image KNN Similarity Search' tool:</p> <ul> <li>Through the <code>Tools</code> menu &gt; <code>ML Image Similarity Search</code></li> <li>Directly at <code>/search_ml_images</code> </li> </ul> <p>Uses these pre-trained models:</p> <ul> <li>YOLO v8 (Structural/Layout)</li> <li>MobileNet V3 (Pattern, scale agnostic)</li> <li>InsightFace (Face feature encoding)</li> </ul> <p>Also uses IIIF Content Search API Object detection of Integrated Annotations</p>","tags":["Experimental ML Tools","Machine Learning","ML"]},{"location":"experimental_tools/#how-could-i-test-this","title":"How could I test this?","text":"<p>Conduct a search for an image in your Archipelago repository, then click on one of the images found in the results set. In the section below the top search results, you can review the output of the Image Similarity searches. You could also select a particular image Annotation to use for the Image Similarity search and comparison.</p>","tags":["Experimental ML Tools","Machine Learning","ML"]},{"location":"experimental_tools/#experimental-text-similarity-search","title":"Experimental Text Similarity Search","text":"<p>As a logged in user, you can find the 'Sbert KNN search on OCR-ed Pages Content' tool:</p> <ul> <li>Through the <code>Tools</code> menu &gt; <code>ML Text Similarity Search</code></li> <li>Directly at <code>/search_sbert/</code> </li> </ul> <p>Uses SBert/384 Vector Size embedding with assigned Task on short sentences over uncorrected OCR</p>","tags":["Experimental ML Tools","Machine Learning","ML"]},{"location":"experimental_tools/#how-could-i-test-this_1","title":"How could I test this?","text":"<p>Conduct a search for a particular text fragment, written in plain language style, such as \"Show me resources related to dogs\". In the section below the text-based search, you can review the output of the Text Similarity searches. You will only see matches against documents that have OCR values.</p>","tags":["Experimental ML Tools","Machine Learning","ML"]},{"location":"experimental_tools/#into-the-future","title":"Into the future","text":"<p>We would like to reiterate that these are early explorations of ML tools within the Archipelago repository architecture. Expect changes, enhancements, and even removal of certain aspects of these experimental tools in future releases. </p> <p>__</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Experimental ML Tools","Machine Learning","ML"]},{"location":"export-to-csv/","title":"Export ADOs to CSV Action","text":"<p>Archipelago has a specialty Action, labeled <code>Export Archipelago Digital Objects to CSV content item</code>, which enables you to generate a CSV file for a selected group of Archipelago Digital Objects (ADOs). </p>"},{"location":"export-to-csv/#where-to-find","title":"Where to Find","text":"<p>In default Archipelago deployments, this speciality action is found in the <code>Action</code> menu for the main Content page:</p> <ul> <li>Accessed through the <code>Content</code> menu item</li> <li>Directly at <code>/admin/content</code></li> </ul> <p></p> <p>And in the <code>Action</code> menu for the Advanced Batch Find and Replace view:</p> <ul> <li>Accessed through the <code>Tools</code> menu &gt; <code>Advanced Batch Find and Replace</code> </li> <li>Directly at <code>/search-and-replace</code></li> </ul>"},{"location":"export-to-csv/#export-ados-to-csv-configuration-options","title":"Export ADOs to CSV Configuration Options","text":"<p>After selecting a group of ADOs through either the 'Content' or 'Find and Replace' pages, you will have the following options available for using this speciality Action.</p> <ol> <li>Expand related ADOs to UUIDs: when enabled, all related ADOs (ismemberof, etc) are going to be expanded to their UUIDs. This allows changes to parentship to be made on the CSV.</li> <li>Do not export Media/Files: when enabled, file references and their associated technical Metadata (as:filetype JSON keys, e.g as:image) will be skipped. This allows pure Descriptive Metadata to be exported to CSV.</li> <li>Convert Media to Portable Absolute URLs: when enabled, all File references will be converted to absolute URLs and their associated technical Metadata (as:filetype JSON keys, e.g as:image) will be skipped. This allows CSVs to be used to ingest new ADOs in other repositories.</li> <li>Attach CSV to a new AMI Set: when checked, a new AMI set with the exported data will be created and configured for \"Updating\" existing ADOs.</li> <li>If attaching to a new AMI Set, you will have the option to Please Give your AMI Set a name: If empty, Archipelago will create a (quite) generic one for you (defaults to 'CSV Export/Import AMI Set').</li> </ol> <p></p> <p>Select <code>Apply</code> when you are ready to move forward to the next step of using this Action.</p>"},{"location":"export-to-csv/#executing-the-action-and-downloading-the-csv","title":"Executing the Action and Downloading the CSV","text":"<p>After selecting your Export ADOs to CSV Configuration Options, you will be directed to a page showing a summary list of the 'Items selected' (list may be truncated depending on the amount of items you selected). You will then need to select <code>Execute Action</code> to proceed.</p> <p>Selecting <code>Execute Action</code> will trigger the generation of the CSV in real time, directly in your browser session. This may take a few seconds to a few minutes, depending on the amount of items you selected to include. You will see a progress bar updating while this Action is running.</p> <p>When the Action is finished, you will see a temporary Status Message stating: \"Your source data was saved and is available as CSV at\", followed by a link to the CSV file generated. Be sure to click on the CSV link and download before navigating away from this temporary Status message. No worries if you accidently navigate away from this page before downloading the generated CSV, you can find a temporary copy of the CSV file generated at: - Accessed through the <code>Content</code> menu &gt; <code>Media Files</code> - Directly at <code>/admin/content/files</code></p> <p>Or: you can simply configure and run the Action again to generate another copy of the CSV if needed.</p> <p>If you selected the option to attach to <code>Attach CSV to a new AMI Set</code>, you will also see a status message stating: \"Well Done! New AMI Set was created and you can see it here\" (and 'see it here' will be linked to the corresponding AMI Set). You will be able to access the AMI Set from the <code>AMI Sets</code> tab on the main Content page found at <code>/admin/content</code> or directly at <code>/amiset/list</code>.</p> <p></p>"},{"location":"export-to-csv/#recommendations","title":"Recommendations","text":"<p>This specialty Action is quite useful for generating CSVs to use for reviewing select batches of materials found in your repository, but it is not intended to be used to generate a complete CSV containing all of your Archipelago repository's digital objects and collections. Depending on the size of your repository, generating a CSV for all of your assets may exceed your browser's timeout processing limits/bandwidth (see 'Known Solr-Related Issue' note below) and also result in a CSV file that is unable to be opened by standard spreadsheet software such as Google Sheets or MS Excel.</p> <p>Known Solr-Related Issue</p> <p>Solr has a fixed limits configured for the maxiumum number of results able to retrieved, so if you make a very large query/select a very large number of ADOs to include that exceeds your fixed limits for Solr, you will only retrieve up to the maximum specified in your Solr limits.</p> <p>You can find the related Solr configurations sections the <code>Advanced</code> section of the 'Edit search server' page found at <code>/admin/config/search/search-api/server/esmero_solr/edit</code>. Take caution with making any changes here, and be aware that changing the maximum query settings will impact your Solr query setup sitewide, not just as pertains to the 'Export ADOs to CSV' Action.</p> <p></p> <p>We would recommend executing this specialty Action using Archipelago's Find and Replace to narrow down your selected ADOs more easily using the Facets available for the Find and Replace interface. We would also recommend keeping your selected batches limited to an amount of objects you can reasonably assess and review using your normal spreadsheet review workflows and software.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"find_and_replace/","title":"Advanced Batch Find and Replace","text":"<p>Archipelago's Advanced Batch Find and Replace functionality provides different ways for you to efficiently Find/Search and Replace metadata values found in the raw JSON of your Digital Objects and Collections. Advanced Batch Find and Replace makes use of customized Actions that extend Drupal's VBO module to enable these powerful batch metadata replacement Actions in your Archipelago environment. </p>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace/#where-to-find","title":"Where to Find","text":"<p>In default Archipelagos, you can find Advanced Batch Find and Replace:</p> <ul> <li>Through the <code>Tools</code> menu &gt; <code>Advanced Batch Find and Replace</code> </li> <li>Directly at <code>/search-and-replace</code> </li> </ul> <p></p> Site Administrator Note <ul> <li>The View driving this can be found at <code>/admin/structure/views/view/solr_search_content_with_find_and_replace</code>. The default Facets referenced above can be found at <code>/admin/structure/block/list/archipelago_subtheme</code> in the <code>Sidebar Second</code> section. Please proceed with caution if making any changes to the default configurations for this View or the Facets referenced on this View Page.     </li> </ul>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace/#main-page-overview","title":"Main Page Overview","text":"<p>From the main page (display title 'Search and Replace'), you will see:</p> <ul> <li>A <code>Fulltext Search</code> box</li> <li>Dropdown list of available <code>Actions</code></li> <li>Listing of all the Digital Objects and Collections found in your Archipelago repository<ul> <li>Option to <code>Select/deselect all results in this view (all pages)</code> via toggle switch</li> <li>Same toggle switch option beside each individual Object/Collection to select one-at-a-time</li> <li>Expandable <code>\u25ba Raw Metadata (JSON)</code> section beneath each each individual Object/Collection containing the full Raw JSON metadata record for reference</li> </ul> </li> <li>Expandable section to show all the <code>Selected items in this view</code> (will be 0 items to start).<ul> <li>Individual selections made on different results pages will be preserved in the overall total of <code>Selected items</code> available for preview on this main/top page.</li> </ul> </li> </ul> <p></p>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace/#default-facets-configured","title":"Default Facets Configured","text":"<p>You will also see a listing of a few different default Facets configured to help guide your selection of potential Digital Objects/Collections:</p> <ul> <li>Object Type<ul> <li>the Archipelago Digital Object/Collection Type</li> <li>JSON key: <code>type</code></li> </ul> </li> <li>JSON keys in your metadata<ul> <li>all of the potential JSON keys that are present in your repository</li> <li>includes 'flattened' keys that may not be readily accessible via Webform elements (such as Archipelago-generated technical and administrative keys)</li> </ul> </li> <li>Ingest Method Service URL<ul> <li>The URL of the Digital Object/Collection Webforms and AMI Sets present in your repository that were used to create Digital Objects/Collections.</li> <li>Please note that using the Find and Replace Webform functionality to update ADOs will cause the original AMI Set URL identified in this Facet to be overwritten and replaced with the specified Webform used during the replacement execution.</li> </ul> </li> </ul>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace/#available-actions","title":"Available Actions","text":"<p>The default options available through the Action dropdown menu include:</p> <ul> <li>*<code>Export Archipelago Digital Objects to CSV content item</code></li> <li><code>Text based find and replace Metadata for Archipelago Digital Objects content item</code></li> <li><code>Webform find-and-replace Metadata for Archipelago Digital Objects content item</code></li> <li><code>JSON Patch Metadata for Archipelago Digital Objects content item</code></li> <li>*<code>Publish Digital Object</code></li> <li>*<code>Unpublish Digital Object</code></li> <li>*<code>Change the author of content</code></li> <li>**<code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code></li> <li>*<code>Delete selected entities/translations</code></li> </ul> <p>* denotes Action options that are also shared with the main <code>Content</code> Page Action Menu</p> <p>**You can read more about Strawberry Runners Post-Processing Actions here</p> <p></p>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace/#find-and-replace-specific-actions","title":"Find and Replace Specific Actions","text":"<p>After reviewing the 'Important Notes &amp; Workflow Recommendations' below, please see the following separate pages for detailed examples walking through the usage of the three different Find and Replace specific actions. </p> <ul> <li>Text Based Find and Replace<ul> <li>Enables you to execute simple but powerful text based search and replacement operations.</li> </ul> </li> <li>Webform Find and Replace<ul> <li>Enables you to search against values found within defined Webform elements to apply metadata replacements with targeted care.</li> </ul> </li> <li>JSON Patch Find and Replace<ul> <li>Enables you to carry out advanced JSON Patch operations within your metadata.</li> </ul> </li> </ul>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace/#important-notes-workflow-recommendations","title":"Important Notes &amp; Workflow Recommendations","text":"<p>Important Note</p> <p>The Actions available through Archipelago's Advanced Batch Find and Replace can potentially have repository-wide effects. It is strongly recommended that you proceed with caution when executing any of the available Actions. </p> <p>Adding New Facets</p> <p>The default Facets available through Archipelago's Advanced Batch Find and Replace have an important configuration selection made on each individual Facet. For every new Facet you add for Find and Replace, you need to select the checkboxes for both the 'VBO batch handler' settings to use the <code>VBO Batch Facet processor</code>, and the selection within the 'VBO batch handler settings' to <code>Use URL based facets in VBO Batches</code>. You need to make sure these are selected so that the \"visible\" list/count of objects you filter using a Facet is respected during actual VBO process execution of batch changes you make for any Find and Replace Actions. Also, please be aware that Drupal's VBO does not pass a \"limit\" (except if your VIEW has actually a \"SHOW\" a defined number of results which most users will never use). Because of that, when you run a VBO-based action, the default batch limitation will be set to the Search API/Solr defined Limit. You can view this Limit information at</p> <p>'~yoursite/admin/config/search/search-api/server/esmero_solr/edit', under the Advanced Tab. This all means that if you first set a Limit of 100 in your Search API/SOLR defined Limit, then you see 1000 objects in your Find and Replace results and select all 1000 results for batch change operations, when you run your Find and Replace action only 100 changes will be processed. There is no way Archipelago can work around that VBO related behavior (for now, except open an ISSUE, perhaps a way can be found!).</p>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace/#simulation-mode","title":"Simulation Mode","text":"<p>Before executing any of the available Find and Replace Actions, the best-practice workflow recommendation is to always first run in Simulation Mode:</p> <ul> <li>Before the final 'Execute Action' step of your Find and Replace operation, select the option to '\u2611\ufe0f only simulate and debug affected JSON'. This will run a quick check against your Action specifications and the potentially impacted Digital Objects and Collections.</li> <li>You can then double check that the total effected changes shown reflect your intended amount of changes. <ul> <li>The total number of changes will always be multipled by a factor of 2, as the Actions count a first step of checking against your data, then the second step of applying the change.</li> <li>If your Action specifications do not match against any JSON metadata values in your specified results, you will also see that no matches were applicable.</li> </ul> </li> </ul> <p></p> <ul> <li>Once you have reviewed the results of the Simulation Mode and confirmed the simulated results match your intentend amount of changes, proceed with executing the Action you first simulated.</li> </ul>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace/#checking-your-changes","title":"Checking Your Changes","text":"<p>After applying any of the Find and Replace Actions, you can review the specific changes that were made within the Revision history of the impacted Digital Objects and Collections.</p> <ul> <li>Through the <code>Find and Replace</code> page results listing or the main <code>Content</code> page, navigate to the Digital Object/Collection you wish to review.</li> <li>Open the 'Revision' tab.</li> <li>The details for the specific Action executed will be visible. All of the Find and Replace Actions will result in slightly different operation notes within the the Revision history.</li> </ul> <p></p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace_action_json_patch/","title":"JSON Patch Find and Replace","text":"<p>Enables you to carry out advanced JSON Patch operations within your metadata.</p> <p>Note</p> <p>Please refer to the main Find and Replace documentation page for a general overview of where to find within your Archipelago, a general overview of default options and important notes and workflow recommendations.</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#what-is-a-json-patch-and-when-to-use-it","title":"What is a JSON Patch and when to use it?","text":"<p>Before we dive into the mechanics of doing JSON Patching Batch in Archipelago we need to learn what a JSON Patch is and of course when applying this action is useful, possible (or not).</p> <p>A JSON Patch is a <code>JSON</code> Document containing precise operations that can heavily modify the structure and values of an existing JSON Document, in this case the RAW JSON found inside a strawberryfield of our ADOs. </p> <p>The operations available for modifications of an JSON document are:</p> <ul> <li>\u201cadd\u201d</li> <li>\u201cremove\u201d</li> <li>\u201creplace\u201d,</li> <li>\u201cmove\u201d</li> <li>\u201ccopy\u201d</li> </ul> <p>And there is also one (very important) used to check/validate the existence of values/keys:</p> <ul> <li>\u201ctest\u201d</li> </ul> <p>Even if you can have multiple operations in a single JSON Patch Document, they are always applied as a whole. Means if any of those fail nothing will be applied and in concrete, in our VBO action, no change will be done to your ADO. This in combination with the \u201ctest\u201d operation gives you a lot of safety and a way of discerning/skipping completely a complex set of operations on e.g. a failed \u201dtest\u201d.</p> <p>JSON Patch uses <code>JSON Pointers</code> as arguments in all of these operations to target a specific key/value of your JSON.</p> <p>Using the following JSON Snippet as example:</p> <pre><code>\"subject_lcgft_terms\": [\n    {\n        \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/genreForms\\/gf2017027249\",\n        \"label\": \"Photographs\"\n    }\n]\n</code></pre> <p>These JSON Pointers will resolve in the following manner:</p> <ul> <li><code>/subject_lcgft_terms</code> </li> </ul> <pre><code>[\n    {\n        \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/genreForms\\/gf2017027249\",\n        \"label\": \"Photographs\"\n    }\n]\n</code></pre> <ul> <li><code>/subject_lcgft_terms/0</code></li> </ul> <pre><code>{\n    \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/genreForms\\/gf2017027249\",\n    \"label\": \"Photographs\"\n}\n</code></pre> <ul> <li><code>/subject_lcgft_terms/0/label</code></li> </ul> <pre><code>Photographs\n</code></pre> <p>AS you can see a JSON Pointer is very precise , allowing you to target complete structures and values but it does not allow Wildcard Operations. Means you can not \"search\" or do loosy comparissons. This very fact limits many times the use case. E.g. if you have a list of terms like this:</p> <pre><code>\"terms\": [\n    \"term1\",\n    \"term2\",\n    \"term3\"\n]\n</code></pre> <p>and you want to \"test\" for the existence of <code>\"term3\"</code> before applying a change, you would need to know exactly at what position inside the <code>terms Array</code> (Starting from 0) it will/should be found. And that might not be consistent across every ADO. </p> <p>So how do we use these pointers in an operation inside a JSON Patch document? Using the same fake \"terms\" JSON snippet the previously listed, operations are written like this:</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#add","title":"add","text":"<pre><code>{ \"op\": \"add\", \"path\": \"/terms/0\", \"value\": \"term_another\" }\n</code></pre> <p>This will add before the first term (in this case \"term1\" ) \"term_another\" as a value.</p> At the end <p>you can use a dash (<code>-</code>) (e.g. \"/terms/-\") instead of the numeric position of an array entry to denote that the \"value\" needs to be added at the end. This is needed specially for empty lists. You can not target <code>0</code> position on an empty array.</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#remove","title":"remove","text":"<pre><code>{ \"op\": \"remove\", \"path\": \"/terms/1\"}\n</code></pre> <p>This will remove the second term (in this case \"term2\" )</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#replace","title":"replace","text":"<p><pre><code>{ \"op\": \"replace\", \"path\": \"/terms/1\", \"value\": \"term_again\" }\n</code></pre> This will remove the second term (in this case \"term2\" ) and put in its place \"term_again\" as a value. Basically two operationes, \u201cremove\u201c and \u201cadd \u201c in a single step</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#move","title":"move","text":"<pre><code>{ \"op\": \"move\", \"from\": \"/terms\", \"path\": \"/terms_somewhere_else\"}\n</code></pre> <p>This will copy all values inside top JSON key <code>terms</code> into a new top JSON key named <code>terms_somewhere_else</code> and remove then the old <code>terms</code> key.</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#copy","title":"copy","text":"<pre><code>{ \"op\": \"copy\", \"from\": \"/terms\", \"path\": \"/terms_somewhere_else\"}\n</code></pre> <p>Similar to \u201cmove\u201c, it will copy values inside top JSON key <code>terms</code> into a new top JSON key named <code>terms_somewhere_else</code> without removing then the old <code>terms</code> key or its content!</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#test","title":"test","text":"<pre><code>{ \"op\": \"test\", \"path\": \"/terms/0\", \"value\": \"term1\"}\n</code></pre> <p>Finally, \u201ctest\u201c will check if on position <code>0</code> of terms there is a value of \"term1\". If not, the test will fail. If a single test fails the whole JSON Patch will be cancelled. Tests can not be concatenated via OR bolean operators. So they always act individually. Two tests with one failing is a failed JSON Patch.</p> <p>There is more of course! The Complete documentation can be found here</p> <p>So. When to use JSON Patch? There are a few general rules/suggestions:</p> <ul> <li>When you can generate one or more precise \u201ctest\u201c operations. If you need to test against data that might exists but its position is not clear, a single JSON Patch will not do the trick. That said you could run the same JSON Patch document against the same set of ADOs more than once modifying slightly the \u201ctest\u201c to cover all variations (check for value at positition <code>0</code>, then again at position <code>1</code>, etc)</li> <li>When you need to use data that is already inside the JSON Document and move it around. Sometimes <code>fixing</code> a value, in the sense of putting a static replacement, is not what you need. Other <code>Actions</code> documented will allow you to replace one value with another fixed one. But JSON Patch will allow you to use existing data inside your target JSON and move it/copy it.</li> <li>When you operate on multiple JSON Keys and can target / match complete values. You can not replace a single word inside a value via a JSON Patch. But you can apply multiple precise operations in a single pass.</li> <li>When you need to change data types. e.g. convert a single value into an array.</li> <li>When you need safety. The fact that a single failed \u201ctest\u201c will cancel a whole JSON Patch allows you to operate with safety and ensure the final destination will always be a JSON</li> </ul> <p>Now that we know what it is and when we should do it, we can make a small exercise. The goal:</p> <ul> <li>For all ADO's with json key with value <code>photograph</code> that are member of collection with Node ID <code>16</code><ul> <li>convert the <code>description</code> key from a single value into an array.</li> <li>add an extra LOD entry</li> <li>Copy a Value into a new Key</li> </ul> </li> </ul>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#step-1-select-the-ados-to-be-modified","title":"Step 1: Select the ADOs to be Modified","text":"<p>As with every other VBO action described in our documentation, start by selecting the ADOs you want to modify using the exposed Search Field(s) and/or Facets present in the Search and Replace <code>View</code> found at <code>/search-and-replace</code> . </p> <p>Once you have filtered down the list to manageable size, containing at least the ADOs you plan on modifying (but for JSON Patch operation could be more too because you can \u201ctest\u201c and match ), press either on <code>Select / deselect all results in this view</code> to pass all the result (this includes all pages, not only the currently visible one) or go selectively one by one by checking on the <code>toggle</code> found beside each ADO's title. You will see how the number in <code>Selected 0 item in this view</code> increases. Now press <code>Apply to select Items</code>. We will use for this example <code>Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?]</code>, an ADO we provide in our DEMO sets. </p> <p>Redundant to say, Batch Actions are intended to be used when a modification needs to be applied to more than one item, implying there is a pattern. For single ADOs you can always do this faster directly via the EDIT tab.</p> Keep your JSON Patches (and friends) around <p>Since JSON Patching involves writing a, sometimes complex, JSON Document, please keep around an Application (or Text File) where you can copy/paste and save your JSON Patches for resuse or future references. Archipelago will not store nor remember between runs the JSON Patch document you submitted. It is also very useful to copy and have at hand the <code>RAW JSON</code> of one of the ADOs you plan to modify as a reference/aid while building the given JSON Patch document.</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#step-2-json-patch-metadata-for-archipelago-digital-objects-action-configuration","title":"Step 2: JSON Patch Metadata for Archipelago Digital Objects Action Configuration","text":"<p>The default config JSON Patch form will contain an example JSON Patch set of Commands (Document)</p> <p></p> <p>We are going to replace this one with a valid JSON document. Notice that it does not require a root <code>{}</code> Object wrapper and it is really a list (or array) of operations. </p> <p>A bit of repetition but needed to explain the JSON Patch document. You can see on the following Note box how we copyed the RAW JSON of one ADO to be Patched to have a reference while building the JSON PATCH. for the sake of brevity we remove here the longer Image File technical Metadata.</p> RAW JSON of <code>Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?]</code> Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?] | before Patching<pre><code>{\n    \"note\": \"\",\n    \"type\": \"Photograph\",\n    \"viaf\": \"\",\n    \"label\": \"Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?]\",\n    \"model\": [],\n    \"owner\": \"New-York Historical Society, 170 Central Park West, New York, NY 10024, 212-873-3400.\",\n    \"audios\": [],\n    \"images\": [\n        26\n    ],\n    \"models\": [],\n    \"rights\": \"This digital image may be used for educational or scholarly purposes without restriction. Commercial and other uses of the item are prohibited without prior written permission from the New-York Historical Society. For more information, please visit the New-York Historical Society's Rights and Reproductions Department web page at [http:\\/\\/www.nyhistory.org\\/about\\/rights-reproductions](http:\\/\\/www.nyhistory.org\\/about\\/rights-reproductions).\",\n    \"videos\": [],\n    \"creator\": \"\",\n    \"ap:tasks\": {\n        \"ap:sortfiles\": \"index\"\n    },\n    \"duration\": \"\",\n    \"ispartof\": [],\n    \"language\": \"English\",\n    \"documents\": [],\n    \"edm_agent\": \"\",\n    \"publisher\": \"\",\n    \"ismemberof\": [\n        16\n    ],\n    \"creator_lod\": [\n        {\n            \"name_uri\": \"\",\n            \"role_uri\": \"http:\\/\\/id.loc.gov\\/vocabulary\\/relators\\/pht\",\n            \"agent_type\": \"personal\",\n            \"name_label\": \"Stonebridge, George Ehler\",\n            \"role_label\": \"Photographer\"\n        }\n    ],\n    \"description\": \"George Ehler Stonebridge (d. 1941) was an amateur photographer who lived and worked in the Bronx, New York. He left little record of himself, but an invaluable one of his surroundings and interests. Stonebridge lived at several locations in the Bronx with his wife Bella, and their three children Grace, George, and William. He worked at the Northern Gaslight Company, although the position he held is unknown. In addition to taking photographs, Stonebridge wrote poetry and prose about his love of the Bronx, his children, and in honor of military victories. Some of Stonebridge's photographs appeared in local papers. In 1898, he was an authorized reporter and photographer for the North Side News; in 1905 he was an authorized reporter for the Bronx Borough Record and Times, and probably took photographs for that paper as well. Stonebridge was fascinated with the subject of military preparedness. Training rituals and staged battles were one of his favorite photographic subjects. His 1898 poem, \\\"Remember the Maine,\\\" celebrates the United States' victory in the Spanish-American War. He was especially proud of soldiers from the Bronx, and photographed historical tablets throughout the Borough commemorating previous military victories. Stonebridge also used his photographs to illustrate lectures. In 1907, he gave several lectures on \\\"The Training of War,\\\" using colored lantern slides.\",\n    \"interviewee\": \"\",\n    \"interviewer\": \"\",\n    \"pubmed_mesh\": null,\n    \"sequence_id\": \"\",\n    \"subject_loc\": [\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/subjects\\/sh85038796\",\n            \"label\": \"Dogs\"\n        }\n    ],\n    \"website_url\": \"\",\n    \"as:generator\": {\n        \"type\": \"Update\",\n        \"actor\": {\n            \"url\": \"http:\\/\\/localhost:8001\\/form\\/descriptive-metadata\",\n            \"name\": \"descriptive_metadata\",\n            \"type\": \"Service\"\n        },\n        \"endTime\": \"2022-12-05T09:19:37-05:00\",\n        \"summary\": \"Generator\",\n        \"@context\": \"https:\\/\\/www.w3.org\\/ns\\/activitystreams\"\n    },\n    \"date_created\": \"1910-01-01\",\n    \"issue_number\": null,\n    \"date_published\": \"\",\n    \"subjects_local\": null,\n    \"term_aat_getty\": \"\",\n    \"ap:entitymapping\": {\n        \"entity:file\": [\n            \"model\",\n            \"audios\",\n            \"images\",\n            \"videos\",\n            \"documents\",\n            \"upload_associated_warcs\"\n        ],\n        \"entity:node\": [\n            \"ispartof\",\n            \"ismemberof\"\n        ]\n    },\n    \"europeana_agents\": \"\",\n    \"europeana_places\": \"\",\n    \"local_identifier\": \"nyhs_PR066_6136\",\n    \"subject_wikidata\": [\n        {\n            \"uri\": \"http:\\/\\/www.wikidata.org\\/entity\\/Q3381576\",\n            \"label\": \"black-and-white photography\"\n        },\n        {\n            \"uri\": \"http:\\/\\/www.wikidata.org\\/entity\\/Q60\",\n            \"label\": \"New York City\"\n        }\n    ],\n    \"date_created_edtf\": \"\",\n    \"date_created_free\": null,\n    \"date_embargo_lift\": null,\n    \"physical_location\": null,\n    \"related_item_note\": null,\n    \"rights_statements\": \"In Copyright - Educational Use Permitted\",\n    \"europeana_concepts\": \"\",\n    \"geographic_location\": {\n        \"lat\": \"40.8466508\",\n        \"lng\": \"-73.8785937\",\n        \"city\": \"New York\",\n        \"state\": \"New York\",\n        \"value\": \"The Bronx, Bronx County, New York, United States\",\n        \"county\": \"\",\n        \"osm_id\": \"9691916\",\n        \"country\": \"United States\",\n        \"category\": \"boundary\",\n        \"locality\": \"\",\n        \"osm_type\": \"relation\",\n        \"postcode\": \"\",\n        \"country_code\": \"us\",\n        \"display_name\": \"The Bronx, Bronx County, New York, United States\",\n        \"neighbourhood\": \"Bronx County\",\n        \"state_district\": \"\"\n    },\n    \"note_publishinginfo\": null,\n    \"subject_lcgft_terms\": [\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/genreForms\\/gf2017027249\",\n            \"label\": \"Photographs\"\n        }\n    ],\n    \"upload_associated_warcs\": [],\n    \"physical_description_extent\": \"\",\n    \"subject_lcnaf_personal_names\": \"\",\n    \"subject_lcnaf_corporate_names\": \"\",\n    \"subjects_local_personal_names\": \"\",\n    \"related_item_host_location_url\": null,\n    \"subject_lcnaf_geographic_names\": [\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/names\\/n81059724\",\n            \"label\": \"Bronx (New York, N.Y.)\"\n        },\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/names\\/n79007751\",\n            \"label\": \"New York (N.Y.)\"\n        }\n    ],\n    \"related_item_host_display_label\": null,\n    \"related_item_host_local_identifier\": null,\n    \"related_item_host_title_info_title\": \"\",\n    \"related_item_host_type_of_resource\": null,\n    \"physical_description_note_condition\": null\n}\n</code></pre> <p>Based on our own plan:</p> <ol> <li>Check first if of type <code>photograph</code> and <code>memberof</code> Node ID <code>16</code></li> </ol> <pre><code>{ \"op\": \"test\", \"path\": \"/type\", \"value\": \"Photograph\"},\n{ \"op\": \"test\", \"path\": \"/ismemberof\", \"value\": [16]}\n</code></pre> <p>Notice that to be really sure we also match the data type, an <code>array</code> with a single value <code>16</code> of type integer (makes sense since the Operation is also a JSON and will be evaluated in the same way as the source RAW JSON). This is a precise match. if the ADO belongs to multiple Collections it will fail of course.</p> <ol> <li>Now we are going to convert <code>description</code> key from a single value into an array.</li> </ol> <pre><code>{\"op\": \"add\",\"path\": \"/temp_description_array\",\"value\": []},\n{\"op\": \"move\",\"from\": \"/description\",\"path\": \"/temp_description_array/-\"},\n{\"op\": \"move\",\"from\": \"/temp_description_array\",\"path\": \"/description\"},\n</code></pre> <p>This is a multi step operation. Given that JSON Patch can not \"cast\" types and depends on a given datatype to be present before, e.g. adding a new value to it, we use here a temporary key. Notice that you can not \u201cadd\u201c or \u201cmove\u201c to e.g. the position <code>0</code> because the destination array is indeed empty (that will fail). But by using the <code>dash</code> you can command it to put it at the end, which on an empty list is also at the beginning (we are starting to understand this!). </p> <ol> <li>And the extra LOD entry for wikidata, in this case the Q Item for collie (type of herding dog)</li> </ol> <pre><code>{ \"op\": \"add\", \"path\": \"/subject_wikidata/-\", \"value\": {\n        \"uri\": \"https://www.wikidata.org/wiki/Q1196071\",\n        \"label\": \"collie\"\n    }\n}\n</code></pre> <ol> <li>Finally we are going to copy the <code>geographic_location.state</code> value and put it into <code>subjects_local</code>:</li> </ol> <pre><code>{\"op\": \"remove\", \"path\": \"/subjects_local\"},\n{\"op\": \"add\", \"path\": \"/subjects_local\", \"value\": []},\n{\"op\": \"copy\", \"from\": \"/geographic_location/state\", \"path\": \"/subjects_local/-\"}\n</code></pre> <p>Why so many operations? Because initially <code>\"subjects_local\"</code> had a value of <code>null</code> and it is not suited to generate/add to it as a multivalued key because of that. So we need to remove it first, recreate it as an empty list and then we can copy. Pro Note: you could partially rewrite this as <code>replace</code> operation!</p> what if subjects_local has data? <p>You will lose it! you can add a \u201ctest\u201c or move data instead of recreating. Many choices.</p> <p>The final JSON Patch will look like this. Copy it into the Configuration JSON Patch commands form field: </p> <pre><code>[\n    {\"op\": \"test\", \"path\": \"/type\", \"value\": \"Photograph\"},\n    {\"op\": \"test\", \"path\": \"/ismemberof\", \"value\": [16]},\n    {\"op\": \"add\",\"path\": \"/temp_description_array\",\"value\": []},\n    {\"op\": \"move\",\"from\": \"/description\",\"path\": \"/temp_description_array/-\"},\n    {\"op\": \"move\",\"from\": \"/temp_description_array\",\"path\": \"/description\"},\n    {\"op\": \"add\",\"path\": \"/subject_wikidata/-\",\"value\": \n        {\n        \"uri\": \"https://www.wikidata.org/wiki/Q1196071\",\n         \"label\": \"collie\"\n         }\n    },\n    {\"op\": \"remove\", \"path\": \"/subjects_local\"},\n    {\"op\": \"add\", \"path\": \"/subjects_local\", \"value\": []},\n    {\"op\": \"copy\", \"from\": \"/geographic_location/state\", \"path\": \"/subjects_local/-\"}\n]\n</code></pre> The inverse process online <p>Now that you know (or are starting to understand) the manual process you can also try this online tool that allows you, based on a source and a destination JSON, generate the needed JSON Patch to mutate one JSON into the another. The logic might not be always what you need and most likely it will not take in account that you actually need to move values and will prefer to fix values to be added via an add operation</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#step-3-run-the-json-patch-action-in-simulation-mode","title":"Step 3: Run the JSON Patch Action in simulation mode","text":"<p>Ready. Now check the \"only simulate and debug affected JSON\" checkbox. We want to see if we did well but not yet modify any ADOs. Press <code>Apply</code> button. You will get another confirmation screen. Press <code>Execute Action</code>.</p> <p>It will run quick (on this example) and you will redirected back on the original Drupal Views of Step 1. If your source <code>ADO</code> is actually the one from our Demo Collection you might see a <code>diff</code>, something very similar to this:</p> Text based diff after a successfull Patch<pre><code>129d129 &lt; \"description\": \"George Ehler Stonebridge (d. 1941) was an amateur photographer who lived and worked in the Bronx, New York. He left little record of himself, but an invaluable one of his surroundings and interests. Stonebridge lived at several locations in the Bronx with his wife Bella, and their three children Grace, George, and William. He worked at the Northern Gaslight Company, although the position he held is unknown. In addition to taking photographs, Stonebridge wrote poetry and prose about his love of the Bronx, his children, and in honor of military victories. Some of Stonebridge's photographs appeared in local papers. In 1898, he was an authorized reporter and photographer for the North Side News; in 1905 he was an authorized reporter for the Bronx Borough Record and Times, and probably took photographs for that paper as well. Stonebridge was fascinated with the subject of military preparedness. Training rituals and staged battles were one of his favorite photographic subjects. His 1898 poem, \\\"Remember the Maine,\\\" celebrates the United States' victory in the Spanish-American War. He was especially proud of soldiers from the Bronx, and photographed historical tablets throughout the Borough commemorating previous military victories. Stonebridge also used his photographs to illustrate lectures. In 1907, he gave several lectures on \\\"The Training of War,\\\" using colored lantern slides.\", 155d154 &lt; \"subjects_local\": null, 182a180,183 &gt; }, &gt; { &gt; \"uri\": \"https:\\/\\/www.wikidata.org\\/wiki\\/Q1196071\", &gt; \"label\": \"collie\" 236c238,244 &lt; \"physical_description_note_condition\": null --- &gt; \"physical_description_note_condition\": null, &gt; \"description\": [ &gt; \"George Ehler Stonebridge (d. 1941) was an amateur photographer who lived and worked in the Bronx, New York. He left little record of himself, but an invaluable one of his surroundings and interests. Stonebridge lived at several locations in the Bronx with his wife Bella, and their three children Grace, George, and William. He worked at the Northern Gaslight Company, although the position he held is unknown. In addition to taking photographs, Stonebridge wrote poetry and prose about his love of the Bronx, his children, and in honor of military victories. Some of Stonebridge's photographs appeared in local papers. In 1898, he was an authorized reporter and photographer for the North Side News; in 1905 he was an authorized reporter for the Bronx Borough Record and Times, and probably took photographs for that paper as well. Stonebridge was fascinated with the subject of military preparedness. Training rituals and staged battles were one of his favorite photographic subjects. His 1898 poem, \\\"Remember the Maine,\\\" celebrates the United States' victory in the Spanish-American War. He was especially proud of soldiers from the Bronx, and photographed historical tablets throughout the Borough commemorating previous military victories. Stonebridge also used his photographs to illustrate lectures. In 1907, he gave several lectures on \\\"The Training of War,\\\" using colored lantern slides.\" &gt; ], &gt; \"subjects_local\": [ &gt; \"New York\" &gt; ]\n</code></pre> <p>Which means your Patch would have been applied!</p> <p>In case something went wrong, e.g. any of the operations did not match your source data, you will see a <code>WARNING</code> like this</p> <pre><code>Patch could not be applied for Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?]\n</code></pre>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_json_patch/#step-4-run-the-json-patch-action-but-for-real","title":"Step 4: Run the JSON Patch Action but for real!","text":"<p>Now that actual patching. Repeat from Step 1 to Step 3 but keep \"only simulate and debug affected JSON\" unchecked and follow the steps again. You ADO will be modified and you will get almost no notifications except of an action completed notice (in soothing green). If you check Laddie's ADO RAW json (expand in the same resulting view) it will look now like this</p> Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?] JSON after patching<pre><code>{ \n    \"note\": \"\",\n    \"type\": \"Photograph\",\n    \"viaf\": \"\",\n    \"label\": \"Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?]\",\n    \"model\": [],\n    \"owner\": \"New-York Historical Society, 170 Central Park West, New York, NY 10024, 212-873-3400.\",\n    \"audios\": [],\n    \"images\": [\n        26\n    ],\n    \"models\": [],\n    \"rights\": \"This digital image may be used for educational or scholarly purposes without restriction. Commercial and other uses of the item are prohibited without prior written permission from the New-York Historical Society. For more information, please visit the New-York Historical Society's Rights and Reproductions Department web page at [http:\\/\\/www.nyhistory.org\\/about\\/rights-reproductions](http:\\/\\/www.nyhistory.org\\/about\\/rights-reproductions).\",\n    \"videos\": [],\n    \"creator\": \"\",\n    \"ap:tasks\": {\n        \"ap:sortfiles\": \"index\"\n    },\n    \"duration\": \"\",\n    \"ispartof\": [],\n    \"language\": \"English\",\n    \"documents\": [],\n    \"edm_agent\": \"\",\n    \"publisher\": \"\",\n    \"ismemberof\": [\n        16\n    ],\n    \"creator_lod\": [\n        {\n            \"name_uri\": \"\",\n            \"role_uri\": \"http:\\/\\/id.loc.gov\\/vocabulary\\/relators\\/pht\",\n            \"agent_type\": \"personal\",\n            \"name_label\": \"Stonebridge, George Ehler\",\n            \"role_label\": \"Photographer\"\n        }\n    ],\n    \"description\": [\n        \"George Ehler Stonebridge (d. 1941) was an amateur photographer who lived and worked in the Bronx, New York. He left little record of himself, but an invaluable one of his surroundings and interests. Stonebridge lived at several locations in the Bronx with his wife Bella, and their three children Grace, George, and William. He worked at the Northern Gaslight Company, although the position he held is unknown. In addition to taking photographs, Stonebridge wrote poetry and prose about his love of the Bronx, his children, and in honor of military victories. Some of Stonebridge's photographs appeared in local papers. In 1898, he was an authorized reporter and photographer for the North Side News; in 1905 he was an authorized reporter for the Bronx Borough Record and Times, and probably took photographs for that paper as well. Stonebridge was fascinated with the subject of military preparedness. Training rituals and staged battles were one of his favorite photographic subjects. His 1898 poem, \\\"Remember the Maine,\\\" celebrates the United States' victory in the Spanish-American War. He was especially proud of soldiers from the Bronx, and photographed historical tablets throughout the Borough commemorating previous military victories. Stonebridge also used his photographs to illustrate lectures. In 1907, he gave several lectures on \\\"The Training of War,\\\" using colored lantern slides.\"\n    ],\n    \"interviewee\": \"\",\n    \"interviewer\": \"\",\n    \"pubmed_mesh\": null,\n    \"sequence_id\": \"\",\n    \"subject_loc\": [\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/subjects\\/sh85038796\",\n            \"label\": \"Dogs\"\n        }\n    ],\n    \"website_url\": \"\",\n    \"as:generator\": {\n        \"type\": \"Update\",\n        \"actor\": {\n            \"url\": \"http:\\/\\/localhost:8001\\/form\\/descriptive-metadata\",\n            \"name\": \"descriptive_metadata\",\n            \"type\": \"Service\"\n        },\n        \"endTime\": \"2022-12-05T09:19:37-05:00\",\n        \"summary\": \"Generator\",\n        \"@context\": \"https:\\/\\/www.w3.org\\/ns\\/activitystreams\"\n    },\n    \"date_created\": \"1910-01-01\",\n    \"issue_number\": null,\n    \"date_published\": \"\",\n    \"subjects_local\": [\n        \"New York\"\n    ],\n    \"term_aat_getty\": \"\",\n    \"ap:entitymapping\": {\n        \"entity:file\": [\n            \"model\",\n            \"audios\",\n            \"images\",\n            \"videos\",\n            \"documents\",\n            \"upload_associated_warcs\"\n        ],\n        \"entity:node\": [\n            \"ispartof\",\n            \"ismemberof\"\n        ]\n    },\n    \"europeana_agents\": \"\",\n    \"europeana_places\": \"\",\n    \"local_identifier\": \"nyhs_PR066_6136\",\n    \"subject_wikidata\": [\n        {\n            \"uri\": \"http:\\/\\/www.wikidata.org\\/entity\\/Q3381576\",\n            \"label\": \"black-and-white photography\"\n        },\n        {\n            \"uri\": \"http:\\/\\/www.wikidata.org\\/entity\\/Q60\",\n            \"label\": \"New York City\"\n        },\n        {\n            \"uri\": \"https:\\/\\/www.wikidata.org\\/wiki\\/Q1196071\",\n            \"label\": \"collie\"\n        }\n    ],\n    \"date_created_edtf\": \"\",\n    \"date_created_free\": null,\n    \"date_embargo_lift\": null,\n    \"physical_location\": null,\n    \"related_item_note\": null,\n    \"rights_statements\": \"In Copyright - Educational Use Permitted\",\n    \"europeana_concepts\": \"\",\n    \"geographic_location\": {\n        \"lat\": \"40.8466508\",\n        \"lng\": \"-73.8785937\",\n        \"city\": \"New York\",\n        \"state\": \"New York\",\n        \"value\": \"The Bronx, Bronx County, New York, United States\",\n        \"county\": \"\",\n        \"osm_id\": \"9691916\",\n        \"country\": \"United States\",\n        \"category\": \"boundary\",\n        \"locality\": \"\",\n        \"osm_type\": \"relation\",\n        \"postcode\": \"\",\n        \"country_code\": \"us\",\n        \"display_name\": \"The Bronx, Bronx County, New York, United States\",\n        \"neighbourhood\": \"Bronx County\",\n        \"state_district\": \"\"\n    },\n    \"note_publishinginfo\": null,\n    \"subject_lcgft_terms\": [\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/genreForms\\/gf2017027249\",\n            \"label\": \"Photographs\"\n        }\n    ],\n    \"upload_associated_warcs\": [],\n    \"physical_description_extent\": \"\",\n    \"subject_lcnaf_personal_names\": \"\",\n    \"subject_lcnaf_corporate_names\": \"\",\n    \"subjects_local_personal_names\": \"\",\n    \"related_item_host_location_url\": null,\n    \"subject_lcnaf_geographic_names\": [\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/names\\/n81059724\",\n            \"label\": \"Bronx (New York, N.Y.)\"\n        },\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/names\\/n79007751\",\n            \"label\": \"New York (N.Y.)\"\n        }\n    ],\n    \"related_item_host_display_label\": null,\n    \"related_item_host_local_identifier\": null,\n    \"related_item_host_title_info_title\": \"\",\n    \"related_item_host_type_of_resource\": null,\n    \"physical_description_note_condition\": null\n}\n</code></pre> <p>That is all. Again, keep your JSON Patches safe in a text document, test/try simple things first, look for patterns, look for No-Nos that can become \"tests\" to avoid touching ADOs that do not need to be updated and always remember that the destination type (single value, array or object) of an existing Key might affect your complex logic. Happy Patching!</p> <p>Thank you for reading! Please contact us on our\u00a0Archipelago Commons Google Group\u00a0with any questions or feedback.</p> <p>Return to the main\u00a0Find and Replace documentation page\u00a0or the\u00a0Archipelago Documentation main page.</p>","tags":["Advanced Batch Find and Replace","JSON Patch Find and Replace","Search and Replace","JSON Patch","JSON Pointer"]},{"location":"find_and_replace_action_text/","title":"Text Based Find and Replace","text":"<p>The text-based find and replace is case-sensitive and space-sensitive, and while it's the most simple of the actions, it's quite powerful. For this reason it's important to be very precise and target only what's intended. Below are a guide and some examples of use cases for this action.</p> <p>Note</p> <p>Please refer to the main Find and Replace documentation page for a general overview of where to find within your Archipelago, a general overview of default options and important notes and workflow recommendations.</p>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace_action_text/#step-by-step-guide","title":"Step-by-Step Guide","text":"<ol> <li>Go to <code>Tools &gt; Advanced Batch Find and Replace</code>.</li> <li>Identify and/or Filter the Digital Objects (ADOs) that need updates by Searching and/or using the available Facets.</li> <li>Either select all (includes results that appear on additional pages) by toggling <code>Select / deselect all results (all pages, x total)</code> or toggle the buttons for individual objects.</li> <li>Expand the <code>\u25ba Raw Metadata (JSON)</code> for some of the objects and double-check that the text being searched is only targeting what is intended and that the replacement text makes sense.</li> <li>Select <code>Text based find and replace Metadata for Archipelago Digital Objects content item</code> from the <code>Action</code> dropdown.</li> <li>If selecting objects individually, expand the <code>Selected X items</code> and review the list.</li> <li>Press the <code>Apply to selected items</code> button (don't worry, nothing will happen yet).</li> <li>Add the values for search (<code>JSON Search String</code>) and replace (<code>JSON Replacement String</code>).</li> <li> <p>If you're absolutely certain about the replacement you have targeted, uncheck the 'only simulate and debug affected JSON' option and select <code>Apply</code>.</p> <p>only simulate and debug affected JSON</p> <p>This option, which is selected by default, will simulate the action and show the list of objects that would be affected, along with the number of modifications for each object and a total number of results processed. For each JSON key and value affected the modifications will count 2:</p> <p>1 for the deletion of the current key and value</p> <p>+</p> <p>1 for the creation of the modified key and value </p> </li> </ol>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace_action_text/#use-cases-and-examples","title":"Use Cases and Examples","text":"<p>Replacing a JSON key</p> <p>Use Case: A JSON key is currently singular but should be plural.</p> JSON key example with empty array value<pre><code>...\n\"myKey\": [],\n...\n</code></pre> JSON key example with array values<pre><code>...\n\"myKey\": [\"strawberries\",\"blueberries\",\"blackberries\"],\n...\n</code></pre> <p>Follow the steps above and use the following for the search and replace values:</p> Search Value<pre><code>\"myKey\":\n</code></pre> Replace Value<pre><code>\"myKeys\":\n</code></pre> <p>Tip</p> <p>By using quotes and the colon instead of <code>myKey</code> only, we avoid unintentionally replacing other instances of the text within the JSON.</p> <p>After applying the changes, we have the following key:</p> JSON key example with empty array value after update<pre><code>...\n\"myKeys\": [],\n...\n</code></pre> JSON key example with array values after update<pre><code>...\n\"myKeys\": [\"strawberries\",\"blueberries\",\"blackberries\"],\n...\n</code></pre> <p>Replacing a JSON value</p> <p>Use Case: After a batch ingest, it was discovered that JSON values across ADOs in multiple keys contain the same typo: <code>Agnes Meyerhoff</code> (two fs) instead of <code>Agnes Meyerhof</code>.</p> JSON value example with typo<pre><code>...\n\"creator_lod\": [\n    {\n        \"name_uri\": \"\",\n        \"role_uri\": \"http:\\/\\/id.loc.gov\\/vocabulary\\/relators\\/art\",\n        \"agent_type\": \"personal\",\n        \"name_label\": \"Meyerhoff, Agnes\",\n        \"role_label\": \"Artist\"\n    },\n    {\n        \"name_uri\": \"\",\n        \"role_uri\": \"http:\\/\\/id.loc.gov\\/vocabulary\\/relators\\/col\",\n        \"agent_type\": \"personal\",\n        \"name_label\": \"Messenger, Maria, , 1849-1937\",\n        \"role_label\": \"Collector\"\n    }\n],\n\"description\": \"Inscription on mount: \\\"Meyerhoff, Agnes \\\\ Frankfurt - a\\/M. \\\\ Inv el-lith \\\\ Painter.\\\" Inscription on verso: \\\"Agnes Meyerhoff \\\\ Frankfurt a\\/M \\\\ inv. [at?] lith. \\\\ [maker in?]\\\".\",\n...\n</code></pre> <p>Follow the steps above and use the following for the search and replace values:</p> Search Value<pre><code>Meyerhoff, Agnes\n</code></pre> Replace Value<pre><code>Meyerhof, Agnes\n</code></pre> <p>After applying the changes, we have the following values:</p> JSON values after update<pre><code>...\n\"creator_lod\": [\n    {\n        \"name_uri\": \"\",\n        \"role_uri\": \"http:\\/\\/id.loc.gov\\/vocabulary\\/relators\\/art\",\n        \"agent_type\": \"personal\",\n        \"name_label\": \"Meyerhof, Agnes\",\n        \"role_label\": \"Artist\"\n    },\n    {\n        \"name_uri\": \"\",\n        \"role_uri\": \"http:\\/\\/id.loc.gov\\/vocabulary\\/relators\\/col\",\n        \"agent_type\": \"personal\",\n        \"name_label\": \"Messenger, Maria, , 1849-1937\",\n        \"role_label\": \"Collector\"\n    }\n],\n\"description\": \"Inscription on mount: \\\"Meyerhof, Agnes \\\\ Frankfurt - a\\/M. \\\\ Inv el-lith \\\\ Painter.\\\" Inscription on verso: \\\"Agnes Meyerhoff \\\\ Frankfurt a\\/M \\\\ inv. [at?] lith. \\\\ [maker in?]\\\".\",\n...\n</code></pre> <p>Replacing a JSON value with escape characters</p> <p>Use Case: The URL for a website that appears in multiple keys needs to be updated from <code>http://hubblesite.org</code> to <code>https://hubblesite.org</code>.</p> JSON value example with URL after update<pre><code>...\n\"rights\": \"This digital image may be used for educational or scholarly purposes without restriction. Commercial and other uses of the item are prohibited without prior written permission from the NASA and the Space Telescope Science Institute (STScI). For more information, please visit the NASA and the Space Telescope Science Institute's Copyright web page at [http:\\/\\/hubblesite.org\\/copyright](http:\\/\\/hubblesite.org\\/copyright).\",\n...\n\"description\": \"\\\"The largest NASA Hubble Space Telescope image ever assembled, this sweeping bird\u2019s-eye view of a portion of the Andromeda galaxy (M31) is the sharpest large composite image ever taken of our galactic next-door neighbor. Though the galaxy is over 2 million light-years away, The Hubble Space Telescope is powerful enough to resolve individual stars in a 61,000-light-year-long stretch of the galaxy\u2019s pancake-shaped disk. ... The panorama is the product of the Panchromatic Hubble Andromeda Treasury (PHAT) program. Images were obtained from viewing the galaxy in near-ultraviolet, visible, and near-infrared wavelengths, using the Advanced Camera for Surveys and the Wide Field Camera 3 aboard Hubble. This cropped view shows a 48,000-light-year-long stretch of the galaxy in its natural visible-light color, as photographed with Hubble's Advanced Camera for Surveys in red and blue filters July 2010 through October 2013.\\\" -full description available at: [http:\\/\\/hubblesite.org\\/image\\/3476\\/gallery\\/73-phat](http:\\/\\/hubblesite.org\\/image\\/3476\\/gallery\\/73-phat).\",\n...\n</code></pre> <p>Follow the steps above and use the following for the search and replace values:</p> Search Value<pre><code>http://hubblesite.org\n</code></pre> Replace Value<pre><code>https://hubblesite.org\n</code></pre> <p>Note</p> <p>You'll notice that the escape characters for the forward slash (<code>\\/</code>), which appear in the raw JSON, do not need to be included in the search or replace values.</p> <p>After applying the changes, we have the following values:</p> JSON value example with URL after update<pre><code>...\n\"rights\": \"This digital image may be used for educational or scholarly purposes without restriction. Commercial and other uses of the item are prohibited without prior written permission from the NASA and the Space Telescope Science Institute (STScI). For more information, please visit the NASA and the Space Telescope Science Institute's Copyright web page at [https:\\/\\/hubblesite.org\\/copyright](https:\\/\\/hubblesite.org\\/copyright).\",\n...\n\"description\": \"\\\"The largest NASA Hubble Space Telescope image ever assembled, this sweeping bird\u2019s-eye view of a portion of the Andromeda galaxy (M31) is the sharpest large composite image ever taken of our galactic next-door neighbor. Though the galaxy is over 2 million light-years away, The Hubble Space Telescope is powerful enough to resolve individual stars in a 61,000-light-year-long stretch of the galaxy\u2019s pancake-shaped disk. ... The panorama is the product of the Panchromatic Hubble Andromeda Treasury (PHAT) program. Images were obtained from viewing the galaxy in near-ultraviolet, visible, and near-infrared wavelengths, using the Advanced Camera for Surveys and the Wide Field Camera 3 aboard Hubble. This cropped view shows a 48,000-light-year-long stretch of the galaxy in its natural visible-light color, as photographed with Hubble's Advanced Camera for Surveys in red and blue filters July 2010 through October 2013.\\\" -full description available at: [https:\\/\\/hubblesite.org\\/image\\/3476\\/gallery\\/73-phat](https:\\/\\/hubblesite.org\\/image\\/3476\\/gallery\\/73-phat).\",\n...\n</code></pre> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the main Find and Replace documentation page or the Archipelago Documentation main page.</p>","tags":["Advanced Batch Find and Replace","Advanced Find and Replace","Find and Replace","Search and Replace","Metadata","Review"]},{"location":"find_and_replace_action_webform/","title":"Webform Find and Replace","text":"<p>Webform Find and Replace enables you to search against values found within defined Webform elements to apply metadata replacements with targeted care. Below are a guide and an example use case for this Action.</p> <p>Note</p> <p>Please refer to the main Find and Replace documentation page for a general overview of where to find within your Archipelago, a general overview of default options and important notes and workflow recommendations.</p>","tags":["Advanced Batch Find and Replace","Webform Find and Replace","Search and Replace"]},{"location":"find_and_replace_action_webform/#important-notes-about-different-webform-elements","title":"Important Notes about Different Webform Elements","text":"<p>Maximum Length as Defined by your Webform Element Configuration OR Theme Defaults</p> <p>For certain text-based webform element types, the maximum field length (<code>maxlength</code>) defined in your specific webform element configurations will be enforced during Webform Find and Replace operations. If no maximum length is defined, the Admin Theme will enforce a maximum length of 128 characters. Please see our main Webforms documentation for information about configuring webforms in Archipelago.</p> <p>Some Complex Webform Elements Not Available</p> <p>Please note that some complex webform elements are not available for use with Webform Find and Replace. Any webform element that requires user interactions (such as the Nominatim Open Street Maps lookup/query and selection) is not available for usage. The different file upload webform elements are also not available for use with Webform Find and Replace.</p>","tags":["Advanced Batch Find and Replace","Webform Find and Replace","Search and Replace"]},{"location":"find_and_replace_action_webform/#step-by-step-guide","title":"Step by Step Guide","text":"<ol> <li>Go to <code>Tools &gt; Advanced Batch Find and Replace</code>.</li> <li>Identify and/or Filter the Digital Objects (ADOs) that need updates by Searching and/or using the available Facets.</li> <li>If using the 'Ingest Method Service URL' for Faceting, please note that using the Find and Replace Webform functionality to update ADOs will cause the original AMI Set URL identified in this Facet to be overwritten and replaced with the specified Webform used during the replacement execution.</li> <li>Either select all (includes results that appear on additional pages) by toggling <code>Select / deselect all results (all pages, x total)</code> or toggle the buttons for individual objects.</li> <li>Expand the <code>\u25ba Raw Metadata (JSON)</code> for some of the objects and double-check that the metadata field and value you are targeting for replacement is present.</li> <li>Select <code>Webform find-and-replace Metadata for Archipelago Digital Objects content item</code> from the <code>Action</code> dropdown.</li> <li>If selecting objects individually, expand the <code>Selected X items</code> and review the list.</li> <li>Press the <code>Apply to selected items</code> button (don't worry, nothing will happen yet).  </li> <li>On the Webform Find and Replace Action Configuration page, you will need to select the configuration options to be applied for your selected ADOs.</li> <li>From the dropdown:<ol> <li>Select which Webform you want to use</li> <li>Select which Form element you want to use</li> <li>Select which value to search for in the [chosen form element] JSON key</li> <li>Select which value to replace with in the [chosen form element] JSON key</li> </ol> </li> <li>See the Simulation Mode notes here for information about this optional simulation/test mode.</li> <li>If you're absolutely certain about the replacement you have targeted, uncheck the 'only simulate and debug affected JSON' option and select <code>Apply</code>.</li> <li>After the operation is executed, check your changes.</li> </ol>","tags":["Advanced Batch Find and Replace","Webform Find and Replace","Search and Replace"]},{"location":"find_and_replace_action_webform/#use-case-example","title":"Use Case Example","text":"<p>In the following example configuration, for the selected 'Senju no oubashi (Senju great bridge)' object, the Media Type (type JSON key) value of \"Visual Artwork\" will be replaced with the <code>type</code> JSON key value of \"Photograph\".</p> <ul> <li> <p>Selection of Single ADO and <code>Webform find-and-replace</code> Action</p> <p></p> </li> <li> <p>Webform Find and Replace Form</p> <p></p> </li> <li> <p>Confirmation of Successfully Executed Changes</p> <p></p> </li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the main Find and Replace documentation page or the Archipelago Documentation main page.</p>","tags":["Advanced Batch Find and Replace","Webform Find and Replace","Search and Replace"]},{"location":"firstobject/","title":"Your First Digital Object","text":"<p>You followed every Deployment step and you have now a local <code>Archipelago</code> instance. Great!</p> <p>So what now? It is time to give your new repository a try and we feel the best way is to start by ingesting a simple Digital Object.</p> <p>Note</p> <p>This guide will assume Archipelago is running on <code>http://localhost:8001</code>, so if you wizardly deployed everything in a different location, please replace all <code>URIs</code> with your own setup while following this guide.</p>"},{"location":"firstobject/#requirements","title":"Requirements","text":"<ul> <li>Running Archipelago (http://localhost:8001)</li> <li>20 minutes of your time.</li> <li>Open Mind.</li> </ul>"},{"location":"firstobject/#welcome","title":"Welcome!","text":"<p>Start by opening <code>http://locahost:8001</code> in your favourite Web Browser.</p> <p></p> <p>Your Demo deployment will have a fancy Home page with some banners and a small explanation of what Archipelago is and can do. Feel free to read through that now or later.</p> <p>Click on <code>Log in</code> in the top left corner and use your <code>demo</code> credentials from the deployment guide.</p> <ul> <li>user: demo</li> <li>pass: demo</li> </ul> <p>(or whatever password you decided was easy for you to remember during the deployment phase)</p> <p>Press the <code>Log in</code> button.</p> <p></p> <p>Great, welcome <code>demo</code> user! This users has limited credentials and uses the same global theme as any anonymous user would. Still, <code>demo</code> can create content, so let's use those super powers and give that a try.</p> <p>You will see a new <code>Menu item</code> under <code>Tools</code> on the top navigation bar named <code>Add Content</code>. Click it!</p> <p></p>"},{"location":"firstobject/#brief-background","title":"Brief Background","text":"<p>As you already know Archipelago is build on <code>Drupal 8/9</code>, a very extensible <code>CMS</code>. In practice that means you have (at least) the same functionality any Drupal deployment has and that is also true for Content Managment.</p> <p>Drupal ships by default with a very flexible <code>Content Entity Type</code> named <code>Node</code>. <code>Nodes</code> are used for creating Articles and simple Pages but also in Archipelago as <code>Digital Objects</code>. Drupal has a pretty tight integration with <code>Nodes</code> and that means you get a lot of fun and useful functionality by default by using them.</p> Want to know more about Entities? <p>An <code>Article</code> and a <code>Digital Object</code> are both of type <code>Nodes</code>, but each one represents a different <code>Content Type</code>. <code>Content Types</code> are also named <code>Bundles</code>. An individual Content, like \"Black and White photograph of a kind Dog\" is named a <code>Content Entity</code> or more specific in this case a <code>Node</code>.</p> <p>What have <code>Article</code> and <code>Digital Object</code> Content types in common and what puts the apart?</p> <ul> <li>Each Content Type or Bundle has a set of <code>Base Fields</code> and also user configurable set of <code>Fields</code> attached (or bundled together).<ul> <li>E.g. <code>Article</code> has a title, a Body and the option to add an image.</li> <li><code>Digital Object</code> has a title but also a special, very flexible one named <code>Strawberry Field</code> (more about that later).</li> </ul> </li> <li> <p>Fields are where you put your data into and also where your data comes from when you expose it to the world.</p> <ul> <li><code>Nodes</code>, as any other Content entity have Base Fields (which means you can't remove or configure them) that are used all over the place. Good examples are the <code>title</code> and also the owner, named <code>uid</code> (you!).</li> <li>Other Fields, specific to a Content Type, can be added and configured per Bundle.</li> <li>A <code>Field Widget</code> is used to input data into a Field.</li> <li>Each field can have a <code>Field Formatter</code> that allows you to setup how it is displayed to the World.</li> <li>A set of <code>Field Formatters</code> (the way you want to show your content formatted to the world) is named a <code>Display Mode</code>. You can have many, create new ones and remove them, but only use one at the time.</li> <li>A set of <code>Field Widgets</code> (the way you want to Create and Edit a <code>Node</code>) is named a <code>Form Mode</code>. You can also have many, create new ones but only use one at the time.</li> </ul> </li> <li> <p>Each Content Type can have different Permissions (using the build in <code>User Roles</code> system).</p> </li> <li>Each Content Type can have one or more <code>Display Modes</code>. In Practice this means <code>Display Modes</code> are attached to <code>Content Types</code>.<ul> <li>Each display modes can have its own Permissions</li> </ul> </li> <li>Each Content Type can have one or more <code>Form Modes</code>. In Practice this means <code>Form Modes</code> are attached to <code>Content Types</code>.<ul> <li>Each <code>Form Mode</code> can have its own Permissions.</li> </ul> </li> </ul> <p>There is of course a lot more to Nodes, Content Types, Formatters, Widgets and in general Content Entities but this is a good start to understand what will happen next.</p>"},{"location":"firstobject/#adding-content","title":"Adding Content","text":"<p>Below you see all the <code>Content Types</code> defined by default in Archipelago. Let's click on <code>Digital Object</code> to get your first Digital Object Node.</p> <p></p>"},{"location":"firstobject/#my-metadata","title":"My Metadata","text":"<p>What you see below is a <code>Form Mode</code> in action. A multi-step Webform that will ingest metadata into a field of type Strawberry Field (where all the magic happens) attached to that field using a <code>Webform Field Widget</code>, an editorial/advanced Block on the right side, and a <code>Quick Save</code> button at the bottom for saving the session.</p> <p>Let's fill out the form to begin our ingest. We recommend using similar values as the ones shown in the screen capture to make following the tutorial easier.</p> <p>Make sure you select <code>Photograph</code> as <code>Media Type</code> and all the fields with a red <code>*</code> are filled up. Then press <code>Move on to next step</code> at the bottom of the webform to load the next step in line.</p> <p></p>"},{"location":"firstobject/#collections","title":"Collections","text":"<p>Since this is our first digital object we do not yet have a Digital Object Collection for which <code>My First Digital Object</code> could be a member of. In other words, you can leave <code>Collection Membership</code> blank and click <code>Next: Upload Files</code>.</p> <p></p> Why does this look different than Repository X? <p>We assume you come from a world where repositories define different Content types and the shape, the fields and values (Schema) are fixed and set by someone else or at least quite complicated to configure. This is where Archipelago differs and starts to propose its own style. You noticed that there is a single Content Type named <code>Digital Object</code> and you have here a single Web Form. So how does this allow you to have images, sequences, videos, audio, 3D images, etc?</p> <p>There are many ways of answering that, Archipelago works under the idea of an (or many) Open Schema(s), and that notion permeates the whole environment. Practical answer and simplest way to explain based on this demo is:</p> <ul> <li>The <code>Digital Object</code> is a generic container for any shape of metadata. Metadata is generated either via this Webform-based widget you're currently using, manually (power-user need only) or via APIs. Because of this, Metadata can take any shape to express your needs of Digital Objects and therefore we do not recommend making multiple Digital Object types. However, if you ever do need more Digital Object types, the option is available.</li> <li>The Strawberry field Field Widget allows you to attach any <code>Webform</code>, built using the <code>Webform Module</code> and Webforms can be setup in almost infinite ways. Any field, combo, or style can be used. Multi Step, single step - we made sure they always only touch/modify data they know how to touch, so even a single input element webform would ensure any previous metadata to persist even if not readable by itself (See the potential?). And Each Webform can be also quite smart!</li> <li>The Strawberry field Field Widget will take all your Webform input, process any uploaded files, generate a JSON representation, enrich and complement it with Archipelago specific data and save it for you inside the <code>Strawberry field</code>.</li> </ul> <p>We will come back to this later.</p>"},{"location":"firstobject/#linked-data","title":"Linked Data","text":"<p>As the name of this step suggests; you will be adding all your Linked Data elements here. This step showcases some of the autocomplete Linked Data Webform elements we built for Archipelago. We truly believe in Wikidata as an open, honest, source of Linked Open Data and also one where you can contribute back. But we also have LoC autocompletes and Getty.</p> <p>Again, enter all fields with a red <code>*</code> and when you are finished, click <code>Move on to next step</code></p> <p></p> <p>Tip</p> <p>When entering a location, place or address you will need to click on the <code>Search OpenStreet Maps</code> button, which is what that big red arrow is pointing to in the screenshot below.</p> <p></p>"},{"location":"firstobject/#upload-files","title":"Upload Files","text":"<p>Now we will upload our <code>Photograph</code>. Click <code>Choose Files</code> to open your file selector window and choose which file you would like to ingest.</p> <p></p> <p>Once you've uploaded your file, you will see all the Exif data extracted from the image, like so...</p> <p></p> <p>Once you've mentally digested all of that data, let's go ahead and click <code>Save Metadata</code>.</p> Wait! Why only the metadata? I'm ready for this Digital Object to be shared with the world! <p>By clicking <code>Save Metadata</code> we are simply persisting all the metadata in the current webform session. The actual ingest of the Object happens when you click <code>Save</code> on the next and final step, <code>Complete</code>.</p>"},{"location":"firstobject/#complete","title":"Complete","text":"<p>Alright, we've made it. We've added metadata, linked Data, uploaded our files and now... we're ready to save! Go ahead and change the status from Draft to Published and click <code>Save</code>.</p> <p></p> <p>Once you hit save you should see the following green message and your first Archipelago Digital Object!</p> <p></p> <p>Congratulations on creating your first digital object! \ud83c\udf53</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"fragaria/","title":"Fragaria Redirects Module","text":"<p>Archipelago's Fragaria Redirect Module is a Drupal 9/10 module that provides dynamic redirect routes matched against existing Search API field values. This module will also provide future Unique IDs (API) integrations and PURLs.</p>","tags":["Fragaria","Redirects"]},{"location":"fragaria/#prerequisites","title":"Prerequisites","text":"<p>Before proceeding with the following configuration steps, you need to first create the Strawberry Key Name Provider and Solr Field that corresponds to the Field that will be matched against the variable part of the route exists.</p> <p>In other words, if your Digital Objects have a field and value such as:</p> <pre><code>    \"legacy_PID\": [\n        \"mylegacyrepo:1234\"\n    ],  \n</code></pre> <p>You need to make sure the values from the <code>legacy_PID</code> JSON key are indexed (as a Solr/Search API Field) and ready for use as part of the Fragaria Redirects configuration. </p> <p>Best Practice</p> <p>Your new Solr field should to be of field type \"String\" for a perfect match and best results. Using \"Full Text\" or a related variant Solr field type will allow for a partial match, which might lead multiple original URLs redirecting to the first match in Archipelago.</p>","tags":["Fragaria","Redirects"]},{"location":"fragaria/#fragaria-redirect-entity-configuration","title":"Fragaria Redirect Entity Configuration","text":"<ol> <li> <p>Navigate to <code>/admin/config/archipelago/fragariaredirect</code>.</p> </li> <li> <p>Select the <code>Add a Redirect Config</code> button.</p> </li> <li> <p>Enter a label for the Fragaria Redirect Entity you are configuring.</p> </li> <li> <p>Enter the Prefix (that follows your domain) for the Redirect Route.</p> <ul> <li>Do not use <code>node/</code> or <code>do/</code> as a Prefix. Even if these will technically work (redirect), using either of these Prefix paths will override your existing Paths defined by Drupal and Archipelago.</li> </ul> </li> <li> <p>If applicable, enter the the Suffixes (that follow the prefix + the variable part) for the Redirect Route.</p> <ul> <li>Enter one by line. This configuration option is not required.</li> <li>Check/uncheck the option to <code>Instead of fixed Prefixes add a single {catch_all} variable suffix at the end</code> as needed. Checking this will disable any entered static suffixes.</li> </ul> </li> <li> <p>Select the Search API Index where the Field that will be matched against the variable part of the route exists.</p> </li> <li> <p>Select the Search API Field that will be matched against the variable part of the route.</p> </li> <li> <p>If applicable, add static prefixes for to the variable part/argument of the path.</p> <ul> <li>Enter one by line. This is useful when the variable part of the ROUTE does not match 1:1 the actual indexed data. e.g the route is /oldrepo/1 and the indexed value is \"namespace:1\". In that case add \"namespace:\" here.</li> </ul> </li> <li> <p>Add static suffixes for to the variable part/argument of the path.</p> <ul> <li>Enter one by line. This is useful when the variable part of the ROUTE does not match 1:1 the actual indexed data. e.g the route is /oldrepo/namespace:1 and the indexed value is \"namespace:1-page\". In that case add \"-page\" here.</li> </ul> </li> <li> <p>Select the Type of HTTP redirect to perform.</p> <ul> <li>Option 1: Temporary Redirect (forced GET)</li> <li>Option 2: Permanent Redirect</li> </ul> </li> <li> <p>Lastly, select the box next to <code>Is this Fragaria Redirect Route active?</code> to set your Redirect to <code>active</code>, and Save your configuration.</p> </li> </ol>","tags":["Fragaria","Redirects"]},{"location":"fragaria/#example-redirects-configuration","title":"Example Redirects Configuration","text":"<p>The above example configuration would enable a Temporary redirect from a legacy repository site with a URL of https://mylegacyrepo.edu//mylegacyrepo/object/mylegacyrepo:1234 to your new Archipelago PURL of https://mynewarchipelagorepo.edu/do/mynewADOUUID.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Fragaria","Redirects"]},{"location":"generalqa_minio_logging/","title":"Min.io Logging","text":"<p>Q: How can I see my minio (S3) docker container's realtime traffic and requests?</p> <p>A: For standard demo deployments, mini.io storage server runs on the <code>esmero-minio</code> docker container. Steps are:</p> <ol> <li> <p>Install the <code>mc</code> binaries (minio client) for your platform following this instructions. e.g for OSX run on your terminal:</p> <pre><code>brew install minio/stable/mc\nmc alias set esmero-minio http://localhost:9000 user password\n</code></pre> <p>with <code>http://localhost:9000</code> being your current machines mini.io URL and exposed port,  <code>user</code> being your username (defaults to <code>minio</code>) and your original choosen <code>password</code> (defaults to <code>minio123</code>)</p> </li> <li> <p>Run a <code>trace</code> to watch realtime activity on your terminal:</p> <pre><code>mc admin trace -v -a --debug  --insecure --no-color esmero-minio\n</code></pre> </li> </ol> <p>Note: <code>mc</code> client is also AWS S3 compatible and can be used to move/copy/delete files on the local instance and to/from a remote AWS storage.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"generalqa_smtp_configuration/","title":"SMTP Configuration","text":"<p>Q: How can I enable SMTP for Archipelago?</p> <p>A: For standard demo deployments, SMTP is not setup to send emails. To enable SMTP:</p> <ol> <li> <p>Enter the following commands in your terminal. Note: make sure docker is running. Optionally, you can verify that all Archipelago containers are present by entering the <code>docker ps</code> command first.</p> <pre><code>docker exec -ti esmero-php bash -c 'php -dmemory_limit=-1 /usr/bin/composer require drupal/smtp:^1.0'\ndocker exec -ti esmero-php bash -c 'drush en -y smtp'\n</code></pre> </li> <li> <p>Check that the SMTP module has been enabled by navigating (as admin user) to the EXTEND module menu item (<code>localhost:8001/admin/modules</code>). You should see \"SMTP Authentication Support\" listed.</p> </li> <li> <p>Navigate to <code>localhost:8001/admin/config/system/smtp</code> to configure the SMTP settings.</p> This screenshot shows settings if a GMAIL account is used. <p></p> </li> <li> <p>Save your settings, then test by adding a recipient address in the \u201cSEND TEST E-MAIL\u201d field.</p> </li> </ol> <p>Note: Depending on your email provider, you may also need to enable \u201cless secure\u201d applications in your account settings (such as here for Google email accounts: https://myaccount.google.com/lesssecureapps)</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"generalqa_twig_modules_configuration/","title":"Twig Modules Configuration","text":"<p>Q: When attempting to save a Twig template for a Metadata Display, I receive an error message related to an <code>Unknown \"bamboo_load_entity\" function</code>.</p> <p></p> <p>A: You need to enable the necessary Twig modules.</p> <ol> <li> <p>Navigate to: <code>yoursite/admin/modules</code></p> </li> <li> <p>In the \u201cEnter a part of the module name or description\u201d box, enter \u201cbam\u201d to filter for the related Bamboo Twig modules. Alternatively, scroll down to the Bamboo Twig modules section on this page.</p> <p></p> </li> <li> <p>Check the box next to each of the following to enable (some may already be enabled):</p> <ul> <li>Bamboo Twig</li> <li>Bamboo Twig - Loaders</li> <li>Bamboo Twig - Path &amp; Url</li> <li>Bamboo Twig - Token</li> </ul> <p></p> </li> <li> <p>Click <code>Install</code>.</p> </li> <li> <p>After receiving the successful installation confirmation, check to make sure you are now able to save your Twig template without receiving an error message.</p> </li> </ol> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"giveortake/","title":"Archipelago Contribution Guide","text":"<p>Contributing Documentation</p> <p>Looking to contribute documentation? Start here.</p> <p>Archipelago welcomes and appreciates any type of contribution, from use cases and needs, questions, documentation, devops and configuration and -- of course -- code, fixes, or new features. To make the process less painful, we recommend you first to read our documentation and deploy a local instance. After that please follow the guidelines below to help you get started.</p> <p><code>Archipelago</code> welcomes, appreciates, and recognizes any and all types of contribution. This includes input on all use cases and needs, questions or answers, documentation, DevOps, and configurations. We also welcome general ideas, thoughts, and even dreams for the future of our repository! Of course, we also invite you to contribute PHP code, including fixes and new features.</p> <p>We will be helpful, kind, and open. We encourage discussions and always respect one another's opinions, language, gender, style, backgrounds, origins, and destinations, provided they come from the same root values of respect, as stated here. We support conflict resolution using nothing more than basic common sense. We value diversity in all its shapes, forms, colors, epoches, numbers, and kinds, with or without labels, including in-between and evolving. We always assume we can do better and that you have done a lot. Under this very basic social framework, this is how we hope you can contribute:</p>"},{"location":"giveortake/#where-the-wild-things-live","title":"Where The Wild Things Live","text":"<p>Archipelago has 5 active GitHub repositories</p> <ul> <li>Strawberryfield   What? Code: Deals with metadata storage and exposure to Drupal. Events/Subscribers that trigger when content is modified. The way internal pieces of JSON are exposed to the rest of the ecosystem. Core to all of what Archipeleago is as a concept. One of its Drupal forms is a Field.</li> <li>Webform Strawberryfield   What? Code: Deals with UI based ingest of content using webforms. How files and media are attached to JSON, how tech md is extracted, and any interaction that happens during the edit and ingest processes via a form. In its Drupal form it provides a Field Widget for <code>Strawberryfield</code>.</li> <li>Format Strawberryfield   What? Code: Deals with exposing and transforming the JSON when navigating the site. What is displayed, how it is displayed. Provides templating, metadata display entities via Twig, and direct file downloads. In its Drupal form it provides many field formatters for Strawberryfield` and a content entity for the Twig templates.</li> <li>Archipelago Deployment   What? DevOps: Docker-compose deployment strategy, including a full skeleton project with persistence folders for Min.io, DB, Solr, Cantaloupe and Drupal 8. Includes initial deployment configurations, which modules are enabled, how things look in Drupal 8 and some scripts plus the deployment documentation for both OSX and Linux.</li> <li>Archipelago Documentation   What? Documentation: This guide and whatever we manage to write to explain Archipelago goes here.</li> </ul>"},{"location":"giveortake/#questions-answers-and-in-transit-between-both-ends","title":"Questions, Answers and In Transit Between Both Ends","text":"<p>We host a community interaction channel, our google group. This is the best place to ask questions and make suggestions that are not specific to a single module, and/or if you would like to contribute to a larger conversation within our community. Discussions work best in this forum (not excluding GitHub of course), and our official announcements are posted there too.</p>"},{"location":"giveortake/#documentation-workflow","title":"Documentation Workflow","text":"<p>Documentation is an evolving effort, and we need help. This guide lives in GitHub in Archipelago Documentation. Documentation and Development Worklfow both work the same way, so keep reading!</p>"},{"location":"giveortake/#development-workflow","title":"Development Workflow","text":"<p>Start by reading open ISSUES (so you don't end up redoing what someone else is already working on) and looking at our Roadmap for version 1.0.0. If the solution to your problem is not there or if there is an unchecked element in the roadmap, this is a great opportunity to help by creating a new ISSUE.</p> <p>Next, start by opening an GitHub ISSUE in any of the 5 GitHub repositories, depending on what it is you are trying to do.</p> <p>Please be concise with the title of your ISSUE so that it is easy to understand. Use Markdown to explain the what, how, etc, of your contribution. Note: Even if something related is already in the works, you can still contribute. Just add your comments on any open ISSUE. Or, if you think you want to contribute with a totally different perspective, feel free to open a new ISSUE anyway. We can always discuss next steps starting from there. Every community has its rhythm and style and our style is just beginning to develop. We are still figuring out what works best for everyone.</p> <p>Once you are done and you feel comfortable working to make a change yourself, take note of the <code>ISSUE number</code> (lets name it <code>#issuenumber</code>).</p> <p>The gist is:</p> <ul> <li>Fork the GitHub repository where you created the ISSUE, decide which branch you want to change.</li> <li>Make a new branch out of that one and name it ISSUE-#issuenumber. E.g if #issuenumber is 6, name your branch ISSUE-6.</li> <li>Make changes in that branch and send a pull request.</li> </ul> <p>As a best practice, we encourage pull requests to discuss/fix existing code, new code, and documention changes.</p> <p>For the full step-by-step workflow, we will use Archipelago Documentation and the <code>1.0.0</code> branch as example. The same applies to any of the other repositories: just change the remote urls and use the most current branch name.</p>"},{"location":"giveortake/#example-set-up-archipelago-documentation-github-repository","title":"Example: Set Up Archipelago Documentation GitHub Repository","text":"<p>Fork the Archipelago Documentation Upstream source repository to your own personal GitHub account (e.g. YOU). Copy the URL of your Archipelago Documentation fork (you will need it for the <code>git clone</code> command below).</p> <pre><code>$ git clone https://github.com/YOU/archipelago-documentation\n$ cd archipelago-documentation\n$ git checkout 1.0.0\n</code></pre>"},{"location":"giveortake/#set-up-git-remote-as-upstream","title":"Set Up Git Remote As <code>upstream</code>","text":"<pre><code>$ git remote add upstream https://github.com/esmero/archipelago-documentation\n$ git fetch upstream\n$ git merge upstream/1.0.0\n...\n</code></pre>"},{"location":"giveortake/#create-your-issue-branch","title":"Create Your ISSUE Branch","text":"<p>Before making changes, make sure you create a branch using the ISSUE number you created for these contributions.</p> <pre><code>$ git checkout -b ISSUE-6\n</code></pre>"},{"location":"giveortake/#do-some-clean-up-and-test-locally","title":"Do Some Clean Up and Test Locally","text":"<p>After your code changes, make sure</p> <ul> <li>If modifying <code>PHP</code>, run <code>phpcs --standard=Drupal yourchanged.file.php</code>. We (try our best to) use Drupal 8 coding standards.</li> <li>If modifying a <code>MARKDOWN</code> file, make sure it renders well (you can use Textmate, Atom, Textile, etc to preview) and that links are not broken.</li> <li>If writing large pieces of code, add PHP Tests. We can help if you don't know how (tell us in the ISSUE). Tests will be enforced starting with the first stable release.</li> <li>If modifying <code>PHP</code>, please test your changes live on your local instance of Archipelago. All non-documentation modules are already inside <code>web/modules/contrib/</code>.</li> </ul>"},{"location":"giveortake/#commit-changes","title":"Commit Changes","text":"<p>After verification, commit your changes. This is very good post on how to write commit messages.</p> <pre><code>$ git commit -am 'Fix that Strawberry'\n</code></pre>"},{"location":"giveortake/#push-to-the-branch","title":"Push To The Branch","text":"<p>Push your locally committed changes to the remote origin (your fork)</p> <pre><code>$ git push origin ISSUE-6\n</code></pre>"},{"location":"giveortake/#create-a-pull-request","title":"Create A Pull Request","text":"<p>Pull requests can be created via GitHub. This document explains in detail how to create one. After your Pull Request gets peer reviewed and approved, it can be merged. Discussion can happen and peers can ask you for modifications, fixes or more information on how to test. We will be respectful. You will be given credit for all your contributions and shown appreciation. There is no wrong and never too little. There could never be too much!</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"googleapi/","title":"Configuration for Google Sheets API","text":"<p>To allow the Archipelago Multi Importer (AMI) to read from Google spreadsheets, you first need to configure the Google Sheets API as outlined in the following instructions.</p> <p>Please note:</p> <ul> <li>Frequent changes to the Google Sheets API specifications may impact the configurations needed. </li> <li>This set of instructions will only work for individuals using Google accounts affiliated with Organizations.</li> <li>Please contact us on our Archipelago Commons Google Group with any questions/issues.</li> </ul>"},{"location":"googleapi/#generating-google-oauth2-credentials","title":"Generating Google OAuth2 Credentials","text":"<ol> <li> <p>Login to the Google Developer Console. You will see the API &amp; Services Dashboard.</p> <p></p> </li> <li> <p>If you have not created Credentials or a Project before, you will need to first create a Project.  </p> <ul> <li>Recommended Project Name: \"Archipelago Multi Importer\" or \"AMI\".</li> <li>The Organization and Location information should be specific to you and your organization/institution.</li> </ul> <p></p> </li> <li> <p>Next, click the <code>Create credentials</code> select box and select <code>OAuth client ID</code></p> <p></p> </li> <li> <p>You will now need to Configure the Consent Screen.</p> <p></p> </li> <li> <p>On the initial OAuth Consent Screen setup, select <code>Internal</code> for User Type.</p> <p></p> </li> <li> <p>Now enter <code>AMI</code> as the App name, and your email address in the User support email. You may also wish to add Authorized domains (bottom of image below) as well.</p> <p></p> </li> <li> <p>On the Scopes page, select <code>Add or Remove Scopes</code>. Then either search/filter the API table for the Google Sheets API. Or, under <code>Manually add scopes</code> enter: https://www.googleapis.com/auth/spreadsheets.readonly</p> <p></p> </li> <li> <p>After selecting or entering in the Google Sheets API, you should see this listed under <code>Sensitive Scopes</code>.</p> <p></p> </li> <li> <p>Review the information on the <code>Summary</code> page, then Save.</p> <p></p> </li> <li> <p>You will now be able to <code>Create Oauth client ID</code>. Select <code>Web Application</code> as the <code>Application type</code></p> <p></p> </li> <li> <p>Enter \"AMI\" under 'Name' and add any URIs you will be using below.</p> <ul> <li>For using AMI within your local Archipelago environment, enter <code>http://localhost:8001/google_api_client/callback</code></li> <li> <p>All URIs need to include <code>/google_api_client/callback</code></p> <p></p> </li> </ul> </li> <li> <p>After Saving, you will see a message notifying you that the OAuth client was created. You can copy the <code>Client ID</code> and <code>Client Secret</code> directly from this confirmation message into a text editor. You can also access the information from <code>Credentials</code> in the <code>APIs &amp; Services</code> section in the Developer console, where you will have additional options for downloading, copying, and modifying if needed.</p> <p></p> <p></p> </li> <li> <p>On the 'Add Google Api Client account' configuration page, enter the following information using your <code>Client ID</code> and <code>Client Secret</code>. 'Developer Key' is optional. Select <code>Google Sheets API</code> under 'Services' and <code>https://www,googleapis.com/auth/spreadsheets.readonly</code> under 'Scopes'. Check the box for <code>Is Access Type Offline</code>. Select the Save button.</p> <p></p> </li> <li> <p>You will now need to Authenticate your AMI Google API Client. Return to the Google API Client Listing page. Under the Operation menu on the right-hand side of the AMI client listing, select <code>Authenticate</code>.</p> <p></p> </li> <li> <p>You will be directed to the Google Consent Screen. You may need to login to your corresponding Google Account before proceeding. When loged in, you will see the following screen requesting that AMI is allowed to \"View your Google Spreadsheets\".  Click <code>Allow</code>.</p> <p></p> </li> <li> <p>On the Google API Client Listing page, your AMI client listing should now have 'Yes' under 'Is Authenticated'. You are now ready to use Google Sheets with AMI! Return to the main AMI documentation page to get started.</p> </li> </ol> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"iiif-content-search/","title":"IIIF Content Search API Integration","text":"<p>Beginning in release 1.3.0 and now fully mature in 1.4.0, Archipelago features IIIF Content Search API integration with attendant default configurations and settings.</p> <p>Through a non-trifling amount of code and maths, Archipelago speaks the IIIF Content Search API language using data from your Archipelago's Digital Objects, to enable you to search within Mirador (or other supported viewers) for specific hits within OCR, VTT file, or manually created textual annotations. </p> <p>Please also see the related IIIF Server Settings Form, and Strawberry Runners guides for Reviewing and adjusting the <code>pager</code> and <code>ocr</code> Post-Processor operations and Reviewing and Adjusting the <code>subtitle</code> Post-Processor operations.</p>","tags":["IIIF","IIIF Server Settings Form","IIIF Content Search API","Solr","Solr Fields","Solr Index"]},{"location":"iiif-content-search/#1-iiif-manifest-templates","title":"1. IIIF Manifest Templates","text":"<p>First, Archipelago's default IIIF Manifest templates explicitly state that they support the 3 versions of IIIF Content Search APIS in the 'service' key.</p> <pre><code>\"service\": [ \n        { \n            \"id\": \"{{ baseurl }}iiifcontentsearch/v2/do/{{ node.uuid.value }}/metadatadisplayexposed/iiifmanifest/mode/advanced/page/0\", \n            \"type\": \"SearchService2\" \n        }, \n        { \n            \"id\": \"{{ baseurl }}iiifcontentsearch/v1/do/{{ node.uuid.value }}/metadatadisplayexposed/iiifmanifest/mode/advanced/page/0\", \n            \"type\": \"SearchService1\", \n            \"@context\": \"http://iiif.io/api/search/1/context.json\", \n            \"profile\": \"http://iiif.io/api/search/1/search\" \n        }, \n        { \n            \"@id\": \"{{ baseurl }}iiifcontentsearch/v1/do/{{ node.uuid.value }}/metadatadisplayexposed/iiifmanifest/mode/advanced/page/0\", \n            \"@context\": \"http://iiif.io/api/search/0/context.json\", \n            \"profile\": \"http://iiif.io/api/search/0/search\" \n        } \n    ], \n</code></pre>","tags":["IIIF","IIIF Server Settings Form","IIIF Content Search API","Solr","Solr Fields","Solr Index"]},{"location":"iiif-content-search/#2-api-endpoints-exposure","title":"2. API Endpoints Exposure","text":"<p>Next, in the default Exposed Metadata Endpoints API Endpoints (generated from the IIIF templates), Archipelago provides the specific structure needed for the IIIF Content Search API. Archipelago passes the data about \u201cthe template containing it\u201d, the IIIF API version, if simple or advanced, and the Archipelago Digital Object resource UUID we are searching against (the one that contains the RAW data feeding the template, or at least the Top level/parent one of that).</p>","tags":["IIIF","IIIF Server Settings Form","IIIF Content Search API","Solr","Solr Fields","Solr Index"]},{"location":"iiif-content-search/#3-pathway-into-and-out-of-the-solr-index","title":"3. Pathway into and out of the Solr Index","text":"<p>Then, Archipelago's backend recreates an ADO's IIIF manifest using this data (basically repeats what the client did before), but uses JMESPATHs to extract just what is needed, flipping the order of the structure and putting IIIF Image IDs, as \"top keys\" referencing canvases and their #xywh selectors (for the annotation text), if present.</p> <p>Using this transformed data, Archipelago's backend search is able to be limited to OCR generated only by those images (importantly, as Archipelago repositories can contain millions of OCR'd documents). Archipelago's internal search then returns natively, via the Bavarian State Library\u2019s Solr OCR highlight plugin, the relevant hits within a specified ADO. These are then reprocessed to be IIIF compliant (W3C) annotations and then reverted back to results as \u201ccanvases with images\u201d.</p>","tags":["IIIF","IIIF Server Settings Form","IIIF Content Search API","Solr","Solr Fields","Solr Index"]},{"location":"iiif-content-search/#things-to-keep-in-mind","title":"Things to keep in mind","text":"<ul> <li> <p>To make this performant, Archipelago uses two levels of caches that get invalidated automatically on any \"ingredient\" used modification.</p> </li> <li> <p>Archipelago can also tell the backend to use a \"different\" template than the one used at the front (Mirador), allowing you to define which \"canvases\" are searchable. This is not a normal use case, but still a valid one. And you can, per resource, have complex logic and/or different Viewers, even on a one by one basis.</p> </li> </ul>","tags":["IIIF","IIIF Server Settings Form","IIIF Content Search API","Solr","Solr Fields","Solr Index"]},{"location":"iiif-content-search/#acknowledgements","title":"Acknowledgements","text":"<p>Archipelago's developers would like to extend our gratitude to our community, especially to Mike and Johannes for their work and help, and everyone else in the IIIF and repository communities for all the amazing tools, viewers, specs and cookbook examples.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p>","tags":["IIIF","IIIF Server Settings Form","IIIF Content Search API","Solr","Solr Fields","Solr Index"]},{"location":"iiif_server_settings/","title":"IIIF Server Settings Form Default Settings","text":"<p>The IIIF Server Settings Form is used to configure different IIIF related settings used throughout your Archipelago environment. We strongly advise keeping the default settings intact. The necessary Solr Fields listed below should be setup by default.</p> <p>You can find the IIIF Configuration Form:</p> <ul> <li>Through the <code>Manage</code> menu &gt; <code>Configuration</code> &gt; <code>Archipelago</code> &gt; <code>Configure Strawberry Runners Post Processors</code></li> <li>Directly at <code>/admin/config/archipelago/iiif</code> </li> </ul> <p></p> <p>On the IIIF Server Settings Form page, you will see the following:</p> <ol> <li> <p>Note that these 'IIIF Server configuration URLs are used as defaults for field formatters using IIIF, but can be overridden on a one by one basis when setting up your formatters for each Display Mode.'</p> </li> <li> <p>Base URL of your IIIF Media Server public accessible from the Outside World.</p> <ul> <li>Please provide a publicly accessible IIIF server URL. This URL will be used for AJAX and JS calls. Trailing Slashes will be removed.</li> <li>Set to <code>http://localhost:8183/iiif/2</code> by default.</li> <li>We do not recommend changing this selection. </li> </ul> </li> <li> <p>Base URL of your IIIF Media Server accessible from inside this Webserver.</p> <ul> <li>Please provide Internal IIIF server URL. This URL will be used by Internal Server calls and needs to be locally accessible by your server, e.g 127.0.0.1 or an local Docker alias. Trailing Slashes will be removed.</li> <li>Set to <code>http://esmero-cantaloupe:8182/iiif/2</code> by default.</li> <li>We do not recommend changing this selection.</li> </ul> </li> <li> <p>Checkbox to 'Enable IIIF Content Search API V1 and V2 endpoints'.</p> <ul> <li>Checked by default in later (1.4.0+) versions of Archipelago.</li> <li>See the related (and essential) IIIF Manifest snippet shared here</li> <li>APIs are accesible at the following path: \"/iiifcontentsearch/{version}/do/{node_uuid}/metadatadisplayexposed/{metadataexposeconfig_entity}/mode/{mode}/page/{page}\" with:</li> <li>{version} one of [v1,v2]</li> <li>{node_uuid} the UUID of the ADO whose Manifest you want to search inside</li> <li>{metadataexposeconfig_entity} the machine name of the exposed Metadata Display endpoint used to render the Manifest that is calling the API (e.g iiifmanifest)</li> <li>{mode} one of [simple,advanced]. Advanced is the smartest choice. Simple is faster, but requires your Canvas ids to be exactly in this pattern http(s)://domain.ext/do/{node_uuid}/{file_uuid}/canvas/{internal_to_the_file_sequence_order}</li> <li>{page} 0 to N depedening on the Number of results. By default please use 0</li> </ul> </li> <li> <p>Checkbox to 'Only allow searches inside a Manifest If the Manifest itself (for an ADO) defines the Search Endpoints as a Service'</p> <ul> <li>Checked by default in later (1.4.0+) versions of Archipelago.</li> <li>If enabled we will double check if the calling IIIF Manifest defines the Endpoint(s) in the <code>service</code> key. If unchecked any Manifest will be searchable by calling an API URL directly.</li> </ul> </li> <li> <p>IIIF Content Search API: field(s) that holds Parent Nodes</p> <ul> <li>Strawberry Flavor Data Source Search API Fields that can be used to connect a Strawberry Flavor to a Parent AD0.</li> <li>Default specified fields are: <code>Strawberryfield Flavor Datasource &gt;&gt; SBF Parent ID</code> and <code>Strawberryfield Flavor Datasource &gt;&gt; SBF Parent Node &gt;&gt; isPartOf &gt;&gt; ID</code></li> </ul> </li> <li> <p>Strawberry Runner processors that should be searched against for visual highlights.</p> <ul> <li>e.g Strawberry Flavor Data might have been generated by the \"ocr\" strawberry runners processor. A comma separated list of processors (machine names) that generated miniOCR.</li> <li>Default is: <code>ocr</code></li> <li>If you are using the Strawberry Runners <code>pager</code> and <code>ocr</code> post-processors, you should always keep this enabled.</li> </ul> </li> <li> <p>Strawberry Runner processors that should be searched against for time based media.</p> <ul> <li>e.g Strawberry Flavor Data might have been generated by the \"subtitle\" strawberry runners processor. These will have time based fragments and will match IIIF Annotations with motivation supplementing and target the time based media on the parent Canvas. A comma separated list of processors (machine names) that generated time based transcripts encoded as miniOCR.</li> <li>Default is: <code>subtitle</code></li> </ul> </li> <li> <p>Check to 'Target the VTT Supplementing Annotation'</p> <ul> <li>If enabled (aligned with the specs) the target of a hit result will point to the supplementing Annotation containing in its body the VTT file. If not the Canvas containing in its body a Media Resource (less precise but more compatible with Viewers</li> <li>If you are using the Strawberry Runners <code>subtitle</code> post-processor, you should always keep this enabled.</li> </ul> </li> <li> <p>Strawberry Runner processors that should be searched against plain text extractions.</p> <ul> <li>e.g Strawberry Flavor Data might have been generated by the \"text\" strawberry runners processor. These will not have coordinates but will match IIIF Annotations with motivation supplementing and target the whole canvas. A comma separated list of processors (machine names) that generated time based transcripts encoded as miniOCR.</li> <li>Default is: <code>text</code></li> <li>If you are using the Strawberry Runners <code>subtitle</code> post-processor, you should always keep this enabled.</li> </ul> </li> <li> <p>IIIF Content Search API: field(s) that hold the URI of the File that produced the Searchable content</p> <ul> <li>Strawberry Flavor Data Source Search API Fields that hold the URI of the File that generated its content.</li> <li>Default specified fields are: <code>Strawberryfield Flavor Datasource &gt;&gt; Parent File</code>, <code>Strawberryfield Flavor Datasource &gt;&gt; SBF source or related URI/URL</code>, and<code>Strawberryfield Flavor Datasource &gt;&gt; Parent File &gt;&gt; URI</code></li> </ul> </li> <li> <p>IIIF Content Search API: Max Results per Page</p> <ul> <li>Default is: <code>25</code></li> </ul> </li> <li> <p>IIIF Content Search API: Max allowed characters/length for a Search term</p> <ul> <li>Default is: <code>64</code></li> </ul> </li> </ol> <p>Return to the main Strawberry Runners or the Archipelago Documentation main page.</p>","tags":["IIIF","IIIF Server Settings Form","IIIF Content Search API","Solr","Solr Fields","Solr Index"]},{"location":"inthewild/","title":"Archipelagos in the Wild \ud83d\uddfa\ufe0f","text":"<p>Explore Archipelago instances running free across digital realms.</p> <p>Note</p> <p>*Please be aware that some of the following Archipelago instances are still brewing and these links may change.</p>"},{"location":"inthewild/#metro-archipelago","title":"METRO + Archipelago","text":"<p>The Archipelagos listed below are supported by the Digital Services Team at the Metropolitan New York Library Council. \ud83e\uddd1\u200d\ud83c\udf3e \ud83d\udc1d \ud83c\udf53</p> <ul> <li> <p>Archipelago (Early/Legacy) Playground and Studio Site</p> <ul> <li>METRO's public (play..) and internal (studio..) Archipelago playgrounds to experiment, learn, and evaluate.</li> </ul> </li> <li> <p>Barnard College</p> </li> <li> <p>Digital Culture of Metropolitan New York (DCMNY)</p> </li> <li> <p>Empire Archival Discovery Cooperative (EADC) Finding Aid Toolkit</p> <ul> <li>Supported by the Southeastern New York Library Resources Council (SENYLRC) </li> </ul> </li> <li> <p>Empire Immersive Experiences</p> <ul> <li>Supported by the Western New York Library Resources Council (WNYLRC) </li> </ul> </li> <li> <p>Frick Collection and Webrecorder Team Web Archives Collaboration</p> </li> <li> <p>Hamilton College Library &amp; IT Services</p> </li> <li> <p>Olin College Library Phoenix Files</p> <ul> <li>*Early adopter - live since Summer 2020</li> </ul> </li> <li> <p>New York State Archives Finding Aids Discovery Portal</p> </li> <li> <p>New York State COVID-19 Personal History Initiative</p> </li> <li> <p>Rensselaer Polytechnic Institute Libraries</p> </li> <li> <p>San Diego State University Libraries Digital Collections</p> </li> <li> <p>Union College Library</p> </li> <li> <p>Western Washington University</p> </li> </ul>"},{"location":"inthewild/#neighborhood-archipelagos","title":"Neighborhood Archipelagos","text":"<p>From all around our beautiful shared world. \ud83c\udfe1 \ud83c\udfeb \ud83c\udfdb\ufe0f </p> <ul> <li> <p>Amherst College</p> </li> <li> <p>Association Montessori Internationale</p> </li> <li> <p>California Revealed</p> </li> <li> <p>Christian Observatory of the Pro Civitate Christiana</p> </li> <li> <p>Consiglio Nazionale delle Ricerche / National Research Council of Italy</p> <ul> <li>https://dbopen.ba.cnr.it/</li> <li>https://archiplavit.to.cnr.it/</li> <li>https://vitisgrinzane.ipsp.cnr.it/</li> <li>http://archipelago.byterfly.eu/ \ud83e\udd8b<ul> <li>Virtual Tour Santuario Paola</li> </ul> </li> </ul> </li> <li> <p>Universidad Nacional Aut\u00f3noma de M\u00e9xico</p> </li> <li> <p>University of Edinburgh Libraries</p> <ul> <li>*Development of Archipelago environment began late 2022/3</li> </ul> </li> <li> <p>Vipassana Treasures</p> </li> </ul>"},{"location":"inthewild/#we-should-be-here","title":"We should be here","text":"<p>If you have a public Archipelago instance you'd like to share on this page \ud83c\udfdd\ufe0f\ud83d\udccd, please contact us. We would love to add your great work to this list! \ud83d\udc9a </p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"metadata_display_preview/","title":"Metadata Display Preview","text":"<p>Archipelago's Metadata Display Preview is a very handy tool for your repository toolkit that enables you to preview the output of your Metadata Display (Twig) Templates (found at <code>/metadatadisplay/list</code>). You can use the Metadata Display Preview to test and check the results of any type of Template (HTML Display, JSON Ingest, IIIF JSON, XML, etc.) against both Archipelago Digital Objects (ADOs) and AMI Sets (rows within). </p> <p>Prerequisite Note</p> <p>Before diving into Metadata Display (Twig) Template changes, we recommend reading our Twigs in Archipelago documentation overview guide and also our Working with Twig primer.</p>","tags":["Metadata Display","Twig Template","Preview"]},{"location":"metadata_display_preview/#step-by-step","title":"Step-by-Step","text":"<ol> <li> <p>Navigate to the Metadata Display list at <code>/admin/content/metadatadisplay/list</code> (or through the admin menu via  <code>Manage &gt; Content &gt; Metadata Displays</code>). From the main Metadata Display List page, you can access all of the different display, rendering, and processing templates found in your Archipelago.</p> <p>Selecting a Metadata Display Template</p> <p></p> </li> <li> <p>Open and select 'Edit' for the Template you wish to Edit and/or Preview.</p> <p>Editing a Metadata Display Template</p> <p></p> </li> <li> <p>You will now be able to select either an Archipelago Digital Object (ADO) or AMI Set to Preview. Both selection types will use an autocomplete search (make sure the autocomplete matches fully against your selection before proceeding).</p> <p>Archipelago Digital Object (ADO) selection</p> <p></p> <p>AMI Set and Row selection</p> <p>For the Row, you can enter either a (CSV row) number</p> <p></p> <p>AMI Set and Row selection</p> <p>Or a label found within the Source Data CSV:</p> <p></p> </li> <li> <p>After you select your ADO or AMI Set and press the <code>Show Preview</code> button, the fuller Preview section will open up on the right side of the screen. The left side will continue to show the Metadata Display Template you originally selected to Edit.</p> <p>Tip</p> <p>It is strongly recommended to always select the option to \"Show Preview using native Output Format (e.g. HTML)\".</p> <p>Archipelago Digital Object (ADO) selection against an HTML Display template</p> <p></p> <p>AMI Set and Row selection against a JSON Ingest template</p> <p></p> </li> <li> <p>To keep track of the JSON keys used in your template select the <code>Show Preview with JSON keys used in this template</code> option before pressing <code>Show Preview</code>. For more details see below.</p> </li> <li> <p>Within the Preview Section on the right side of the screen:</p> <ul> <li>The top section contains full JSON metadata record for the selected digital object or AMI Set + specified row.</li> <li>If previewing against an AMI Set + specified row, the middle section will show the 'Reconciliated LoD' for the selected row if you have used AMI LoD Reconciliation for the selected AMI Set.</li> <li>The bottom section will show the rendered Output from the Template using the metadata from the selected ADO or AMI Set + specified row.</li> </ul> </li> <li> <p>From the Edit + Preview mode, you can:</p> <ul> <li>Add and edit additional JSON keys to an HTML-output Display Template, such as subjects from LoD sources, found in your digital objects and collections data.</li> <li>Preview your incoming AMI Sets against your Ingest Template to ensure all your Source Data CSV columns and values are being being mapped properly to their Archipelago destination JSON keys; And make adjustments as needed.</li> <li>Enrich a provided schema-based XML template to incorporate more elements found in your Archipelago environment.</li> <li>And more \ud83e\uddd1\u200d\ud83c\udf73\ud83c\udfa8\ud83c\udfc4     </li> </ul> </li> <li> <p>Select the <code>Show Preview</code> button as you make changes to refresh the Preview output and check your work. After saving any changes you may have made to your selected Template, all of the displays/AMI Sets/other outputs that reference this same Template will reflect the changes made.</p> </li> </ol>","tags":["Metadata Display","Twig Template","Preview"]},{"location":"metadata_display_preview/#metadata-display-preview-json-key-variables","title":"Metadata Display Preview JSON Key Variables","text":"<p>Note</p> <p>This feature is available as of <code>strawberryfield/format_strawberryfield:1.2.0.x-dev</code> and <code>archipelago/ami:0.6.0.x-dev</code>. To make use of it before the official 0.6.0/1.2.0 release you can run the following commands:</p> <ol> <li>Alias the next release branches with the current dependency requirement versions:     <pre><code>docker exec -ti esmero-php bash -c \"composer require 'archipelago/ami:0.6.0.x-dev as 0.5.0.x-dev' 'strawberryfield/format_strawberryfield:1.2.0.x-dev as 1.1.0.x-dev'\"\n</code></pre></li> <li>Then run any database updates:     <pre><code>docker exec -ti esmero-php bash -c \"drush updb\"\n</code></pre></li> <li>And finally, clear the cache:     <pre><code>docker exec -ti esmero-php bash -c \"drush cr\"\n</code></pre></li> </ol> <p>When creating or editing a Metadata Display Twig template, you can keep track of the JSON keys being used in the template by enabling the option after selecting an Archipelago Digital Object (ADO) or AMI Set row before pressing <code>Show preview</code>:</p> <p>Enable Metadata Display Preview Variables</p> <p></p> <p>The last two tabs in the Preview section above expand to show two tables listing the JSON keys that are used and unused by the template. The used keys are sorted by first instance line number (from the template) and the unused keys are sorted alphabetically.</p> <p>Metadata Display Preview Variables Used JSON Keys</p> <p></p> <p>Metadata Display Preview Variables Unused JSON Keys</p> <p></p> <p>The JSON Keys that appear in these tables will vary based on changes to the template and the selected ADO or AMI set row.</p>","tags":["Metadata Display","Twig Template","Preview"]},{"location":"metadata_display_preview/#warnings-and-errors","title":"Warnings and Errors","text":"<p>Warnings and Errors encountered during the processing will be shown at the top of the Preview section. A line number (from the template) will be included in the message if available.</p> <p>Warning</p> <p>A Warning will be generated if output can be rendered, and the output will be displayed below it.</p> <p></p> <p>Error</p> <p>An Error will be generated if no output can be rendered, and no output will be displayed.</p> <p></p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Metadata Display","Twig Template","Preview"]},{"location":"metadata_display_usage/","title":"Metadata Display Usage","text":"<p>Archipelago's Metadata Display Usage tab is an extension of Metadata Display Preview that provides a convenient way for you to check where a Metadata Display Template is currently being used across your Archipelago. You can review the direct and indirect usage of each Metadata Display Template (found at <code>/metadatadisplay/list</code>) across your whole system, and quickly jump to corresponding usage areas to make adjustments if needed. </p>","tags":["Metadata Display","Twig Template","Usage"]},{"location":"metadata_display_usage/#step-by-step","title":"Step-by-Step","text":"<ol> <li> <p>Navigate to the Metadata Display list at <code>/admin/content/metadatadisplay/list</code> (or through the admin menu via  <code>Manage &gt; Content &gt; Metadata Displays</code>). From the main Metadata Display List page, you can access all of the different display, rendering, and processing templates found in your Archipelago. You can also quickly review the 'In use' status from this page.</p> <p></p> </li> <li> <p>Click on the Template Name or select 'Edit' for the Template you wish to review and navigate to the 'Usage' tab.</p> </li> <li> <p>On the Usage tab for every Template in your Archipelago, you will be able to see the 'Label' and 'How' (direct or indirect) usage information for the following four areas across your whole system:</p> <ul> <li>Exposed Metadata Display Entities (such as an exposed Dublin Core XML endpoint)</li> <li>AMI sets (Archipelago Multi Importer)</li> <li>Entity View Modes (Primer on View/Display Modes)</li> <li>Views (Templates can be used by Views for rendering outputs, amongst other uses)</li> </ul> <p></p> <p> </p> </li> <li> <p>For each usage section noted above, you will be able to navigate to any corresponding Exposed Metadata Display Entity, AMI Set, Entity View Mode, or View to make changes to the template usage.</p> </li> </ol> <p>Before Making Any Changes</p> <p>Before diving into Metadata Display Usage review and making changes to what/where templates are used, we recommend reading our Twigs in Archipelago documentation overview guide and also our Working with Twig primer.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Metadata Display","Twig Template","Usage"]},{"location":"metadatainarchipelago/","title":"Your JSON, our JSON - RAW Metadata in Archipelago","text":"<p>From the desk of Diego Pino</p> <p></p> <p>Archipelago's RAW metadata is stored as JSON and this is core to our architecture. To avoid writing RAW over and over, this document will refer to RAW Metadata simply as Metadata.</p> <p>Data and Metadata can be extremely complex and extensive. The use cases that define what Data, Media and Metadata to collect, to catalog and expose, to use during search and discovery or to enable interactive functionality, including questions like  \"what public facing schemas, formats and serializations I need or want to be compliant with\" are as diverse and complex as the Metadata driving them.</p> <p>But also Metadata, in specific, is plastic and evolving as are use cases. And more over, some Metadata is descriptive and some Metadata is technical and there are other types of Metadata too, e.g Control Metadata.</p> <p>Finally Metadata is very close to their generators. Means you and your peers will know, better than any Software Development team, what is needed, useful and, many times also, available given what use case you have, end users needs and resources at hand, your In real life workflows and future expectations.</p>"},{"location":"metadatainarchipelago/#reason-behind-using-json","title":"Reason behind using JSON","text":"<ul> <li>Complex hierarchical Metadata needs to be machinable (and fast)</li> <li>All types of Metadata might need to be \"easy to access\" while generating output, visualizations, search and discovery.</li> <li>The shape of this Metadata might need to change and adapt in time. Idea that supports Open Schema v/s a highly structured and fixed Schema, e.g Relational database (RDB) for most of the use cases we think Archipelago can cover.</li> <li>JSON is shareable, portable, easy to validate online and offline.</li> <li>JSON is compact.</li> <li>JSON is searchable/filterable using Query Languages like JSONPATH and JMESPATH.</li> <li>JSON is easy to patch and modify in batches.</li> <li>JSON is platform agnostic and has been around for decades and has not changed since then.</li> <li>JSON is typed and allows ordering/sorting of data easily.</li> </ul>"},{"location":"metadatainarchipelago/#drupal-and-json","title":"Drupal and JSON","text":"<p>Drupal, the OSS CMS system Archipelago uses and extends, is RDB driven. This means that <code>Content Types</code> normally follow the idea of an Entity with Fields attached. Each of these Fields becomes then a Database Table and the sum of all these fields living under a <code>Content Type</code> definition, a fixed schema. </p> <p>For integration and interoperability reasons with the larger Drupal ecosystem, we inherit in Archipelago the idea of an Entity, in specific, a Content Entity (Node) and <code>Content Type</code> (Bundled fields for a Node). But instead of generating (and encouraging) the use of hundreds of fixed fields to describe your Digital Objects we put all Metadata as JSON, means a JSON BLOB, into a single smart Field of type <code>Strawberry Field</code>. \ud83c\udf53 </p> <p>We go a long way of making as much as possible flexible and dynamic. This also implies the definition (and separation) of what an Archipelago Digital Object (ADO) is in our Architecture v/s what a general Drupal Content Type (e.g a static page or a blog post) is defined in code as: \"Any Content type that uses a Strawberry Field is an ADO and will be processed as such\". No configuration is needed. In other words, all is a NODE but any node that uses a Strawberry Field gets a different treatment and will be named in Archipelago an ADO.</p> <p>One of the challenges of our flexible approach is how to allow Drupal to access the JSON in a way, as native as possible, to generate filtered listings via Drupal Views, free text Search and Faceting. To make this happen Strawberry Field uses a JSON Querying and Exposing as \"Native Field Properties\" logic. Through a special type of Plugin system named Strawberry Key Name Providers and associated Configuration Entities (can be found at <code>/admin/structure/strawberry_keynameprovider</code>), you have control on which keys and values of your JSON are going to be exposed as field properties of any Strawberry Field, allowing Drupal through this to access values in a flat manner and expose them to the Search API natively. The access to the values of any JSON is done via JMESPATH expressions and then transformed either to a list of values or even \"cast\" into more complex data Data types, like an Entity Reference (means a connection to another Entity).</p> <p>This gives you a lot of power and control and makes a lot of very heavy operations lighter. You can even plan upfront or evolve these properties in time. </p> <p>In other words, you control how storage is mapped to Discovery and this allows Drupal Views to work that way too. Of course this also means traditional SQL based Drupal Views won't have access to these internals (for filtering) given that your JSON data nor the virtual Properties generated via Strawberry Key Name Providers are not accessible as individual RDB tables to generate SQL joins and that is why we heavily depend on the Search API (Solr).</p>"},{"location":"metadatainarchipelago/#open-schema-what-is-yours-what-is-archipelagos","title":"Open Schema. What is yours, what is Archipelago's","text":"<p>What can you add to an ADO's Strawberry Field? As long as it is valid JSON, Archipelago can store it, display it, transform it and search across it in Archipelago. The way you manage Metadata can be as \"intime\" or \"aligned\" to other schemas as you want. Still, there are a few suggested keys/functional ideas:</p>"},{"location":"metadatainarchipelago/#suggested-json-keys","title":"Suggested JSON keys","text":""},{"location":"metadatainarchipelago/#the-type-key","title":"The <code>type</code> key","text":"<pre><code>{\n  \"type\": \"Photograph\"\n}\n</code></pre> <p>The <code>type</code> JSON key has a semantic and functional importance in Archipelago. Given that we don't use multiple Drupal Content Types to denote the difference between e.g. a Photograph or a Painting (which would also mean you would be stuck with one or other if we did), we use this key's value to allow Archipelago to select/swap View Modes. This approach also allows for your own needs to define what an ADO in real life or digital realm is (the WHAT). This key is also important when doing <code>AMI</code> based batch ingests since many of the mappings and decisions (e.g. what Template to use to transform your CSV or if the Destination Drupal Content Type is going to be a Digital Object or a Digital Object Collection) will depend on this.</p> <p>Note: Archipelago does something extra fun too when using <code>type</code> value for View Mode Selection (and this is also a feature of one of the Key Name provider Plugins). It will flatten the JSON first and then fetch all <code>type</code> keys. How does this in practice work?</p> <pre><code>{\n    \"type\": \"Photograph\",\n    \"subtypes\": [\n        {\n            \"type\": \"125 film\"\n        },\n        {\n            \"type\": \"Instant film\"\n        }\n    ]  \n}\n</code></pre> <p>Means, while doing a View Mode Selection Archipelago will bring all found <code>type</code> key values together and will have ['Photograph', '125 film', 'Instant film'] available as choices, meaning you will be able to make even finer decisions on how to display your ADOs. View Mode selection is based on order or evaluation, means we recommend putting the more specific mappings first.</p>"},{"location":"metadatainarchipelago/#the-label-key","title":"The <code>label</code> key","text":"<pre><code>{\n    \"label\": \"Black and White Photograph of Cataloger working with JSON\"\n}\n</code></pre> <p>Archipelago will use the <code>label</code> key's value to populate the ADO's (Drupal Node) Title. Drupal has a length limit for its native build in Node Entity Title but JSON has not, so in case of more than 255 characters Archipelago will truncate the Title (not the <code>label</code> key's value) adding an ellipsis (...) as suffix.</p>"},{"location":"metadatainarchipelago/#archipelago-drupal-entities-integration-keys","title":"Archipelago Drupal Entities integration keys","text":"<p>Because of the need of having Technical Metadata, Descriptive Metadata and Semantic Metadata while generating different representations of your JSON via Metadata Display Entities (Twig templates) transformations, we store and characterize Files attached to an ADO as part of the JSON. We also use a set of special keys to map and cast JSON keys and values to Drupal's internal Entities system via their Numeric and/or UUID IDs. </p> <p>Through this, Archipelago will also move files between upload locations and permanent storage, execute Technical metadata extraction, keep track of ADO to ADO relationships (e.g ispartof or ismemberof) and emulate what a traditional <code>Drupal Entity Reference field</code> would do without the limitations (speed and immutability) a static RDB definition imposes.</p>"},{"location":"metadatainarchipelago/#the-apentitymapping-key","title":"The <code>ap:entitymapping</code> key","text":"<pre><code>{\n    \"ap:entitymapping\":{\n        \"entity:file\": [\n            \"model\",\n            \"audios\",\n            \"images\",\n            \"videos\",\n            \"documents\",\n            \"upload_associated_warcs\"\n            ],\n        \"entity:node\": [\n            \"ispartof\",\n            \"ismemberof\"\n        ]\n    }\n}\n</code></pre> <p>the <code>ap:entitymapping</code> is a hint for Archipelago. With this key we can treat certain keys and their values as Drupal Numeric Entity IDs instead of semantically unknown values.</p> <p>In the presence of the structure exemplified above the following JSON snipped:</p> <pre><code>    \"images\": [\n        1,\n        2,\n        3\n    ]   \n</code></pre> <p>Will tell Archipelago that the JSON key <code>images</code> should be treated as containing Entity IDs for a Drupal Entity of type (<code>entity:file</code>) File. This has many interessting consequences. Archipelago, on edit/update/ingest will try (hard) to get a hold of Files with ID 1, 2 and 3. If in temporary storage Archipelago will move them to its final Permanent Location, will make sure Drupal knows those files are being used by this ADO, will run multiple Technical Metadata Extractions and classify internally the Files, adding everything it could learn from them. In practice, this means that Archipelago will write for you additional structures into the JSON enriching your Metadata.</p> <p>Without this structure, the <code>images</code> key would not trigger any logic but will of course still exist and can always still be used as a list of numbers while templating.</p> <p>This also implies that for a persisted ADO with those values, if you edit the JSON and delete e.g. the number (<code>integer</code> or <code>string</code> representation of an <code>integer</code>) <code>3</code>, Archipelago will disconnect the File Entity with ID 3 from this ADO, remove the enriched metadata and mark the File as not being anymore used by this ADO. If nobody else is using the File it will become <code>temporary</code> and eventually be automatically removed from the system, if that is setup at the Drupal - Filesystem - level.</p> <p>Using the same example <code>ap:entitymapping</code> structure, the following snippet:</p> <pre><code>    \"ispartof\": [\n        2000\n    ]   \n</code></pre> <p>Will hint to Archipelago on assumed connection between this ADO and another ADO with Drupal Entity ID <code>2000</code>. This will drive other functionality in Archipelago (semantic), allowing for example a Navigation Breadcrumb to be built using all connections found in its hierarchical path.</p> <p>In Archipelago ADO to ADO relationships are normally from Child to Parent and hopefully (but not enforced!) building an Acyclic graph, from leaves to trunk. This will also allow inheritance to happen. This means also that a Parent ADO needs to exist before connecting/relating to it (chicken first). But if it does not, the system will not fail and assume a temporarily broken relationship (egg stays safely intact). </p> <p>Entity mapping key also drives a very special compatibility addition to any ADO. Archipelago will populate Native Computed Drupal fields (attached at run time to each ADO) with these values loading and exposing them as Drupal Entities, processing both Files and Node Entities and making them visible outside the scope of a <code>Strawberry Field</code> to the whole CMS.</p> <p>The following Computed fields are provided:</p> <ul> <li><code>field_file_drop</code>: Computed Entity Reference Field. Needed also for JSON API level upload of Files to an ADO (Drupal need). It will expose all File Entities referenced in an ADO, independently of the type of the File.</li> <li><code>field_sbf_nodetonode</code>: Computed Entity Reference Field. It will expose all Nodes (other ADOs) Entities referenced in an ADO, independently of the Content type and/or the semantic predicate (ismemberof, ispartof, etc) used.</li> </ul> <p>These Fields, because of their native Drupal nature, can be used directly everywhere, e.g. in the Search API to index all related ADOs (or any of their Fields and subproperties, even deeply chained, tree down) without having to specify what predicate is used. Said differently, they act as aggregators, as a generic \"isrelatedto\" property bringing all together.</p>"},{"location":"metadatainarchipelago/#the-asas_file_type-keys","title":"The <code>as:{AS_FILE_TYPE}</code> keys","text":"<p>As explained in the <code>ap:entitymapping</code> section above, when Archipelago gets hold of a File entity it will enrich your JSON with its extracted data. Archipelago will compute and append to your JSON a set of controlled <code>as:{AS_FILE_TYPE}</code> keys containing a classified File's Metadata. The naming will be automatic based on grouping Files by their Mime Types.</p> <p>The possible values for <code>as:{AS_FILE_TYPE}</code> are</p> <ul> <li><code>as:image</code></li> <li><code>as:document</code> </li> <li><code>as:video</code> </li> <li><code>as:audio</code> </li> <li><code>as:application</code></li> <li><code>as:text</code></li> <li><code>as:model</code></li> <li><code>as:multipart</code> </li> <li><code>as:message</code> </li> </ul> <p>An example for an Image attached to an ADO:</p> <pre><code>{\n    \"as:image\": {\n        \"urn:uuid:ef596613-b2e7-444e-865d-efabbf1c59b0\": {\n            \"url\": \"s3:\\/\\/de2\\/image-f6268bde41a39874bc69e57ac70d9764-view-ef596613-b2e7-444e-865d-efabbf1c59b0.jp2\",\n            \"name\": \"f6268bde41a39874bc69e57ac70d9764_view.jp2\",\n            \"tags\": [],\n            \"type\": \"Image\",\n            \"dr:fid\": 7461,\n            \"dr:for\": \"images\",\n            \"dr:uuid\": \"ef596613-b2e7-444e-865d-efabbf1c59b0\",\n            \"checksum\": \"de2862d4accf5165d32cd0c3db7e7123\",\n            \"flv:exif\": {\n                \"FileSize\": \"932 KiB\",\n                \"MIMEType\": \"image\\/jp2\",\n                \"ImageSize\": \"1375x2029\",\n                \"ColorSpace\": \"sRGB\",\n                \"ImageWidth\": 1375,\n                \"ImageHeight\": 2029\n            },\n            \"sequence\": 1,\n            \"flv:pronom\": {\n                \"label\": \"JP2 (JPEG 2000 part 1)\",\n                \"mimetype\": \"image\\/jp2\",\n                \"pronom_id\": \"info:pronom\\/x-fmt\\/392\",\n                \"detection_type\": \"signature\"\n            },\n            \"dr:filesize\": 954064,\n            \"dr:mimetype\": \"image\\/jp2\",\n            \"crypHashFunc\": \"md5\",\n            \"flv:identify\": {\n                \"1\": {\n                    \"width\": \"1375\",\n                    \"format\": \"JP2\",\n                    \"height\": \"2029\",\n                    \"orientation\": \"Undefined\"\n                }\n            }\n        }\n    }\n}\n</code></pre> <p>That is a lot of Metadata! But to understand what is happening here, we need to dissect this into more readable chunks. Let's start with the basics from root to leaves of this hierarchy.</p>"},{"location":"metadatainarchipelago/#direct-file-level-metadata","title":"Direct File level Metadata","text":"<p>Every Classified File inside the <code>as:{AS_FILE_TYPE}</code> key will be contained in a unique URN JSON Object property:</p> <pre><code>\"urn:uuid:ef596613-b2e7-444e-865d-efabbf1c59b0\": {}\n</code></pre> <p>We use a Property instead of a \"List or Array\" of Technical Metadata because this allows us (at code level) to access quickly from e.g. <code>as:image</code> structure all the data for a File Entity with UUID <code>ef596613-b2e7-444e-865d-efabbf1c59b0</code> without iterating. (Also now you know what urn:uuid:ef596613-b2e7-444e-865d-efabbf1c59b0 means.)</p> <p>Next, inside that property, the following Data provides basic Information about the File so you can access/make decisions when Templating. Notice the duplication of similar data at different levels. Duplication is on purpose and again, allows you to access certain JSON values (or filter) quicker without having to go to other keys or hierarchies to make decisions.</p> <pre><code>{\n    \"url\": \"s3:\\/\\/de2\\/image-f6268bde41a39874bc69e57ac70d9764-view-ef596613-b2e7-444e-865d-efabbf1c59b0.jp2\",\n    \"name\": \"Original Name of my Image.jp2\",\n    \"tags\": [],\n    \"type\": \"Image\",\n    \"dr:fid\": 3,\n    \"dr:for\": \"images\",\n    \"dr:uuid\": \"ef596613-b2e7-444e-865d-efabbf1c59b0\",\n    \"crypHashFunc\": \"md5\",\n    \"checksum\": \"de2862d4accf5165d32cd0c3db7e7123\",\n    \"dr:filesize\": 954064,\n    \"dr:mimetype\": \"image\\/jp2\",\n    \"sequence\": 1\n</code></pre> <ul> <li><code>\"url\"</code>: Contains the Final Storage location/URI of the File. It's prefixed with the configured Streamwrapper, a functional symbolic link to the underlying complexities of the backend storage. e.g <code>s3://</code> implies an S3 API backend with a (hidden/abstracted) set of credentials, Bucket and Prefixes inside the bucket. This value is also used in Archipelago's IIIF Cantaloupe Service as the Image <code>id</code> when building a IIIF Image API URL.</li> <li><code>\"name\"</code>: The Original Name of the File. Can be used to give a Download a human readable name or as an internal hint/preservation for you.</li> <li><code>\"tags\"</code>: Unused by default.  You can use this for your own logic if needed.</li> <li><code>\"type\"</code>: A redundant (contextual, at this level) key whose value will match <code>{AS_FILE_TYPE}</code> already found at 2 levels before. Allows you to know what File type this is when iterating over this File's data (without having to look back, or on our Code, when dealing with Flattened JSON).</li> <li><code>\"dr:fid\"</code>:  The Drupal Entity Numeric ID.</li> <li><code>\"dr:for\"</code>:  Where in your JSON (top level key) this File ID was stored (or in other words where you can find the value of <code>\"dr:fid\"</code>. All this will match / was be driven of course by <code>ap:entitymapping</code>. Sometimes (try uploading a WARC file and run the queue) this key might contain  <code>flv:{ACTIVE_STRAWBERRY_RUNNERS_CONFIG_ID}</code>. This means the File will have been generated by an active Strawberry Runners Processor and not uploaded by you. <code>ACTIVE_STRAWBERRY_RUNNERS_CONFIG_ID</code> will be the Machine name (or ID) of a given Strawberry Runners Processor Configuration Entity.</li> <li><code>\"dr:uuid\"</code>: A redundant (contextual, at this level) key whose value will match the Drupal File entity UUID for this File.</li> <li><code>\"crypHashFunc\"</code>: What Cryptographic function was used for generating the checksum. By default Archipelago will do MD5 (faster but also because S3 APIs use that to ensure upload consistency and E-tag). In the future others can be enabled and made configurable</li> <li><code>\"checksum\"</code>: The Checksum (calculated) of this File via <code>\"crypHashFunc\"</code></li> <li><code>\"dr:filesize\"</code>:  The File size in Bytes.</li> <li><code>\"dr:mimetype\"</code>:  The Drupal level infered Mime Type. Archipelago extends this list. This is based on the File Extension.</li> <li><code>\"sequence\"</code>:  A number (integer) denoting order of this file relative to other files of the same type inside the JSON. Which default type ordering is used will depend on how the ADO was created/edited, but can be overriden using Control Metadata.</li> </ul>"},{"location":"metadatainarchipelago/#technical-file-level-metadata","title":"Technical File level Metadata","text":"<p>Deeper inside this structure Archipelago will produce Extracted Technical Metadata. Some of this Metadata will be common to every File Type, some will be specific to a subset, like Moving Media or PDFs. What runs and how it runs can be configured at the File Persister Service Settings configuration form found at<code>/admin/config/archipelago/filepersisting</code>. Why there? These are service that run syncroniusly on ADO save (Create/Edit) and in while doing  File persistance.</p> <p><code>\"flv:exif\"</code>: EXIF Tool extraction for a file. The number of elements that come out might vary, for an Image file it might be normally short, but a PDF might have a very extensive and long list. The above mentioned File Persister Service Settings form allows you to also set a Files Cap Number, that will, once reached, limit and reduce the EXIF. This is very useful if you want to control the size of your complete JSON for any reason you feel that is needed (performance, readability, etc).</p> <pre><code>{\n    \"flv:exif\": {\n                \"FileSize\": \"932 KiB\",\n                \"MIMEType\": \"image\\/jp2\",\n                \"ImageSize\": \"1375x2029\",\n                \"ColorSpace\": \"sRGB\",\n                \"ImageWidth\": 1375,\n                \"ImageHeight\": 2029\n            },\n}\n</code></pre> <p><code>\"flv:identify\"</code>: Graphics Magic Identity binary will run on every file and format it knows how to run (and will try even on the ones it does not). Will give you data similar to EXIF but processed based on the actual File and not just extracted from the EXIF data found at the header. Notice that the details will be inside a \"1\", \"2\", etc property.  This is because Identify might also go deeper and for e.g a Multi Layer Tiff extract different sequences on the same File.</p> <pre><code>{\n    \"flv:identify\": {\n                \"1\": {\n                    \"width\": \"1375\",\n                    \"format\": \"JP2\",\n                    \"height\": \"2029\",\n                    \"orientation\": \"Undefined\"\n                }\n            }\n}\n</code></pre> <p><code>\"flv:pronom\"</code>: Droid, a File Signature detection tool will find a  matching <code>pronom_id</code>for your File based on  https://www.nationalarchives.gov.uk/aboutapps/pronom/droid-signature-files.htm. This detection type is deeper that EXIF or the mime type based on extension, reading from binary data. It allows you to get small differences between formats (even if e.g both are JP2) and thus make decisions like \"Will <code>Cantaloupe IIIF Image Server</code> be able to handle this type?\". This has also positive Digital Preservation consequences.</p> <pre><code>{\n    \"flv:pronom\": {\n                \"label\": \"JP2 (JPEG 2000 part 1)\",\n                \"mimetype\": \"image\\/jp2\",\n                \"pronom_id\": \"info:pronom\\/x-fmt\\/392\",\n                \"detection_type\": \"signature\"\n            },\n}\n</code></pre> <p><code>\"flv:mediainfo\"</code>: Media Info works on Video and Audio. It goes very detailed into <code>codecs</code> and<code>streams</code> and the output added to your JSON might look massive. This is also very needed when working with IIIF Manifests and deciding if a certain Video will be able to play natively on a browser or if <code>Cantaloupe IIIF Image Server</code> will be able to extract individual frames as images. This again has positive Digital Preservation consequences. The Following is an example of an MP4 file generated via Quicktime on an Apple MacOS computer.</p> <pre><code>{\n    \"flv:mediainfo\": {\n                \"menus\": [],\n                \"audios\": [\n                    {\n                        \"id\": {\n                            \"fullName\": \"1\",\n                            \"shortName\": \"1\"\n                        },\n                        \"count\": \"282\",\n                        \"title\": \"Core Media Audio\",\n                        \"format\": {\n                            \"fullName\": \"AAC LC\",\n                            \"shortName\": \"AAC\"\n                        },\n                        \"bit_rate\": {\n                            \"textValue\": \"85.3 kb\\/s\",\n                            \"absoluteValue\": 85264\n                        },\n                        \"codec_id\": \"mp4a-40-2\",\n                        \"duration\": {\n                            \"milliseconds\": 21215\n                        },\n                        \"channel_s\": {\n                            \"textValue\": \"1 channel\",\n                            \"absoluteValue\": 1\n                        },\n                        \"frame_rate\": {\n                            \"textValue\": \"43.066 FPS (1024 SPF)\",\n                            \"absoluteValue\": 43\n                        },\n                        \"format_info\": \"Advanced Audio Codec Low Complexity\",\n                        \"frame_count\": \"914\",\n                        \"stream_size\": {\n                            \"bit\": 226109\n                        },\n                        \"streamorder\": \"0\",\n                        \"tagged_date\": \"UTC 2017-12-05 17:14:10\",\n                        \"encoded_date\": \"UTC 2017-12-05 17:14:07\",\n                        \"source_delay\": \"-0\",\n                        \"bit_rate_mode\": {\n                            \"fullName\": \"Variable\",\n                            \"shortName\": \"VBR\"\n                        },\n                        \"samples_count\": \"935582\",\n                        \"sampling_rate\": {\n                            \"textValue\": \"44.1 kHz\",\n                            \"absoluteValue\": 44100\n                        },\n                        \"channel_layout\": \"C\",\n                        \"kind_of_stream\": {\n                            \"fullName\": \"Audio\",\n                            \"shortName\": \"Audio\"\n                        },\n                        \"commercial_name\": \"AAC\",\n                        \"source_duration\": [\n                            \"21269\",\n                            \"21 s 269 ms\",\n                            \"21 s 269 ms\",\n                            \"21 s 269 ms\",\n                            \"00:00:21.269\"\n                        ],\n                        \"compression_mode\": {\n                            \"fullName\": \"Lossy\",\n                            \"shortName\": \"Lossy\"\n                        },\n                        \"channel_positions\": {\n                            \"fullName\": \"1\\/0\\/0\",\n                            \"shortName\": \"Front: C\"\n                        },\n                        \"samples_per_frame\": \"1024\",\n                        \"stream_identifier\": \"0\",\n                        \"source_frame_count\": \"916\",\n                        \"source_stream_size\": [\n                            \"226460\",\n                            \"221 KiB (1%)\",\n                            \"221 KiB\",\n                            \"221 KiB\",\n                            \"221 KiB\",\n                            \"221.2 KiB\",\n                            \"221 KiB (1%)\"\n                        ],\n                        \"source_delay_source\": \"Container\",\n                        \"format_additionalfeatures\": \"LC\",\n                        \"proportion_of_this_stream\": \"0.01178\",\n                        \"count_of_stream_of_this_kind\": \"1\",\n                        \"source_streamsize_proportion\": \"0.01180\"\n                    }\n                ],\n                \"images\": [],\n                \"others\": [\n                    {\n                        \"type\": \"meta\",\n                        \"count\": \"188\",\n                        \"duration\": {\n                            \"milliseconds\": 21215\n                        },\n                        \"frame_count\": \"1\",\n                        \"kind_of_stream\": {\n                            \"fullName\": \"Other\",\n                            \"shortName\": \"Other\"\n                        },\n                        \"stream_identifier\": [\n                            \"0\",\n                            \"1\"\n                        ],\n                        \"count_of_stream_of_this_kind\": \"2\"\n                    },\n                    {\n                        \"type\": \"meta\",\n                        \"count\": \"188\",\n                        \"duration\": {\n                            \"milliseconds\": 21215\n                        },\n                        \"frame_count\": \"1\",\n                        \"kind_of_stream\": {\n                            \"fullName\": \"Other\",\n                            \"shortName\": \"Other\"\n                        },\n                        \"stream_identifier\": [\n                            \"1\",\n                            \"2\"\n                        ],\n                        \"count_of_stream_of_this_kind\": \"2\"\n                    }\n                ],\n                \"videos\": [\n                    {\n                        \"id\": {\n                            \"fullName\": \"2\",\n                            \"shortName\": \"2\"\n                        },\n                        \"count\": \"380\",\n                        \"title\": \"Core Media Video\",\n                        \"width\": {\n                            \"textValue\": \"1 280 pixels\",\n                            \"absoluteValue\": 1280\n                        },\n                        \"format\": {\n                            \"fullName\": \"AVC\",\n                            \"shortName\": \"AVC\"\n                        },\n                        \"height\": {\n                            \"textValue\": \"720 pixels\",\n                            \"absoluteValue\": 720\n                        },\n                        \"bit_rate\": {\n                            \"textValue\": \"7 144 kb\\/s\",\n                            \"absoluteValue\": 7144261\n                        },\n                        \"codec_id\": \"avc1\",\n                        \"duration\": {\n                            \"milliseconds\": 21215\n                        },\n                        \"rotation\": \"0.000\",\n                        \"bit_depth\": {\n                            \"textValue\": \"8 bits\",\n                            \"absoluteValue\": 8\n                        },\n                        \"scan_type\": {\n                            \"fullName\": \"Progressive\",\n                            \"shortName\": \"Progressive\"\n                        },\n                        \"format_url\": \"http:\\/\\/developers.videolan.org\\/x264.html\",\n                        \"frame_rate\": {\n                            \"textValue\": \"29.970 (29970\\/1000) FPS\",\n                            \"absoluteValue\": 29\n                        },\n                        \"buffer_size\": \"768000\",\n                        \"color_range\": \"Limited\",\n                        \"color_space\": \"YUV\",\n                        \"format_info\": \"Advanced Video Codec\",\n                        \"frame_count\": \"636\",\n                        \"stream_size\": {\n                            \"bit\": 18951244\n                        },\n                        \"streamorder\": \"1\",\n                        \"tagged_date\": \"UTC 2017-12-05 17:14:10\",\n                        \"encoded_date\": \"UTC 2017-12-05 17:14:07\",\n                        \"bit_rate_mode\": {\n                            \"fullName\": \"Variable\",\n                            \"shortName\": \"VBR\"\n                        },\n                        \"codec_id_info\": \"Advanced Video Coding\",\n                        \"framerate_den\": \"1000\",\n                        \"framerate_num\": \"29970\",\n                        \"sampled_width\": \"1280\",\n                        \"format_profile\": \"Main@L3.1\",\n                        \"kind_of_stream\": {\n                            \"fullName\": \"Video\",\n                            \"shortName\": \"Video\"\n                        },\n                        \"sampled_height\": \"720\",\n                        \"color_primaries\": \"BT.709\",\n                        \"commercial_name\": \"AVC\",\n                        \"format_settings\": \"CABAC \\/ 2 Ref Frames\",\n                        \"frame_rate_mode\": {\n                            \"fullName\": \"Variable\",\n                            \"shortName\": \"VFR\"\n                        },\n                        \"bits_pixel_frame\": \"0.259\",\n                        \"maximum_bit_rate\": {\n                            \"textValue\": \"768 kb\\/s\",\n                            \"absoluteValue\": 768000\n                        },\n                        \"stream_identifier\": \"0\",\n                        \"chroma_subsampling\": [\n                            \"4:2:0\",\n                            \"4:2:0\"\n                        ],\n                        \"maximum_frame_rate\": [\n                            \"30.000\",\n                            \"30.000 FPS\"\n                        ],\n                        \"minimum_frame_rate\": [\n                            \"28.571\",\n                            \"28.571 FPS\"\n                        ],\n                        \"pixel_aspect_ratio\": \"1.000\",\n                        \"colour_range_source\": \"Stream\",\n                        \"format_settings_gop\": \"M=2, N=30\",\n                        \"internet_media_type\": \"video\\/H264\",\n                        \"matrix_coefficients\": \"BT.709\",\n                        \"original_frame_rate\": [\n                            \"25.000\",\n                            \"25.000 FPS\"\n                        ],\n                        \"display_aspect_ratio\": {\n                            \"textValue\": \"16:9\",\n                            \"absoluteValue\": 1.778\n                        },\n                        \"format_settings_cabac\": {\n                            \"fullName\": \"Yes\",\n                            \"shortName\": \"Yes\"\n                        },\n                        \"codec_configuration_box\": \"avcC\",\n                        \"colour_primaries_source\": \"Container \\/ Stream\",\n                        \"transfer_characteristics\": \"BT.709\",\n                        \"proportion_of_this_stream\": \"0.98734\",\n                        \"colour_description_present\": \"Yes\",\n                        \"matrix_coefficients_source\": \"Container \\/ Stream\",\n                        \"count_of_stream_of_this_kind\": \"1\",\n                        \"transfer_characteristics_source\": \"Container \\/ Stream\",\n                        \"format_settings_reference_frames\": [\n                            \"2\",\n                            \"2 frames\"\n                        ],\n                        \"colour_description_present_source\": \"Container \\/ Stream\"\n                    }\n                ],\n                \"general\": {\n                    \"count\": \"336\",\n                    \"format\": {\n                        \"fullName\": \"MPEG-4\",\n                        \"shortName\": \"MPEG-4\"\n                    },\n                    \"codec_id\": [\n                        \"qt  \",\n                        \"qt   0000.00 (qt  )\"\n                    ],\n                    \"datasize\": \"19177730\",\n                    \"duration\": {\n                        \"milliseconds\": 21215\n                    },\n                    \"file_name\": \"c98e7bc52e4bd3fe5681a746f2d9c76f_diego4\",\n                    \"file_size\": {\n                        \"bit\": 19194157\n                    },\n                    \"footersize\": \"0\",\n                    \"frame_rate\": {\n                        \"textValue\": \"29.970 FPS\",\n                        \"absoluteValue\": 29\n                    },\n                    \"headersize\": \"16427\",\n                    \"othercount\": \"2\",\n                    \"folder_name\": \"\\/tmp\\/ami\\/setfiles\\/cb606b13b823eaea784dc77c460f3baf\",\n                    \"frame_count\": \"636\",\n                    \"stream_size\": {\n                        \"bit\": 16804\n                    },\n                    \"tagged_date\": \"UTC 2017-12-05 17:14:10\",\n                    \"audio_codecs\": \"AAC LC\",\n                    \"codec_id_url\": \"http:\\/\\/www.apple.com\\/quicktime\\/download\\/standalone.html\",\n                    \"codecs_video\": \"AVC\",\n                    \"encoded_date\": \"UTC 2017-12-05 17:14:07\",\n                    \"isstreamable\": \"Yes\",\n                    \"complete_name\": \"\\/tmp\\/ami\\/setfiles\\/cb606b13b823eaea784dc77c460f3baf\\/c98e7bc52e4bd3fe5681a746f2d9c76f_diego4.m4v\",\n                    \"file_extension\": \"m4v\",\n                    \"format_profile\": \"QuickTime\",\n                    \"kind_of_stream\": {\n                        \"fullName\": \"General\",\n                        \"shortName\": \"General\"\n                    },\n                    \"codecid_version\": \"0000.00\",\n                    \"commercial_name\": \"MPEG-4\",\n                    \"writing_library\": {\n                        \"fullName\": \"Apple QuickTime\",\n                        \"shortName\": \"Apple QuickTime\"\n                    },\n                    \"overall_bit_rate\": {\n                        \"fullName\": \"7 238 kb\\/s\",\n                        \"shortName\": \"7237957\"\n                    },\n                    \"audio_format_list\": \"AAC LC\",\n                    \"stream_identifier\": \"0\",\n                    \"video_format_list\": \"AVC\",\n                    \"codecid_compatible\": \"qt  \",\n                    \"file_name_extension\": \"c98e7bc52e4bd3fe5681a746f2d9c76f_diego4.m4v\",\n                    \"internet_media_type\": \"video\\/mp4\",\n                    \"encoded_library_name\": \"Apple QuickTime\",\n                    \"comapplequicktimemake\": \"Apple\",\n                    \"overall_bit_rate_mode\": {\n                        \"fullName\": \"Variable\",\n                        \"shortName\": \"VBR\"\n                    },\n                    \"comapplequicktimemodel\": \"iPhone SE\",\n                    \"count_of_audio_streams\": \"1\",\n                    \"count_of_video_streams\": \"1\",\n                    \"comapplequicktimesoftware\": \"10.3.2\",\n                    \"proportion_of_this_stream\": \"0.00088\",\n                    \"audio_format_withhint_list\": \"AAC LC\",\n                    \"video_format_withhint_list\": \"AVC\",\n                    \"file_last_modification_date\": {\n                        \"date\": \"2022-10-19 20:02:32.000000\",\n                        \"timezone\": \"UTC\",\n                        \"timezone_type\": 3\n                    },\n                    \"count_of_stream_of_this_kind\": \"1\",\n                    \"comapplequicktimecreationdate\": \"2017-10-25T16:58:17-0400\",\n                    \"format_extensions_usually_used\": \"braw mov mp4 m4v m4a m4b m4p m4r 3ga 3gpa 3gpp 3gp 3gpp2 3g2 k3g jpm jpx mqv ismv isma ismt f4a f4b f4v\",\n                    \"comapplequicktimelocationiso6709\": \"+40.6145-074.2678+020.977\\/\",\n                    \"file_last_modification_date_local\": {\n                        \"date\": \"2022-10-19 20:02:32.000000\",\n                        \"timezone\": \"America\\/New_York\",\n                        \"timezone_type\": 3\n                    }\n                },\n                \"version\": \"21.09\",\n                \"subtitles\": []\n            }\n        }\n}\n</code></pre> <p><code>\"flv:pdfinfo\"</code>: PDF Info will get Page level Information for a PDF or Ghostscript document. The dimensions displayed in the following example are not in pixels but points (resolution independent) and are also used for IIIF generation when deciding at what rasterized pixel size a given PDF document page will be rendered. Same as with <code>flv:identify</code>, the technical metadata will be contained inside a keyed (string but semantically an integer) property. In this particular case each number is a page sequence in the original PDF order.</p> <pre><code>{\n    \"flv:pdfinfo\": {\n                \"1\": {\n                    \"width\": \"612\",\n                    \"height\": \"792\",\n                    \"rotation\": \"0\",\n                    \"orientation\": \"TopLeft\"\n                }\n            }\n}\n</code></pre> <p>@TODO: add the extra special key used by Strawberry Runners when it attaches a file. e.g WARC to WACZ</p>"},{"location":"metadatainarchipelago/#did-you-know","title":"Did you #know?","text":"<p>If you delete a whole <code>as:{AS_FILE_TYPE}</code> structure or one of the File level structures (a <code>urn:uuid:{uuid}</code> key and its children), Archipelago will recreate it. If you modify any internal value contained in it, Archipelago will do nothing and will trust you (and if you do strange things like modifying the <code>url</code> something might even fail e.g in a IIIF Metadata Display Entity Twig Template). No data edit there will trigger a modification/moving/deletion of a File (or e.g write back EXIF to be binary). You will have time to revert to a previous revision (version) of the ADO if any manual change was done. So, should you modify/delete this structures? Almost never. Ever. But you might find needs for that someday. Also to be noted. Producing this structure for a large file in S3:// is intensive. It needs to be downloaded to a local path and if the File is a few Gigabytes in size Archipelago might even run out of PHP processing time. If that ever happens you can also copy/paste from a previous revision of the ADO the relevant piece. If archipelago finds it (implied in the previous explanation) it will not have to regenerate it. The AMI module does this in an async/enqueued way to avoid time out issues and can reuse a cached metadata extraction between runs, but when working directly on an ADO via e.g a webform or via RAW edit, take that in account.  More work is being done to allow also one on one async File operations and larger uploads via the web.</p>"},{"location":"metadatainarchipelago/#the-aptasks-keys","title":"The <code>ap:tasks</code> keys","text":"<p>As mentioned briefly before, there is also Control Metadata. What do we mean with that? Control metadata in Archipelago's way of allowing you to give, through metadata, (that you might want to preserve or not) instructions to Archipelago that relate to processing. Let's start with the basic one:</p> <pre><code>{\n    \"ap:tasks\": {\n        \"ap:sortfiles\": \"index\"\n    }\n}\n</code></pre> <p><code>\"ap:sortfiles\"</code> key will instruct Archipelago to sort (create a <code>sequence</code> key and a sequential number (integer value) inside each Metadata File entry of a <code>as:{AS_FILE_TYPE}</code> structure. Values can be one of <code>['natural', 'index', 'manual']</code> defaulting, if absent or has an invalid value, to <code>natural</code>.  - <code>natural</code>: files will be sorted by File Name, the <code>filename</code> key found at the same level of <code>sequence</code> in the previously mentioned <code>as:{AS_FILE_TYPE}</code> structure.  a <code>Photograph_1.jpeg</code> will come before a <code>Photograph_10.jpeg</code>. The way a human being naturally would order by name. - <code>index</code> files will be sorted by the order in which they appear inside the upload JSON key (the <code>dr:for</code> key, one of the keys mapped in the <code>ap:entitymapping</code> structure under <code>entity:file</code> explained before. e.g. <code>images</code>:[5, 10 , 1 ], would imply the File Entity with Drupal ID 5 would get <code>\"sequence\": 1</code>, the one with Drupal ID 10 will get <code>\"sequence\": 1</code>, etc. This is the default when ingesting via the AMI module  given the need to preserve file order that has/might have unknown names or names you don't have control of (thus natural won't work) coming from e.g a Remote, HTTP/HTTPS location - <code>manual</code>: You can modify the values manually for any <code>sequence</code> key inside <code>as:{AS_FILE_TYPE}</code> structure and those values will stick.</p> <p>What do we mean with stick? Well, everytime archipelago gets a change in this <code>\"ap:sortfiles\"</code>, e.g a new File is added, a File is deleted, automatic re-sorting will happen. </p> <pre><code>{\n    \"ap:tasks\": {\n        \"ap:forcepost\": true\n    }\n}\n</code></pre> <p><code>\"ap:forcepost\"</code>: A boolean. The functionality of this key is provided by the Strawberry Runners Module. Will force Strawberry Runners Post processing for this ADO. </p> <p>Each Configured and active Postprocessor provided by the <code>Strawberry Runners</code> module might or not <code>kick in</code> by evaluating a set of rules. If rule evaluates to TRUE, the PostProcessor will generate a certain output., e.g a Solr Indexed <code>Strawberry Flavor</code> Data Source containing OCR, HOCR and NLP metadata for one or more pages of a PDF. </p> <p>Everytime a Create or Update operation on an ADO happens, these rules will be evaluated and the Processor will be enqueued as a future task. But at the moment of executing, when the queue workers take one item, a check will be made and if the result of a previous run (e.g HOCR) is already present in the system (e.g in Solr) and it's veryfied to be belonging to the same source, the actual heavyload of processing the PDF will be skipped. </p> <p>While testing, coding, doing complex changes in the system (like modifying largely the settings for one processor) or even in the case of an ISSUE (e.g HOCR was wrongly run with a setting that made all look like garbage!) you can instruct Archipelago run again without checks. And again. And again. basically everytime you Save an ADO by setting <code>ap:forcepost</code> to <code>true</code>. This can also be used batch and is already implied (means it does it for you but only once, without modifying the JSON) in the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects</code> VBO action we provide.</p> <p>In the absence of <code>\"ap:forcepost\"</code> the  value is implicitly <code>false</code>, same as setting it explicitly  to <code>false</code>.</p> <pre><code>{\n    \"ap:tasks\": {\n        \"ap:nopost\": [\n            'pager'\n        ]\n    }\n}\n</code></pre> <p><code>\"ap:nopost\"</code>:  an array or list of <code>ACTIVE_STRAWBERRY_RUNNERS_CONFIG_ID</code> entries, means Machine names (or IDs) Strawberry Runners Processor Configuration Entities. The functionality of this key is provided by the Strawberry Runners. If not an array it will be ignored. Any value present not matching an active Strawberry Runners Processor Configuration Entity ID will also be ignored. Effectively, any post processors in this list will be skipped for this ADO. This allows a finer grained avoidance of some expensive processing that might lead to unsuable data. E.g a particular Manuscript ADO that Tesseract won't be able to OCR correctly.  Adding this key to an ADO that was already processed won't remove existing generated/stored processing.</p> <pre><code>{\n    \"ap:tasks\": {\n        \"ap:ami\": {\n            \"metadata_display\": 7\n\n        }\n    }\n}\n</code></pre> <p><code>\"ap:ami\"</code> is a newer key ( as of Archipelago 1.1.0 and AMI 0.5.0) and for now  can only contain another single key named <code>\"metadata_display\"</code>. The value of this one can be either a single Integer, the Drupal ID of a Metadata Display Entity or a string, the <code>UUID</code> of a Metadata Display Entity.  The functionality triggered by this key is provided by the AMI module and will do something extremely powerfull: it will take the complete JSON and process through the Twig or Metadata Display Entity refererenced in its value, IF, and only IF, the output of that template is JSON. This runs before any other event (Archipelago runs a ton of events that validate, enrich, check, etc your ADOs from the moment you SAVE or Create it) and because of that allows you to totally pivot, transform, change RAW data coming into Archipelago, e.g via the JSON:API into the structure you need/want.  Said differently, you could push JSON from a totally different system and if the referenced  Metadata Display Entity is well written, end with a perfectly aligned JSON matching your internal structure without modifying the INPUT manually. Because Twig is very powerful you can also do cleanups, complex logic, etc. More over, you can transform any existing ADO via Batch by adding this key(s) and values using the JSON Patch VBO action. Once processed and if all went well, meaning the output of the Template is valid JSON, the key itself will be removed. This, to avoid running over and over (invisibly to you) on further operations/edits/etc.  This is a one time operation that does not stick. What happens if it does not run well, fails, errors out or the Template referenced does not exist? You get a second change (everyone deserves one), the Original ingested JSON, without transformations is kept. All this is very similar to what the AMI module does via a CSV but in this case its atomic. We know what you are thinking. You can process data twice, via AMI and then at the end pass it again through another template based on a certain logic coming from the first? yes. you can!</p> <p>In the future <code>\"ap:ami\"</code> might contain more keys to do more advanced File level actions. Archipelago is being constantly enhanced!</p>"},{"location":"metadatainarchipelago/#activity-stream","title":"Activity Stream","text":"<p>Archipelago also keeps information about who/how a certain JSON was generated. Depending on how the Ingest/Edit of an ADO happened, this can be automatically generated or added manually (the case for AMI ingests).</p> <p>The structure is simple and not accumulative because there is also versioning at the ADO (Drupal) level that allows you to look back if needed.</p> <pre><code>{\n     \"as:generator\": {\n        \"type\": \"Update\",\n        \"actor\": {\n            \"url\": \"http:\\/\\/localhost:8001\\/form\\/default-descriptive-metadata-ami\",\n            \"name\": \"default_descriptive_metadata_ami\",\n            \"type\": \"Service\"\n        },\n        \"endTime\": \"2022-03-16T15:51:24-04:00\",\n        \"summary\": \"Generator\",\n        \"@context\": \"https:\\/\\/www.w3.org\\/ns\\/activitystreams\"\n    },\n}\n</code></pre> <p>The <code>\"as:generator\"</code> Conforms to the Activity Stream Vocabulary Version 2.0 and keeps track of the last Operation executed on an ADO. Edits and Ingests via the Webform Widget will create this automatically using the Canonical URL of the Webform That generated the content and <code>\"type\"</code> might be either <code>\"Update\"</code> or \"<code>Create</code>\". ADOs that were processed via <code>\"ap:ami\"</code> will have automatically one generated to express the fact that the Original JSON was modified by a Metadata Display Entity.  Objects created via AMI and using a <code>Metadata Display Entity</code> can also add via the Twig template syntax the AMI Set ID used to generate the ADO (or Update) allowing the Service URL (<code>\"url\"</code>) to be faceted/searched for (e.g show me all objects ingested via AMI Set with ID 4).</p>"},{"location":"metadatainarchipelago/#all-the-other-keys-lists-objects-your-metadata","title":"All the other keys, lists, objects. Your Metadata","text":"<p>Anything or everything else (including unknow data, future data, upcoming data) belongs to you. How you name it, how you structure, how it evolves is up to you and the functionality and integration you want. That said, as someone (the writer) that enjoys cooking and had to learn the hardway (experimenting and failing sometmes) the basics before doing proper meals for others to enjoy, we suggest you plan on this before inventing the next Open Schema. </p> <p>Note: Why the out-of-context Cooking Analogy?</p> <p>This idea is deeply embedded in our Architecture. We see Metadata as ingredients. Your JSON is your fridge (or pantry or both). Metadata Display Entities, and their Twig Templates, recipes that allow you to pick and choose Ingredients and your Twig coding skills (and filters, functions, loops and conditionals) your basic cooking skills. This analogy has many consequences:</p> <ul> <li>You can plan upfront and have more ingredients (metadata keys and JSON structures) than what you need/know how to cook, your current Recipes allow. Planning for \"your future\" needs and skills (and imagination) is a big part of this</li> <li>Reading, understanding and testing small changes on the existing recipes will give provide you with future skills and the ability to do similar, incremental better, meals before going for something totally new and complex</li> <li>Keeping your Ingredientes RAW and unprocessed is a good idea. You might be tempted to store canned refried beans in the fridge, but that might limit your future options. Maybe try both until you feel more secure, the canned ones AND the dried, single beans too.</li> <li>Your use cases might dictate where you start. You cook visually, starting with a very concrete Meal in mind? (means your plan is to make a certain Caserolle - in a less metaphoricall way, a certain well defined output Schema like MODS 3.7). So you need to be sure you have the ingredients that Schema at least requires and know how (or look and copy, MODS is provided already) to process the data. Or you are into a certain type of food? And want to have the most common ingredientes needed around, you might not know how to make all the different dishes but you for sure know Onions (and cutting those) is needed. This are just 2. So many ways of approaching Cooking.</li> </ul> <p>The Open Schema you will get from a Vanilla Archipelago already covers many many uses cases and was developed by a caring team of metadata professionals and practitioners working with Archipelago for a while already. It covers LoD and most Description needs for your Whys, Wheres, When, Who/Whom. Some tips:</p> <ul> <li>Start by adding new Keys instead of removing existing ones. Existing keys might already be used in your Vanilla Archipelago (in their Processed form) in, e.g the Search API, as Key Name providers (means the target of a JMESPATH query that will be exposed to Drupal as a Field Property), in Views (Filters, Sorting of data). Or in Twig templates (as part of Recipees) that Provide data for Viewers, IIIF V2 and V3, or in your HTML Object Descriptions. Or a Webform configured to Edit/Create new ADOs. All these can be of course modified and adapted, but before spending too much time doing that experiment with adding a new Ingredient and incorporating it to either one of the many Outputs of Archipelago or writing a simple new Recipee that uses it.</li> <li>Not all Metadata needs to be used. Want to keep track of your Workflows? Contextual cataloging data? Notes for other metadata professionals in your team? The command line used to capture a WACZ file? You can add that to your JSON. Might all come handy in the future. You don't need to display it at all if you don't want to.</li> <li>Experiment with the Existing Webforms and Webform Elements. If you need to allow Metadata that is very unique in structure and values (and validation), try first generating the simplest structure via the shipped Webforms and their elements via the UI. If you feel Webform can not handle then maybe you are structuring it in a too complex way. Test adding and editing. Check the \"What if/what ifs\". Is your value correctly a string? Would it be better as a boolean or an integer? Think of the \"i want many values\" v/s this is a single value differences. Because you are in the presence of an OpenSchema you can always change your mind, still better to start on the correct track.</li> <li>Give your keys meaningfull names for you/use case/others: <code>property_1</code> might be hard to document for you. But <code>original_artifact_in_collection</code> might be better (and denotes semantically the value might be a boolean, true of false). Use plural and singular in your naming to denote that something might contain more than one entry. Try to be generic but assertive. <code>mods_modsinfo_namepart</code> is tempting but is already hinting a single original fixed schema. And you might end using the same value (the who) in Dublin Core, IIIF, schema.org, etc outputs. So mybe <code>author</code> instead? This also leads to: sometimes multiple keys are better than many deeply nested ones where understanding. You can keep authors and contributors in separate keys.</li> <li>Document your keys in the Metadata Display Templates (JSON does not allow comments but also why would you want to document the same in every ADO, would be like writing notes on your Potatoes. The most obvious place to document are your recipees. If adding a new Key add a small note using <code>{# #}</code> explaining why/what it holds. You can also add Help/Extra info when designing your schema via a the Webform. Each element has extra properties to do so and that way you can also explain others (the ones using the Webform to add/edit) what the purpose of your metadata is.</li> <li>Use a (or many) local Archipelago Deployment as your experimental Kitchen</li> </ul> <p>Do you have your own Kitchen/cooking tips you want to share? We hope you enjoy the learning process and the many choices Archipelago provides.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"metadatatwigs/","title":"Twig Templates and Archipelago","text":"<p>Archipelago uses a fast, cached templating system that is core to Drupal, called <code>Twig</code>. In its guts (or its heart?) Archipelago uses this system to transform the close to your needs open schema metadata that lives in every <code>strawberryfield</code> as JSON into close to other one's fixed schema needs metadata. This is quite simple, but it is an essential component of our vision of how a repository should manage metadata.</p>"},{"location":"metadatatwigs/#what-is-twig","title":"What is Twig?","text":"<p>Twig is a template engine for PHP part of Symfony framework.</p> <ul> <li>Twig in Symfony</li> <li>Twig in Drupal</li> <li>A template engine is a processor. It allows you to mix and process templates with data to generate an output document.<ul> <li>Template: Some type of static Document (we name this a \u201cFrame\u201d)</li> <li>Data: Your Archipelago Digital Object (ADO) info and your Metadata</li> <li>Processor: Allows you to use a rich and expressive language to pick, check, iterate, transform and output your data inside the Template. We refer to this as \u201ccasting\u201d.</li> </ul> </li> </ul>"},{"location":"metadatatwigs/#where-is-twig-used-in-archipelago","title":"Where is Twig used in Archipelago?","text":"<p>This templating system is exposed to Archipelago users through the UI, and is stored in the repository as content. This setup empowers users to fully control how metadata is transformed and published without touching their individual sources or needing to manage hard-coded configurations. We named these readily accessible and powerful templates <code>Metadata Display entities</code>, but they serve more than just display needs.</p> <p>Twig drives every Page in a Drupal 8/9/10 environment.</p> <ul> <li>Twig templates are normally files (.twig.html) that live in your Code.</li> <li>Modules provide Templates, Themes provide Templates</li> </ul> <p>Twig drives every aspect of your ADO exposure to the world in Archipelago and even batch Ingest.</p> <ul> <li>Strawberryfield Metadata (JSON, your Data) is passed through a Metadata Display Entity which holds:<ul> <li>A Twig template (so you do not need to edit Files)</li> <li>A desired output serialization format (the Output Document)</li> </ul> </li> </ul>"},{"location":"metadatatwigs/#twig-templates-as-metadata-display-entities","title":"Twig Templates as Metadata Display Entities","text":"<p>Templates or recipes can be shared, exported, ingested, updated, and adapted in many ways. This means you can make changes quickly without having to wait for the next major release of Archipelago or your favorite Metadata Schema Specs Committee\u2019s agreement to implement the next or the last version. This module not only handles metadata but media assets as well. It will extract local or remote URIs and files from your metadata and render them as media viewers: books, 3D models, images, panoramas, A/V, all with IIIF in its soul.</p> <p>Metadata Display Entities are used for:</p> <ul> <li>Display:<ul> <li>ADO landing pages (via Drupal Field Formatter)</li> <li>IIIF or JSON driven viewers (via Drupal Field Formatter and using Exposed Metadata Endpoints)</li> <li>Map Formatter (Drupal Field Formatter)</li> <li>Custom Blocks (Drupal Views)</li> <li>Search Result Displays (Drupal Views)</li> <li>Collection and Creative Work Series (old compound) displays (Drupal Views)</li> </ul> </li> <li>Machinable Output<ul> <li>Exposed Metadata Endpoints (Standalone URLs to access metadata)</li> </ul> </li> <li>Batch Ingest<ul> <li>AMI Ingest: To transform your CSV data (one row == DATA) to Strawberry field JSON to generate an ADO</li> </ul> </li> </ul>"},{"location":"metadatatwigs/#twig-templates-shipped-with-archipelago","title":"Twig Templates Shipped with Archipelago","text":"<p>Archipelago Ships with:</p> <ul> <li>IIIF Manifest V3 for Images (JSON-LD) Metadata Display</li> <li>IIIF Manifest V2 for Images and Documents (JSON-LD) Metadata Display</li> <li>IIIF Manifest V3 for Collections (JSON-LD) Metadata Display</li> <li>IIIF Manifest V3 for Creative Work Series/Compound Objects Parent and Children (JSON-LD) Metadata Displays</li> <li>A General ADO Description (HTML) Metadata Display</li> <li>A Linked Data Display (HTML) Metadata Display</li> <li>GEOJSON (JSON) Metadata Display</li> <li>An AMI (JSON) Ingest Template</li> <li>A Multiple Thumbnails via IIIF and Fontawesome (HTML) Metadata Display</li> <li>A Metadata Abstract for Search Results (HTML) Metadata Display</li> <li>A Simple Dublin Core (XML) Metadata Display</li> <li>MODS 3.7 (XML) Metadata Display</li> <li>A Schema.org (JSON-LD) Metadata Display</li> <li>Carousel (in Bootstrap) for Images (HTML) Metadata Display </li> </ul> <p>You can find these templates here:</p> <ul> <li>On Github:<ul> <li>Local Deployment</li> <li>Live/Production Deployment</li> </ul> </li> <li>In your local instance: http://localhost:8001/metadatadisplay/list</li> <li>In your live instance: https://yourdomain.org/metadatadisplay/list</li> </ul> <p>Archipelago (the humans) will keep adding and refining these with every release.</p>"},{"location":"metadatatwigs/#instructions-and-examples","title":"Instructions and Examples","text":"<p>While a lot of core needs and use cases are covered with the Twig Templates shipped with Archipelago, you may want to add more Input elements to your Webforms, which in turn will generate new JSON Values, which in turn you may want to show/expose to end users.</p> <p>Knowing (even if you do not plan to) how to edit or create your own Twig templates is important.</p> <ul> <li>This guide covers the Basics of Working With Twig in Archipelago</li> <li>This section contains Full Examples of Common Use Cases</li> <li>This section covers a Recommended Workflow</li> <li>You may also want learn more about what <code>format_strawberryfield</code> can do and what many other possibilities are exposed through our templating system in this guide: Strawberryfield Formatters.</li> <li>Metadata Display Preview: use Preview to check your template outputs against Archipelago Digital Objects or AMI Sets.</li> <li>Metadata Display Usage: use the Usage tab to check where a Metadata Display Template is currently being used across your Archipelago.</li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"modifyingfileextensionsinwebform/","title":"Modifying allowable file extensions","text":"<ul> <li>Webform</li> <li>Webform Elements</li> <li>File Extensions</li> </ul>"},{"location":"modifyingfileextensionsinwebform/#customizing-webforms-modifying-allowable-file-extensions","title":"Customizing Webforms: Modifying allowable file extensions","text":"<p>A guide to walk users through how to modify the Webform <code>Descriptive Metadata</code> to allow additional file extensions to be ingested into Archipelago. This is the default Webform with Archipelago by following archipelago-deployment.</p>"},{"location":"modifyingfileextensionsinwebform/#context","title":"Context","text":"<p>When creating an Archipelago Digital Object (ADO), on Step 4 of the ingest, <code>Attach Files</code>, there is a step during the ingest to upload the files associated with your ADO. There will be a section on the Webform outlining the maximum number of files allowed, the maximum file size allowed, and the allowed file extensions that can be uploaded.</p> <p>Let's say we are creating an ADO with the media type <code>DigitalDocument</code> and this ADO contains a data set saved as a <code>csv</code> file, but when we get to Step 4 of the ingest workflow we find that <code>csv</code> is not an allowed file extension. Fortunately, Archipelago has no restrictions on what file extensions can be uploaded, but some use cases will require a little configuring to fit a specific need. This guide will walk users through the steps to modify the default Webform, <code>Descriptive Metadata</code>, to allow additional file extensions to be included during an ingest.</p> <p>Prerequisites for following this guide:</p> <ul> <li>Running instance of Archipelago (on http://localhost:8001 if you followed the deployment guide verbatim)</li> <li>Admin credentials</li> </ul>"},{"location":"modifyingfileextensionsinwebform/#lets-begin","title":"Let's begin!","text":""},{"location":"modifyingfileextensionsinwebform/#managing-webforms","title":"Managing Webforms","text":"<p>Once logged in as <code>admin</code>, the first thing we need to do is navigate to the Webforms page so we can edit the Webform <code>Descriptive Metadata.</code> Click on <code>Manage</code>, then <code>Structure</code> and when the page loads, scroll down and click <code>Webforms</code>.</p> <p></p> <p>This is where all of the Webforms inside your Archipelago live. For this guide we're going to edit the Webform <code>Descriptive Metadata</code>. Go ahead and click <code>Build</code> under the <code>OPERATIONS</code> column for <code>Descriptive Metadata</code>.</p> <p></p>"},{"location":"modifyingfileextensionsinwebform/#step-3-editing-elements","title":"Step 3: Editing Elements","text":"<p>Here we see all of the elements in <code>Descriptive Metadata</code>; Title, Media type, Description, Linked Data elements, etc. The element that we want to edit is <code>Upload Associated Documents</code> as this is the field you will use to upload <code>pdf</code>, <code>doc</code>, <code>rtf</code>, <code>txt</code>, etc. files during the ingest workflow. Click on <code>Edit</code> under the <code>OPERATIONS</code> column.</p> <p></p> <p>A new screen will pop up named <code>Edit Upload Associated Documents element</code>. This is where you can configure the maximum number of values (under <code>ELEMENT SETTINGS</code>), the maximum file size and also edit the allowed file extensions for this element, which is what we'll be doing. The latter both exist under <code>FILE SETTINGS</code> section, highlighted in the screenshot below.</p> <p></p> <p>When you scroll down you'll see the <code>Allowed file extensions</code> field. This is where we will add the <code>csv</code> file extension. Please note: All file extensions are separated by a space; no <code>,</code> or <code>.</code> between the values.</p> <p>Once you've added all the file extensions your project needs, scroll down to the bottom of <code>Edit Upload Associated Documents element</code> and click <code>Save</code>.</p> <p></p> <p>This next step is imperative for saving your changes, scroll to the bottom of your elements list page and click <code>Save elements</code> in order to persist all changes made.</p> <p></p>"},{"location":"modifyingfileextensionsinwebform/#complete","title":"Complete","text":"<p>Woohoo! Now when you are ingesting a <code>DigitalDocument</code> object, you will be able to add <code>csv</code> files! \ud83c\udf53</p> <p></p>"},{"location":"modifyingfileextensionsinwebform/#recap","title":"Recap","text":"<p>When logged in as an admin, we go to Manage &gt; Structure &gt; Webforms and click on <code>Build</code> under the <code>OPERATIONS</code> column of <code>Descriptive Metadata</code> (shortcut: /admin/structure/webform/manage/descriptive_metadata). Then we click on <code>Upload Associated Documents</code> to edit the element, scroll down to the Allowed file extensions field and add <code>csv</code> without <code>.</code> or <code>,</code> separating the values. Click <code>Save</code> at the bottom of the <code>Edit Upload Associated Documents element</code> page and then <code>Save elements</code> at the bottom of the Webform page.</p>"},{"location":"modifyingfileextensionsinwebform/#that-was-helpful-but","title":"That was helpful, but...","text":"How do I upload a <code>wav</code> or <code>aiff</code> file for \"MusicRecording\" or an <code>mov</code> file for a \\\"Movie\\\"? <p>The steps are virtually the same as what is outlined in this guide! The difference here is that instead of editing <code>Upload Associated Documents</code>, you will need to edit the field element that is associated with your ADO's media type. For example, with Media type <code>MusicRecording</code>, you will edit <code>Upload Audio File</code>, for <code>Movie</code>, will edit <code>Videos</code>.</p> How do I know which element in Descriptive Metadata to edit per media type? <p>When editing an element inside <code>Descriptive Metadata</code>, at the top of the window <code>Edit Upload Associated Documents element</code> (see Step 3 for a recap on how to get here) there is a tab next to <code>General</code> titled <code>Conditions</code>. Inside of <code>Conditions</code> we have <code>CONDITIONAL LOGIC</code> which is where the Webform is told which <code>Media type</code> needs this element to be visible in the Webform. In the example below, we know that the field element <code>Upload Associated Documents</code> will be visible when <code>DigitalDocument</code>, <code>Thesis</code> and <code>Book</code> are the selected <code>Media type</code>.</p> <p>This is also the place you can add new logic or delete present logic by clicking the <code>+</code> or <code>-</code> next to the <code>TRIGGER/VALUE</code> to create new conditionals.</p> <p></p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"open_api/","title":"Metadata API Module","text":"<p>Archipelago's Metadata API Module is a special Strawberryfield Formatter Module that implements Metadata API Endpoints and uses Views and Metadata Display Entities as a configuration option to provide dynamic APIs based on Metadata present in each Archipelago Digital Object (or Node) that contains a Strawberryfield type of field (JSON). </p> <p>Beginning with Archipelago 1.4.0, Archipelago is shipped with a standard implementation of a set of OAI-PMH functions. This includes a full Open API Configuration for OAI-PMH in the Metadata API Module, a corresponding OAI-PMH View, and two corresponding OAI-PMH Metadata Display Templates (Wrapper and Item to output object data into Dublin Core XML). All of these components are necessary to enable OAI-PMH functionality in your Archipelago. Please see the notes below that further explain the specific OAI-PMH <code>verbs</code> (request and response types) supported by Archipelago's default configuration. Please see the OAI Protocol for Metadata Harvesting Specifications for more information about OAI-PMH functions.</p> <p>This module could be extended to be used with other Open APIs, in addition to the default OAI-PMH configurations.</p>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"open_api/#quick-local-test-drive-for-oai-pmh","title":"Quick Local Test Drive for OAI-PMH","text":"<p>The default configurations (specifically the View, see more details below) limits the OAI-PMH queries to members of the 'Biodiversity Heritage Library Collection' (Node #25 as ingested via the default AMI Demo Set).</p> <p>To give this a quick test drive in your running local 1.4.0 Archipelago deployment, copy and paste the following query into your browser:</p> <pre><code>http://localhost:8001/ap/api/oai_pmh/oai?verb=ListRecords&amp;set=c80703db-4644-4f9c-99d4-46a12c500870&amp;metadataPrefix=oai_dc\n</code></pre> Click to see the Example Test Drive Results <p></p>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"open_api/#1-default-oai-pmh-configuration-form","title":"1. Default OAI-PMH Configuration Form","text":"<p>You can find the Metadata API Module Configuration Form:</p> <ul> <li>Through the <code>Manage</code> menu &gt; <code>Configuration</code> &gt; <code>Archipelago</code> &gt; <code>Configure Metadata APIs</code></li> <li>Directly at <code>/admin/config/archipelago/metadataapi</code> </li> </ul> <p></p>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"open_api/#default-supported-oai-pmh-verbs-configuration","title":"Default Supported OAI-PMH Verbs &amp; Configuration","text":"<p>In Archipelago's default <code>oai_pmh</code> Metadata API Endpoint, the following OAI-PMH verbs (requests and responses) are supported:</p> <ul> <li>GetRecord</li> <li>Identify</li> <li>ListIdentifiers</li> <li>ListMetadataFormats</li> <li>ListRecords</li> <li>ListSets</li> </ul> Click to see the Full Default OAI-PMH Open API Config JSON of Default OAI-PMH Open API Config<pre><code>{\n    \"openapi\": \"3.0.2\",\n    \"info\": {\n        \"title\": \"Test API\",\n        \"version\": \"1.0.0\"\n    },\n    \"paths\": {\n        \"http://localhost:8001/ap/api/oai_pmh/{oai}\": {\n            \"get\": {\n                \"parameters\": [\n                    {\n                        \"name\": \"verb\",\n                        \"in\": \"query\",\n                        \"description\": \"\",\n                        \"required\": true,\n                        \"deprecated\": false,\n                        \"style\": \"form\",\n                        \"explode\": true,\n                        \"schema\": {\n                            \"enum\": [\n                                \"GetRecord\",\n                                \"Identify\",\n                                \"ListIdentifiers\",\n                                \"ListMetadataFormats\",\n                                \"ListRecords\",\n                                \"ListSets\"\n                            ],\n                            \"type\": \"string\"\n                        }\n                    },\n                    {\n                        \"name\": \"set\",\n                        \"in\": \"query\",\n                        \"description\": \"\",\n                        \"required\": false,\n                        \"deprecated\": false,\n                        \"style\": \"form\",\n                        \"explode\": true,\n                        \"schema\": {\n                            \"type\": \"string\",\n                            \"format\": \"uuid\"\n                        }\n                    },\n                    {\n                        \"name\": \"identifier\",\n                        \"in\": \"query\",\n                        \"description\": \"\",\n                        \"required\": false,\n                        \"deprecated\": false,\n                        \"style\": \"form\",\n                        \"explode\": true,\n                        \"schema\": {\n                            \"type\": \"string\",\n                            \"format\": \"uuid\"\n                        }\n                    },\n                    {\n                        \"name\": \"oai\",\n                        \"in\": \"path\",\n                        \"description\": \"\",\n                        \"required\": true,\n                        \"deprecated\": false,\n                        \"style\": \"simple\",\n                        \"explode\": false,\n                        \"schema\": {\n                            \"type\": \"string\"\n                        }\n                    },\n                    {\n                        \"name\": \"page\",\n                        \"in\": \"query\",\n                        \"description\": \"\",\n                        \"required\": false,\n                        \"deprecated\": false,\n                        \"style\": \"form\",\n                        \"explode\": true,\n                        \"schema\": {\n                            \"type\": \"integer\"\n                        }\n                    },\n                    {\n                        \"name\": \"metadataPrefix\",\n                        \"in\": \"query\",\n                        \"description\": \"\",\n                        \"required\": false,\n                        \"deprecated\": false,\n                        \"style\": \"form\",\n                        \"explode\": true,\n                        \"schema\": {\n                            \"enum\": [\n                                \"oai_dc\",\n                                \"mods\"\n                            ],\n                            \"type\": \"string\"\n                        }\n                    },\n                    {\n                        \"name\": \"from\",\n                        \"in\": \"query\",\n                        \"description\": \"\",\n                        \"required\": false,\n                        \"deprecated\": false,\n                        \"style\": \"form\",\n                        \"explode\": true,\n                        \"schema\": {\n                            \"type\": \"string\",\n                            \"format\": \"date\"\n                        }\n                    },\n                    {\n                        \"name\": \"until\",\n                        \"in\": \"query\",\n                        \"description\": \"\",\n                        \"required\": false,\n                        \"deprecated\": false,\n                        \"style\": \"form\",\n                        \"explode\": true,\n                        \"schema\": {\n                            \"type\": \"string\",\n                            \"format\": \"date\"\n                        }\n                    }\n                ]\n            }\n        }\n    }\n}\n</code></pre>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"open_api/#reviewing-and-editing-the-default-oai-pmh-configuration-form","title":"Reviewing and Editing the Default OAI-PMH Configuration Form","text":"<p>You can make changes to the above configurations by selecting <code>Edit</code> on the right-hand side of the configuration overview.</p> <p></p> <p>Caution with making changes</p> <p>Proceed with caution before making changes to the default OAI-PMH Configuration Form. This functionality relies on the interconnected components described in this documentation. Before making changes to the configuration form itself, first backup your configurations so you can revert your changes if needed. It is also recommended to test your changes in a local environment before implementing in a production instance.</p>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"open_api/#2-default-oai-pmh-view","title":"2. Default OAI-PMH View","text":"<ul> <li>Through the <code>Manage</code> menu &gt; <code>Structure</code> &gt; <code>Views</code> &gt; <code>OAI_exposed_entity_reference</code></li> <li>Directly at <code>/admin/structure/views/view/oai_exposed_entity_reference</code> </li> </ul> <p>As stated above, the Default OAI-PMH View is configured to limit the OAI-PMH queries to members of the 'Biodiversity Heritage Library Collection' (Node #25 as ingested via the default AMI Demo Set). To change the collection this View limits queries and results for, change the Node ID listed in the <code>Content datasource: # Strawberry_(Descriptive Metadata source) \u00bb entity_sbf_entity_reference_ismemberof</code> under the Filter Criteria section.</p> <p></p> <ul> <li>Please note: this View is using the Full Pager, and this is required for properly outputting the fuller lists of results for OAI-PMH queries.</li> </ul>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"open_api/#3-default-oai-pmh-templates","title":"3. Default OAI-PMH Templates","text":"<p>The default OAI-PMH functionality relies on using two Metadata Display Templates, one for wrapping or setting the query results within a standard structure, and one for outputting individual query results (such as the Dublin Core XML for a specific Archipelago Digital Object). The two different templates are specified at the bottom of the OAI-PMH Configuration Form described above.</p> <p></p>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"open_api/#oai-pmh-wrapper-template","title":"OAI-PMH Wrapper Template","text":"<ul> <li>Found at: http://localhost:8001/metadatadisplay/17</li> <li>The Metadata display Entity (Twig) used to generate data for the API wrapper response.</li> </ul>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"open_api/#oai-pmh-item-with-dc-template","title":"OAI-PMH Item with DC Template","text":"<ul> <li>Found at: http://localhost:8001/metadatadisplay/18</li> <li>The Metadata display Entity (Twig) to be used to generate data at this endpoint.</li> </ul> <p>Caution with making changes</p> <p>Please also proceed with caution before making changes to the default OAI-PMH Metadata Display Templates. Before saving any changes you make to the templates, mark to save as a 'Revision' at the bottom of the template form, so you can revert your template to a previous version if needed.</p> <p>__</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Metadata API Module","Open API","OAI-PMH"]},{"location":"ourtake/","title":"Archipelago's Philosophy &amp; Guiding Principles","text":"<p>Archipelago operates under a  different concept than the one we all have become used to in recent times. We like to think this is not done by re-inventing the wheel, but by making sure the road is clean, level, and with fewer obstacles than before. We do this by removing some heavy weight from the top, some unneeded ballast, plus, of course, some well positioned innovations to make the ride enjoyable.</p> <p>We also like to say that Archipelago is like a Metadata Synthetizer (LFO anyone?) and we want to give you all the knobs, parameters, inputs and outputs to make the best out of it. Still, you can make \"music\" by just tapping the keyboard.</p> <p>To get here we had to do a full stop first. Look around. Questioning everything we knew. Research and test (repeat) and then re-architect slowly on new and old assumptions, and especially new community values.</p>"},{"location":"ourtake/#whys-and-whats-of-archipelago","title":"Whys and Whats of Archipelago","text":"<p>Because this topic is near and dear to our hearts, we are taking extra care with writing this important document. Please stay tuned for the full, verbose, heartfelt, and detailed long story of Archipelago's origins, development, future hopes and dreams.</p> <p>In the meantime, please consider reviewing this presentation created by Archipelago's Lead Architect Diego Pino which captures the essence of Archipelago's philosophy and guiding principles:</p> <ul> <li>Archipelago : an empathic Digital Repository Architecture</li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"presentations_events/","title":"Archipelago Presentations, Events, and Additional Resources","text":"<p>Important General &amp; Internal Recordings Notes</p> <p>Please be aware that some of the presentation documents shared above may contain links to older documentation resources that have since changed or are no longer available. We recommend referring to the latest documentation versions available on this site whenever needed.</p> <p>METRO's Digital Services Team facilitated many different internal training sessions throughout 2020-2022. If you and your team need access to any of these sessions that were recorded, please contact us. Thank you!</p>"},{"location":"presentations_events/#2024","title":"2024","text":"<ul> <li> <p>Archipelago Summer Workshop Series (August 2024) : registration details here</p> <ul> <li>Session 1: Archipelago 1.4.0 Local Deployment &amp; Features Tour<ul> <li>\ud83c\udfa5 Recording available (registration required)</li> </ul> </li> <li>Session 2: Review of Display/View Modes and Strawberryfield Formatters<ul> <li>\ud83c\udfa5 Recording available (registration required)</li> </ul> </li> <li>Session 3: Search &amp; Solr Overview<ul> <li>\ud83c\udfa5 Recording available (registration required)</li> </ul> </li> </ul> </li> <li> <p>IIIF AI/ML Community Group (July 2024)</p> <ul> <li>Creating a Better Balance: Respectful Reuse &amp; ML/AI Tags in IIIF Manifests. Allison Sherrick.</li> </ul> </li> <li> <p>IIIF Annual Conference (June 2024)</p> <ul> <li>Creating a Better Balance: Respectful Reuse &amp; ML/AI Tags in IIIF Manifests. Allison Sherrick, Diego Pino Navarro.</li> <li>Transdimensional mutations for audio/moving media annotations in IIIF Content Search. Allison Sherrick, Diego Pino Navarro.</li> <li>\ud83d\udcfa Recordings made available on the IIIF Youtube channel.</li> </ul> </li> <li> <p>Open Repositories Annual Conference (June 2024):</p> <ul> <li>Hybrid ML/AI driven search as cataloging aid in Archipelago Commons. Diego Pino Navarro.</li> <li>Creating a better balance: the need for tools and practices to combat AI harvests and resource flooding in repository environments. Diego Pino Navarro, Allison Sherrick.</li> <li>Collaboration Across Borders. Jessica Barlow, Lisa Lamont, Matt Ferrill, Hilario Castillo Castillo, Kristofer Patr\u00f3n Soberano.</li> </ul> </li> <li> <p>Public Release of IOI's InfraFinder Tool (Spring 2024): Archipelago Commons on Infrafinder</p> </li> </ul>"},{"location":"presentations_events/#2023","title":"2023","text":"<ul> <li> <p>IIIF Search API and Dynamic/evolving Manifest Generation: Facing the Unknown. Diego Alberto Pino Navarro, Allison Sherrick.</p> <ul> <li>\ud83d\udcfa Recording available</li> </ul> </li> <li> <p>DLF Forum (November 2023)</p> <ul> <li>Working and Learning the IIIF Search API in Archipelago. Diego Pino Navarro, Allison Sherrick.</li> <li>Working with Open-Schema JSON in Archipelago. Allison Sherrick, Diego Pino Navarro, Martha Tenney, Joanna DiPasquale, Corinne Chatnik.</li> <li>Slaying the Migration Dragon: Approaches to Navigating an Open Source System Migration. Lisa McFall, Sarah Walden McGowan, Brenden McCarthy, Shay Foley.</li> </ul> </li> <li> <p>Archipelago 1.3.0 Release Announcement (October 31, 2023) </p> </li> <li> <p>IIIF Annual Conference (June 2023)</p> <ul> <li>Experimental IIIF Kitchen using Archipelago. Pino Navarro, Diego; Sherrick, Allison.</li> <li>Mapping an Engineer Through IIIF. Monger, Jenifer J.; McCarthy, Brenden; Pino Navarro, Diego; Sherrick, Allison.</li> </ul> </li> <li> <p>Into Archipelago Commons: Access, Innovation and Community in Modern Archives.  Monger, Jenifer J.; McCarthy, Brenden; Corinne Chatnik. (June 2023)</p> </li> <li>Implementing Archipelago: An Innovative, Community Driven, Open-Source Repository. Corinne Chatnik, Union College; Martha Tenney, Barnard College. (June 2023)</li> <li>For the Love of Data and Ourselves: The Bumpy, Technical Road to Modern Archives. Monger, Jenifer J.; McCarthy, Brenden. (January/February 2023)</li> </ul>"},{"location":"presentations_events/#2022","title":"2022","text":"<ul> <li> <p>Archipelago Late 2022 Workshop Series:</p> <ul> <li>Session 1 : AMI Essentials and Tricks of the Trade. Pino Navarro, Diego; Sherrick (Lund), Allison; Romabiles, Katie. (November 2022)<ul> <li>\ud83c\udfa5 Recording available (registration required) </li> </ul> </li> <li>Session 2: Twig Templating and Metadata Display Preview for AMI Ingest. Pino Navarro, Diego; Sherrick (Lund), Allison; Romabiles, Katie; Min, Albert. (December 2022)<ul> <li>\ud83c\udfa5 Recording available (registration required)</li> </ul> </li> <li>Session 3: AMI Set Processing and Advanced Find + Replace (December 2022)<ul> <li>\ud83c\udfa5 Recording available (registration required)</li> </ul> </li> </ul> </li> <li> <p>McCarthy, B. J. (2022). Archipelago Commons: Using the Archipelago and AMI software to provide access to Rensselaer Polytechnic Institute's engineering drawings, a pilot project. Issues in Science and Technology Librarianship, 101. https://doi.org/10.29173/istl2717</p> </li> <li> <p>Open Perspectives Forum. Monger, Jenifer J.; McCarthy, Brenden. (November 2022) </p> </li> <li> <p>Migration, Collaboration and Innovation with Archipelago Commons. Monger, Jenifer J. (September 2022)</p> </li> <li> <p>\ud83c\udf53 Archipelago 1.0.0 - August 2022 Release Announcement (August 2022) and updated Specs and Features List </p> </li> <li> <p>Open Repositories June 2022</p> <ul> <li>Collaborative W3C Web Annotations using Annotorious in Archipelago and computer vision explorations as cataloger aids. Pino Navarro, Diego; Simon, Rainer.</li> <li>Modern Web Archiving with Archipelago and Webrecorder : WACZ, Replay.web and Deep Discovery in Digital Repositories. Pino Navarro, Diego; Kreymer, Ilya.</li> <li>Working with IIIF Manifests in Archipelago. Sherrick (Lund), Allison.</li> </ul> </li> <li> <p>Formation of the Archipelago Working Group (April 2022)</p> <ul> <li>In the Spring of 2022, METRO supported the creation of a select group of both early adopters and longtime members of the Archipelago community to provide a dedicated space for Archipelago power users to build upon their demonstrated use-explorations, contribute further to the platform and have a direct influence on roadmap code, direction, and timeline. This group will also work on documentation needs, use cases and outreach (including public showcases, trainings/workshops, and other events).</li> </ul> </li> <li> <p>Toward Empathetic Digital Repositories: An Interview with Diego Pino Navarro (January 2022)</p> </li> </ul>"},{"location":"presentations_events/#2021","title":"2021","text":"<ul> <li> <p>\ud83c\udf53 Archipelago 1.0.0-RC3 and 1.0.0 Release Announcement - November 2021</p> </li> <li> <p>AMIA Conference Workshop: Building a Web Archive-Capable Digital Repository with Webrecorder and Archipelago. Kreymer, Ilya; Ramirez-Lopez, Lorena; Dickson, Emma; Pino Navarro, Diego; Sherrick (Lund), Allison. (November 2021)</p> </li> <li> <p>Solr Importer AMI Migrations, Showcase and Roundtable. Pino Navarro, Diego; Sherrick (Lund), Allison. (July 2021)</p> <ul> <li>Please see latest I7 Solr Importer and AMI LoD Reconciliation documentation.</li> </ul> </li> <li> <p>IIIF Annual 2021 Conference:</p> <ul> <li>Learning &amp; Working with IIIF in Archipelago - 2021 IIIF Annual Conference. Pino Navarro, Diego; Sherrick (Lund), Allison. (June 2021)</li> <li>Editing a IIIF Manifest in Archipelago. Pino Navarro, Diego; Sherrick (Lund), Allison.</li> </ul> </li> <li> <p>June 2021 Open Repositories Conference:</p> <ul> <li>Europeana Data Model (EDM) Workflows in Archipelago. Pino Navarro, Diego; Sherrick (Lund), Allison.<ul> <li>\ud83d\udcfa Recording available</li> </ul> </li> <li>'Broken for All' Persistent Identifiers Panel Discussion. Kunze, John; Holmes-Wong, Deborah; Rafique, Zahid; Lohnash, Megan; Sherrick (Lund), Allison; Turner, Adrian; McKinley, Matthew.<ul> <li>\ud83d\udcfa Recording available </li> </ul> </li> </ul> </li> <li> <p>WebRecorder + Archipelago Workshop. Pino Navarro, Diego; Sherrick (Lund), Allison; Kreymer, Ilya; Ramirez-Lopez, Lorena; Dickson, Emma. (May 2021)</p> <ul> <li>\ud83d\udcfa Recording available</li> </ul> </li> <li> <p>Twig Templates and Archipelago. Pino Navarro, Diego; Sherrick (Lund), Allison. (May 2021)</p> <ul> <li>Review of the core role Twig templating plays in Archipelago, introduction to the basics of Twig, and demonstration of editing Twig templates in Archipelago to refine metadata displays and AMI ingests.</li> <li>\ud83c\udfa5 Recording available (registration required)</li> </ul> </li> <li> <p>\ud83c\udf53Archipelago 1.0.0-RC2 Release Announcement (May 2021) and Archipelago RC2 Specs and Features List</p> </li> <li> <p>Working with Archipelago Multi-Importer (AMI). Pino Navarro, Diego; Sherrick (Lund), Allison. (April 2021)</p> <ul> <li>Introduction to AMI, discussion of ingest strategies and options, and demonstration of an AMI ingest.</li> <li>\ud83d\udcfa Recording available</li> </ul> </li> <li> <p>Archipelago Digital Objects Repository (an) architecture to last. Pino Navarro, Diego. (DrupalCon North America 2021)</p> <ul> <li>\ud83d\udcfa Recording available</li> </ul> </li> <li> <p>Metadata, Schemas and Media in Archipelago. Pino Navarro, Diego; Sherrick (Lund), Allison (February 2021)</p> <ul> <li>Exploration of the flexible and extensible ways Archipelago manages Metadata, Schemas, and Media.</li> <li>\ud83c\udfa5 Recording available (registration required)</li> </ul> </li> <li> <p>Deploying Archipelago 1.0.0-RC1. Pino Navarro, Diego; Sherrick (Lund), Allison. (February 2021)</p> <ul> <li>Demonstration of a complete walkthrough of a local Archipelago 1.0.0-RC1 deployment.</li> <li>\ud83c\udfa5 Recording available (registration required)</li> </ul> </li> </ul>"},{"location":"presentations_events/#2020","title":"2020","text":"<ul> <li> <p>\ud83c\udf53 Archipelago 1.0.0-RC1 Release Announcement (December 2020)</p> </li> <li> <p>Webforms in Archipelago. Pino Navarro, Diego; Sherrick (Lund), Allison; Palmentiero, Jennifer. (December 2020)</p> </li> <li> <p>IIIF and Archipelago - Community Call. Pino Navarro, Diego. (October 2020)</p> </li> <li> <p>Archipelago : an empathic Digital Repository Architecture (September 2020)</p> </li> <li> <p>\ud83c\udf53 Archipelago 8.x-1.0-beta3 Release Announcement (July 2020)</p> </li> </ul>"},{"location":"presentations_events/#we-should-be-here","title":"We should be here","text":"<p>If you have a public Archipelago presentation, recording, or other resource you'd like to share on this page \ud83c\udfdd\ufe0f\ud83d\udccd, please contact us. We would love to add your great work to this list! \ud83d\udc9a </p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"search-within-collection/","title":"How to Add a 'Search Within Collection' Block","text":"<p>This guide covers how to add a 'Search Within Collection' Exposed Form Block to the default Archipelago Collection Display Page. </p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#preamble-prerequisites","title":"Preamble + prerequisites","text":"<p>Before diving into any Search and Solr configuration related changes, we strongly recommend that you read our Metadata in Archipelago overview documentation, which provides important context for understanding how the shape of your Archipelago Digital Objects/Collections (ADOs) metadata will inform your Search and Solr options and outcomes. If you don't have the bandwidth to read the (stellar) Metadata in Archipelago documentation, we recommend you read through our in-a-nutshell overview.</p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#step-1-open-the-collection-membership-view","title":"Step 1:  Open the Collection Membership View","text":"<p>Navigate to the Collection Membership view found at:</p> <ul> <li><code>/admin/structure/views/view/collection_membership</code></li> <li>Through the <code>Structure</code> menu &gt; <code>Views</code> &gt; <code>Collection Membership</code></li> </ul> <p>This View is setup to list the member Digital Objects of a Collection and is driven by Solr.</p> <p></p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#step-2-adjust-the-views-advanced-tab-settings","title":"Step 2. Adjust the View's Advanced tab settings","text":"<p>Open the 'Advanced' tab settings and change the 'Exposed form in block' to 'Yes'.</p> <p></p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#step-3-add-the-fulltext-search-filter-criteria-for-the-view","title":"Step 3. Add the Fulltext Search Filter Criteria for the View","text":"<p>On the left-hand side of the Collection Membership View form, under the 'Filter criteria' section, add and configure the 'Fulltext search' Criteria.</p> <p></p> <p>At the top of the 'Configure filter criterion: Search: Fulltext search' form that opens:</p> <ul> <li>Select the option to 'Expose this filter to visitors, to allow them to change it'.</li> </ul> <p>On the lower section of the form, adust the configuration options as follows:</p> <ul> <li>Remove any text from 'Label' or 'Description</li> <li>Operator: select/check 'Contains all of these words'</li> <li>Allow multiple selections : leave unchecked</li> <li>Remember the last selection : leave unchecked</li> <li>Filter identifier: leave default of 'search_api_fulltext'</li> <li>Placeholder: enter 'Search within Collection' (or your preferred text)</li> <li>Search field character limit: set to '128'</li> <li>Expose searched fields : leave unchecked</li> <li>Searched fields : leave all unselected, so \"If no fields are selected, all available fulltext fields will be searched.\"</li> <li>Minimum keyword length : set to '1'</li> </ul> <p>Select 'Apply' and continue to the next Step.</p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#step-4-reviewadjust-options-and-save-the-updated-view","title":"Step 4. Review/adjust options and Save the Updated View","text":"<p>Review the changes you made to the Collection Membership View. Optionally further adjust the options if desired. For example, you may choose to change the <code>Submit button text</code> to 'Search' instead of 'Apply'). If you make any further changes select, 'Apply' before exiting.</p> <p></p> <p>'Save' your changes made to the View before proceeding.</p> <p>Your updated Collection Membership View should look like the following now:</p> <p></p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#step-5-place-the-exposed-form-block","title":"Step 5. Place the Exposed Form Block","text":"<p>Navigate to the Block Layout found at:</p> <ul> <li><code>/admin/structure/block</code></li> <li>Through the <code>Structure</code> menu &gt; <code>Block layout</code></li> </ul> <p>Select the <code>Archipelago Base Theme</code> (or whatever theme you are using):</p> <ul> <li><code>admin/structure/block/list/archipelago_subtheme</code></li> </ul> <p>Navigate to the 'Content' section of the theme and select 'Place Block'.</p> <p></p> <p>Search for the 'Exposed form: collection_membership-block_1' and select 'Place Block'.</p> <p></p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#step-6-configure-the-blocks-settings","title":"Step 6. Configure the Block's Settings:","text":"<p>Configure the different settings for the Block. </p> <p>You will need to specify both of the following options under the <code>Visibility</code> section:</p> <ul> <li>ADO Type: specify Collection (and any other Collection types you may also have such as 'Newspaper')</li> </ul> <p></p> <ul> <li>Content type: select 'Digital Object Collection'</li> </ul> <p></p> <p>In the top section of the form, we recommend the following settings:</p> <ul> <li>Deselect 'Display title'</li> <li>Under <code>Exposed Form element and component Visibility</code>, deselect the following:<ul> <li>Show filter components of type select if exposed</li> <li>Show filter components of type checkbox/options if exposed</li> <li>Override Submit button Label</li> </ul> </li> </ul> <p></p> <p>Select 'Save block' when you are finished configuring the Block's Settings.</p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#step-7-position-the-block-and-save","title":"Step 7. Position the Block and Save.","text":"<p>Drag to re-order and position the Exposed Form Block to sit above the Collection Membership block in the <code>Content</code> section. This will position the Exposed Form Block above the list of Collection Member Objects on the display pages for Collections.</p> <p>After you have positioned the blocks, scroll down to the bottom of the <code>Block layout</code> page and select 'Save blocks'.</p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#step-8","title":"Step 8.","text":"<p>Navigate to a Collection and test out a Search in the search box that is now in place above the 'Objects in this Collection' listing.</p> <p>In this screenshot, you can see a demonstrative Search for 'map' within one of the Archipelago Demo Collections.</p> <p></p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search-within-collection/#additional-considerations","title":"Additional Considerations","text":"<p>You may also wish to pair this 'Search Within Collection' Exposed Form Block with related Facets (setup on the same corresponding collection membership View). You can find follow the step-by-step instructions in our Strawberry Key Name Providers, Solr Field, and Facet Configuration documentation.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Search","Search Within","Collections","Views","Exposed Forms","Blocks"]},{"location":"search_advanced/","title":"Advanced Search","text":"<p>To provide Advanced Search functionality, Archipelago extends a custom Search API based Fulltext Filter for (Drupal) Views. This special 'Search: Advanced Fulltext search' filter is capable of handling multiple inputs with a combination of Boolean Operators, and features additional customizable options for tailoring your Archipelago's Advanced Search setup.</p>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#default-advanced-search","title":"Default Advanced Search:","text":"<p>The default Advanced Search setup shipped with standard Archipelago deployments is intended to serve as an initial blueprint for your own learning and further customizations. Your Archipelago instance, whether you are working in a modified local or production deployment, may not have the same corresponding Solr Fields and/or Facets as present in the default Advanced Search configuration. This guide covers the default Advanced Search Setup, and offers general guidance for customizing to match your desired Advanced Search setup for your unique environment variables (metadata elements/schema found in your JSON data, Keyname combinations, Solr fields, etc.).</p>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#where-to-find-and-corresponding-configurations","title":"Where to Find and Corresponding Configurations","text":"<p>In default Archipelagos, you can find Archipelago's default Advanced Search page:</p> <ul> <li>Through the top navigation menu &gt; <code>Advanced Search</code></li> <li>Directly at <code>/advanced-search</code></li> </ul> <p></p> <p>The default Advanced Search Page includes the following:</p> <ul> <li>Advanced Fulltext Search form (more information on the out-of-the-box View settings associated with this form below)</li> <li>Five Facets for narrowing your results (Date of Original, Digital Object Type, Collection Membership, Subjects, and Agents)</li> <li>A Facet Summary section (visible only after selecting Facets)</li> <li>Custom Block containing a Default Advanced Search Note</li> </ul> <p></p> <p>The five Facets associated with the default Advanced Search can be found at <code>Administration &gt; Configuration &gt; Search and metadata &gt; Facets</code> under the <code>Facet source - search_api:views_page__advanced_search__page_1</code>.</p> <p>The Custom Block containing the Default Advanced Search Note can be found at <code>/admin/content/block</code> or as a Tab on the main Content page.</p> <p>The corresponding Facet Blocks and Custom Block can be found at <code>/admin/structure/block/list/archipelago_subtheme</code> in the <code>Sidebar Second</code> section. Please see the this documentation for more information about Keyname Providers, Solr Fields, and Facet Configurations</p>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#advanced-search-view","title":"Advanced Search View","text":"<p>The default Advanced Search View drives the Advanced Search Form. This View can be found at:</p> <ul> <li><code>/admin/structure/views/view/advanced_search</code></li> <li>Through the <code>Structure</code> menu &gt; <code>Views</code> &gt; <code>Advanced Search</code></li> </ul> <p></p>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#configuring-the-filter-criteria-for-the-search-advanced-fulltext-search","title":"Configuring the filter criteria for the 'Search: Advanced Fulltext search'","text":"<p>To view or adjust the configurations click on 'Search: Advanced Fulltext search (and)' in the 'Filter criteria section in the View. You will then see the first part of the following configuration form:</p> <p></p>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#part-1-of-the-advanced-search-filter-form","title":"Part 1 of the Advanced Search Filter Form","text":"<p>Beginning at the top of the form, you will see:</p> <ul> <li>Brief note explaining that the Advanced Fulltext search filter can 'Search several or all fulltext fields at once allowing also multiple operator separated entries'</li> <li>Option to 'Expose this filter to visitors, to allow them to change it'<ul> <li>checked in default</li> </ul> </li> <li>Option to marked as 'Required'<ul> <li>unchecked in default</li> </ul> </li> <li>Label: Text field to enter Label for this filter<ul> <li>set to 'Advanced Fulltext search' in default</li> </ul> </li> <li>Description: Text field to enter brief description for this filter.<ul> <li>Tokens are allowed in this field. Replacement options can be found in the \"Global replacement patterns\" section, below.</li> <li>empty in default</li> </ul> </li> </ul> <p>You will then see a section for configuring the starting Operator to use for the form:</p> <ul> <li>Contains all of these words<ul> <li>checked in default</li> </ul> </li> <li>Contains any of these words</li> <li>Contains none of these words</li> <li>A note that 'Depending on the parse mode set, some of these options might not work as expected. Please either use \"Multiple words\" as the parse mode or make sure that the filter behaves as expected for multiple words.'</li> <li>An optional starting <code>Value</code><ul> <li>empty in default</li> </ul> </li> </ul> <p>Next is the beginning of the fuller Operator and Additional Options:</p> <ul> <li>Option to 'Expose operator' <ul> <li>Allow the user to choose the operator</li> <li>checked in default</li> </ul> </li> <li>Option to 'Limit the available operators'<ul> <li>Limit the available operators to be shown on the exposed filter</li> <li>unchecked checked in default</li> <li>recommended to enable for Classic Mode (see further notes related to Classic Mode below)</li> </ul> </li> <li>Operator identifier: Text field to specify 'the URL after the ? to identify this operator'<ul> <li>set to 'sbf_advanced_search_api_fulltext_op' in default</li> </ul> </li> <li>Option to 'Allow multiple selections' (Enable to allow users to select multiple items)</li> <li>Option to 'Remember the last selection' (Enable to remember the last selection made by the user)</li> <li>Parse mode : dropdown menu to choose how the search keys will be parsed<ul> <li>Direct query</li> <li>Single phrase</li> <li>Multiple words *selected by default and strongly recommended to keep</li> <li>Multiple words with EDisMax</li> <li>Multiple words with fuzziness</li> <li>Phrase search with sloppiness Multiple words with sloppiness</li> </ul> </li> <li>Filter identifier : Text field to specify what will appear in the URL after the ? to identify this filter<ul> <li>Cannot be blank. Only letters, digits and the dot (\".\"), hyphen (\"-\"), underscore (\"_\"), and tilde (\"~\") characters are allowed.</li> <li>set to 'sbf_advanced_search_api_fulltext' in default</li> </ul> </li> </ul>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#part-2-of-the-advanced-search-filter-form","title":"Part 2 of the Advanced Search Filter Form","text":"<ul> <li>Placeholder : Text field to enter hint text that appears inside the field when empty<ul> <li>empty in default</li> </ul> </li> <li>Note about the 'Multiple words' parse mode (selected earlier in the form)<ul> <li>The query is interpreted as multiple keywords separated by spaces. Keywords containing spaces may be \"quoted\". Quoted keywords must still be separated by spaces. Keywords can be negated by prepending a minus sign (-) to them.</li> </ul> </li> <li>Search field character limit : specify the maximum number of characters to allow as keywords input.<ul> <li>set to 128 in default</li> </ul> </li> <li>Option to 'Expose searched fields', which allows users to narrow the search to the desired fields.<ul> <li>checked in default</li> </ul> </li> <li>Searched fields identifier : text field to specify the URL after the ? to identify this searched fields form element.<ul> <li>set to 'sbf_advanced_search_api_fulltext_searched_fields' in default</li> </ul> </li> <li>Option to have 'Multiple/add more Search Fields'<ul> <li>This allows users to add more Search Fields with the same general exposed settings. All identifiers passed by this filter in the URL after the ? will get an incremental suffix.</li> <li>checked in default</li> </ul> </li> <li> <p>Max number of Multiple/add more Search Fields the user can expose.</p> <ul> <li>The number of additional search Fields with the same general exposed settings the user will be able to expose.</li> <li>set to a value of '4' in default</li> </ul> </li> <li> <p>Searched fields</p> <ul> <li>Select the fields that will be searched. If no fields are selected, all available fulltext fields will be searched.</li> <li>The options exposed here will be dependent on your configured Solr Fields.</li> <li>Fields selected by default include: <ol> <li>Content &gt; Strawberry (Descriptive Metadata source) &gt; local_identifier</li> <li>Content &gt; Strawberry (Descriptive Metadata source) &gt; name</li> <li>Rendered HTML output</li> <li>Content &gt; Title FullText</li> </ol> </li> </ul> </li> <li> <p>Min number of Multiple/add more Search Fields the user will see.</p> <ul> <li>Number must be less or equal to the max. If not it will cap automatically.</li> <li>The number of search Fields with the same general exposed settings the user will see by default.</li> <li>set to a value of '1' in default</li> </ul> </li> <li> <p>Minimum keyword length</p> <ul> <li>Minimum length of each word in the search keys. Leave empty to allow all words.</li> <li>empty in default</li> </ul> </li> </ul>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#part-3-of-the-advanced-search-filter-form","title":"Part 3 of the Advanced Search Filter Form","text":"<p>In the last part of the form, you will see:</p> <ul> <li> <p>Label to be used for the \"add one\" button</p> <ul> <li>\"Label to be used for the \"add more\" button. By default it is \"add one\" if left empty</li> <li>empty in default</li> </ul> </li> <li> <p>Replacement pattern for user facing Fields.</p> <ul> <li>In this text area, you will have the option to supply your preferred user-facing label for your selected search fields.</li> <li>You will see all of the fields available in your Solr.</li> <li>You supply your preferred labels and set the user-facing dropdown ordering for your targeted fields being used.</li> <li>Use a Pipe (|) to separate value from desired label. One per line</li> <li>To adjust the label for a field, replace the text following the pipe (|) symbol in the string to your preferred label.</li> <li>Field label adjustments and order specified selected by default include:<ol> <li>rendered_item|Rendered HTML Output</li> <li>title|Title (full text)</li> <li>name_1|Names/Agents</li> <li>local_identifier|Local Identifier</li> </ol> </li> </ul> </li> </ul> <p></p> <ul> <li>Label to be used for the \"remove one\" button<ul> <li>\"Label to be used for the \"add one\" button. By default it is \"remove one\" if left empty</li> <li>set to 'remove' in default</li> </ul> </li> <li>Option to 'Expose between search fields operator'<ul> <li>Allow the user to choose the operator between Multiple Search Fields (AND/OR)</li> <li>checked in default    </li> </ul> </li> </ul>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#classic-mode","title":"Classic Mode","text":"<ul> <li>Option to 'Use Classic Mode' : When you select the 'Classic Mode', your Advanced Search form will interact in a way that mimics older catalog search interfaces. This means that:<ul> <li>when you add/remove fields, this does not trigger automatic refreshing of the Search results</li> <li>search only triggers on the default Form Filter submit button</li> <li>this mode fully depends on Javascript to get around Views Exposed Filters in forms always submitting on any interaction.</li> <li>unchecked in default</li> </ul> </li> </ul> <p>If you specify to 'Use Classic Mode', you will also see:</p> <ul> <li> <p>Option to  'Add a Remove button to every Advanced Search Field combo.'</p> <ul> <li>This will allow a user to remove a specific Advanced Search Field/And/or/Text. Only works on Classic Mode.</li> <li>only appears on form when 'Use Classic Mode' is enabled, hidden and unchecked in default</li> <li>strongly recommended to enable for Classic Mode setups</li> </ul> </li> <li> <p>Please see above note in Part 1 of the Advanced Search Filter Form to also select the option to 'Limit the available operators' when using Classic Mode.  </p> </li> </ul>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#end-of-advanced-search-filter-form","title":"End of Advanced Search Filter Form","text":"<p>For both the normal, modern Advanced Search setup and also with Classic Mode enabled, you will also see:</p> <ul> <li> <p>Multiple/add more Search Fields operator (AND/OR) fields identifier</p> <ul> <li>Text field to specify what will appear in the URL after the ? to identify the operator used in the filter when multiple Search Fields and their extra options are exposed.</li> <li>set to 'sbf_advanced_search_api_fulltext_group_operator' in default</li> </ul> </li> <li> <p>You will also see a section for 'Global replacement patterns (for description field only), where you can Brose available tokens for this purpose.</p> <ul> <li>No replacement patterns specified for the View Description field in default. Not recommended.</li> </ul> </li> </ul>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_advanced/#example-query-test","title":"Example Query Test","text":"<p>In your local or fresh live Archipelago deployment, populated with the Archipelago Demo Set Objects and Collections, test your Advanced Search setup using the following query: - 'Contains all of these words' + the term 'Art' in the 'Rendered HTML Output' Field - 'OR' operator + 'Contains all of these words' + the term 'Diego' in the 'Names/Agents' field - Select the 'Archipelago Demo Collection' from the Collection Membership Facet</p> <p>You should see the following results:</p> <p></p> <p>The corresponding advanced search url should be: - http://localhost:8001/advanced-search?sbf_advanced_search_api_fulltext_op=and&amp;sbf_advanced_search_api_fulltext=Art&amp;sbf_advanced_search_api_fulltext_searched_fields=rendered_item&amp;sbf_advanced_search_api_fulltext_advanced_search_fields_count=2&amp;sbf_advanced_search_api_fulltext_group_operator_1=or&amp;sbf_advanced_search_api_fulltext_op_1=and&amp;sbf_advanced_search_api_fulltext_1=Diego&amp;sbf_advanced_search_api_fulltext_searched_fields_1=name_1&amp;f%5B0%5D=is_member_of_content_title%3AArchipelago+Demo+Collection&amp;op=Search</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Search","Search API","Solr","Solr Fields","Solr Index","Facets","Strawberry Key Name Providers"]},{"location":"search_solr_index/","title":"Search and Solr","text":"<p>Archipelago's default Search and Solr configurations are intended to cover the most common needs for a typical repository search experience.</p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#preamble-prerequisites","title":"Preamble + prerequisites","text":"<p>Before diving into any Search and Solr configuration changes, we strongly recommend that you read our Metadata in Archipelago overview documentation, which provides important context for understanding how the shape of your Archipelago Digital Objects/Collections (ADOs) metadata will inform your Search and Solr options and outcomes.</p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#archipelago-and-solr","title":"Archipelago and Solr","text":"<p>Archipelago's latest Release (1.4.0) uses Apache Solr 9.2, which incorporates some major improvements and changes from Solr 8. Please refer to the primary Solr documentation for the most comprehensive and in-depth information about Solr's wide breadth of functionality and configuration options.</p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#ocr-highlights","title":"OCR Highlights","text":"<p>Archipelago uses solr-ocrhighlighting v0.8.4, built by the Development Team at the Bavarian State Library. See our Strawberry Runners Post-Processing documentation for more information about configuring page-based HOCR/OCR extraction for image and pdf-based ADOs and options for sending that output to the Search API.</p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#in-a-nutshell-json-data-to-strawberry-keyname-providers-to-solr","title":"In-a-nutshell : JSON data to Strawberry Keyname Providers to Solr","text":"<p>If you don't have the bandwidth to read the (stellar) Metadata in Archipelago documentation, focus on the following in-a-nutshell understanding of the way Archipelago's Search and Solr is crafted. Then follow the step-by-step instructions found in Strawberry Key Name Providers, Solr Field, and Facet Configuration to get started customizing your Search setup.</p> <p></p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#1-json-data-for-your-archipelago-digital-objects","title":"1. JSON Data for your Archipelago Digital Objects","text":"<p>You need to have your descriptive metadata in JSON keys and values for ADOs and Collections in your Archipelago. Your JSON metadata and (extracted technical) data is the crucial source for Search and Solr.</p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#2-strawberry-keyname-providers","title":"2. Strawberry Keyname Providers","text":"<p>You need to have corresponding Strawberry Keyname Providers configured to feed the values from your desired JSON keys to Solr fields. One of the most powerful tools in Archipelago's Search functionality comes from the extensibility of Strawberry Keyname Providers, which can be used to source specific data points from a variety of JSON keys found in your repository.</p> More about Strawberry Keyname Providers","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#excerpted-from-metadata-in-archipelago","title":"Excerpted from Metadata in Archipelago","text":"<p>One of the challenges of our flexible approach is how to allow Drupal to access the JSON in a way, as native as possible, to generate filtered listings via Drupal Views, free text Search and Faceting. To make this happen Strawberry Field uses a JSON Querying and Exposing as \"Native Field Properties\" logic. Through a special type of Plugin system named Strawberry Key Name Providers and associated Configuration Entities (can be found at /admin/structure/strawberry_keynameprovider), you have control on which keys and values of your JSON are going to be exposed as field properties of any Strawberry Field, allowing Drupal through this to access values in a flat manner and expose them to the Search API natively. The access to the values of any JSON is done via JMESPATH expressions and then transformed either to a list of values or even \"cast\" into more complex data Data types, like an Entity Reference (means a connection to another Entity). This gives you a lot of power and control and makes a lot of very heavy operations lighter. You can even plan upfront or evolve these properties in time.</p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#3-solr-fields","title":"3. Solr Fields","text":"<p>You need to configure your desired Solr Fields to source from the Strawberry Keynames you have configured. By default, Archipelago also provider Solr Fields sourced from your HOCR data and the Rendered HTML output of your ADOs.</p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#4-drupal-views","title":"4. Drupal Views","text":"<p>For your regular Search, Advanced Search, and potentially other specialized Views, you can configure to search within specific and/or a variety of Solr Fields. </p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#5-search-results-facets","title":"5. Search Results &amp; Facets","text":"<p>Your metadata and data, as configured through Keyname Providers and Fields, Facets and Facet Blocks, indexed into your Solr. </p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"search_solr_index/#instructions-and-guides","title":"Instructions and Guides","text":"<ul> <li>Strawberry Key Name Providers, Solr Field, and Facet Configuration </li> <li>Advanced Search</li> <li>How to Add a 'Search Within Collection' Block</li> <li>IIIF Content Search</li> <li>For Live Archipelago Deployment Site Administrators: Upgrading Solr</li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Search","Search API","Solr","Solr Index","Facets","Strawberry Key Name Providers","OCR","HOCR"]},{"location":"security_bots/","title":"Managing Bots","text":"<p>A public-facing production instance will likely encounter bad bots and other malicious traffic that will consume resources. There are many solutions available that address a variety of different needs, but we provide basic configurations and a Docker image for integrating the NGINX Ultimate Bad Bot &amp; Referrer Blocker.</p> <p>Warning</p> <p>Before proceeding, please be sure to familiarize yourself with the NGINX Ultimate Bad Bot &amp; Referrer Blocker README.</p>","tags":["Security","Bots"]},{"location":"security_bots/#deployment","title":"Deployment","text":"<ol> <li>Uncomment or add the following docker-compose environment variables, replacing any appropriate values with your own and leaving the blocker and cron disabled to start (see highlighted lines):    .env<pre><code>MSMTP_ACCOUNT=SMTP_ACCOUNT_NAME\nMSMTP_EMAIL=repositorysupport@metro.org\nMSMTP_HOST=smtp.metro.org\nMSMTP_PASSWORD=YOUR_SMTP_PASSWORD\nMSMTP_PORT=SMTP_PORT\nMSMTP_STARTTLS=on\nNGXBLOCKER_ENABLE=false\nNGXBLOCKER_CRON=00 22 * * *\nNGXBLOCKER_CRON_COMMAND=/usr/local/sbin/update-ngxblocker -x\nNGXBLOCKER_CRON_START=false\n</code></pre></li> <li>Uncomment or add the following lines and comment out the line for the original NGINX image:    docker-compose.yml<pre><code># Run docker-compose up -d\n# Docker file for Arm64 and Apple M1 machines\nversion: '3.5'\nservices:\n  web:\n    container_name: esmero-web\n    # image: jonasal/nginx-certbot\n    image: esmero/nginx-bot-blocker:1.1.0-multiarch\n    restart: always\n    environment:\n      CERTBOT_EMAIL: ${ARCHIPELAGO_EMAIL}\n      ENVSUBST_VARS: FQDN\n      FQDN: ${ARCHIPELAGO_DOMAIN}\n      NGINX_ENVSUBST_OUTPUT_DIR: /etc/nginx/user_conf.d\n      MSMTP_ACCOUNT: ${MSMTP_ACCOUNT}\n      MSMTP_EMAIL: ${MSMTP_EMAIL}\n      MSMTP_HOST: ${MSMTP_HOST}\n      MSMTP_PASSWORD: ${MSMTP_PASSWORD}\n      MSMTP_PORT: ${MSMTP_PORT}\n      MSMTP_STARTTLS: ${MSMTP_STARTTLS}\n      NGXBLOCKER_CRON: ${NGXBLOCKER_CRON}\n      NGXBLOCKER_CRON_COMMAND: ${NGXBLOCKER_CRON_COMMAND}\n      NGXBLOCKER_CRON_START: ${NGXBLOCKER_CRON_START}\n      NGXBLOCKER_ENABLE: ${NGXBLOCKER_ENABLE}\n    ports:\n      - \"80:80\"\n      - \"443:443\"\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/config_storage/nginxconfig/template:/etc/nginx/templates\n      - ${ARCHIPELAGO_ROOT}/drupal:/var/www/html:cached\n      - ${ARCHIPELAGO_ROOT}/data_storage/ngnixcache:/var/cache/nginx\n      - ${ARCHIPELAGO_ROOT}/data_storage/letsencrypt:/etc/letsencrypt\n      - ${ARCHIPELAGO_ROOT}/config_storage/nginxconfig/bots.d:/etc/nginx/bots.d\n</code></pre></li> <li> <p>First pull the new image:    <pre><code>docker compose pull\n</code></pre></p> <p>Note</p> <p>If using an older version of docker, don't forget the hyphen: <pre><code>docker-compose pull\n</code></pre></p> </li> <li> <p>Now bring the Docker ensemble down and up again:    <pre><code>docker compose down &amp;&amp; docker compose up -d\n</code></pre></p> <p>Note</p> <p>If using an older version of docker, don't forget the hyphen: <pre><code>docker-compose down &amp;&amp; docker-compose up -d\n</code></pre></p> </li> <li> <p>Run the install script for the bot blocker in the default dry run mode and review the output:    <pre><code>docker exec -ti esmero-web bash -c \"/usr/local/sbin/install-ngxblocker\"\n</code></pre></p> </li> <li>The script will output the changes that are going to be made. Review them carefully and ensure that they are ok to make. Then run the command with the execute flag:    <pre><code>docker exec -ti esmero-web bash -c \"/usr/local/sbin/install-ngxblocker -x\"\n</code></pre></li> <li>Run the setup script for the bot blocker in the default dry run mode and review the output:    <pre><code>docker exec -ti esmero-web bash -c \"/usr/local/sbin/setup-ngxblocker -v /etc/nginx/templates -e .copy\"\n</code></pre></li> <li>The script will output the NGINX configuration changes that are going to be made. Review them carefully and ensure that they are ok to make. Then run the command with the execute flag:    <pre><code>docker exec -ti esmero-web bash -c \"/usr/local/sbin/setup-ngxblocker -v /etc/nginx/templates -e .copy -x\"\n</code></pre></li> <li> <p>Enable the bot blocker and cron (if applicable):    .env<pre><code>MSMTP_ACCOUNT=SMTP_ACCOUNT_NAME\nMSMTP_EMAIL=repositorysupport@metro.org\nMSMTP_HOST=smtp.metro.org\nMSMTP_PASSWORD=YOUR_SMTP_PASSWORD\nMSMTP_PORT=SMTP_PORT\nMSMTP_STARTTLS=on\nNGXBLOCKER_ENABLE=true\nNGXBLOCKER_CRON=00 22 * * *\nNGXBLOCKER_CRON_COMMAND=/usr/local/sbin/update-ngxblocker -x\nNGXBLOCKER_CRON_START=true\n</code></pre></p> <p>Note</p> <p>If <code>MSMTP_EMAIL</code> is blank and cron is enabled the flag for sending email notifications will be skipped.</p> </li> <li> <p>Bring the Docker ensemble down and back up again:    <pre><code>docker compose down &amp;&amp; docker compose up -d\n</code></pre></p> <p>Note</p> <p>If using an older version of docker, don't forget the hyphen: <pre><code>docker-compose down &amp;&amp; docker-compose up -d\n</code></pre></p> </li> <li> <p>Test that it is working by following the \"TESTING\" section (STEP 10) in the official documentation: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker</p> </li> </ol>","tags":["Security","Bots"]},{"location":"security_bots/#deployment-with-upgrade","title":"Deployment with Upgrade","text":"<p>If looking to use this solution as part of an upgrade (from 1.0.0 to 1.1.0, for example) we recommend coming back to the above steps after successfully completing the upgrade. After the upgrade, you will only need to add the environment variables and docker compose configurations and follow the steps as detailed above.</p>","tags":["Security","Bots"]},{"location":"security_bots/#advanced-configuration","title":"Advanced Configuration","text":"<p>Because our Docker containers only persist our mounted files and folders, any advanced configurations may require overriding the files generated by our esmero-web container on boot. For example, the above <code>setup-ngxblocker</code> script is normally responsible for writing the following include lines:</p> <pre><code>include /etc/nginx/bots.d/blockbots.conf;\ninclude /etc/nginx/bots.d/ddos.conf;\n</code></pre> <p>Because the script is unable to place them in the correct part of our <code>nginx.conf.template</code> file, which in turn generates our <code>nginx.conf</code> file (see Using environment variables in nginx configuration), our own script adds (when <code>NGINXBLOCKER_ENABLE=true</code>) or removes (when <code>NGINXBLOCKER_ENABLE=false</code>) the lines to an empty file, which in turn is statically included in our main <code>nginx.conf.template</code> file. One option provided by <code>setup-ngxblocker</code> is to exclude (<code>-d</code>) the DDOS rule. In our case, we need to manually override the lines in our template file to reproduce this behavior:</p> <p>Example</p> nginx.conf.template<pre><code>upstream cantaloupe {\n  server esmero-cantaloupe:8182;\n}\n\nserver {\n    listen              443 ssl;\n    server_name         ${FQDN};\n    ssl_certificate     /etc/letsencrypt/live/${FQDN}/fullchain.pem;\n    ssl_certificate_key /etc/letsencrypt/live/${FQDN}/privkey.pem;\n    client_max_body_size 1536M; ## Match with PHP from FPM container\n\n    root /var/www/html/web; ## &lt;-- Your only path reference.\n\n    fastcgi_send_timeout 120s;\n    fastcgi_read_timeout 120s;\n    fastcgi_pass_request_headers on;\n\n    fastcgi_buffers 16 16k;\n    fastcgi_buffer_size 32k;\n\n    # Please adapt to your needs\n    proxy_buffers 16 16k;  \n    proxy_buffer_size 16k;\n\n    #include /etc/nginx/conf.d/bots.include;\n    include /etc/nginx/bots.d/blockbots.conf;\n</code></pre> <p>Note</p> <p>Keep in mind that from this point, when disabling/enabling the bot blocker via the environment variable, you'll also need to uncomment/comment the added line. </p> <p>Another more generally applicable approach is to override files that are part of the docker image:</p> <p>Example</p> <p>Our bash script (setup_bot_blocker.sh) is triggered by and runs just before the startup script (start_nginx_certbot.sh) for the NGINX Certbot image. For any advanced needs involving custom startup behavior, our script can be modified and overwridden:</p> <ol> <li>First, we'll copy the script from the running Docker container onto our host:    <pre><code>docker cp esmero-web:/scripts/setup_bot_blocker.sh drupal/scripts/archipelago/\n</code></pre></li> <li>Then we mount the file to override the container's file:    docker-compose.yml<pre><code># Run docker-compose up -d\n# Docker file for Arm64 and Apple M1 machines\nversion: '3.5'\nservices:\n  web:\n    container_name: esmero-web\n    # image: jonasal/nginx-certbot\n    image: esmero/nginx-bot-blocker:1.1.0-multiarch\n    restart: always\n    environment:\n      CERTBOT_EMAIL: ${ARCHIPELAGO_EMAIL}\n      ENVSUBST_VARS: FQDN\n      FQDN: ${ARCHIPELAGO_DOMAIN}\n      NGINX_ENVSUBST_OUTPUT_DIR: /etc/nginx/user_conf.d\n      MSMTP_ACCOUNT: ${MSMTP_ACCOUNT}\n      MSMTP_EMAIL: ${MSMTP_EMAIL}\n      MSMTP_HOST: ${MSMTP_HOST}\n      MSMTP_PASSWORD: ${MSMTP_PASSWORD}\n      MSMTP_PORT: ${MSMTP_PORT}\n      MSMTP_STARTTLS: ${MSMTP_STARTTLS}\n      NGXBLOCKER_CRON: ${NGXBLOCKER_CRON}\n      NGXBLOCKER_CRON_COMMAND: ${NGXBLOCKER_CRON_COMMAND}\n      NGXBLOCKER_CRON_START: ${NGXBLOCKER_CRON_START}\n      NGXBLOCKER_ENABLE: ${NGXBLOCKER_ENABLE}\n    ports:\n      - \"80:80\"\n      - \"443:443\"\n    volumes:\n      - ${ARCHIPELAGO_ROOT}/config_storage/nginxconfig/template:/etc/nginx/templates\n      - ${ARCHIPELAGO_ROOT}/drupal:/var/www/html:cached\n      - ${ARCHIPELAGO_ROOT}/data_storage/ngnixcache:/var/cache/nginx\n      - ${ARCHIPELAGO_ROOT}/data_storage/letsencrypt:/etc/letsencrypt\n      - ${ARCHIPELAGO_ROOT}/config_storage/nginxconfig/bots.d:/etc/nginx/bots.d\n      - ${ARCHIPELAGO_ROOT}/drupal/scripts/archipelago/setup_bot_blocker.sh:/scripts/setup_bot_blocker.sh\n</code></pre></li> <li>Then we modify our script copy (we'll reproduce the same behavior from the previous example but incorporate it into our script): drupal/scripts/archipelago/setup_bot_blocker.sh<pre><code>#!/bin/bash\n\nset -e\n\nif [ ! -z \"${MSMTP_EMAIL}\" ]; then\n    envsubst &lt; /root/.msmtprc.template &gt; /root/.msmtprc\nfi\n\nif [ \"${NGXBLOCKER_CRON_START}\" = true ]; then\n    if [ ! -z \"${MSMTP_EMAIL}\" ]; then\n      CRON_COMMAND=\"${NGXBLOCKER_CRON} ${NGXBLOCKER_CRON_COMMAND} -e ${MSMTP_EMAIL}\"\n    else\n      CRON_COMMAND=\"${NGXBLOCKER_CRON} ${NGXBLOCKER_CRON_COMMAND} -n\"\n    fi\n    echo \"${CRON_COMMAND}\" | crontab - &amp;&amp;\n    /etc/init.d/cron start\nfi\n\nif [ ! -f /etc/nginx/templates/bots.include.copy ]; then\n    touch /etc/nginx/templates/bots.include.copy\nfi\nif [ ! -f /etc/nginx/templates/bots.include.template ]; then\n    touch /etc/nginx/templates/bots.include.template\nfi\n\nif [ \"${NGXBLOCKER_ENABLE}\" = true ]; then\n    if [ ! -L /etc/nginx/conf.d/botblocker-nginx-settings.conf ]; then\n        ln -s /etc/nginx/bots_settings_conf.d/botblocker-nginx-settings.conf /etc/nginx/conf.d/botblocker-nginx-settings.conf\n    fi\n    if [ ! -L /etc/nginx/conf.d/globalblacklist.conf ]; then\n        ln -s /etc/nginx/bots_settings_conf.d/globalblacklist.conf /etc/nginx/conf.d/globalblacklist.conf\n    fi\n    if ! grep -q blockbots.conf /etc/nginx/templates/bots.include.copy; then\n        echo \"include /etc/nginx/bots.d/blockbots.conf;\" &gt;&gt; /etc/nginx/templates/bots.include.copy\n    fi\n    #if ! grep -q ddos.conf /etc/nginx/templates/bots.include.copy; then\n    #    echo \"include /etc/nginx/bots.d/ddos.conf;\" &gt;&gt; /etc/nginx/templates/bots.include.copy\n    #fi\n    if ! grep -q blockbots.conf /etc/nginx/user_conf.d/bots.include; then\n        echo \"include /etc/nginx/bots.d/blockbots.conf;\" &gt;&gt; /etc/nginx/user_conf.d/bots.include\n    fi\n    #if ! grep -q ddos.conf /etc/nginx/user_conf.d/bots.include; then\n    #    echo \"include /etc/nginx/bots.d/ddos.conf;\" &gt;&gt; /etc/nginx/user_conf.d/bots.include\n    #fi\n    cp /etc/nginx/templates/bots.include.copy /etc/nginx/templates/bots.include.template\nelse\n    &gt;|/etc/nginx/templates/bots.include.template\n    &gt;|/etc/nginx/user_conf.d/bots.include\n    if [ -L /etc/nginx/conf.d/botblocker-nginx-settings.conf ]; then\n        rm /etc/nginx/conf.d/botblocker-nginx-settings.conf\n    fi\n    if [ -L /etc/nginx/conf.d/globalblacklist.conf ]; then\n        rm /etc/nginx/conf.d/globalblacklist.conf\n    fi\nfi\n</code></pre></li> <li>Next we remove the <code>include</code> line from the existing files:    <pre><code>docker exec -ti esmero-web bash -c \"sed -i '/include \\/etc\\/nginx\\/bots.d\\/ddos.conf/d' /etc/nginx/templates/bots.include.copy\"\n</code></pre> <pre><code>docker exec -ti esmero-web bash -c \"sed -i '/include \\/etc\\/nginx\\/bots.d\\/ddos.conf/d' /etc/nginx/templates/bots.include.template\"\n</code></pre> <pre><code>docker exec -ti esmero-web bash -c \"sed -i '/include \\/etc\\/nginx\\/bots.d\\/ddos.conf/d' /etc/nginx/user_conf.d/bots.include\"\n</code></pre></li> <li> <p>Finally we bring the Docker ensemble down and back up again to propagate the changes in our container:    <pre><code>docker compose down &amp;&amp; docker compose up -d\n</code></pre></p> <p>Note</p> <p>If using an older version of docker, don't forget the hyphen: <pre><code>docker-compose down &amp;&amp; docker-compose up -d\n</code></pre></p> </li> </ol> <p>The above is an example of a more complicated customization, but it's a pattern that can be used more generally throughout the Docker containers, i.e.:</p> <ol> <li>Copy the file that needs to be overwridden from the Docker container to the host and make custom changes.</li> <li>Mount the file from the host to the location within the docker container, e.g.:    <pre><code>- ${ARCHIPELAGO_ROOT}/LOCATION_ON_HOST/CUSTOMIZED_FILE_ON_HOST:/LOCATION_IN_DOCKER_CONTAINER/FILE_IN_DOCKER_CONTAINER\n</code></pre></li> <li>Bring the Docker ensemble down and bring it up again.</li> </ol>","tags":["Security","Bots"]},{"location":"sslsetup/","title":"How to Setup SSL for Docker/Archipelago","text":"<p>Work-In-Progress Note This documentation page is still under construction and content may change with future updates. Please use caution when implementing any instructions referenced herein, as there may be missing steps or corresponding configuration files. Thank you for your patience as we continue to update Archipelago's documentation.</p> <p>The steps found below describe one potential manual SSL configuration for Archipelago deployments. A <code>git clone</code> deployment option will be available for future releases.</p>"},{"location":"sslsetup/#manual-configuration-steps-for-an-ec2-aws-server","title":"Manual Configuration Steps for an EC2 AWS Server","text":"<p>This process takes less than 10 minutes of reading YML files and editing a few files (described below) to get SSL running and setup with auto-renewal.</p> <ol> <li> <p>First, configure Certbot, following the instructions found on https://certbot.eff.org.</p> </li> <li> <p>Inside a /persistent partition, establish the following folder structure. Note: you can keep the existing folder structure if you so choose. A benefit of the following structure is that it decouples the git clone of archipelago-deployment, which is made to be self sustainable and good for coding or smaller deployments.</p> <pre><code>[ec2-user@ip-17x-xx-x-xxx persistent]$ ls -lah\ntotal 64K\ndrwxr-xr-x 14 root           root  4.0K Oct  5 23:11 .\ndr-xr-xr-x 19 root           root  275 Dec 15  2019 ..\ndrwxr-xr-x  8       999  999   4096 Oct 13 20:07 db\ndrwxr-xr-x 13 root           root  4.0K Oct  5 23:03 drupal8\ndrwxr-xr-x  5           8183  8183   4.0K Feb 23  2020 iiifcache\ndrwxr-xr-x  2 root           root  4.0K Feb 23  2020 iiifconfig\ndrwxr-xr-x  4 root           root  4.0K Oct  5 22:45 nginx_conf\ndrwxr-xr-x  3 root           root  4.0K Feb 26  2019 solrconfig\ndrwxr-xr-x  3           8983 8983  4.0K Feb 26  2019 solrcore\n</code></pre> <p>To get to this point, create a git clone of archipelago deployment and then copy the content of the /persistent out of the repo folder into this structure. The original (or what is left) archipelago-deployment ends inside a drupal8 folder here.</p> </li> <li> <p>Copy and paste the following to create a local copy of this file:</p> docker-compose.yml <p>**Be sure to replace youremail@gmail.com with your email address.</p> <pre><code> version: '3.5'\n services:\n   web:\n     container_name: esmero-web\n     image: staticfloat/nginx-certbot\n     restart: always\n     environment:\n       CERTBOT_EMAIL: \"youremail@gmail.com\"\n     ports:\n       - \"80:80\"\n       - \"443:443\"\n     volumes:\n       - /persistent/nginx_conf/conf.d:/etc/nginx/user.conf.d:ro\n       - /persistent/nginx_conf/certbot_extra_domains:/etc/nginx/certbot/extra_domains:ro\n       - /persistent/drupal8:/var/www/html:cached\n     depends_on:\n       - solr\n       - php\n     tty: true\n     networks:\n       - host-net\n       - esmero-net\n   php:\n     container_name: esmero-php\n     restart: always\n     image: \"esmero/php-7.3-fpm:latest\"\n     tty: true\n     networks:\n       - host-net\n       - esmero-net\n     volumes:\n       - ${PWD}:/var/www/html:cached\n   solr:\n     container_name: esmero-solr\n     restart: always\n     image: \"solr:7.5.0\"\n     tty: true\n     ports:\n       - \"8983:8983\"\n     networks:\n       - host-net\n       - esmero-net\n     volumes:\n       - /persistent/solrcore:/opt/solr/server/solr/mycores:cached\n       - /persistent/solrconfig:/drupalconfig:cached\n     entrypoint:\n       - docker-entrypoint.sh\n       - solr-precreate\n       - drupal\n       - /drupalconfig\n   # see https://hub.docker.com/_/mysql/\n   db:\n     image: mysql:5.7\n     command: --max_allowed_packet=256M\n     container_name: esmero-db\n     restart: always\n     environment:\n       MYSQL_ROOT_PASSWORD: esmerodb\n     networks:\n       - host-net\n       - esmero-net\n     volumes:\n       - /persistent/db:/var/lib/mysql:cached\n   iiif:\n     container_name: esmero-cantaloupe\n     image: \"esmero/cantaloupe-s3:4.1.6\"\n     restart: always\n     ports:\n       - \"8183:8182\"\n     networks:\n       - host-net\n       - esmero-net\n     volumes:\n       - /persistent/iiifconfig:/etc/cantaloupe\n       - /persistent/iiifcache:/var/cache/cantaloupe\n networks:\n   host-net:\n     driver: bridge\n   esmero-net:\n     driver: bridge\n     internal: true\n</code></pre> <p>Note: This file shows how the folders in Step 1 are being used, and how SSL is being automatically deployed and renewed (without any human interaction other than starting the docker-compose and watching the logs).</p> </li> <li> <p>Now copy and paste the following to create a local copy of this file:</p> ngnix.conf <p>**Be sure to replace all instances of yoursite.org with your own domain.</p> <pre><code> # goes into /persistent/nginx_conf/conf.d/nginx.conf\n upstream cantaloupe {\n  server  esmero-cantaloupe:8182;\n  }\n\n  server {\n    listen              443 ssl;\n    server_name         yoursite.org;\n    ssl_certificate     /etc/letsencrypt/live/yourstie.org/fullchain.pem;\n    ssl_certificate_key /etc/letsencrypt/live/yoursite.org/privkey.pem;\n\n     client_max_body_size 512M; ## Match with PHP from FPM container\n\n    root /var/www/html/web; ## &lt;-- Your only path reference.\n\n    fastcgi_send_timeout 120s;\n    fastcgi_read_timeout 120s;\n    fastcgi_pass_request_headers on;\n\n    fastcgi_buffers 16 16k;\n    fastcgi_buffer_size 32k;\n\n    # Cantaloupe proxypass\n    location /cantaloupe/ {\n       proxy_set_header X-Forwarded-Proto $scheme;\n       proxy_set_header X-Forwarded-Host $host;\n       proxy_set_header X-Forwarded-Port $server_port;\n       proxy_set_header X-Forwarded-Path /cantaloupe/;\n       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n       if ($request_uri ~* \"/cantaloupe/(.*)\") {\n         proxy_pass http://cantaloupe/$1;\n       }\n    }\n\n    location = /favicon.ico {\n        log_not_found off;\n        access_log off;\n    }\n\n    location = /robots.txt {\n        allow all;\n        log_not_found off;\n        access_log off;\n    }\n\n    # Very rarely should these ever be accessed outside of your lan\n    location ~* \\.(txt|log)$ {\n        deny all;\n    }\n\n    location ~ \\..*/.*\\.php$ {\n        return 403;\n    }\n\n    location ~ ^/sites/.*/private/ {\n        return 403;\n    }\n\n    # Allow \"Well-Known URIs\" as per RFC 5785\n    location ~* ^/.well-known/ {\n        allow all;\n    }\n\n    # Block access to \"hidden\" files and directories whose names begin with a\n    # period. This includes directories used by version control systems such\n    # as Subversion or Git to store control files.\n    location ~ (^|/)\\. {\n        return 403;\n    }\n\n    location / {\n        try_files $uri /index.php?$query_string; # For Drupal &gt;= 7\n    }\n\n    location @rewrite {\n        rewrite ^/(.*)$ /index.php?q=$1;\n    }\n\n    # Don't allow direct access to PHP files in the vendor directory.\n    location ~ /vendor/.*\\.php$ {\n        deny all;\n        return 404;\n    }\n\n    # Allow Modules to be updated via UI (still we believe composer is the way)    \n    rewrite ^/core/authorize.php/core/authorize.php(.*)$ /core/authorize.php$1;\n\n    # In Drupal 8, we must also match new paths where the '.php' appears in\n    # the middle, such as update.php/selection. The rule we use is strict,\n    # and only allows this pattern with the update.php front controller.\n    # This allows legacy path aliases in the form of\n    # blog/index.php/legacy-path to continue to route to Drupal nodes. If\n    # you do not have any paths like that, then you might prefer to use a\n    # laxer rule, such as:\n    #   location ~ \\.php(/|$) {\n    # The laxer rule will continue to work if Drupal uses this new URL\n    # pattern with front controllers other than update.php in a future\n    # release.\n    location ~ '\\.php$|^/update.php' {\n        fastcgi_split_path_info ^(.+?\\.php)(|/.*)$;\n        include fastcgi_params;\n        # Block httpoxy attacks. See https://httpoxy.org/.\n        fastcgi_param HTTP_PROXY \"\";\n        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;\n        fastcgi_param PATH_INFO $fastcgi_path_info;\n        fastcgi_param PHP_VALUE \"upload_max_filesize=512M \\n post_max_size=512M\";\n        proxy_read_timeout 900s;\n        fastcgi_intercept_errors on;\n        fastcgi_pass esmero-php:9000;\n    }\n\n     # Fighting with Styles? This little gem is amazing.\n    location ~ ^/sites/.*/files/styles/ { # For Drupal &gt;= 7\n        try_files $uri @rewrite;\n    }\n\n    # Handle private files through Drupal.\n    location ~ ^/system/files/ { # For Drupal &gt;= 7\n        try_files $uri /index.php?$query_string;\n    }\n}\n</code></pre> </li> <li> <p>Create the following folder:</p> <pre><code>/persistent/nginx_conf/conf.d/\n</code></pre> </li> <li> <p>Place the ngnix.conf file inside the <code>/conf.d/</code> folder.</p> </li> <li> <p>Create also this other folder:</p> <pre><code>/persistent/nginx_conf/certbot_extra_domains/\n</code></pre> </li> <li> <p>Inside the <code>/certbot_extra_domains/</code> folder, create a text file named the same way as your domain (which can/or not contain additional subdomains but needs to exist).</p> <pre><code>cat  /persistent/nginx_conf/certbot_extra_domains/yoursite.org\n</code></pre> <pre><code>drwxr-xr-x 2 root root 4.0K Oct  5 22:46 .\ndrwxr-xr-x 4 root root 4.0K Oct  5 22:45 ..\n-rw-r--r-- 1 root root   48 Oct  5 22:46 yoursite.org\n</code></pre> <p>Optionally, create additional subdomains if needed.</p> <pre><code>cat  /persistent/nginx_conf/certbot_extra_domains/yoursite.org\nsubdomain.yoursite.org\nanothersub.yoursite.org\n</code></pre> </li> <li> <p>Make sure you have edited the <code>docker-compose.yml</code> and <code>ngnix.conf</code> files you created to match your own information. Also make sure to also adjust the paths if you do not want the /persistent approach described in Step 1.</p> </li> <li> <p>Run the following commands:</p> <pre><code>docker -compose up -d\ndocker ps\n</code></pre> <p>You should see this:</p> <pre><code>b5a04747ee06        staticfloat/nginx-certbot    \"/bin/bash /scripts/\u2026\"   8 days ago          Up 8 days           0.0.0.0:80-&gt;80/tcp, 0.0.0.0:443-&gt;443/tcp   esmero-web\n84afae094b57        esmero/php-7.3-fpm:latest    \"docker-php-entrypoi\u2026\"   8 days ago          Up 8 days           9000/tcp                                   esmero-php\n13a9214acfd0        esmero/cantaloupe-s3:4.1.6   \"sh -c 'java -Dcanta\u2026\"   8 days ago          Up 8 days           0.0.0.0:8183-&gt;8182/tcp                     esmero-cantaloupe\n044dd5bc7245        mysql:5.7                    \"docker-entrypoint.s\u2026\"   8 days ago          Up 8 days           3306/tcp, 33060/tcp                        esmero-db\n31f4f0f45acc        solr:7.5.0                   \"docker-entrypoint.s\u2026\"   8 days ago          Up 8 days           0.0.0.0:8983-&gt;8983/tcp                     esmero-solr\n</code></pre> </li> <li> <p>SSL has now been configured for your Archipelago instance.</p> </li> </ol>"},{"location":"sslsetup/#user-contributed-documentation","title":"User contributed documentation:","text":"<p>Adding SSL to Archipelago running docker by Zachary Spalding: https://youtu.be/rfH5TLzIRIQ</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"strawberry_key_name_providers/","title":"Strawberry Key Name Providers, Solr Field, and Facet Configuration","text":"<p>For an overview of how Strawberry Key Name Providers fit within the context of the rest of Archipelago, please see the Drupal and JSON section in our Metadata in Archipelago overview documentation.</p> <p>In order to expose the Strawberry Field JSON keys (and values) for Archipelago Digital Objects (ADOs) to Search/Solr, Views, and Facets, we need to make use of a plugin system called Strawberry Key Name Providers. The following guide covers:</p> <ul> <li>Creating first a Strawberry Key Name Provider</li> <li>Creating a corresponding Solr Field necessary for Search and Views exposure</li> <li>Creating of Facets</li> <li>Creating of Facet blocks on your theme as needed </li> </ul>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#creating-a-strawberry-key-name-provider","title":"Creating a Strawberry Key Name Provider","text":"<ol> <li> <p>First, we'll start with an example of a Strawberry Field JSON key that we would like to expose:</p> <p>date_created_edtf</p> <pre><code>...\n\"subject_wikidata\": [\n    {\n        \"uri\": \"http:\\/\\/www.wikidata.org\\/entity\\/Q55488\",\n        \"label\": \"railway station\"\n    }\n],\n\"date_created_edtf\": {\n    \"date_to\": \"\",\n    \"date_free\": \"2016~\\/2017~\",\n    \"date_from\": \"\",\n    \"date_type\": \"date_free\"\n},\n\"date_created_free\": null,\n...\n</code></pre> </li> <li> <p>Next, we are going to create a new Strawberry Key Name Provider by going to <code>Administration &gt; Structure &gt; Strawberry Key Name Providers</code>, pressing the <code>+ Add Strawberry Key Name Provider</code> button, filling in the fields as follows, and saving:</p> <ul> <li><code>Label</code>: <code>Date Created EDTF</code></li> <li><code>Strawberry Key Name Provider Plugin</code>: <code>JmesPath Strawberry Field Key Name Provider</code></li> <li><code>One or more comma separated valid JMESPaths</code>: <code>date_created_edtf.date_free</code></li> <li>Confirm that the value for <code>Exposed Strawberry Field Property</code> (under the <code>One or more comma separated valid JMESPaths</code> field) is set to <code>date_created_edtf_date_free</code>. This is the <code>Strawberry Field Property</code> that will hold the data coming from the JMESPath Query when evaluated against and ADO's JSON and will be visible as a Strawberry Field Property to Drupal and the Search API. When doing this in a production environment, you might want to change the automatically generated value and assign a simpler one to remember. You can always do this by pressing <code>Edit</code>. But for the purpose of this documentation please keep <code>date_created_edtf_date_free</code>.</li> <li><code>Is Date?</code>: <code>\u2611</code></li> </ul> </li> </ol>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#strawberry-key-name-provider-plugins","title":"Strawberry Key Name Provider Plugins","text":"<p>You'll notice that there are four plugins, each with different options, available for different use cases. Below you'll find each plugin with examples from the providers that come with a default deployment.</p>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#entity-reference-jmespath-strawberry-field-key-name-provider","title":"Entity Reference JmesPath Strawberry Field Key Name Provider","text":"<p>ismemberof</p> <p>One or more comma separated valid JMESPaths: <code>ismemberof</code></p> <p>Entity type: <code>node</code></p>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#flavorembedded-json-service-strawberry-field-key-name-provider","title":"Flavor/Embedded JSON Service Strawberry Field Key Name Provider","text":"<p>hoCR Service</p> <p>Source JSON Key used to read the Service/Flavour: <code>ap:hocr</code></p>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#jmespath-strawberry-field-key-name-provider","title":"JmesPath Strawberry Field Key Name Provider","text":"<p>Subject Labels</p> <p>One or more comma separated valid JMESPaths: <code>subject_loc[*].label, subject_wikidata[*].label, subject_lcnaf_geographic_names[*].label,subject_temporal[*].label, subject_lcgft_terms[*].label, term_aat_getty[*].label, pubmed_mesh[*].label</code></p>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#jsonld-strawberry-field-key-name-provider","title":"JSONLD Strawberry Field Key Name Provider","text":"<p>Best Practice</p> <p>As in the example below, if there are a group of flat and unique keys that you want to expose, we recommend creating one provider with this plugin and using a list of keys instead of creating multiple providers. This Provider will also auto assign Lists of Properties from an external JSON-LD ontology/vocabulary (e.g Schema.org). It uses direct access approach, e.g. <code>type</code> will get all values for any JSON Key named <code>type</code> at any hierarchy level (across the whole JSON document) and it will also use the same exact name (<code>type</code>) for the <code>Exposed Strawberry Field Property</code>.</p> <p>schema.org</p> <p>Additional keys separated by commas: <code>ismemberof,type,hocr,city,category,country,state,display_name,author,license</code></p>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#creating-a-solr-field","title":"Creating a Solr Field","text":"<ol> <li>Go to <code>Administration &gt; Configuration &gt; Search and metadata &gt; Search API &gt; Drupal Content to Solr 9 &gt; Fields</code>.</li> <li>Press the <code>Add fields</code> button.</li> <li>Search for the field created above (expand the <code>\ud83c\udf53 Strawberry (Descriptive Metadata source) (field_descriptive_metadata)</code>, e.g. for the key mapped above, look for <code>field_descriptive_metadata:date_created_edtf_date_free</code>.</li> <li>Scroll down after adding to make sure the <code>Type</code> for the field is correct (<code>date</code> for the example in this guide).</li> <li>Reindex Solr.<ol> <li>Go to <code>Administration &gt; Configuration &gt; Search and metadata &gt; Search API</code> and click on the link to the index for your Drupal data.</li> <li>Press the <code>Queue all items for reindexing</code> button.</li> <li>Let cron reindex or press the <code>Index now</code> button.</li> </ol> </li> </ol> <p>Important Note About Facets and Facet Blocks</p> <p>You cannot reuse the same Facets or Facet Blocks for different Views display outputs. You need to create a unique Facet and corresponding Facet Block for every distinct <code>Facet Source</code> you will be using in your Archipelago. For example, if you have both <code>List</code> and <code>Grid</code> style Views setup for your Search display outputs, you will need to create Facets for your desired Subjects, etc. for both the <code>List</code> and <code>Grid</code> View Sources. You will then need to place your Facet Blocks in the appropriate region and make sure each Facet Block corresponds to the correct underlying <code>View</code> using the <code>View inclusion</code> settings for the Facet Block.</p>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#creating-a-facet","title":"Creating a Facet","text":"<ol> <li>Go to <code>Administration &gt; Configuration &gt; Search and metadata &gt; Facets</code>.</li> <li>Press the <code>+ Add facet</code> button.</li> <li>Select your facet settings. For the example in this guide, we'll select the following:<ul> <li><code>Facet source</code>: <code>View Solr search content, display Page</code></li> <li><code>Field</code>: <code>\ud83c\udf53 Strawberry (Descriptive Metadata source) &gt;&gt; date_created_edtf_date_free (field_descriptive_metadata:date_created_edtf_date_free)</code></li> <li><code>Name</code>: <code>\ud83c\udf53 Strawberry (Descriptive Metadata source) &gt;&gt; date_created_edtf_date_free</code></li> </ul> </li> <li>Save.</li> <li>Continue with the facet configuration by pressing <code>Edit</code> for the facet we just created and adjusting the many options available as needed. For the example in this guide, we'll adjust the below from the default settings:<ul> <li><code>Facet settings</code><ul> <li><code>\u2611</code> <code>Date item processor</code><ul> <li><code>Date display</code><ul> <li><code>\ud83d\udd18</code> <code>Actual date with granularity</code></li> </ul> </li> <li><code>Granularity</code><ul> <li><code>\ud83d\udd18</code> <code>Year</code></li> </ul> </li> </ul> </li> <li><code>URL alias</code>: <code>sbf_date_created_edtf</code></li> </ul> </li> </ul> </li> <li>Save.</li> </ol>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberry_key_name_providers/#creating-a-block-for-the-facet","title":"Creating a Block for the Facet","text":"<ol> <li>Go to <code>Administration &gt; Structure &gt; Block layout</code>.</li> <li>Select the appropriate theme. For the example in this guide, we'll select <code>Archipelago Base Theme</code>.</li> <li>Press the <code>Place block</code> button next to the appropriate region. For the example in this guide, we'll be placing the block in the <code>Sidebar second</code> region.</li> <li>Select your facet from the list. For the example in this guide, we'll select <code>\ud83c\udf53 Strawberry (Descriptive Metadata source) &gt;&gt; date_created_edtf_date_free</code></li> <li>Press the <code>Place block</code> button next to the facet. Once the block is added, you can drag and drop it to change its position among the existing blocks and saving.</li> </ol> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Strawberry Key Name Providers","Solr","Facets","Search"]},{"location":"strawberryfield-formatters/","title":"Strawberryfield Formatters","text":"<p>This documentation will give a brief overview of Archipelago's Strawberryfield Formatters and how they work using the default View mode <code>Digital Object Full View</code> as an example.</p>"},{"location":"strawberryfield-formatters/#at-a-glance","title":"At a glance","text":"<p>When taking a look at your First Digital Object note that multiple formatters are working together to create this <code>Display</code> ( or <code>View mode</code>). Since \"My First Digital Object\" is a <code>Photograph</code> the <code>Display</code> being used is <code>Digital Object Full View</code> which, by default, uses formatters to:</p> <ul> <li>(Red) Create the image viewer where users can zoom in, zoom out, fullscreen and rotate all the images associated with the ADO.</li> <li>(Blue) Display the <code>Object Description</code> and <code>Type of Resource</code>.</li> <li>(Green) Display the Raw JSON Metadata and IIIF Presentation Manifest.</li> </ul> <p></p>"},{"location":"strawberryfield-formatters/#in-greater-detail","title":"In Greater Detail","text":"<p>When editing an ADO, at the top of the Webform page there is a tab titled <code>Manage display</code> which will take us to where all the Formatters live. Take note that the <code>DISPLAY SETTINGS</code> shown in the screenshot below are using the Default View mode.</p> <p></p> <p>Once the page loads the <code>Default</code> View mode is automatically selected. However, because we are editing an object with the <code>Media type</code> <code>Photograph</code>, we need to edit the View mode <code>Digital Object Full View</code> since it is the Default View mode for this <code>Media type</code>.</p>"},{"location":"strawberryfield-formatters/#how-to-find-and-configure-which-view-mode-is-default-per-media-type","title":"How to find and configure which View mode is Default per Media type","text":"<p>The ADO Type to View Mode Mapping page tells the ADOs which View mode to use by default per Media type. You can find the ADO Type to View Mode Mapping Form at <code>~yoursite/admin/config/archipelago/viewmode_mapping</code></p> <p></p> Strawberryfield Formatters Shipped and Configured with Archipelago <ol> <li>Default</li> <li>Collection listing</li> <li>Digital Object Full View</li> <li>Digital Object Image Only for Carousel</li> <li>Digital Object with 3D Viewer</li> <li>Digital Object with Audio Player</li> <li>Digital Object with Book Reader</li> <li>Digital Object with Mirador Viewer</li> <li>Digital Object with PDF Viewer</li> <li>Digital Object with Pannellum Panorama</li> <li>Digital Object with Video Player</li> <li>Digital Object with Replay.web WARC Replay.web Widget</li> <li>Digital Object with thumbnail and abstract</li> <li>Digital Object with thumbnail for Grid For Digital Object Collections/Compound Objects/CreativeWorkSeries</li> <li>Digital Object Collection with Mirador Viewer</li> <li>Digital Object Creative Work Series with Mirador Viewer</li> </ol> Default View Modes <p>You can see the full list of Default View Modes included in Archipelago here, and you can access your Archipelago's ADO Type to View Mode Mapping at <code>~yoursite/admin/config/archipelago/viewmode_mapping</code>.</p> <p></p> <p>There are two sections in <code>Manage display</code> for <code>Digital Object Full View</code>: 1) Content and 2) Disabled. Moving a field into Content means this formatter will be used to the display the ADO in some way. The formatters moved to Disabled are inactive and are subsequently not being used for displaying the ADO.</p> <p>There are four fields named <code>\ud83c\udf53Strawberry</code> and each one is a copy of the field <code>\ud83c\udf53Strawberry (Descriptive Metadata source)</code>. Since the names of the fields do not imply their function, they have been named Strawberry in four different ways (Italiano, Deutsch, Din\u00e9 Bizaad, and English) in order to organize and help users visually remember which field is doing what for the <code>Display</code>.</p> <p>Recall My First Digital Object at beginning of this document where there were 3 sections highlighted in Red, Blue, and Green.</p> <ul> <li>In Red (<code>\ud83c\udf53Fragola</code>) there is the Strawberry Field Formatter for IIIF media which takes the image stored in S3 to display the photograph with the image viewer.</li> <li>In Blue (<code>\ud83c\udf53Erdbeere</code>) there is the Strawberry Field Formatter for Custom Metadata Templates which displays the raw JSON metadata using configurable Twig templates. In this example, the default Twig template uses the JSON key <code>type</code> to display the <code>Type of Resource</code>.</li> <li>In Green (<code>\ud83c\udf53Strawberry (Descriptive Metadata)</code>) there is the Strawberry Default Formatter which is used to display the Raw JSON Metadata.</li> </ul> <p></p>"},{"location":"strawberryfield-formatters/#at-the-end-of-the-day","title":"At the end of the day","text":"<p>The decision for how your metadata is displayed is totally in your control.</p> <p>Under the <code>WIDGET</code> column, there is a quick description/overview of what the formatter is doing.</p> <p></p> <p>And by clicking on the gear icon under the <code>OPERATIONS</code> column, all of the options for configuring the formatter are revealed. To use <code>\ud83c\udf53Fragola</code> as an example (the Formatter for IIIF media), we can choose which JSON Key is being used to fetch the IIIF Media URLs (found inside the raw JSON being played with <code>Strawberry Default Formatter</code>), the maximum height and width of the viewer, etc.</p> <p></p> <p>And then with <code>\ud83c\udf53Erdbeere</code> (the Formatter for Custom Metadata Templates) there is the option, among many others, to configure which Twig template the formatter will use for displaying your Metadata.</p> <p></p> <p>More information about Managing Metadata Displays with Twig Templates can be found here.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"strawberryfields/","title":"Strawberryfields Forever","text":""},{"location":"strawberryfields/#what-strawberry-fields-does-why-we-built-it-and-what-issues-it-addresses","title":"What Strawberry fields does, why we built it, and what issues it addresses","text":"<p>Archipelago integrates transparently into the Drupal 8 ecosystem using its Core Content Entity System (Nodes), Discovery (Search API) and in general all its Core Components plus a few well maintained external ones.</p> <p>By design (and because we think its imperative), Archipelago takes full charge of the metadata layer and associated media assets by implementing a highly configurable, smart Drupal field written in JSON named <code>Strawberryfield</code> that attaches to any content.</p> <p>All of JSON's internals, keys, paths, and values are dynamically exposed to the rest of the ecosystem. Strawberryfield even remembers its structure as data evolves by storing JSON paths of every little detail.</p>"},{"location":"strawberryfields/#nothing-is-real","title":"Nothing Is Real","text":"<p>Archipelago includes additional companion modules, <code>Webform_strawberryfield</code> and <code>Format_strawberryfield</code> that extend the core metadata capabilities of the main <code>Strawberryfield</code> module and allow the same flexibility to be exposed during ingest and viewing of digital objects.</p> <p>The in-development <code>Strawberry Runners</code> and <code>AMI</code> modules further extend Archipelago's capabilities. Additional information related to these modules will be made available following initial public releases.</p>"},{"location":"strawberryfields/#ingesting","title":"Ingesting","text":"<p><code>Webform Strawberryfield</code> (we had a better name) extends and integrates into the <code>amazing Drupal Webform module</code> to allow Archipelago users to build any possible metadata and media, ingest and edit, workflows directly via the UI using webforms.</p> <p>By not having a hardcoded ingest method, Archipelago can be used outside the GLAM community too, as a pure data repository in biological sciences, digital humanities, archives, or even as a mixed, multidisciplinary/cross-domain system.</p> <p>We also added <code>WIKIDATA</code>, <code>LoC</code>, <code>Getty</code>, and <code>VIAF</code> authority querying elements to aid in linking to external Linked Open Data sources.</p> <p>All these integrations are made to help local needs and community identities to survive the never-ending race for the next metadata schema. They are made to prototype, plan, and grow independently of how metadata will need to be exposed yesterday or tomorrow. And we plan to add more.</p> <p>Explore what other features <code>webform_strawberryfield</code> provides to help with ingesting, reading, and interacting with your metadata during that process.</p>"},{"location":"strawberryfields/#exposing","title":"Exposing","text":"<p><code>Format Strawberryfield</code> (we had even a better name but...) deals with taking your JSON based metadata and <code>casting</code>, mashing, mixing, exposing, displaying, and transforming it to allow rich interaction for users and other systems with your digital objects.</p> <p>In its guts (or heart?), Archipelago does something quite simple but core to our concept of repository: it transforms in realtime the close to your needs open schema metadata that lives in strawberryfield as JSON into close to other one's fixed schema needs metadata; any destination format, using a fast, cached templating system. A templating system that is core to Drupal, called <code>Twig</code>:</p> <ul> <li>Twig in Symfony</li> <li>Twig in Drupal</li> </ul> <p>This templating system is exposed to Archipelago users through the UI and stored side by side in the repository as content (we named them <code>Metadata Display entities</code>, but they not only serve display needs!) so users can fully control how metadata is transformed and published without touching their individual sources.</p> <p>Templates or recipes can be shared, exported, ingested, updated, and adapted in many ways. Fast changes are possible without having to wait for the next mayor release of Archipelago or your favorited Metadata Schema Specs Committee agreeing on the next or the last version. Of course, this module not only handles metadata but media assets too, extracting local or remote URIs and files from your metadata and rendering them as media viewers: books, 3D models, images, panoramas, A/V with IIIF in its soul.</p> <p>You can learn more about what format_strawberryfield can do and what many other possibilities are exposed through our templating system.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"strawberryrunners/","title":"Strawberry Runners Post-Processing Configuration","text":"<p>Archipelago's Strawberry Runners (SBR) module provides provides a set of post-processing capabilities for the JSON based metadata, files and entities that comprise your Archipelago Digital Objects (ADOs). These post-processing actions are based on dispatched events, direct http calls, and invoked webhooks from partner services (such as Min.io, AWS S3 or self-invoked). </p> <p>The default Archipelago SBR post-processor configurations include operations that: - perform page-based HOCR/OCR for image and pdf-based ADOs, send the output to the Search API, and use Natural Language Processing to extract entities from the output - extract text from pages within a Webarchives File and send the output to the Search API - convert WARC format Webarchives Files into WACZ format and attach the new WACZ file to the original source ADO to complement the WARC original - extract textual values from subtitle/transcript VTT files and generates time/space transmuted OCR</p> <p>SBR actions can be chained and nested to enable ordered operations, such as first extract individual pages in an ordered sequence and then run HOCR/OCR across the individual pages.</p>","tags":["Strawberry Runners","HOCR","OCR","Post-processing","Background Processing"]},{"location":"strawberryrunners/#strawberry-runners-settings-overview","title":"Strawberry Runners Settings Overview","text":"<p>You can access the Strawberry Runners Settings:</p> <ul> <li>Through the <code>Manage</code> menu &gt; <code>Configuration</code> &gt; <code>Archipelago</code> &gt; <code>Configure Strawberry Runners Post Processors</code></li> <li>Directly at <code>/admin/config/archipelago/strawberry_runners</code> </li> </ul> <p>On the Strawberry Runners Settings page, you will see the Archipelago default post processor configurations (unless modified).</p> <p></p> <ol> <li>The <code>pager</code> action uses the 'Post processor that extracts/generates Ordered Sequences of files/pages/children using Files present in an ADO' plugin.</li> <li>Nested one level in, the <code>ocr</code> action uses the 'Post processor that Runs OCR/HORC against files' plugin. The <code>ocr</code> operations will be executed after the completion of the <code>pager</code> operations.</li> <li>The <code>wacz_page_extractor</code> action uses the 'Post processor that extracts/generates Indexed Page Content from WACZ files in an ADO' plugin.</li> <li>Nested one level in, the <code>webpage</code> action uses the 'Post processor that Indexes WACZ Frictionless data Search Index to Search API' plugin. The <code>webpage</code> operations will be executed after the completion of the <code>wacz_page_extractor</code> operations.</li> <li>The <code>warc_to_wacz</code> action uses the 'Post processor that uses a System Binary to process * files' operations.</li> <li>The <code>subtitle</code> action extracts textual values from subtitle/transcript VTT files and generates time/space transmuted OCR. This transmuted OCR can be used to search within a time-based video or audio file's corresponding subtitle/transcript VTT file(s), then navigate to the matching time of the video or audio file within a media viewer.</li> </ol>","tags":["Strawberry Runners","HOCR","OCR","Post-processing","Background Processing"]},{"location":"strawberryrunners/#reviewing-and-adjusting-the-default-post-processors","title":"Reviewing and Adjusting the default Post-Processors","text":"<p>From the main Strawberry Runner Settings page, you can review and adjust the settings for the default Archipelago configurations by selecting <code>Edit</code> from the `Operations`` menu. </p> <p>Please see the following guides for:</p> <ul> <li>Adjusting the <code>pager</code> and <code>ocr</code> operations</li> <li>Adjusting the <code>wacz_page_extractor</code> and <code>webpage</code> operations</li> <li>Adjusting the <code>warc_to_wacz</code> operation</li> <li>Adjusting the <code>subtitle</code> operation</li> </ul>","tags":["Strawberry Runners","HOCR","OCR","Post-processing","Background Processing"]},{"location":"strawberryrunners/#triggering-post-processing-actions-manually","title":"Triggering Post-Processing Actions Manually","text":"<p>After making adjustments to Strawberry Runners Post-Processing configurations, you may want to trigger/re-trigger a particular action manually.</p> <p>You can use Archipelago's Find and Replace to first select a specific group of Digital Objects you wish to target for Post-Processing, then select the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code> from the Find and Replace <code>Actions menu</code>.</p>","tags":["Strawberry Runners","HOCR","OCR","Post-processing","Background Processing"]},{"location":"strawberryrunners/#additional-post-processor-operations","title":"Additional Post Processor Operations","text":"<p>Archipelago also includes the <code>Post processor that writes/reads Frictionless Data Packages</code> plugin. Please keep a lookout for future documentation related to using this plugin.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Strawberry Runners","HOCR","OCR","Post-processing","Background Processing"]},{"location":"strawberryrunners_pager_ocr/","title":"Reviewing and adjusting the <code>pager</code> and <code>ocr</code> Post-Processor operations","text":"<p>The <code>pager</code> and <code>ocr</code> Post-processor operations are likely the most important pair of Strawberry Runners in your Archipelago.</p> <p>As stated on the Strawberry Runners overview page, the <code>pager</code> action uses the 'Post processor that extracts/generates Ordered Sequences of files/pages/children using Files present in an ADO' plugin. Nested one level in, the <code>ocr</code> action uses the 'Post processor that Runs OCR/HORC against files' plugin. The <code>ocr</code> operations will be executed after the completion of the <code>pager</code> operations.</p> <p>Common changes you may wish to make for the <code>pager</code> and/or <code>ocr</code> operations include adding or removing particular types of Archipelago Digital Objects to apply these operations to.</p>","tags":["Strawberry Runners","HOCR","OCR","Pager","Post-processing","Background Processing"]},{"location":"strawberryrunners_pager_ocr/#pager-settings","title":"Pager Settings","text":"<p>To review or adjust the configurations for the <code>pager</code> operation, select <code>Edit</code> from the <code>Operations</code> menu.</p> <p>In the <code>pager</code> settings, you will see the following configuration options:</p> <p></p> <ol> <li> <p>Label: </p> <ul> <li>Label for this Processor; which should be a unique machine-readable name</li> <li>Can only contain lowercase letters, numbers, and underscores</li> <li>We do not recommend changing this Label from the default <code>pager</code>. </li> </ul> </li> <li> <p>Strawberry Runner Post Processor Plugin:</p> <ul> <li>The <code>Post processor that extracts/generates Ordered Sequences of files/pages/children using Files present in an ADO</code> should be selected.</li> <li>We do not recommend changing this Plugin selection.</li> </ul> </li> <li> <p>Checkbox to mark this processor plugin as active </p> <ul> <li>We recommend keeping this checked as <code>active</code> at all times, but you may wish to temporarily disable this if you are performing certain types of administrative review tasks such as running large test ingests where you plan on deleting the ADOs before a final ingest.</li> <li>If you accidentally uncheck this and need to re-trigger the <code>pager</code> (and corresponding nested <code>ocr</code> action), you can use Archipelago's Find and Replace to first select a specific group of Digital Objects you wish to target for Post-Processing, then select the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code> from the Find and Replace <code>Actions menu</code>.</li> </ul> </li> <li> <p>ADO type(s) to limit this processor to:</p> <ul> <li>A single ADO type or a comma delimited list of ado types that qualify to be Processed.</li> <li>Leave empty to apply to all ADOs. If you do not provide any specific ADO types here, the processor will be applied for all ADOs with the JSON keys selected in the next step.</li> <li>Default ADO types specified are: 'Document,Book,Article'  </li> <li>You may wish to add additional types of document/multiple-paged type of ADOs to this list that are custom to you Archipelago environment. </li> </ul> </li> <li> <p>The JSON key that contains the desired source files:</p> <ul> <li>By default, the <code>as:image</code> and <code>as:document</code> keys are selected.</li> <li>We do not recommend changing this selection.</li> </ul> </li> <li> <p>Mimetypes(s) to limit this Processor to:</p> <ul> <li>A single Mimetype type or a comma separated list of mimetypes that qualify to be Processed.</li> <li>Leave empty to apply any file.</li> <li>Default mimetypes are: 'application/pdf,image/tiff,image/jpeg,image/jp2'  </li> </ul> </li> <li> <p>Within the ADO's metadata, the JSON key that contains the language in ISO639-3 (3 letter) format to be used for OCR/NLP processing via Tesseract.</p> <ul> <li>Default JSON key specified is: 'language_iso639_3'</li> </ul> </li> <li> <p>Please provide a default language in ISO639-3 (3 letter) format. If none is provided we will use 'eng'.</p> <ul> <li>Default language specified is: 'eng'</li> </ul> </li> <li> <p>Timeout in seconds for this process.</p> <ul> <li>If the process runs out of time it can still be processed again</li> <li>Default selection is: 10</li> </ul> </li> <li> <p>Order or execution in the global chain.</p> <ul> <li>Default selection is: 0</li> </ul> </li> </ol>","tags":["Strawberry Runners","HOCR","OCR","Pager","Post-processing","Background Processing"]},{"location":"strawberryrunners_pager_ocr/#ocr-hocr-settings","title":"OCR / HOCR Settings","text":"<p>To review or adjust the configurations for the <code>ocr</code> operation, select <code>Edit</code> from the <code>Operations</code> menu.</p> <p>In the <code>pager</code> settings, you will see several different configuration options.</p> <p></p> <ol> <li> <p>Label: </p> <ul> <li>Label for this Processor; which should be a unique machine-readable name</li> <li>Can only contain lowercase letters, numbers, and underscores</li> <li>We do not recommend changing this Label from the default <code>ocr</code>. </li> </ul> </li> <li> <p>Strawberry Runner Post Processor Plugin:</p> <ul> <li>The <code>Post processor that Runs OCR/HORC against files</code> should be selected.</li> <li>We do not recommend changing this Plugin selection.</li> </ul> </li> <li> <p>Checkbox to mark this processor plugin as active </p> <ul> <li>We recommend keeping this checked as <code>active</code> at all times, but you may wish to temporarily disable this if you are performing certain types of administrative review tasks such as running large test ingests where you plan on deleting the ADOs before a final ingest.</li> <li>If you accidentally uncheck this and need to re-trigger the <code>pager</code> (and corresponding nested <code>ocr</code> action), you can use Archipelago's Find and Replace to first select a specific group of Digital Objects you wish to target for Post-Processing, then select the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code> from the Find and Replace <code>Actions menu</code>.</li> </ul> </li> <li> <p>The type of source data this processor works on:</p> <ul> <li>Select from where the source file this processor needs is fetched.</li> <li>Default selection of 'File entities referenced in the as:filetype JSON structure'.</li> <li>You also have the option of selecting 'Full file paths passed by another processor', but we do not recommend using this option as it is less granular in its application.      </li> </ul> </li> <li> <p>ADO type(s) to limit this processor to:</p> <ul> <li>A single ADO type or a comma delimited list of ado types that qualify to be Processed</li> <li>Leave empty to apply to all ADOs. If you do not provide any specific ADO types here, the processor will be applied for all ADOs with the JSON keys selected in the next step.</li> <li>Default ADO types specified are: 'Document,Book,Article'  </li> <li>You may wish to add additional types of document/multiple-paged type of ADOs to this list that are custom to you Archipelago environment. </li> </ul> </li> <li> <p>The JSON key that contains the desired source files:</p> <ul> <li>By default, the <code>as:image</code> and <code>as:document</code> keys are selected.</li> <li>We do not recommend changing this selection.</li> </ul> </li> <li> <p>Mimetypes(s) to limit this Processor to:</p> <ul> <li>A single Mimetype type or a comma separated list of mimetypes that qualify to be Processed.</li> <li>Leave empty to apply any file.</li> <li>Default mimetypes are: 'application/pdf,image/tiff,image/jpeg,image/jp2'  </li> </ul> </li> </ol> <p>Advanced OCR/HOCR Settings</p> <p>We do not recommend making changes to the follow settings unless you are the System Administrator.</p> <ol> <li> <p>The system path to the ghostscript (gs) binary that will be executed by this processor.</p> <ul> <li>A full system path to the gs binary present in the same environment your PHP runs</li> <li>Default path specified is: '/usr/bin/gs'</li> </ul> </li> <li> <p>Any additional argument your executable binary requires.</p> <ul> <li>Any arguments your ghostscript (gs) binary requires to run. Use %file as replacement for the file if the executable requires the filename to be passed under a specific argument. We recommend testing with -r150 (150dpi image extraction) for better performance but -r300 can be also used if source Images in a PDF are small</li> <li>Default argument specified is: -r150 %file</li> </ul> </li> <li> <p>The system path to the Tesseract binary that will be executed by this processor.</p> <ul> <li>A full system path to the Tesseract binary present in the same environment your PHP runs</li> <li>Default path specified is: '/usr/bin/tesseract'</li> </ul> </li> <li> <p>Within the ADO's metadata, the JSON key that contains the language in ISO639-3 (3 letter) format to be used for OCR/NLP processing via Tesseract.</p> <ul> <li>Default JSON key specified is: 'language_iso639_3'</li> </ul> </li> </ol> <p></p> <ol> <li> <p>Please provide a default language in ISO639-3 (3 letter) format. If none is provided we will use 'eng' </p> <ul> <li>Default language specified is: 'eng'</li> </ul> </li> <li> <p>Any additional argument for your tesseract binary.</p> <ul> <li>Any arguments your binary requires to run. Use %file as replacement for the file that is output by the GS binary. Use %language as replacement for the chosen language.</li> <li>Default arguments specified are: '%file stdout -l %language hocr'</li> </ul> </li> <li> <p>The data/languages folder for Tesseract</p> <ul> <li>Absolute path where the Languages are stored in the Server. This will be used in --tessdata-dir</li> <li>Default path specified is: '/usr/share/tessdata'  </li> </ul> </li> <li> <p>The system path to the pdfalto binary that will be executed by this processor.</p> <ul> <li>A full system path to the pdfalto binary present in the same environment your PHP runs, e.g /usr/local/bin/pdfalto</li> <li>Default path specified is: '/usr/bin/pdfalto'</li> </ul> </li> <li> <p>Any additional argument for your pdfalto binary.</p> <ul> <li>Any arguments your binary requires to run. Use %file as replacement for the file that is output by the pdfalto binary.</li> <li>Default arguments specified are: '%file -q</li> </ul> </li> <li> <p>The expected and desired output of this processor.</p> <ul> <li>If the output is just data and \"One or more Files\" is selected all data will be dumped into a file and handled as such.</li> <li>Default selection is: 'Data/Values that can be serialized to JSON'</li> <li>Additional optional is to select 'One or more Files', but it is not recommended unless to use this for the default <code>ocr</code> operation since this will alter how the data is incorporated in the Search API (Solr index).</li> </ul> </li> <li> <p>Where and how the output will be used.</p> <ul> <li>Default select is: 'In a Search API Document using the Strawberryfield Flavor Data Source (e.g used for HOCR highlight)'  </li> <li>Additional option to select 'As Input for another processor Plugin' --which will only have an effect if another Processor is setup to consume this output.</li> </ul> </li> <li> <p>The queue to use for this processor.</p> <ul> <li>The primary queue will be execute in realtime while the Secondary will be execute in background</li> <li>Default selection is for the 'Secondary queue in background'</li> </ul> </li> <li> <p>Checkbox to Use NLP (Natural Language Processing) to extract entities from Text</p> <ul> <li>If checked Full text will be processed for Natural language Entity extraction using Polyglot.</li> <li>Default option is to have the option checked. </li> </ul> </li> <li> <p>The URL location of your NLP64 server.</p> <ul> <li>Defaults to http://esmero-nlp:6400</li> </ul> </li> <li> <p>Which method(NER) to use</p> <ul> <li>The NER NLP method to use to extract Agents, Places and Sentiment.</li> <li>Default selection: 'Polyglot (faster)'</li> <li>Alternation selection: 'spaCy (more accurate)'</li> </ul> </li> <li> <p>Timeout in seconds for this process.</p> <ul> <li>900</li> <li>If the process runs out of time it can still be processed again.</li> </ul> </li> <li> <p>Order or execution in the global chain.</p> <ul> <li>0</li> </ul> </li> </ol> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the main Strawberry Runners or the Archipelago Documentation main page.</p>","tags":["Strawberry Runners","HOCR","OCR","Pager","Post-processing","Background Processing"]},{"location":"strawberryrunners_subtitle/","title":"Reviewing and adjusting the <code>subtitle</code> Post-Processor operations","text":"<p>As stated on the Strawberry Runners overview page, the <code>subtitle</code> action extracts textual values from subtitle/transcript VTT files and generates time/space transmuted OCR. This transmuted OCR can be used to search within a time-based video or audio file's corresponding subtitle/transcript VTT file(s), then navigate to the matching time of the video or audio file within a media viewer.</p> <p>We strongly recommend using caution when making any adjustments to the default <code>subtitle</code> configurations as this may result in unexpected issues with the transmuted OCR values in your Solr Index. Also, the <code>subtitle</code> Strawberry Runner Post-Processor needs to used with corresponding related default IIIF Configuration Form Settings. </p>","tags":["Strawberry Runners","Subtitles","VTT","HOCR","OCR","Post-processing","Background Processing"]},{"location":"strawberryrunners_subtitle/#subtitle-settings","title":"Subtitle Settings","text":"<p>To review or adjust the configurations for the <code>subtitle</code> operation, select <code>Edit</code> from the <code>Operations</code> menu.</p> <p>In the <code>subtitle</code> settings, you will see the following configuration options:</p> <p></p> <ol> <li> <p>Label: </p> <ul> <li>Label for this Processor; which should be a unique machine-readable name</li> <li>Can only contain lowercase letters, numbers, and underscores</li> <li>We do not recommend changing this Label from the default <code>pager</code>. </li> </ul> </li> <li> <p>Strawberry Runner Post Processor Plugin:</p> <ul> <li>The <code>Post processor that extracts/generates Ordered Sequences of files/pages/children using Files present in an ADO</code> should be selected.</li> <li>We do not recommend changing this Plugin selection.</li> </ul> </li> <li> <p>Checkbox to mark this processor plugin as active </p> <ul> <li>We recommend keeping this checked as <code>active</code> at all times, but you may wish to temporarily disable this if you are performing certain types of administrative review tasks such as running large test ingests where you plan on deleting the ADOs before a final ingest.</li> <li>If you accidentally uncheck this and need to re-trigger the <code>pager</code> (and corresponding nested <code>ocr</code> action), you can use Archipelago's Find and Replace to first select a specific group of Digital Objects you wish to target for Post-Processing, then select the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code> from the Find and Replace <code>Actions menu</code>.</li> </ul> </li> <li> <p>ADO type(s) to limit this processor to:</p> <ul> <li>A single ADO type or a comma delimited list of ado types that qualify to be Processed.</li> <li>Leave empty to apply to all ADOs. If you do not provide any specific ADO types here, the processor will be applied for all ADOs with the JSON keys selected in the next step.</li> <li>Default ADO types specified are: 'Document,Book,Article'  </li> <li>You may wish to add additional types of document/multiple-paged type of ADOs to this list that are custom to you Archipelago environment. </li> </ul> </li> <li> <p>The JSON key that contains the desired source files:</p> <ul> <li>By default, the <code>as:image</code> and <code>as:document</code> keys are selected.</li> <li>We do not recommend changing this selection.</li> </ul> </li> <li> <p>Mimetypes(s) to limit this Processor to:</p> <ul> <li>A single Mimetype type or a comma separated list of mimetypes that qualify to be Processed.</li> <li>Leave empty to apply any file.</li> <li>Default mimetypes are: 'application/pdf,image/tiff,image/jpeg,image/jp2'  </li> </ul> </li> <li> <p>Within the ADO's metadata, the JSON key that contains the language in ISO639-3 (3 letter) format to be used for OCR/NLP processing via Tesseract.</p> <ul> <li>Default JSON key specified is: 'language_iso639_3'</li> </ul> </li> <li> <p>Please provide a default language in ISO639-3 (3 letter) format. If none is provided we will use 'eng'.</p> <ul> <li>Default language specified is: 'eng'</li> </ul> </li> <li> <p>Timeout in seconds for this process.</p> <ul> <li>If the process runs out of time it can still be processed again</li> <li>Default selection is: 10</li> </ul> </li> <li> <p>Order or execution in the global chain.</p> <ul> <li>Default selection is: 0</li> </ul> </li> </ol>","tags":["Strawberry Runners","Subtitles","VTT","HOCR","OCR","Post-processing","Background Processing"]},{"location":"strawberryrunners_subtitle/#related-iiif-configuration-form-and-iiif-content-search-guides","title":"Related IIIF Configuration Form and IIIF Content Search Guides","text":"<ul> <li>IIIF Configuration Form</li> <li>IIIF Content Search API Overview</li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the main Strawberry Runners or the Archipelago Documentation main page.</p>","tags":["Strawberry Runners","Subtitles","VTT","HOCR","OCR","Post-processing","Background Processing"]},{"location":"strawberryrunners_wacz_binary/","title":"Reviewing and adjusting the <code>warc_to_wacz</code> Post-Processor operation","text":"<p>As stated on the Strawberry Runners overview page, the <code>warc_to_wacz</code> action uses the 'Post processor that uses a System Binary to process * files' operations.</p>","tags":["Strawberry Runners","WACZ","WARC","Binary","Post-processing","Background Processing"]},{"location":"strawberryrunners_wacz_binary/#wacz-page-extractor-settings","title":"Wacz Page Extractor Settings","text":"<p>To review or adjust the configurations for the <code>wacz_page_extractor</code> operation, select <code>Edit</code> from the <code>Operations</code> menu.</p> <p>In the <code>wacz_page_extractor</code> settings, you will see the following configuration options:</p> <ol> <li> <p>Label: </p> <ul> <li>Label for this Processor; which should be a unique machine-readable name</li> <li>Can only contain lowercase letters, numbers, and underscores</li> <li>We do not recommend changing this Label from the default <code>pager</code>. </li> </ul> </li> <li> <p>Strawberry Runner Post Processor Plugin:</p> <ul> <li>The <code>Post processor that extracts/generates Indexed Page Content from WACZ in an ADO</code> should be selected.</li> <li>We do not recommend changing this Plugin selection.</li> </ul> </li> <li> <p>Checkbox to mark this processor plugin as active </p> <ul> <li>We recommend keeping this checked as <code>active</code> at all times, but you may wish to temporarily disable this if you are performing certain types of administrative review tasks such as running large test ingests where you plan on deleting the ADOs before a final ingest.</li> <li>If you accidentally uncheck this and need to re-trigger the <code>pager</code> (and corresponding nested <code>ocr</code> action), you can use Archipelago's Find and Replace to first select a specific group of Digital Objects you wish to target for Post-Processing, then select the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code> from the Find and Replace <code>Actions menu</code>.</li> </ul> </li> <li> <p>The JSON key that contains the desired source files:</p> <ul> <li>By default, only the <code>as:document</code> key is selected.</li> <li>We do not recommend changing this selection.</li> </ul> </li> <li> <p>Mimetypes(s) to limit this Processor to:</p> <ul> <li>A single Mimetype type or a comma separated list of mimetypes that qualify to be Processed.</li> <li>Default mimetype for this Processor is: 'application/warc'      </li> </ul> </li> </ol> <p>Advanced OCR/HOCR Settings</p> <p>We do not recommend making changes to the follow settings unless you are the System Administrator.</p> <ol> <li> <p>The system path to the binary that will be executed by this processor.</p> <ul> <li>A full system path to a binary present in the same environment your PHP runs</li> <li>Default is: '/usr/bin/wacz'</li> </ul> </li> <li> <p>Any additional argument your executable binary requires.</p> <ul> <li>Any arguments your binary requires to run. Use %file as replacement for the file if the executable requires the filename to be passed under a specific argument. Use %outfile if the binary is intended to generate a new file and the output is going to be a file entity. If you know the extension please add it in the form of %outfile.extension</li> <li>Default is: 'create %file -t -o %outfile.wacz'</li> </ul> </li> <li> <p>The expected and desired output of this processor.</p> <ul> <li>If the output is just data and \"One or more Files\" is selected all data will be dumped into a file and handled as such.</li> <li>Default is set to 'One or more Files'</li> </ul> </li> <li> <p>Where and how the output will be used.</p> <ul> <li>Default is set to 'A new file to be attached to the source ADO'</li> <li>The 'As Input for another processor Plugin' selection will only have an effect if another Processor is setup to consume this output.</li> <li>Other optional selections include:</li> <li>In the same Source Metadata, as a child structure of each Processed file</li> <li>In the same Source Metadata but inside its own, top level, \"as:flavour\" subkey based on the given machine name of the current plugin</li> <li>A new file to be attached to the source ADO</li> <li>As Input for another processor Plugin</li> <li>In a Search API Document using the Strawberryfield Flavor Data Source (e.g used for HOCR highlight)</li> </ul> </li> <li> <p>Timeout in seconds for this process.</p> <ul> <li>Default is set to: 99</li> <li>If the process runs out of time it can still be processed again.</li> </ul> </li> <li> <p>Order or execution in the global chain.</p> <ul> <li>Default is set to: 0</li> </ul> </li> </ol> <p>Return to the main Strawberry Runners or the Archipelago Documentation main page.</p>","tags":["Strawberry Runners","WACZ","WARC","Binary","Post-processing","Background Processing"]},{"location":"strawberryrunners_webpage_text/","title":"Reviewing and adjusting the <code>wacz_page_extractor</code> and <code>webpage</code> Post-Processor operations","text":"<p>As stated on the Strawberry Runners overview page, the <code>wacz_page_extractor</code> action uses the 'Post processor that extracts/generates Indexed Page Content from WACZ files in an ADO' plugin. Nested one level in, the <code>webpage</code> action uses the 'Post processor that Indexes WACZ Frictionless data Search Index to Search API' plugin. The webpage operations will be executed after the completion of the wacz_page_extractor operations. The <code>ocr</code> operations will be executed after the completion of the <code>pager</code> operations.</p>","tags":["Strawberry Runners","Fulltext Search","WACZ","Webpage","Post-processing","Background Processing"]},{"location":"strawberryrunners_webpage_text/#wacz-page-extractor-settings","title":"Wacz Page Extractor Settings","text":"<p>To review or adjust the configurations for the <code>wacz_page_extractor</code> operation, select <code>Edit</code> from the <code>Operations</code> menu.</p> <p>In the <code>wacz_page_extractor</code> settings, you will see the following configuration options:</p> <ol> <li> <p>Label: </p> <ul> <li>Label for this Processor; which should be a unique machine-readable name</li> <li>Can only contain lowercase letters, numbers, and underscores</li> <li>We do not recommend changing this Label from the default <code>pager</code>. </li> </ul> </li> <li> <p>Strawberry Runner Post Processor Plugin:</p> <ul> <li>The <code>Post processor that extracts/generates Indexed Page Content from WACZ in an ADO</code> should be selected.</li> <li>We do not recommend changing this Plugin selection.</li> </ul> </li> <li> <p>Checkbox to mark this processor plugin as active </p> <ul> <li>We recommend keeping this checked as <code>active</code> at all times, but you may wish to temporarily disable this if you are performing certain types of administrative review tasks such as running large test ingests where you plan on deleting the ADOs before a final ingest.</li> <li>If you accidentally uncheck this and need to re-trigger the <code>pager</code> (and corresponding nested <code>ocr</code> action), you can use Archipelago's Find and Replace to first select a specific group of Digital Objects you wish to target for Post-Processing, then select the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code> from the Find and Replace <code>Actions menu</code>.</li> </ul> </li> <li> <p>The JSON key that contains the desired source files:</p> <ul> <li>By default, only the <code>as:document</code> key is selected.</li> <li>We do not recommend changing this selection.</li> </ul> </li> <li> <p>Mimetypes(s) to limit this Processor to:</p> <ul> <li>A single Mimetype type or a comma separated list of mimetypes that qualify to be Processed.</li> <li>Default mimetype for this Processor is: 'application/vnd.datapackage+zip'   </li> </ul> </li> <li> <p>ADO type(s) to limit this processor to:</p> <ul> <li>A single ADO type or a comma delimited list of ado types that qualify to be Processed.</li> <li>Leave empty to apply to all ADOs. If you do not provide any specific ADO types here, the processor will be applied for all ADOs with the JSON keys selected in the next step.</li> <li>Default ADO type specified for this Processor: 'WebPage'  </li> </ul> </li> <li> <p>The expected and desired output of this processor:</p> <ul> <li>Only option for this Processor is JSON output</li> <li>Default is set to: <code>Data/Values that can be serialized to JSON</code></li> </ul> </li> <li> <p>The queue to use this processor:</p> <ul> <li>The primary queue will be execute in realtime while the Secondary will be execute in background</li> <li>Default selection is for the 'Secondary queue in background' </li> </ul> </li> <li> <p>Timeout in seconds for this process.</p> <ul> <li>Default is set to: 300</li> <li>If the process runs out of time it can still be processed again.</li> </ul> </li> <li> <p>Order or execution in the global chain.</p> <ul> <li>Default is set to: 7</li> </ul> </li> </ol>","tags":["Strawberry Runners","Fulltext Search","WACZ","Webpage","Post-processing","Background Processing"]},{"location":"strawberryrunners_webpage_text/#webpage-settings","title":"Webpage  Settings","text":"<p>To review or adjust the configurations for the <code>webpage</code> operation, select <code>Edit</code> from the <code>Operations</code> menu.</p> <p>In the <code>webpage</code> settings, you will see the following configuration options:</p> <ol> <li> <p>Label: </p> <ul> <li>Label for this Processor; which should be a unique machine-readable name</li> <li>Can only contain lowercase letters, numbers, and underscores</li> <li>We do not recommend changing this Label from the default <code>webpage</code>. </li> </ul> </li> <li> <p>Strawberry Runner Post Processor Plugin:</p> <ul> <li>The <code>Post processor that extracts/generates Indexed Page Content from WACZ in an ADO</code> should be selected.</li> <li>We do not recommend changing this Plugin selection.</li> </ul> </li> <li> <p>Checkbox to mark this processor plugin as active </p> <ul> <li>We recommend keeping this checked as <code>active</code> at all times, but you may wish to temporarily disable this if you are performing certain types of administrative review tasks such as running large test ingests where you plan on deleting the ADOs before a final ingest.</li> <li>If you accidentally uncheck this and need to re-trigger the <code>pager</code> (and corresponding nested <code>ocr</code> action), you can use Archipelago's Find and Replace to first select a specific group of Digital Objects you wish to target for Post-Processing, then select the <code>Trigger Strawberrry Runners process/reprocess for Archipelago Digital Objects content item</code> from the Find and Replace <code>Actions menu</code>.</li> </ul> </li> <li> <p>ADO type(s) to limit this processor to:</p> <ul> <li>A single ADO type or a comma delimited list of ado types that qualify to be Processed.</li> <li>Leave empty to apply to all ADOs. If you do not provide any specific ADO types here, the processor will be applied for all ADOs with the JSON keys selected in the next step.</li> <li>Default ADO type specified for this Processor: 'WebPage'</li> </ul> </li> <li> <p>The expected and desired output of this processor:</p> <ul> <li>Only option for this Processor is JSON output</li> <li>If the output is just data and \"One or more Files\" is selected all data will be dumped into a file and handled as such.</li> <li>Default is set to: <code>Data/Values that can be serialized to JSON</code></li> </ul> </li> <li> <p>Where and how the output will be used.</p> <ul> <li>'As Input for another processor Plugin' selection will only have an effect if another Processor is setup to consume this ouput.</li> <li>Default selection is: 'In a Search API Document using the Strawberryfield Flavor Data Source (e.g used for HOCR highlight)'</li> </ul> </li> <li> <p>The queue to use this processor:</p> <ul> <li>The primary queue will be execute in realtime while the Secondary will be execute in background</li> <li>Default selection is for the 'Secondary queue in background' The queue to use for this processor.</li> </ul> </li> <li> <p>Checkbox option to 'Use NLP to extract entities from Text'</p> <ul> <li>If checked Full text will be processed for Natural language Entity extraction using Polyglot</li> <li>Default is have this option checked</li> </ul> </li> <li> <p>The URL location of your NLP64 server.</p> <ul> <li>Defaults to http://esmero-nlp:6400</li> </ul> </li> <li> <p>Which method(NER) to use</p> <ul> <li>The NER NLP method to use to extract Agents, Places and Sentiment.</li> <li>Default selection: 'Polyglot (faster)'</li> <li>Alternation selection: 'spaCy (more accurate)'</li> </ul> </li> <li> <p>Timeout in seconds for this process.</p> <ul> <li>Default is set to: 25</li> <li>If the process runs out of time it can still be processed again.</li> </ul> </li> <li> <p>Order or execution in the global chain.</p> <ul> <li>Default is set to: 0</li> </ul> </li> </ol> <p>Return to the main Strawberry Runners or the Archipelago Documentation main page.</p>","tags":["Strawberry Runners","Fulltext Search","WACZ","Webpage","Post-processing","Background Processing"]},{"location":"traditional-install/","title":"Traditional install","text":""},{"location":"traditional-install/#traditional-installation-notes","title":"Traditional Installation Notes","text":"<p>For those who prefer classic approaches to system installation and configuration (instead of Dockerized deployment), this page is reserved for notes, recommendations, and guides.</p> <ul> <li>Giancarlo Birello is maintaining and sharing the following documentation:<ul> <li>Dev DBOpen: developer site of the DBOPen project</li> <li>Includes an Architecture overview and Step by Step instructions</li> </ul> </li> </ul> <p>Please stay tuned for additional future updates. Thank you!</p> <p>Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"},{"location":"twig_extensions/","title":"Twig Extensions","text":"<p>One advantage of Drupal's integration of the Twig template engine is the availability of extensions (filters and functions).</p>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON","Markdown","HTML"]},{"location":"twig_extensions/#default-twig-extensions-from-symfony","title":"Default Twig Extensions from Symfony","text":"<p>The Symfony PHP framework, which is integrated into Drupal Core, provides extensions, which we use in our default templates:</p> <ul> <li>Twig Filters from Symfony</li> <li>Twig Functions from Symfony</li> </ul>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON","Markdown","HTML"]},{"location":"twig_extensions/#default-twig-extensions-from-drupal","title":"Default Twig Extensions from Drupal","text":"<p>Additionally, we have some very handy Drupal-specific extensions:</p> <ul> <li>Twig Filters from Drupal</li> <li>Twig Functions from Drupal</li> </ul>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON","Markdown","HTML"]},{"location":"twig_extensions/#default-twig-extensions-from-archipelago","title":"Default Twig Extensions from Archipelago","text":"<p>Finally, we have a growing list of extensions that apply to our own specific use cases:</p>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON","Markdown","HTML"]},{"location":"twig_extensions/#twig-filters-from-archipelago","title":"Twig Filters from Archipelago","text":"<p>edtf_2_human_date</p> <p>The <code>edtf_2_human_date</code> filter takes an EDTF date and an optional language code (defaults to English), and converts it to a human-readable format using the EDTF PHP library. The list of language codes is available here.</p> <p>Let's start with the following metadata fragment: Metadata Fragment<pre><code>...\n\"subject_wikidata\": \"\",\n\"date_created_edtf\": {\n    \"date_to\": \"\",\n    \"date_free\": \"~1899\",\n    \"date_from\": \"\",\n    \"date_type\": \"date_free\"\n},\n\"date_created_free\": null,\n...\n</code></pre></p> <p>Then we pass the <code>date_free</code> field through the <code>trim</code> filter (as a precaution, in case there's any accidental whitespace), and then we finally hand off the field to our <code>edtf_2_human_date</code> filter: edtf_2_human_date<pre><code>{{ data.date_created_edtf.date_free|trim|edtf_2_human_date('en') }}\n\n{# Output: Circa 1899 #}\n</code></pre></p> <p>html_2_markdown</p> <p>The <code>html_2_markdown</code> filter, as the name suggests, converts HTML to Markdown.</p> <p>We start with this string of HTML: HTML string<pre><code>{% set html_string = \"\n  &lt;ul&gt;\n    &lt;li&gt;One thing&lt;/li&gt;\n    &lt;li&gt;Another thing&lt;/li&gt;\n    &lt;li&gt;The last thing&lt;/li&gt;\n  &lt;/ul&gt;\n\" %}\n</code></pre></p> <p>Then we pass it to the filter: html_2_markdown<pre><code>{{ html_string | html_2_markdown }}\n\n{# Output:\n  - One thing\n  - Another thing\n  - The last thing\n#}\n</code></pre></p> <p>markdown_2_html</p> <p>The <code>markdown_2_html</code> filter, as the name suggests, is the reverse of the above and converts Markdown to HTML.</p> <p>We start with this string of Markdown: Markdown string<pre><code>{% set markdown_string = \"\n  - One thing\n  - Another thing\n  - The last thing\n\" %}\n</code></pre></p> <p>Then we pass it to the filter: markdown_2_html<pre><code>{{ markdown_string | markdown_2_html }}\n\n{# Output:\n  &lt;ul&gt;\n    &lt;li&gt;One thing&lt;/li&gt;\n    &lt;li&gt;Another thing&lt;/li&gt;\n    &lt;li&gt;The last thing&lt;/li&gt;\n  &lt;/ul&gt;\n#}\n</code></pre></p> <p>sbf_json_decode</p> <p>The <code>sbf_json_decode</code> filter decodes a JSON-encoded string.</p> <p>We start with this string of JSON string: JSON string<pre><code>{% set json_string = \"\n  {\n    \\\"date_to\\\": \\\"\\\",\n    \\\"date_free\\\": \\\"~1899\\\",\n    \\\"date_from\\\": \\\"\\\",\n    \\\"date_type\\\": \\\"date_free\\\"\n  }\n\" %}\n</code></pre></p> <p>Then we pass it to the filter: sbf_json_decode<pre><code>{% json_decoded = json_string | sbf_json_decode %}\n\n{{ json_decoded.date_free }}\n{# Output:\n  ~1899\n#}\n</code></pre></p>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON","Markdown","HTML"]},{"location":"twig_extensions/#twig-functions-from-archipelago","title":"Twig Functions from Archipelago","text":"<p>clipboard_copy</p> <p>The <code>clipboard_copy</code> function, using the clipboard-copy-element library, takes a provided CSS class for the element(s) whose text we'd like to copy, and targets the CSS class of an existing HTML element on the page or generates an HTML element that can be clicked to copy the text to the user's clipboard.</p> <p>Usage</p> clipboard_copy usage<pre><code>{{ clipboard_copy('CSS CLASS','OPTIONAL CSS CLASS(ES)','OPTIONAL TEXT') }}\n</code></pre> <p>This function takes three arguments:</p> <ul> <li>a CSS class for the element to copy</li> <li>an optional CSS class (the default is <code>clipboard-copy-button</code>) or classes (space-separated) for the copy button if auto-generating or a single, unique class if using your own existing button(s) </li> <li>optional text (the default is <code>Copy to Clipboard</code>) for the copy button if auto-generating</li> </ul> <p>In the examples below, we want users to be able to copy the text from three different kinds of HTML elements: a <code>div</code>, an <code>input</code>, and an <code>a</code> hyperlink href.</p> Copying div element text with auto-generated button <p>First we start by giving the div element(s) we'd like to copy a unique class:</p> div element text<pre><code>&lt;div class=\"csl-bib-body-container chicago-fullnote-bibliography\"&gt;\n  &lt;div id=\"copy-csl\" class=\"csl-bib-body\"&gt;\n    &lt;div class=\"csl-entry\"&gt;\n      New York Botanical Garden. \u201cDescriptive Guide to the Grounds, Buildings and Collections.\u201d\n    &lt;/div&gt;\n  &lt;/div&gt;\n&lt;/div&gt;\n</code></pre> <p>Then we pass the class to the function:</p> clipboard_copy for div element text<pre><code>{{ clipboard_copy('csl-bib-body','','Copy Bibliography Entry') }}\n</code></pre> <p>Note</p> <p>The class can be attached to parent elements of the element we are ultimately targeting if needed, but any intermediate characters may get caught up in the copied text.</p> <p>Or to give the generated button multiple classes (in case they need additional styling):</p> clipboard_copy for div element text<pre><code>{{ clipboard_copy('csl-bib-body','custom custom-button','Copy Bibliography Entry') }}\n</code></pre> <p>The result for the above <code>div</code> example looks as follows:</p> <p></p> <p>The following is the HTML for the auto-generated button with no provided CSS class:</p> <pre><code>&lt;button class=\"clipboard-copy-button\"&gt;\n  &lt;clipboard-copy for=\"copy-csl\" tabindex=\"0\" role=\"button\"&gt;Copy Bibliography Entry&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n</code></pre> <p>And the following is HTML for the auto-generated button with multiple CSS classes provided:</p> <pre><code>&lt;button class=\"custom custom-button\"&gt;\n  &lt;clipboard-copy for=\"copy-csl\" tabindex=\"0\" role=\"button\"&gt;Copy Bibliography Entry&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n</code></pre> <p>Note</p> <p>The clipboard-copy-element library requires an element ID. If the element being copied does not have an ID, one will automatically generated and assigned. </p> Copying input element value with auto-generated button <p>First we start by giving the input element(s) we'd like to copy a unique class:</p> input element value<pre><code>{% if attribute(data, 'as:image')|length &gt; 0  or attribute(data, 'as:document')|length &gt; 0  %}\n  &lt;h2&gt;\n    &lt;span class=\"align-middle\"&gt;Direct Link to Digital Object's IIIF Presentation Manifest V3 &lt;/span&gt;\n    &lt;img src=\"https://iiif.io/img/logo-iiif-34x30.png\"&gt;\n  &lt;/h2&gt;\n  {% set iiifmanifest = nodeurl|render ~ \"/metadata/iiifmanifest/default.jsonld\" %}\n  &lt;input type=\"text\" value=\"{{ iiifmanifest }}\" id=\"iiifmanifest_copy\" size=\"{{ iiifmanifest|length }}\" class=\"col-xs-3 copy-content\"&gt;\n{% endif %}\n</code></pre> <p>Then we pass the class to the function:</p> clipboard_copy for input element value<pre><code>{{ clipboard_copy('copy-content','',\"Copy Link to Digital Object's IIIF Presentation Manifest V3\") }}\n</code></pre> <p>Or to give the generated button multiple classes (in case they need additional styling):</p> clipboard_copy for input element text<pre><code>{{ clipboard_copy('copy-content','custom custom-button',\"Copy Link to Digital Object's IIIF Presentation Manifest V3\") }}\n</code></pre> <p>The result for the above <code>input</code> example looks as follows:</p> <p></p> <p>The following is the HTML for the auto-generated button with no provided CSS class:</p> <pre><code>&lt;button class=\"clipboard-copy-button\"&gt;\n  &lt;clipboard-copy for=\"iiifmanifest_copy\" tabindex=\"0\" role=\"button\"&gt;Copy Link to Digital Object's IIIF Presentation Manifest V3&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n</code></pre> <p>And the following is HTML for the auto-generated button with multiple CSS classes provided:</p> <pre><code>&lt;button class=\"custom custom-button\"&gt;\n  &lt;clipboard-copy for=\"iiifmanifest_copy\" tabindex=\"0\" role=\"button\"&gt;Copy Link to Digital Object's IIIF Presentation Manifest V3&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n</code></pre> <p>Note</p> <p>The clipboard-copy-element library requires an element ID. If the element being copied does not have an ID, one will automatically generated and assigned. </p> Copying anchor element hyperlink href with auto-generated button <p>First we start by giving the <code>a</code> element(s) we'd like to copy a unique class:</p> anchor element hyperlink href<pre><code>&lt;a id=\"copy-documentation-id\" class=\"copy-documentation-class row\" href=\"https://docs.archipelago.nyc\"&gt;Archipelago Documentation&lt;/a&gt;\n</code></pre> <p>Then we pass the class to the function:</p> clipboard_copy for anchor element hyperlink href<pre><code>{{ clipboard_copy('copy-documentation-class','',\"Copy Link to Documentation\") }}\n</code></pre> <p>Or to give the generated button multiple classes (in case they need additional styling):</p> clipboard_copy for anchor element text<pre><code>{{ clipboard_copy('copy-documentation-class','custom custom-button',\"Copy Link to Documentation\") }}\n</code></pre> <p>The result for the above anchor example looks as follows:</p> <p></p> <p>The following is the HTML for the auto-generated button with no provided CSS class:</p> <pre><code>&lt;button class=\"clipboard-copy-button\"&gt;\n  &lt;clipboard-copy for=\"copy-documentation-id\" tabindex=\"0\" role=\"button\"&gt;Copy Link to Documentation&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n</code></pre> <p>And the following is HTML for the auto-generated button with multiple CSS classes provided:</p> <pre><code>&lt;button class=\"custom custom-button\"&gt;\n  &lt;clipboard-copy for=\"copy-documentation-id\" tabindex=\"0\" role=\"button\"&gt;Copy Link to Documentation&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n</code></pre> <p>Note</p> <p>The clipboard-copy-element library requires an element ID. If the element being copied does not have an ID, one will automatically generated and assigned. </p> <p>The above examples automatically generate <code>copy</code> buttons. They can be styled, but if we need more control over the button placement and styling, we can use our own button(s) by ensuring that they meet the following requirements:</p> <ol> <li>A <code>&lt;copy-clipboard&gt;</code> element (this can be hidden) with a <code>for</code> attribute, whose value is the ID of the source element, attached to the element acting as the button.</li> <li>A class on the existing button that can be targeted. The class must either be unique (if a single button) or the number of elements with the class must match the number of source elements.</li> <li>A separate class for the copy source(s) with the same requirements as the previous step.</li> </ol> Copying div element text with custom button <p>First we start by giving the div element(s) we'd like to copy a unique class:</p> div element text<pre><code>&lt;div class=\"csl-bib-body-container chicago-fullnote-bibliography\"&gt;\n  &lt;div id=\"copy-csl\" class=\"csl-bib-body\"&gt;\n    &lt;div class=\"csl-entry\"&gt;\n      New York Botanical Garden. \u201cDescriptive Guide to the Grounds, Buildings and Collections.\u201d\n    &lt;/div&gt;\n  &lt;/div&gt;\n&lt;/div&gt;\n</code></pre> <p>Then we generate the button and pass the class to the function:</p> clipboard_copy custom button for div element text<pre><code>&lt;button class=\"custom-button btn btn-primary btn-sm\"&gt;\n  &lt;clipboard-copy for=\"copy-csl\"&gt;Copy Text&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n\n{{ clipboard_copy('csl-bib-body','custom-button','') }}\n</code></pre> <p>Note</p> <p>The clipboard-copy-element library requires an element ID. If the element being copied does not have an ID, one will automatically generated and assigned. </p> Copying input element value with custom button <p>First we start by giving the input element(s) we'd like to copy a unique class:</p> input element value<pre><code>{% if attribute(data, 'as:image')|length &gt; 0  or attribute(data, 'as:document')|length &gt; 0  %}\n  &lt;h2&gt;\n    &lt;span class=\"align-middle\"&gt;Direct Link to Digital Object's IIIF Presentation Manifest V3 &lt;/span&gt;\n    &lt;img src=\"https://iiif.io/img/logo-iiif-34x30.png\"&gt;\n  &lt;/h2&gt;\n  {% set iiifmanifest = nodeurl|render ~ \"/metadata/iiifmanifest/default.jsonld\" %}\n  &lt;input type=\"text\" value=\"{{ iiifmanifest }}\" id=\"iiifmanifest_copy\" size=\"{{ iiifmanifest|length }}\" class=\"col-xs-3 copy-content\"&gt;\n{% endif %}\n</code></pre> <p>Then we generate the button and pass the class to the function:</p> clipboard_copy custom button for input element value<pre><code>&lt;button class=\"custom-button btn btn-primary btn-sm\"&gt;\n  &lt;clipboard-copy for=\"iiifmanifest_copy\"&gt;Copy Input&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n\n{{ clipboard_copy('copy-content','custom-button','') }}\n</code></pre> <p>Note</p> <p>The clipboard-copy-element library requires an element ID. If the element being copied does not have an ID, one will automatically generated and assigned. </p> Copying anchor element with custom button <p>First we start by giving the <code>a</code> element(s) we'd like to copy a unique class:</p> anchor element hyperlink href<pre><code>&lt;a id=\"copy-documentation-id\" class=\"copy-documentation-class row\" href=\"https://docs.archipelago.nyc\"&gt;Archipelago Documentation&lt;/a&gt;\n</code></pre> <p>Then we generate the button and pass the class to the function:</p> clipboard_copy custom button for anchor element hyperlink href<pre><code>&lt;button class=\"custom-button btn btn-primary btn-sm\"&gt;\n  &lt;clipboard-copy for=\"copy-documentation-id\"&gt;Copy Link&lt;/clipboard-copy&gt;\n&lt;/button&gt;\n\n{{ clipboard_copy('copy-documentation-class','custom-button','') }}\n</code></pre> <p>Note</p> <p>The clipboard-copy-element library requires an element ID. If the element being copied does not have an ID, one will automatically generated and assigned. </p> <p>sbf_entity_ids_by_label</p> <p>The <code>sbf_entity_ids_by_label</code> function, as the name suggests, provides a Drupal entity ID for the following Drupal entity types:</p> <ul> <li>node</li> <li>taxonomy_term</li> <li>group</li> <li>user </li> </ul> <p>If we start with the user entity <code>jsonapi</code>, we can do the following: sbf_entity_ids_by_label<pre><code>{% set jsonapi_user_ids=sbf_entity_ids_by_label('jsonapi','user','') %}\n\n{% for jsonapi_user_id in jsonapi_user_ids %}\n  {{ jsonapi_user_id }}\n{% endfor %}\n\n{# Output:\n  3\n#}\n</code></pre></p> <p>As you can see above, the <code>sbf_entity_ids_by_label</code> function takes three arguments:</p> <ul> <li>the entity label</li> <li>the entity type (see above for supported types)</li> <li>an optional entity bundle</li> </ul> <p>We then loop through the returned result, which is an array of IDs (in this case, just a single one).</p> <p>sbf_search_api</p> <p>The <code>sbf_search_api</code> function executes a search API query against a specified index.</p> sbf_search_api<pre><code>{% set search_results=sbf_search_api('default_solr_index','strawberry',[],{'status':1},[]) %}\n{% set labels=search_results['results']['13']['fields']['label_2'] %}\n&lt;ul&gt;\n  {% for label in labels %}\n    &lt;li&gt;{{ label }}&lt;/li&gt;\n  {% endfor %}\n&lt;/ul&gt;\n</code></pre> <p>As you can see above, the <code>sbf_search_api</code> function takes eight arguments:</p> <ul> <li>The machine name of the Search API index to search against (string)</li> <li>A full text term to search (string)</li> <li>An array of full text fields to search the term against. If empty all will be used. (array)</li> <li>The fields =&gt; filters to match against (associative array)</li> <li>The fields to facet (array)</li> <li>The fields to sort against (associative array)</li> <li>Offset for the results (int)</li> </ul> <p>For this example we end up with the following output:</p> <ul> <li>JPEG File Interchange Format</li> <li>Organic farming--United States</li> <li>Strawberries</li> <li>Strawberry Field at Thorpes Organic Family Farm</li> <li>organic agriculture</li> <li>strawberries</li> </ul>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON","Markdown","HTML"]},{"location":"twig_recipe_cards/","title":"Twig Recipe Cards for Common Use Cases","text":"<p>The Twig Recipe Cards below reference common Metadata transformation, display, or other use cases/needs you may have in your own Archipelago repository.</p>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON"]},{"location":"twig_recipe_cards/#getting-started-working-with-twig-in-archipelago","title":"Getting Started Working with Twig in Archipelago","text":"<p>We recommend reading through our main Metadata Display Preview and Twigs in Archipelago documentation overview guides, and also our Working with Twig primer before diving into applying any of these recipes in your own Archipelago.</p>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON"]},{"location":"twig_recipe_cards/#ami-ingest-template-adaptations-common-use-cases-and-twig-recipe-cards","title":"AMI Ingest Template Adaptations -- Common Use Cases and Twig Recipe Cards:","text":"<p>Use Case #1: I used AMI LoD Reconciliation to reconciliate the values in my AMI Set Source CSV <code>mods_subject_topic</code> column against both LCSH and Wikidata. I would like to map the reconciliated values into the Archipelago default <code>subject_loc</code> and <code>subject_wikidata</code> JSON keys.</p> <p>Twig Recipe Card for Use Case #1:</p> <pre><code>  {#- LCSH -#}\n    {% if data.mods_subject_topic|length &gt; 0 %}\n    \"subject_loc\": {{ data_lod.mods_subject_topic.loc_subjects_thing|json_encode|raw }},\n    {% endif %}\n  {#- Wikidata -#}     \n    {% set subject_wikidata = [] %}\n    {% for source, reconciliated in data_lod %}\n      {% if (('subject' in source) or ('genre' in source)) and reconciliated.wikidata_subjects_thing and reconciliated.wikidata_subjects_thing|length &gt; 0 %}\n         {% set subject_wikidata = subject_wikidata|merge(reconciliated.wikidata_subjects_thing) %}\n      {% endif %}\n    {% endfor %}  \n</code></pre> <p>Use Case #2: I have both columns containing a <code>mods_subject_authority_lcsh_topic</code> (labels) and corresponding <code>mods_subject_authority_lcsh_valueuri</code> (URIs) data in my AMI Set Source Data CSV that I would like to pair and map into the Archipelago default <code>subject_loc</code> JSON key.</p> <p>Twig Recipe Card for Use Case #2:</p> <pre><code>{%- if data['mods_subject_authority_lcsh_topic'] is defined and not empty -%}\n    {% set subjects = data[\"mods_subject_authority_lcsh_topic\"] is iterable ? data[\"mods_subject_authority_lcsh_topic\"] : data[\"mods_subject_authority_lcsh_topic\"]|split('|@|') %} \n    {% set subject_uris = data[\"mods_subject_authority_lcsh_valueuri\"] is defined ? data[\"mods_subject_authority_lcsh_valueuri\"] : '' %} \n    {% set subject_uris_list = subject_uris is iterable ? subject_uris : subject_uris|split('|@|') %}\n    \"subject_loc\": [\n        {% for subject in subjects %}\n        {\n            \"uri\": {{ subject_uris_list[loop.index0]|default('')|json_encode|raw }},\n            \"label\": {{ subject|json_encode|raw }}\n        }\n        {{ not loop.last ? ',' : '' }}\n        {% endfor %}\n    ],\n{%- endif -%}\n</code></pre> <p>Use case #3: I have <code>dc.creator</code> and <code>dc.contributor</code> columns in my AMI Set Source Data CSV with simple JSON-encoded values (e.g. source column cells contain <code>[\"Name 1, Name 2\"]</code>) that I would like to map to the Archipelago default <code>creator_lod</code> JSON key.</p> <p>Twig Recipe Card for Use Case #3:</p> <p><pre><code>{% if data['dc.creator']|length &gt; 0 or data['dc.contributor']|length &gt; 0 %}\n    {% set total_creators = (data[\"dc.creator\"]|length) + (data[\"dc.contributor\"]|length) %}\n    {% set current_creator = 0 %}                \n    \"creator_lod\": [\n        {% for creator in data[\"dc.creator\"] %}\n            {% set current_creator = current_creator + 1 %}\n            {% set creator_source = data[\"dc.creator\"][loop.index0] %}\n        {\n            \"name_uri\": null,\n            \"agent_type\": null,\n            \"name_label\": {{ creator|json_encode|raw }},\n            \"role_label\": \"Creator\",\n            \"role_uri\": \"http://id.loc.gov/vocabulary/relators/cre\"\n        }\n        {{ current_creator != total_creators ? ',' : '' }}\n        {% endfor %}\n        {% for creator in data[\"dc.contributor\"] %}\n            {% set current_creator = current_creator + 1 %}\n            {% set creator_source = data[\"dc.contributor\"][loop.index0] %}\n        {\n            \"name_uri\": null,\n            \"agent_type\": null,\n            \"name_label\": {{ creator|json_encode|raw }},\n            \"role_label\": \"Contributor\",\n            \"role_uri\": \"http://id.loc.gov/vocabulary/relators/ctb\"\n        }\n        {{ current_creator != total_creators ? ',' : '' }}\n        {% endfor %}   \n ],\n{% endif %}  \n</code></pre> Use Case #4: I have a mix of different columns containing Creator/Contributor/Other-Role-Types Name values with or without corresponding URI values that I would like to map to the default Archipelago <code>creator_lod</code> JSON key.</p> <p>Twig Recipe Card for Use Case #4:</p> Click to view the full Recipe Card <pre><code>{#- START Names from LoD and MODS CSV with/without URIS. -#} \n    {# Updated August 26th 2022, by Diego Pino. New checks/logic for mods_name_type_role_composed_or_more_namepart\n    - Check first IF for a given namepart there is already reconciliaton. \n    - IF not i check if there is a matching valueuri, \n    - If not leave the URL empty and use the value in the namepart (label) only?\n    - Only check/use mods_name_corporate/personal_namepart field IF there are no other fields\n    - That specify Roles. Since normally in ISLANDORA that field (no role) is a Catch all names one\n    - And in that case USE creator as the default ROLE\n    #}\n    {%~ set creator_lod = [] -%} \n    {# Used to keep track of parts after the type (corporate, etc) that are no roles\n     but authority properties. Add more if you find them #}\n    {%  set roles_that_are_no_roles = ['authority_naf','authority_marcrelator',''] %}\n    {# Used to keep track of the ones that are reconciled already #}\n    {%- set name_has_creator_lod = [] -%}\n    {%- for key,value in data_lod -%}\n      {%- if key starts with 'mods_name_' and key ends with '_namepart' -%}\n      {# If there is mods_name_SOMETHING_namepart in data_lod we keep track so we \n      do not try afterwards to use that Sources KEY from the CSV.\n      #}\n        {%- set name_has_creator_lod = name_has_creator_lod|merge([key]) -%}\n      {# Now we remove 'mods_name_' and '_namepart' #}\n        {%- set name_type_and_role = key|replace({'mods_name_':'', '_namepart':''}) -%}\n      {# We will only target personal or corporate. If any of those are missing we skip? #}\n        {% set name_type = null %}\n        {%- if name_type_and_role starts with 'personal_' -%}\n           {% set name_type = 'personal' %}\n        {%- elseif name_type_and_role starts with 'corporate_' -%}\n           {%- set name_type = 'corporate' -%}\n        {%- endif -%}\n        {%- if name_type is not empty -%}\n        {#- Now we remove 'type', e.g 'corporate_' -#}\n          {%- set name_role = name_type_and_role|replace({(name_type ~ '_'):''}) -%}\n          {# in case the name_role contains one of roles_that_are_no_roles, e.g\n          something like `creator_authority_marcrelator` we remove that #}\n          {% for role_that_is_no_role in roles_that_are_no_roles %}\n             {%- set name_role = name_role|replace({(role_that_is_no_role):''}) -%}\n          {% endfor %}\n          {# After removing all what can not be a role if we end with an empty #}\n          {% if name_role|trim|length == 0 %}\n             {%- set name_role = \"creator\" %}\n          {% else %}\n            {%- set name_role = name_role|replace({'\\\\/':'//' , '_':' '})|trim -%}\n          {% endif %}\n        {#- we iterate over all possible vocabularies and fetch the reconciliated names from them (if any) -#}\n          {%- for approach, names in value -%} \n        {#- if there are actually name pairs (name and uri) that were reconciliated we use them -#}\n            {%- if names|length &gt; 0 -%}\n              {#- we call the ami_lod_reconcile twig extension with the role label using the LoC Relators endpoint in english and get 1 result -#}\n              {%- set role_uri = ami_lod_reconcile(name_role|lower|capitalize,'loc;relators;thing','en',1) -%}\n              {#- for each found name pair in a list of possible LoD reconciliated elements we generate the final structure that goes into \"creator_lod\" json key -#}\n              {%- for name in names -%} \n                {%- set creator_lod = creator_lod|merge([{'role_label': name_role|lower|capitalize, 'role_uri': role_uri[0].uri, \"agent_type\": name_type, \"name_label\": name.label, \"name_uri\": name.uri}]) -%}     \n             {%- endfor -%}\n            {%- endif -%}\n          {%- endfor -%}\n        {% endif -%}\n      {%- endif -%} \n    {%- endfor -%}\n    {# Now go for the RAW CSV data for names #}\n    {%- for key,value in data -%}\n    {# here we skip values previoulsy fetched from LoD and stored in name_has_creator_lod #}\n      {%- if key not in name_has_creator_lod and key starts with 'mods_name_' and key ends with '_namepart' -%}\n      {# If there is mods_name_SOMETHING_namepart in data_lod we keep track so we \n      do not try afterwards to use that Sources KEY from the CSV.\n      #}\n        {%- set name_has_creator_lod = name_has_creator_lod|merge([key]) -%}\n      {# Now we remove 'mods_name_' and '_namepart' #}\n        {%- set name_type_and_role = key|replace({'mods_name_':'', '_namepart':''}) -%}\n      {# We will only target personal or corporate. If any of those are missing we skip? #}\n        {%- set name_type = null -%}\n        {%- if name_type_and_role starts with 'personal_' -%}\n           {%- set name_type = 'personal' -%}\n        {%- elseif name_type_and_role starts with 'corporate_' -%}\n           {%- set name_type = 'corporate' -%}\n        {%- endif -%}\n        {% if name_type is not empty %}\n        {# Now we remove 'type', e.g 'corporate_' #}\n          {%- set name_role = name_type_and_role|replace({(name_type ~ '_'):''}) -%}\n          {# in case the name_role contains one of roles_that_are_no_roles, e.g\n          something like `creator_authority_marcrelator` we remove that #}\n          {% for role_that_is_no_role in roles_that_are_no_roles %}\n             {%- set name_role = name_role|replace({(role_that_is_no_role):''}) -%}\n          {% endfor %}\n          {# After removing all what can not be a role if we end with an empty #}\n          {% if name_role|trim|length == 0 %}\n             {%- set name_role = \"creator\" %}\n          {% else %}\n            {%- set name_role = name_role|replace({'\\\\/':'//' , '_':' '})|trim -%}\n          {% endif %}\n          {# Now we check if there is a corresponding _valueuri for this #}\n          {% set name_uris = [] %}\n          {%- if data[('mods_name_' ~ name_type_and_role ~ '_valueuri')] is not empty \n          and data[('mods_name_' ~ name_type_and_role ~ '_valueuri')] != '' -%}\n            {%- set name_uris = data[('mods_name_' ~ name_type_and_role ~ '_valueuri')]|split('|@|') -%}\n          {%- endif -%}\n          {%- set role_uri = ami_lod_reconcile(name_role|lower|capitalize,'loc;relators;thing','en',1) -%}\n        {#- we split and iterate over the value of the mods_name key -#}\n        {# NOTE. THIS IS TARGETING Anything after the year 1000, or 2000 #}\n          {%- for index,name in value|replace({'|@|1':', 1', '|@|2':', 2', '|@|-':', -'})|split('|@|') -%}\n            {%- if name is not empty and name|trim != '' -%}\n              {%- set name_uri = null -%}\n              {# Here we can check if one of the names IS not a name (e.g a year? #}\n              {#- we call the ami_lod_reconcile twig extension with the role label using the LoC Relators endpoint in english and get 1 result -#}\n              {%- if name_uris[index] is defined and name_uris[index] is not empty -%}\n                 {%- set name_uri = name_uris[index] -%}\n              {%- endif -%}\n              {%- set creator_lod = creator_lod|merge([{'role_label': name_role|lower|capitalize, 'role_uri': role_uri[0].uri, \"agent_type\": name_type, \"name_label\": name, \"name_uri\": name_uri}]) -%}\n            {%- endif -%}\n          {%- endfor -%}\n        {%- endif -%}\n      {%- endif -%}\n    {%- endfor ~%}\n    {# Use reduce filter + other logic for depulicating #}\n    {% set creator_lod = creator_lod|reduce((unique, item) =&gt; item in unique ? unique : unique|merge([item]), []) %}\n    \"creator_lod\": {{ creator_lod|json_encode|raw -}},\n    {#- END Names from LoD and MODS CSV with/without URIS. -#}\n</code></pre> <p>Use Case #5: I have geographic location information that I would like to reconciliate against Nominatim and map into the default Archipelago 'geographic_location' key. I have AMI Source Data CSVs which contain values/labels and some which contain coordinates.</p> <p>Twig Recipe Card for Use Case #5 with variation notes:</p> <pre><code>{#- &lt;-- Geographic Info and terms:\n  Includes options for geographic info for:\n  - Nominatim lookup by value/label\n  - Nominatim lookup by coordinates \n  -#}\n    {#- use value for Nominatim search -#}\n    {% if data.mods_subject_geographic|length &gt; 0 %}\n      {% set nominatim_from_label = ami_lod_reconcile(data.mods_subject_geographic,'nominatim;thing;search','en') -%}\n    \"geographic_location\": {{ nominatim_from_label|json_encode|raw }},\n    {% endif %}\n    {#- use coordinates for Nominatim search, if provided -#}\n    {% if data.mods_subject_cartographics_coordinates|length &gt; 0 %}\n      {% set nominatim_from_coordinates = ami_lod_reconcile(data.mods_subject_cartographics_coordinates,'nominatim;thing;reverse','en') -%}\n    \"geographic_location\": {{ nominatim_from_coordinates|json_encode|raw }},\n    {% endif %}\n{#- Geographic Info and terms --&gt; #}  \n</code></pre> <p>Use Case #6: I have date values in a <code>dc.date</code> column that contain instances of 'circa' or 'Circa' where I would like to replace with the EDTF-friendly '~' instead and map to the Archipelago default 'date_created_edtf' JSON key.</p> <p>Twig Recipe Card for Use Case #6: </p> <pre><code>    {% if data['dc.date'] is defined %}\n        {% set datecleaned = data['dc.date']|replace({\"circa \":\"~\", \"Circa \":\"~\"}) %}\n        \"date_created_edtf\": {\n        \"date_to\": \"\",\n        \"date_free\": {{ datecleaned|json_encode|raw }},\n        \"date_from\": \"\",\n        \"date_type\": \"date_free\"\n        },        \n    {% endif %}\n</code></pre> <p>More recipe cards will be added over time. Please see our Archipelago Contribution Guide to learn about contributing your own recipe card or other documentation.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Twig","Twig Filters","Twig Functions","Twig Templates","Examples","JSON"]},{"location":"utility_scripts/","title":"Utility Scripts","text":"<p>If you've already followed deployment guides for archipelago-deployment and archipelago-deployment-live, you may have used some shell scripts that archipelago provides. The scripts are available in the <code>scripts/archipelago/</code> and <code>drupal/scripts/archipelago/</code> folders respectively.</p>","tags":["Bash","Scripts","DevOps"]},{"location":"utility_scripts/#importexport-metadata-display-twig-templates","title":"Import/Export Metadata Display Twig Templates","text":"<p>Metadata Display Entity Twig Templates can be exported out of and imported into both local and remote deployments with the following script: <code>import_export.sh</code>. The script can be run interactively or non-interactively.</p> <p>Docker host vs. Docker container</p> <p>Because the script uses the Docker <code>.env</code> file for the JSONAPI user and URL by default, we recommend running this directly on the host.</p>","tags":["Bash","Scripts","DevOps"]},{"location":"utility_scripts/#interactive-mode","title":"Interactive Mode","text":"<p>Running the script interactively will guide you through a number of prompts to configure and import or export to an existing folder or to one which will be created.</p> <pre><code>./import_export.sh -n\n</code></pre>","tags":["Bash","Scripts","DevOps"]},{"location":"utility_scripts/#non-interactive-mode","title":"Non-interactive Mode","text":"<p>To run the command non-interactively provide the required and optional parameters with the necessary arguments as needed.</p> <p>Options for Non-interactive Mode</p> <p><code>-i</code> or <code>-e</code> (required)</p> <p>\u00a0\u00a0\u00a0\u00a0Import or export, respectively, Metadata Entity Display Twig Templates from a local folder.</p> <p><code>-s path</code> (required)</p> <p>\u00a0\u00a0\u00a0\u00a0The absolute path of the local folder to export to or import from.</p> <p><code>-j path/filename</code> (only required if the <code>.env</code> file containing the JSONAPI user and password is in a non-standard location)</p> <p>\u00a0\u00a0\u00a0\u00a0The absolute path to the <code>.env</code> file containing the JSONAPI user and password.</p> <p><code>-d url</code> (required if URL is not in <code>.env</code> file or importing to or exporting from a remote deployment)</p> <p>\u00a0\u00a0\u00a0\u00a0The URL of the archipelago deployment.</p> <p><code>-k</code> (optional)</p> <p>\u00a0\u00a0\u00a0\u00a0Keep any existing files ending with <code>.json</code> in the specified folder (the default is to delete) before exporting.</p> <p>JSONAPI User</p> <p>The JSONAPI user credentials, by default, will be read from the <code>.env</code> files in the following locations (relative to the root of the deployment):</p> Deployment File Location archipelago-deployment-live <code>./deploy/ec2-user/.env</code> archipelago-deployment <code>./.env</code> <p>A separate file can also be passed as an argument using the <code>-j</code> option.</p> .env<pre><code>JSONAPI_USER=jsonapi\nJSONAPI_PASSWORD=jsonapi\n</code></pre> <p>Exporting from local archipelago-deployment-live</p> <p><pre><code>./import_export.sh -e -s /home/ec2-user/metadatadisplay_export\n</code></pre> After logging into the archipelago-deployment-live host, the above command will delete any files with the <code>.json</code> extension if the destination folder exists. Otherwise, the folder will be created. The JSON user credentials and domain from the <code>.env</code> file will then be used to download the files so please make sure these are set.</p> <p>Exporting from local archipelago-deployment</p> <p><pre><code>./import_export.sh -e -s /home/user/metadatadisplay_export -d http://localhost:8001\n</code></pre> This will work the same way as the above example, but the URL is passed as an argument in this case since the <code>.env</code> file will not contain (in most cases) the domain. As above, the JSON user credentials will have to be set in the <code>.env</code> file.</p> <p>Exporting from remote archipelago-deployment-live</p> <p><pre><code>./import_export.sh -e -s /home/user/metadatadisplay_export -d https://archipelago.nyc\n</code></pre> This is essentially the same as the example directly above, except that in this case the JSON user credentials in the <code>.env</code> file will have to be set to the ones used to access the remote instance.</p> <p>Importing locally into archipelago-deployment-live</p> <p><pre><code>./import_export.sh -i -s /home/user/metadatadisplay_import\n</code></pre> This is essentially the same as the first example above, except that the import option (<code>-i</code>) is used. The folder name is changed for the sake of example, but you can use the same folder that was used for exporting.</p> <p>Importing locally into archipelago-deployment</p> <p><pre><code>./import_export.sh -i -s /home/user/metadatadisplay_import -d http://localhost:8001\n</code></pre> As in the example directly above, this corresponds to the example for exporting with a local archipelago-deployment instance, except that the import option (<code>-i</code>) is used.</p> <p>Importing from local instance into remote archipelago-deployment-live</p> <p><pre><code>./import_export.sh -i -s /home/user/metadatadisplay_import -d https://archipelago.nyc\n</code></pre> In this example, the locally exported files are being imported into a remote instance. As in the above examples with remote instances, the JSON user credentials need to be set in the <code>.env</code> file to those with access to the remote instance.</p>","tags":["Bash","Scripts","DevOps"]},{"location":"utility_scripts/#automatic-deployment-script","title":"Automatic Deployment Script","text":"<p>If you're frequently deploying locally with archipelago-deployment, you may want to use the automated deployment script available at <code>scripts/archipelago/devops/auto_deploy.sh</code>. The script is interactive and can be called from the root of the deployment, e.g. <code>/home/user/archipelago-deployment/</code>:</p> <p>Automatic Deployment</p> <p>scripts/archipelago/devops/auto_deploy.sh</p> <p>Follow the prompts and select your options to complete the deployment.</p>","tags":["Bash","Scripts","DevOps"]},{"location":"webformLoDfromCSV/","title":"Using Archipelago's 'Webform LoD from CSV attached to an ADO suggest'","text":"<p>Archipelago's custom webform element 'Webform LoD from CSV attached to an ADO suggest' provides a form element autocomplete labels/urls(values) from a CSV attached to a Digital Object. Using this element affords a way for you to utilize a custom local vocabulary, a subset of labels and URIs from a wider LoD authority source, or an LoD vocabulary that does not have an accessible API or public query service.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#step-by-step","title":"Step-by-step","text":"","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#1-prepare-your-csv-file","title":"1. Prepare your CSV File","text":"<p>To use this element, you need to first have prepared a CSV file containing two columns only:</p> <ul> <li>one column for 'label' containing the labels for your vocabulary</li> <li>one column for 'uris' containing the corresponding uris for your vocabulary</li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#2-new-ado-using-your-csv-file","title":"2. New ADO Using Your CSV File","text":"<p>Create a new Digital Object, and attached the prepared CSV file to the new Digital Object.</p> <ul> <li>Be sure to provide a unique label to help you identify this Object in future steps.</li> <li>It is recommended that you do not Publish this Object.</li> </ul> <p>Note</p> <p>If you are not yet familiar with how to create a Digtial Object, please refer to this guide.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#3-edit-your-webform","title":"3. Edit your Webform","text":"<p>Go to <code>Admin &gt; Structure &gt; Webforms</code> and select the 'Build' button beside the Default Descriptive Metadata Webform.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#4-add-new-custom-webform-element","title":"4. Add New Custom Webform Element","text":"<p>Scroll down to the 'Subjects and Other Classifications' page of the Webform and select 'Add Element'.</p> <p></p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#5-webform-from-lod-from-csv-attached-to-an-ado-suggest","title":"5. Webform from LoD from CSV attached to an ADO suggest","text":"<p>In the 'Select an element to add ..' popup that opens:</p> <ul> <li>Either scroll down to select the 'Composite Element' section or search for the 'Webform LoD from CSV attached to an ADO suggest.' </li> <li> <p>Select 'Add Element'.</p> <p></p> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#6-adjust-element-settings","title":"6. Adjust Element Settings","text":"<p>In the Edit tab that opens for your newly added element, you will need to review the following sections.</p> <ul> <li> <p>Element Settings</p> <ul> <li>provide a Title for element</li> <li>check that the Key generated from the Title you supply is well formed and make changes if needed</li> <li>specify the 'Allowed number of values'</li> </ul> <p></p> </li> <li> <p>Webform LoD from CSV attached to an ADO suggest settings:</p> <ul> <li>It is recommended to keept both 'label' and 'uri' checked as Visible.</li> <li>You may also wish to mark both elements as 'Required'</li> </ul> <p></p> </li> <li> <p>Autocomplete settings</p> <ul> <li>In the 'Choose an ADO' box, begin typing to search for the Digital Object that holds a CSV containing the Vocabulary you want to autocomplete.</li> <li>Under 'The CSV column(header name) that will be used for autocompleting', enter 'label'</li> <li>Under 'The CSV column(header name) that will be used for the URL value', enter 'uri'</li> <li>'Autocomplete limit'<ul> <li>determines the maximum number of matches to be displayed</li> <li>recommended that you set to '10'.</li> </ul> </li> <li>'Autocomplete minimum number of characters'<ul> <li>determines the minimum number of characters a user must type before a search is performed</li> <li>recommended that you set to '3'.</li> </ul> </li> <li>'Autocomplete matching operator'<ul> <li>determines the method used to collect autocomplete suggestions</li> <li>recommended to use 'Starts with'          </li> </ul> </li> </ul> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#7-advanced-tab-for-webform-element","title":"7. Advanced Tab for Webform Element","text":"<p>Navigate to the 'Advanced' tab for this Webform element.</p> <ul> <li>Open the 'Multiple settings' section</li> <li> <p>Deselect the options to 'Allow users to sort elements' and 'Allow users to add more items'</p> <p></p> </li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#8-save-your-work","title":"8. Save Your Work","text":"<p>Save your new form element settings. Then Save your updated Webform.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformLoDfromCSV/#9-use-your-new-webform-element","title":"9. Use Your New Webform Element","text":"<p>Navigate to a Digital Object in your repository that you would like to use this new custom vocabulary element with. </p> <ul> <li>Select 'Edit' for that Digital Object and navigate to the 'Subjects and Other Classifications' page of the webform.</li> <li> <p>Begin typing a label found in your prepared CSV associated with the webform element.</p> <p></p> </li> </ul> <p>You can now begin using this custom vocab vocabulary element when using the corresponding webform (where you added this element) to Edit and Update your Digital Objects. You may also wish to add this same element to the Default Digital Object Collection/Creative Work Series webform.</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webforms/","title":"Webforms in Archipelago","text":"<p>The Webform Strawberryfield module provides Drupal Webform ( == awesome piece of code) integrations for StrawberryField so you can really have control over your Metadata ingests. These custom elements provide Drupal Webform integrations for Archipelago\u2019s StrawberryField so you can have fine grained and detailed control over your Metadata ingests and edits.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webforms/#where-to-find","title":"Where to Find","text":"<p>You can access Webforms in Archipelago at: - <code>Admin &gt; Structure &gt; Webforms</code> - <code>/admin/structure/webform/</code></p> <p></p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webforms/#instructions-and-guides","title":"Instructions and Guides","text":"<ul> <li>How to Create a Webform as an Input Method for Archipelago Digital Objects (ADO)</li> <li>Customizing Webforms: Modifying allowable file extensions</li> <li>Archipelago Custom Webform Elements</li> <li>Using Archipelago's 'Webform LoD from CSV attached to an ADO suggest'</li> </ul>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webforms/#archipelago-default-example-webforms","title":"Archipelago Default Example Webforms","text":"<p>Use these webforms or their elements to create a custom webform for your own repository/project needs:</p> <ul> <li> <p>Archipelago Default Deployment Webforms</p> <ul> <li> <p>Descriptive Metadata</p> <ul> <li>Corresponding Schema.org Type Options</li> </ul> </li> <li> <p>Digital Object Collection</p> <ul> <li>Corresponding Schema.org Type Options</li> </ul> </li> </ul> </li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Webform","Form Mode","Webform Elements"]},{"location":"webformsasinput/","title":"Primer on Display Modes &amp; How to Create a Webform as an Input Method for Archipelago Digital Objects (ADO)","text":"<p>Drupal 9/10 provides a lot of out-of-the-box functionality to setup the way Content Entities (Nodes or in our case ADOs) are exposed to users with the proper credentials. That functionality lives under the \"Display Modes\" &gt;&gt; and can be accessed at <code>yoursite/admin/structure/display-modes</code>.</p> <p>In a few quick words, The Display Mode Concept covers: formatting your Content Entities and their associated Fields so when a user lands on a Content Page, they are displayed in a certain, hopefully pleasing, way and also how users with proper Credentials can fill inputs/edit values for each <code>field</code> a Content Entity provides.</p> <p>First, formatting output (basically building the front facing page for each content entity) is done by a <code>View Mode</code>. Second, defining how/what input method you are going to use to create or edit Content entities, is handled by a <code>Form Mode</code>. Both Modes, are, in Drupal Lingo, Configuration Entities, they provide things you can configure, you can name them and reuse them and those configurations can all be exported and reimported using YAML files. Also both Modes the following in common:</p> <ul> <li>Drupal always provides a \"default\" one that can not be deleted.</li> <li>You can create new ones.</li> <li>You can apply permissions to them.</li> <li>All Modes work on \"fields\", means the tiny little input/output pieces that are either part of a Content Entity or attached to them (the title, the Body, and in our case a Strawberryfield (SBF),</li> <li>They Provide Config/setting options for each Field.</li> <li>They are always associated to Content Types/Bundles. Means all Nodes of the same Content type will share the same modes.</li> </ul> <p>The main difference, other than their purpose (Output v/s Input) is that, on View Modes, the settings you apply to each field are associated to \"Formatters\" and on Form Modes, the settings you apply to each field are connected to \"Widgets\".</p> <p>*Please note that this guides features some older screenshots using earlier versions of Archipelago/Drupal Adminsitrative Theming. Please pardon any jumps between themes.</p> <p>So, resuming, this is what lives under the Concept of a \"Display Mode\"...</p>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#view-mode","title":"View Mode","text":"<ul> <li>Each field attached to a Content Entity can have a Formatter applied and most of them have configuration options.</li> <li>Formatters do one thing right: they take the raw, stored value and make it \"visible\" inside Drupal.</li> <li>Which formatters are available will depend on the \"type\" of field the Content Entity has.<ul> <li>E.g A Node title/Label will have a Title formatter with the option of just displaying a text or a text with a link to the entity.</li> <li>More Complex and fun Fields, like the ones of type <code>SBF</code> will provide a large list of possible <code>Formatters</code>, like IIIF driven viewers, Video formatters, Metadata Display (Twig template driven) ones, etc. This is because a SBF type of field has much more than just a text value, it contains a full graph of metadata and properties, inclusive links to Files and provenance metadata.</li> </ul> </li> </ul>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#all-of-the-default-view-modes-bundled-in-archipelago","title":"All of the Default View Modes Bundled in Archipelago","text":"Click to see the full list of Default View Modes <ul> <li>Collection listing        </li> <li>Digital Object Collection with Mirador Viewer     </li> <li>Digital Object Creative Work Series with Mirador Viewer       </li> <li>Digital Object Full View      </li> <li>Digital Object Image Only for Carousel        </li> <li>Digital Object Oral History with Multiple Media       </li> <li>Digital Object with 3D Viewer     </li> <li>Digital Object with Audio Player      </li> <li>Digital Object with Book Reader       </li> <li>Digital Object with Mirador Viewer        </li> <li>Digital Object with Pannellum Panorama        </li> <li>Digital Object with PDF Viewer        </li> <li>Digital Object with thumbnail and abstract        </li> <li>Digital Object with thumbnail for Grid        </li> <li>Digital Object with Video Player      </li> <li>Digital Object with WARC Replay.web Widget        </li> <li>Search index      </li> <li>Search result highlighting input      </li> <li>Plus Drupal default View Modes: RSS, Teaser, and Token</li> </ul>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#form-mode","title":"Form Mode","text":"<ul> <li>Each field attached to a Content Entity can have a Widget applied and most of them have configuration options.</li> <li>Widgets do one thing right: they expose some type of Form/UI interaction that allows a user to input data into the Entity, under that specific field. And of course they make sure that what you input is validated and saved (if good) correctly.</li> <li> <p>Which Widgets are available will depend on the \"type\" of field the Content Entity has.</p> <ul> <li>Example: A <code>Node</code> Title will have a single Text Input with some options, like the size of the Textfield used to feed it.</li> <li>More Complex and fun fields, like the ones of type <code>SBF</code> (strawberryfield), will provide a larger list of possible Widgets, ranging from raw JSON input (which you could select if your data was already in the right format) to the reason we are reading this: <code>Webform driven Widgets</code>. These Widgets include:<ul> <li>ones the webform_strawberryfield Drupal module provides</li> <li>ones that use an existing Webform (which are also Entitites!) which either 1) you created or 2) we provided as a setting</li> </ul> </li> </ul> <p>If you chose a widget other than the raw JSON, the widget will take the raw JSON to build, massage and enrich the data so that it can be presented in a visual format by the SBF. This is because a SBF type of field has much more than just a text value. It contains a full graph of metadata and properties, inclusive links to Files and provenance metadata, which for example allows us to use an Upload field directly in the attached/configured webform. - Form modes also have an additional benefit. Each one can have fine grained permissions. That way you can have many different Form Modes, but allow only certain ones to be visible, or usable by users of a given Drupal Role.</p> </li> </ul>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#i-think-i-get-thisbut-how-can-i-use-this-knowledge-now","title":"I think I get this...but how can I use this knowledge now?","text":"<p>Good question! So, to enable, configure, and customize these Display Modes you have to navigate to your <code>Content Type</code> Configuration page in your running Archipelago. This is found at <code>/admin/structure/types</code>. Note: the way things are named in Drupal can be confusing to even the most deeply committed Drupal user, so bear in mind some terms will change. Feel free to read and re-read.</p> <p></p> <p>You can see that for every existent Content Type, there is a drop down menu with options:</p> <ul> <li>Manage Display: will lead you to configuration page where you can setup each View/Display Mode and its settings for a given Content Type</li> <li>Manage Form Display: will lead you to configuration page where you can setup each Form Mode and its settings</li> </ul>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#manage-display","title":"Manage Display","text":"<p>For Digital Objects: </p> <p>For Digital Object Collections and Compounds/Creative Work Series </p> <p>On the top you will see all your View Modes Listed, with the <code>Default</code> one selected and expanded. The Table that follows has one row per Field attached/part of this Content Type. Some of the fields are part of the Content Type itself, in this case Digital Object (bundled) and some other ones are common to every Content Entity derived from a Node. The \"Field\" column contains each field name (not their type, reason why you don't see Strawberry Field there!) but we can tell you right now that there is one, named \"Descriptive Metadata\", that is of <code>SBF</code> type.</p> Wait! Which are the fields in my Content Type? <p>How do we know that the field named \"Descriptive Metadata\" is a Strawberryfield? Well, we set-up the Digital Object Content Type for you that way, but also you can know what we know by pressing on \"Manage fields\" Tab on the top (don't forget to come back to \"Manage display\", afterwards!)</p> <p></p> <p>Also Surprise: You Content Entity has really really just 2 fields! And that, friends, is one of the secret ingredients of Archipelago. All goes into a Single Field. But wait: i see more fields in my Manage Display table. Why? Well. Some of them are base fields, part of what a Drupal Node is: base field means you can not remove them, they are part of the Definition itself. One obvious one is the <code>Title</code>.</p> <p>But there are also some fields very particular to Archipelago: You can see there are also ones named \"Formatter Object Metadata\", \"Media\" and one named \"Static Media\"!. Where does come from? Those are also Strawberryfields. It sounds confusing but it is really simple. They are really not \"fields\" in the sense of having different data than \"Descriptive Metadata\". Those are In Memory, realtime, copies of the \"Descriptive Metadata\" SBF field and are there to overcome one limitation of Drupal 8:</p> <p>Each Field can have a single \"Formatter\" setup per field.</p> <p>But we want to re-usue the JSON data to show a Viewer, Show Metadata as HTML directly on the ADO/NODE landing page, and we want also to, for example, format sometimes images as Thumbnails and not using a IIIF viewer only. This CopyFields (Legal term) have also a nice Performance advantage. Drupal needs to fetch only once the data from the real Field, \"Descriptive Metadata\", from the database. And then just makes the data available in real time to its copies. That makes all fast, very very fast! And of course flexible. As you dig more into Archipelago you will see the benefits of this approach. Finally, if you need to, you can make more CopyFields. But the reality is, there is a single, only one, SBF in each Digital Object and its named \"Descriptive Metadata\".</p> <p>You can also simply not care about the type and trust the UI. It will just show Formatters that are right for each type and expose Configuration options (and a little abstract of the current ones) under the Widget Column. Operations Columns allows you to setup each Widget. Widget term here is a bit confusing. These are not really Widget in terms of Data Input, but in terms of \"Configuration\" Input. But D8/9 is evolving and its getting better. Those settings apply always only to the current View Mode.</p> <p>You can play with this, experiment and change some settings to get more comfortable. We humbly propose you that you complete this info with the official Drupal 8 Documentation and also apply custom settings to your own, custom View Mode so you don't end changing base, expected functionality while you are still learning.</p>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#manage-form-display","title":"Manage Form Display","text":"<p>On the top you will see all your Form Modes Listed, with the <code>Default</code> one selected and expanded. The Table that follows has one row per Field attached/partof this Content Type. The list of fields here is shorter, the SBF CopyFields are not present because all data goes really only into real fields. Also some other, display only ones (means you can not modify them) will not appear here. Again, Some of the fields are part of the Content Type itself, in this case Digital Object (bundled) and some other ones are common to every Content Entity derived from a Node. \"Field\" column contains each field name and the Widget Column allows you to select what type of Input you are going to use to feed it on Ingest/edit. On the right you will see again a little gear, that allows you to configure the settings for a particular Widget. Those settings apply always only to the current Form Mode.</p> <p>So. The one we want to understand is the one attached to the \"Descriptive Metadata\" field. Currently one named \"Strawberryfield webform based input with inline rendering\". There are other two. But let's start with this. Press on the Gear to the right on the same row.</p> <p></p> <p>AS you can see there are not too many options. But, the main, first Text input is an Autocomplete field that will resolve against your existing Webforms. So, guess what. If you want to use your own Webform to feed a SBF, what do you do? You type the name, let the autocomplete work, select the right Webform, maybe your own custom one, and the you press \"Update\". Once that is done you need to \"Save\" your Form Mode (hint, button at the bottom of the page).</p> <p>We wish life was that easy (and it will once we are done with refining Drupal's UI) but for now there are some extra things you need to do to make sure the Webform, your custom one, can speak JSON. The default one you get named also \"Descriptive Metadata (descriptive_metadata)\", same as the field, is already setup to be used. Means if you create a new Webform by Copying that one, you can start using it inmediately. But if you created one from scratch (Different tutorial) you need to setup some settings.</p>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#setting-up-a-webform-to-talk-strawberryfield","title":"Setting up a Webform to talk Strawberryfield","text":"<p>Navigate to your Webform Managment form at <code>/admin/structure/webform</code></p> <p></p> <p>If you already created a Webform (different tutorial on how to do that) you will see your own named one in that list. I created for the purpose of this documentation one named \"Diego Test\" (Hi, i'm Diego..) and on the most right Column, \"Operations\" you will haven an Drop Down Menu. On your own Webform row, press on \"Settings\".</p> <p>First time, this can be a little bit intimitading. We recommend going baby steps since the Webform Module is a very powerful one but also exposes you to a lot (and sometimes too many) options. Even more, if you are new to Webforms, we recomment you to copy the \"Descriptive Metadata\" Webform we provided first, and make small changes to it (starting by naming it your own way!) so you can see how that affects your workflow and experience, and how that interacts with the created metadata. The Webform Module provides testing and building capabilities, so you have a Playground there before actually ingesting ADOs. Copying it will also make all the needed settings for SBF interaction to be moved over, so your work will be much easier.</p> <p>But we know you did not do that (where is the fun there right?). So lets setup one from scratch.</p>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#general-settings","title":"General Settings","text":"<p>Gist here is (look at the screenshot and copy the settings):</p> <ul> <li>GENERAL Settings: Check \"Disable saving of submissions\" option. You won't need this form to generate a Native Webform Submission entry.</li> <li>AJAX Settings: Check \"use ajax\" option. We want people to have the experience of staying in a single page while the create a new ADO via a Multi Step Webform Workflow.</li> </ul>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#confirmation-settings","title":"Confirmation Settings","text":"<p>Gist here is (again, look at the screenshot and copy the settings):</p> <ul> <li>Select \"Inline Confirmation\". You don't want Webform to send your user to another page while they are still ingesting their ADOs.</li> </ul>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"webformsasinput/#handler","title":"Handler","text":"<p>The glue, the piece of resistance. The handler is the one that knows how to talk to a SBF. In simple words, the handler (any handler) provides functionality that does something with a Webform Submission. The one that you want to select here, is the \"Strawberryfield harvester\" handler. Add it, name it whatever you like (or copy what you see in the screenshot) and make sure you select, if running using our deployment strategy, \"S3 File System\" as the option for \"Permanent Destination for uploaded files\". The wording is tricky there, its not really Permanent, since that is handled by Archipelago, but more to Temporary, while working in ingesting an Object, destination for the Webform. Its not really wrong neither. Its permanent for the Webform, but we have better plans for the files and metadata!</p> <p>Save your settings. And you are ready to roll. That webform can now be used as a Setting for any of the StrawberryField Widgets that use Webforms.</p> <p>Finally (the real finally). Archipelago encourages at least one Field/JSON key to be present always. One with \"type\" as key value. So make sure that your Custom Webform has that one.</p> <p>There are two ways of doing that:</p> <ul> <li> <p>You can copy how it is setup from the provided Webform's Elements, from the main Descriptive Metadata Webform and then add one \"select\" element to yours using the same \"type\" \"key\".Important in Archipelago is always the key value since that is what builds the JSON for your metadata. The Description can be any, but for UI consistency you could want to keep it the same across all your webforms.</p> <p></p> </li> <li> <p>Or, advanced, you can use the import/export capabilities (Webforms are just YAML files!) and export/copy your custom one as text, add the following element before or after some existing elements there</p> <pre><code> type:\n      '#type': select\n      '#title': 'Media Type'\n      '#options': schema_org_creative_works\n      '#required': true\n      '#label_attributes':\n        class:\n          - custom-form-input-heading\n</code></pre> <p>And then reimport.</p> <p>Having a \"type\" value will make your life easier. You don't need it, but everything works smoother that way.</p> <p>Since you have a single Content Type named Digital Object, having a Webform field that has as key \"type\", which leads to a \"type\" JSON key, allows you to discern the Nature of your Digital Object, book or Podcast, Image or 3D and do smart, nice things with them.</p> </li> </ul> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Webform","Form Mode","View Mode","Display Mode","Manage Display","Manage Form Display","Handler"]},{"location":"workingtwigs/","title":"Working with Twig in Archipelago","text":"<p>The following information can also be found in this Presentation from the \"Twig Templates and Archipelago\" Spring 2021 Workshop:</p> <ul> <li>Twig Templates and Archipelago</li> </ul>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#prerequisites-with-food-analogy","title":"Prerequisites (with food analogy)","text":"<ol> <li>Know your Data/Metadata. What do I have? <ul> <li>What do I have in my Fridge? Do I have Tofu? Do I have Peppermint ? One Bunch?</li> </ul> </li> <li>Know your final desired output Document: MODS, HTML, GEOJSON, etc. <ul> <li>What are you going to cook ? Do you have a picture of the Curry ? Have you ever had Curry ?</li> </ul> </li> <li>Know your Twig Basics<ul> <li>How to cut and dice , steam and saut\u00e9 </li> </ul> </li> <li>Do not be afraid<ul> <li>You can\u2019t get burned  here and Ingredients  do not expire!</li> </ul> </li> <li>Ask for help. Slack/Google Groups/Postcards  <ul> <li>Seeing others cook helps and also motivates. Others may share some spices. </li> </ul> </li> <li>Use and Share your findings!<ul> <li>Eat what you cook.  Share with friends and family. </li> </ul> </li> </ol> <p>Note</p> <p>All examples shown below are using the following JSON snipped from Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?].</p> Click to view image of the JSON snippet. <p></p> Click to view this snippet as JSON. <pre><code>{\n    \"type\": \"Photograph\",\n    \"label\": \"Laddie the dog running in the garden, Bronx, N.Y., undated [c. 1910-1918?]\",\n    \"owner\": \"New-York Historical Society, 170 Central Park West, New York, NY 10024, 212-873-3400.\",\n    \"rights\": \"This digital image may be used for educational or scholarly purposes without restriction. Commercial and other uses of the item are prohibited without prior written permission from the New-York Historical Society. For more information, please visit the New-York Historical Society's Rights and Reproductions Department web page at http:\\/\\/www.nyhistory.org\\/about\\/rights-reproductions\",\n    \"language\": [\n        \"English\"\n    ],\n    \"documents\": [],\n    \"publisher\": \"\",\n    \"ismemberof\": \"111\",\n    \"creator_lod\": [\n        {\n            \"name_uri\": \"\",\n            \"role_uri\": \"http:\\/\\/id.loc.gov\\/vocabulary\\/relators\\/pht\",\n            \"agent_type\": \"personal\",\n            \"name_label\": \"Stonebridge, George Ehler\",\n            \"role_label\": \"Photographer\"\n        }\n    ],\n    \"description\": \"George Ehler Stonebridge (d. 1941) was an amateur photographer who lived and worked in the Bronx, New York.\",\n    \"subject_loc\": [\n        {\n            \"uri\": \"http:\\/\\/id.loc.gov\\/authorities\\/subjects\\/sh85038796\",\n            \"label\": \"Dogs\"\n        }\n    ],\n    \"date_created\": \"1910-01-01\"\n}\n</code></pre>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#first-know-your-data","title":"First: Know Your Data","text":"<p>Understanding the basic structure of your JSON data. </p> <ol> <li> <p>Single JSON Value.</p> <p></p> <ul> <li>For <code>\"type\": \"Photograph\"</code><ul> <li>\"type\" = JSON Key or Property</li> <li>\"Photograph\" = Single JSON Value (string)</li> </ul> </li> </ul> </li> <li> <p>Multiple JSON Values (Array of Enumeration of Strings)</p> <p> - For <code>\"language\": [\"English\",\"Spanish\"]</code>     - \"language\" = JSON Key or Property     - \"[\"English\",\"Spanish\"]\" = Multiple JSON Values (Array of Enumeration of Strings)</p> </li> <li> <p>Multiple JSON Values (Array of Enumeration of Objects)</p> <p></p> <ul> <li>For <code>\"subject_loc\":[{\"uri\":\"http://..\",\"label\":\"Dogs\"},{\"uri\":\"http://..\",\"label\":\"Pets\"}]</code><ul> <li>\"subject_loc\" = JSON Key or Property</li> <li>[{\"uri\":\"http://..\",\"label\":\"Dogs\"},{\"uri\":\"http://..\",\"label\":\"Pets\"}] =<ul> <li>Object with two JSON Keys. Each one with a single Value</li> <li>Multiple JSON Values (Array of Enumeration of Objects)</li> </ul> </li> </ul> </li> </ul> </li> </ol>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#getting-started-with-the-twig-language-in-archipelago","title":"Getting Started with the Twig Language in Archipelago","text":"<ul> <li> <p>Data is known as Context in Twig Lingo.</p> </li> <li> <p>All your JSON Strawberryfield Metadata is accessible inside a Variable named data in your twig template. </p> </li> <li> <p>You can access the values by using <code>data DOT Property (attribute) Name</code>.</p> <ul> <li>In the Laddie the Dog example shown above (originally):<ul> <li><code>data.type</code> will contain \"Photograph\"</li> <li><code>data.language</code> will contain [ \"English\" ]</li> <li><code>data.language[0]</code> will contain \"English\" <ul> <li>0 means first entry in an Array or Enumeration</li> </ul> </li> <li><code>data.subject_loc</code> will contain [{ \"uri\":\"http://..\",\"label\": \"Dog\" }]</li> <li><code>data.subject_loc.uri</code> will contain <code>\"http://..\"</code></li> <li><code>data.subject_loc.label</code> will contain \"Dog\"</li> </ul> </li> </ul> </li> </ul> <p>Note</p> <p>You also have access to other info in your context <code>node</code>: such as<code>node.id</code> is the Drupal ID of your Current ADO; Also <code>is_front</code>, <code>language</code>, <code>is_admin</code>, <code>logged_in</code>; and more!</p>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#twig-statements-and-printing","title":"Twig Statements and Printing","text":"<p>Twig for Template Designers</p> <p>https://twig.symfony.com/doc/3.x/templates.html</p>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#simple-examples-using-printing-statements","title":"Simple examples using Printing Statements","text":"<p>Single JSON Value Example</p> Twig template<pre><code>Hello I am a {{ data.type }} and very happy to meet you\n</code></pre> Rendered output<pre><code>Hello I am a Photograph and very happy to meet you\n</code></pre> <p>Multiple JSON Values Example</p> Twig template<pre><code>Hello I was classified as \"{{ data.subject_loc[0].label }}\" and very happy to meet you\n</code></pre> Rendered output<pre><code>Hello I was classified as \"Dogs\" and very happy to meet you\n</code></pre>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#twig-statements-and-executing","title":"Twig Statements and Executing","text":"<p>If in Twig</p> <p>https://twig.symfony.com/doc/3.x/tags/if.html</p>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#rendered-output-based-upon-different-twig-conditionals-operators-tests-assignments-and-filters","title":"Rendered Output based upon different Twig <code>conditionals</code>, <code>operators</code>, <code>tests</code>, <code>assignments</code>, and <code>filters</code>","text":"<p>Conditionals, Operator, and Test Usage</p> Twig Template<pre><code>{% if data.subject_loc is defined %}\nHey I have a Subject Key\n{% else %}\nUps no Subject Key\n{% endif %}\n</code></pre> Rendered Output<pre><code>Hello I was classified as \"Dogs\" and very happy to meet you\n</code></pre> <ul> <li>if/else are conditionals</li> <li>is is an operator</li> <li>defined is a test </li> </ul> <p>Loop Usage</p> Twig Template<pre><code>{% for key, subject in data.subject_loc %}\n* Subject {{ subject.label }} found at position {{ key }}\n{% endfor %}\n</code></pre> Rendered Output<pre><code>* Subject Dogs found at position 0\n</code></pre> <ul> <li>for is a loop</li> <li>Inside the loop you have access to key, subject</li> </ul> <p>Assignment, Filter, and Loop Usage</p> Twig Template<pre><code>{% for subject in data.subject_loc %}\n{% set label_lowercase = subject.label|lower %}\nMy lower case Subject is {{ label_lowercase }}\n{% endfor %}\n</code></pre> Rendered Output<pre><code>`My lower case Subject is dogs`\n</code></pre> <ul> <li>set is an assignment </li> <li>| is a pipe, used after a value to apply a filter.</li> <li>lower is a filter</li> <li>Inside the loop you have have access to subject and label_lowercase</li> </ul> <p>Loop Scope</p> Twig Template<pre><code>{% for subject in data.subject_loc %}\n  {% set label_lowercase = subject.label|lower %}\nMy lower case Subject is {{ label_lowercase }}\n{% endfor %}\n{# \n The below won\u2019t display because it was assigned inside \n The For Loop\n#}\n{{ label_lowercase }}\n</code></pre> Rendered Output<pre><code>`My lower case Subject is dogs`\n</code></pre>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#full-examples-for-common-uses-cases","title":"Full Examples for Common Uses Cases:","text":"<p>Use Case #1</p> <p>I have multiple LoD Subjects and want to display them in my page as a clickable ordered list but I\u2019m a safe/careful person.</p> Twig Example for Use Case #1<pre><code>{% if data.subject_loc is iterable and data.subject_loc is not empty %}\n&lt;h2&gt;My Subjects&lt;/h2&gt;\n&lt;ul&gt;\n   {% for subject in data.subject_loc %}\n   &lt;li&gt;\n      &lt;a href=\"{{ subject.uri }}\" title=\"{{ subject.label|capitalize }}\" target=\"_blank\"&gt;\n      {{ subject.label }}\n      &lt;/a&gt;\n   &lt;/li&gt; \n   {% endfor %}\n&lt;/ul&gt;\n{% endif %}\n</code></pre> <p>Use Case #2</p> <p>I have sometimes a publication date. I want to show it in beautiful human readable language.</p> Twig Example for Use Case #2<pre><code>{% if data.date_published is not empty %}\n&lt;h2&gt;Date {{ data.label }} was published:&lt;/h2&gt;\n&lt;p&gt;\n{{ data.date_published|date(\"F jS \\\\o\\\\f Y \\\\a\\\\t g:ia\") }}\n&lt;/p&gt;\n{% endif %}\n</code></pre> <p>About <code>date</code></p> <ul> <li>date() is a function</li> <li>It uses a \u201cDate Format Pattern\u201d as argument.</li> </ul> <p>Use Case #3 (Full Curry)</p> <p>{# May 4th 2021 @dpino: I have sometimes a user provided creation date. I want to show it in beautiful human readable language but fallback to automatic date if absent. I also want in the last case to show it was either \u201ccreated\u201d or \u201cupdated\u201d. #}</p> <pre><code>    \"as:generator\": {\n        \"type\": \"Update\",\n        \"actor\": {\n            \"url\": \"https:\\/\\/archipelago.nyc\\/form\\/descriptive-metadata\",\n            \"name\": \"descriptive_metadata\",\n            \"type\": \"Service\"\n        },\n        \"endTime\": \"2021-03-17T13:24:01-04:00\",\n        \"summary\": \"Generator\",\n        \"@context\": \"https:\\/\\/www.w3.org\\/ns\\/activitystreams\"\n    }\n</code></pre> Twig Example for Use Case #3<pre><code>{% if data.date_created is not empty %}\n&lt;h2&gt;Date {{ data.label }} was created:&lt;/h2&gt;\n&lt;p&gt;\n  {{ data.date_created|date(\"F jS \\\\o\\\\f Y \\\\a\\\\t g:ia\") }}\n&lt;/p&gt;\n{% else %}\n&lt;h2&gt;Date {{ data.label }} was {{ attribute(data, 'as:generator').type|lower }}d  in this repository:&lt;/h2&gt;\n&lt;p&gt;\n  {{  attribute(data, 'as:generator').endTime|date(\"F jS \\\\o\\\\f Y \\\\a\\\\t g:ia\") }}\n&lt;/p&gt;\n{% endif %}\n</code></pre>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#a-recommended-workflow","title":"A Recommended Workflow","text":"<p>You want to create a New Metadata Display (HTML) or a new (XML) Schema based format?</p> <ol> <li>Get yourself an example document (Frame). If HTML copy the source. If XML copy the full XML. (Cmd+C or Ctrl+C)</li> <li>Create a new Metadata Display Entity. Copy the content (text) of your Frame into the Edit window. (Cmd+V or Ctrl+V)</li> <li>Select an existing (as complete as possible) ADO to use as preview, press Preview.</li> <li>Put your nice glasses  on. What do you see? What data in your Frame do you have in your ADO (data)?</li> <li>Start nimble. Select the <code>data.label</code> info and check where your Frame uses a Title or a Label. Remove that text (Cmd+X or Ctrl+X) and replace with a <code>{{ data.label }}</code>. Press Preview. Do you see your title?</li> <li>Keep doing 5, over and over. Leave complex values for the end. (e.g <code>data.subject_loc</code>)</li> <li>Document your changes. <code>{# I added this because .. #}</code></li> <li>Save.</li> </ol> <p>Once the Template is in place you can use it in a Formatter, as Endpoint, in your Search Results or just keep it around until you and the world are ready!</p>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"workingtwigs/#and-now-its-your-turn","title":"And now it's your turn!","text":"<p>We hope you found the information presented here to be helpful in getting started working with Twigs in Archipelago. Click here to return to the main Twigs in Archipelago documentation. Happy Twigging!</p> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>","tags":["Twig","Twig Templates","Twig Filters","Twig Functions","Examples","JSON"]},{"location":"xdebug/","title":"Debugging PHP in Archipelago","text":"<p>This document describes how to enable Xdebug for local PHP development using the PHPStorm IDE and a docker container running the Archipelago <code>esmero-php:development</code> image. It involves interacting with the esmero/archipelago-docker-images repo and the esmero/archipelago-deployment repo.</p>"},{"location":"xdebug/#part-1-docker","title":"Part 1: Docker","text":"<ol> <li> <p>Run the following commands from your <code>/archipelago-deployment</code> directory:</p> <p><code>docker-compose down</code> \\ <code>docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d</code></p> <p>This version of <code>docker-compose up</code> uses an override file to modify our services. <code>docker-compose.dev.yml</code> we now have an extra PHP container called <code>esmero-php-debug</code>. </p> <p>To stop the containers in the future, run <code>docker-compose -f docker-compose.yml -f docker-compose.dev.yml down</code>.</p> <p>(To make these commands easier to remember, consider making bash aliases in your .bashrc file.) (If you are running your development on a Linux system, you may need to make a modification to your xdebug configuration file on the esmero-php-dev container. See appendix at the bottom of this page.)</p> <p>So we have reloaded the containers and now you are ready for Part 2.</p> </li> </ol>"},{"location":"xdebug/#part-2-phpstorm","title":"Part 2: PHPStorm","text":"<ol> <li> <p>In PHPStorm, open your <code>archipelago-deployment</code> project.</p> </li> <li> <p>Go to <code>Preferences &gt; Languages &amp; Frameworks &gt; PHP &gt; Debug</code> or <code>Settings &gt; PHP &gt; Servers</code>. In this window there is an Xdebug section. Use these settings:</p> <ul> <li>Debug port: <code>9003</code>. (do NOT use the default, 9000)</li> <li>Can accept external connections: yes, select checkbox</li> <li>(optional) Break at first line in PHP scripts: uncheck. If you leave this selected, you will have to manually step through a breakpoint from Drupal's main index.php file on every request, which is quite annoying. However, leaving this box checked can be useful for making sure the connection is working at first, before you have set any internal breakpoints.</li> </ul> <p>Your settings should look like this. Hit APPLY and OK.  </p> </li> <li> <p>Go to <code>Preferences &gt; Languages &amp; Frameworks &gt; PHP &gt; Servers</code>. We will create a new server here. Use these settings:</p> <ul> <li>Name: <code>docker-debug-server</code></li> <li>Host: <code>localhost</code></li> <li>Port: <code>8001</code></li> <li>Use path mappings: yes, select the checkbox</li> <li>Under project files, select the top-level <code>archipelago-deployment</code> directory in the <code>File/Directory</code> column.</li> <li>In the <code>Absolute path on the server</code> add <code>/var/www/html</code></li> </ul> <p>Hit APPLY and OK and close the window.</p> <p> </p> </li> <li> <p>Go to <code>Run &gt; Edit Configurations</code>. Hit the <code>+</code> Button to create a new PHP Remote Debug. Name whatever you want, I called mine <code>Archipelago</code>. Use these settings:</p> <ul> <li>Filter debug connection by IDE Key: yes, select the checkbox</li> <li>Server: select <code>docker-debug-server</code> from dropdown (we created this in step 3)</li> <li>IDE Key: <code>archipelago</code> (this matches the key set in our container)</li> </ul> <p> </p> </li> <li> <p>Note: If you try to validate your connection, it will fail. But that's ok.</p> </li> <li> <p>Validate your connection. With  <code>Run &gt; Edit Configurations</code> still open, you can hit the link that says \"Validate\". Use these settings in the following validation window:</p> <ul> <li>Path to create validation script <code>&lt;your local path&gt;/archipelago-deployment/web</code></li> <li>Url to validation script: <code>http://localhost:8001</code></li> </ul> <p>Hit VALIDATE. You should get a series of green check marks. If you get a warning about missing <code>php.ini</code> file, that is OK, our file has a different name in the container (<code>xdebug.ini</code>) and is still being read correctly.  </p> </li> </ol>"},{"location":"xdebug/#set-up-browser-integration","title":"Set up Browser Integration","text":"<ol> <li> <p>We have had success using the XDebug Helper extension in Chrome. Once you have the extension installed, right-click on the bug icon in the top right of your chrome browser window and select \"Options\" to configure the IDE key. Under \"IDE\", select \"Other\", and in the text box, enter \"archipelago\"</p> <p> </p> </li> </ol>"},{"location":"xdebug/#actually-debugging","title":"Actually Debugging!","text":"<ol> <li> <p>Hit the button (top right bar of PHPStorm) that looks like a telephone, for <code>Start Listening for PHP Debug Connections</code>.</p> <p></p> </li> <li> <p>Now, you can use <code>Run &gt; Debug</code> and select the <code>Archipelago</code> named configuration that we created in the previous steps. The debugging console will appear. It will say it is waiting for incoming connection from 'archipelago'  .</p> <p> </p> </li> <li> <p>Right now the debugging session is not enabled. Browse to  <code>localhost:8001</code>. Click on the gray XDebug Helper icon at the top right of your window and select the green \"Debug\" button. This will tell chrome to set the xdebug session key when you reload the page.</p> </li> <li> <p>Now set a breakpoint in your code, and refresh the page.  If you have breakpoints set, either manually, or from leaving \"Break at first line in PHP scripts\" checked, you should have output now in the debugger.</p> </li> <li> <p>If you are done actively debugging, it is best to click the green XDebug Helper icon and select \"Disable\". This will greatly improve speed and performance for your app in development. When you need to debug, just turn on debugging using the XDebug Helper button again.</p> </li> <li> <p>If you would like to see the output of your xdebug logs, run the following script:    <code>docker exec -ti esmero-php bash -c 'tail -f /tmp/xdebug.log &gt; /proc/1/fd/2'</code></p> </li> </ol> <p>Then, you can use the typical docker logs command on the <code>esmero-php</code> container, and you will see the xdebug output:    <code>docker logs esmero-php -f</code></p> <p>Xdebug makes accessing variables in Drupal kind of great. Many possibilities, including debugging for Twig templates. Happy debugging!</p>"},{"location":"xdebug/#appendix-xdebug-on-a-linux-host","title":"Appendix: XDebug on a linux host","text":"<p>If you are developing on a linux machine, you may need to make a change to the xdebug configuration file.</p> <ol> <li>Create a new file in the <code>/archipelago-deployment/xdebug</code> folder called <code>xdebug.ini</code> and enter the following text:     <pre><code>zend_extension=xdebug\n\n[xdebug]\nxdebug.mode=develop,debug\nxdebug.discover_client_host = 1\nxdebug.start_with_request=yes\n</code></pre></li> <li>Make a bind mount to this file in your docker-compose.dev.yml file:     <pre><code>  php-debug:\n  ...\n    volumes:\n  - ${PWD}:/var/www/html:cached\n    # Bind mount custom xdebug configuration file...\n  - ${PWD}/xdebug/xdebug.ini:/usr/local/etc/php/conf.d/docker-php-ext-xdebug.ini\n</code></pre></li> <li>Restart your docker containers using the method described at the top of this page.</li> </ol> <p>Thank you for reading! Please contact us on our Archipelago Commons Google Group with any questions or feedback.</p> <p>Return to the Archipelago Documentation main page.</p>"}]}
\ No newline at end of file
diff --git a/1.4.0/search_advanced/index.html b/1.4.0/search_advanced/index.html
index f33e5198..f58e15cc 100644
--- a/1.4.0/search_advanced/index.html
+++ b/1.4.0/search_advanced/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/search_solr_index/index.html b/1.4.0/search_solr_index/index.html
index bb926410..2defec8c 100644
--- a/1.4.0/search_solr_index/index.html
+++ b/1.4.0/search_solr_index/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/security_bots/index.html b/1.4.0/security_bots/index.html
index a449ac6c..af8805f3 100644
--- a/1.4.0/security_bots/index.html
+++ b/1.4.0/security_bots/index.html
@@ -15,10 +15,14 @@
         <link rel="canonical" href="https://docs.archipelago.nyc/1.4.0/security_bots/">
       
       
+        <link rel="prev" href="../utility_scripts/">
+      
+      
+        <link rel="next" href="../xdebug/">
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -500,6 +504,8 @@
       
   
   
+    
+  
   
   
     
@@ -521,11 +527,11 @@
       
     
     
-    <li class="md-nav__item md-nav__item--section md-nav__item--nested">
+    <li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" checked>
         
           
           <label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="">
@@ -539,7 +545,7 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
+        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="true">
           <label class="md-nav__title" for="__nav_3">
             <span class="md-nav__icon md-icon"></span>
             Instructions and Guides
@@ -549,6 +555,8 @@
               
                 
   
+  
+    
   
   
   
@@ -561,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -568,11 +578,11 @@
       
     
     
-    <li class="md-nav__item md-nav__item--nested">
+    <li class="md-nav__item md-nav__item--active md-nav__item--nested">
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1" checked>
         
           
           <label class="md-nav__link" for="__nav_3_1" id="__nav_3_1_label" tabindex="0">
@@ -586,7 +596,7 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_1_label" aria-expanded="false">
+        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_1_label" aria-expanded="true">
           <label class="md-nav__title" for="__nav_3_1">
             <span class="md-nav__icon md-icon"></span>
             DevOps & Deployment
@@ -1123,6 +1133,94 @@
               
             
               
+                
+  
+  
+    
+  
+  
+  
+    <li class="md-nav__item md-nav__item--active">
+      
+      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
+      
+      
+        
+      
+      
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+          
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      
+      <a href="./" class="md-nav__link md-nav__link--active">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+      
+        
+
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+  
+  
+  
+    
+  
+  
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+      
+        <li class="md-nav__item">
+  <a href="#deployment" class="md-nav__link">
+    <span class="md-ellipsis">
+      Deployment
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#deployment-with-upgrade" class="md-nav__link">
+    <span class="md-ellipsis">
+      Deployment with Upgrade
+    </span>
+  </a>
+  
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#advanced-configuration" class="md-nav__link">
+    <span class="md-ellipsis">
+      Advanced Configuration
+    </span>
+  </a>
+  
+</li>
+      
+    </ul>
+  
+</nav>
+      
+    </li>
+  
+
+              
+            
+              
                 
   
   
@@ -1148,10 +1246,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1162,8 +1260,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/sitemap.xml b/1.4.0/sitemap.xml
index 5ba4d6ae..3447cd19 100644
--- a/1.4.0/sitemap.xml
+++ b/1.4.0/sitemap.xml
@@ -2,346 +2,346 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/101_guides_list/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/AMIviaSpreadsheets/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/CODE_OF_CONDUCT/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/I7solrImporter/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/about/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/acknowledgments/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/ami_index/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/ami_lod_rec/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/ami_spreadsheet_overview/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/ami_update/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/annotations/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archifilepersistencestrategy/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archipelag-logo-usage/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archipelago-deployment-UpgradeDrupalD9toD10/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archipelago-deployment-democontent/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archipelago-deployment-live-UpgradeDrupalD9toD10/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archipelago-deployment-live-UpgradeFrom1dot3/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archipelago-deployment-live-gitworkflow/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archipelago-deployment-live-moveToLive/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archipelago-deployment-live-readme/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archipelago-deployment-live-search_solr_index/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archipelago-deployment-live-updatingContainers/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archipelago-deployment-live-upgradeFromD8ToD9/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archipelago-deployment-live-upgradeFromRC3/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archipelago-deployment-osx/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archipelago-deployment-readme/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archipelago-deployment-ubuntu/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archipelago-deployment-windows/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/archipelago-glossary/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/createdisplaymodes/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/customwebformelements/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/devops/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/documentation_about/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/documentation_features/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/documentation_technical/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/documentation_template/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/documentation_workflow/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/drupal_core_update/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/embargo/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/experimental_tools/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/export-to-csv/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/find_and_replace/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/find_and_replace_action_json_patch/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/find_and_replace_action_text/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/find_and_replace_action_webform/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/firstobject/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/fragaria/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/generalqa_minio_logging/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/generalqa_smtp_configuration/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/generalqa_twig_modules_configuration/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/giveortake/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/googleapi/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/iiif-content-search/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/iiif_server_settings/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/inthewild/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/metadata_display_preview/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/metadata_display_usage/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/metadatainarchipelago/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/metadatatwigs/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/modifyingfileextensionsinwebform/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/open_api/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/ourtake/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/presentations_events/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/search-within-collection/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/search_advanced/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/search_solr_index/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/security_bots/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/sslsetup/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/strawberry_key_name_providers/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/strawberryfield-formatters/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/strawberryfields/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/strawberryrunners/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/strawberryrunners_pager_ocr/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/strawberryrunners_subtitle/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/strawberryrunners_wacz_binary/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/strawberryrunners_webpage_text/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/traditional-install/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/twig_extensions/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/twig_recipe_cards/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/utility_scripts/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/webformLoDfromCSV/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/webforms/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/webformsasinput/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/workingtwigs/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://docs.archipelago.nyc/1.4.0/xdebug/</loc>
-         <lastmod>2024-10-18</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
 </urlset>
\ No newline at end of file
diff --git a/1.4.0/sitemap.xml.gz b/1.4.0/sitemap.xml.gz
index 1d922fa9bad42011e6a8c7e8ef8a22a0ae33567e..9f37564ad5113bd585267364b22d3b101408894c 100644
GIT binary patch
delta 1015
zcmV<T0|@+u2!;p;ABzYGfUzo(2Ood5abPPbq#7`g)<9$TN^r>$#hA<4l0&W4Pv4=e
z#10Dd&>n_5=&`%v{KVxv{zF}Td>X>V0aErDuV<_Ia&`gHp!LyR&mQh?y7#k>>o->=
zg$#7T7sq@(+Y^l+7K>?`=GhYrddzc7ZjsHNKjg*v;9@zyoUPwn@S_g!{O*7BxC&_2
zb(GA*STuRAliB+bf_6BMN3&SX-_4hct3|!=_4jn90qy$j^6k5Bwd$5X@+IeE&xRJ?
zEd58DtK~|$;w@wqd?pcwpMSkQc>Q@ypl#j*P!_@1+--kRcQ<Nt_uKYib1%Yh`%}h{
zZif*QK@vgGJr<dDpv_~%DS&@<14N-kpa-vfwD2TmJ)l(yjF|oW;)e=3RvtV|VlZX2
znn1>Uj7XX|QwXiZdUWRlj5S|n;zR|R5>JOa$^o5@b^#rBC2I%5B-mLP10HL@=pIIw
zw1sUdqYm3k!tH9=5}$<u4Mx6`)Dl+k2WWooUPIm_999<ThvopCrwM=4<1XOT9NWOb
z{2o8?+vd<}!Wr7b%23a$T9HZC1ph*KN{=jFHXK#7kDMqPMEvHn+xPdly=<BA_f1nV
z1#?qNu01y`eTh_BBAPf*;3V&cE?~|&$yADlR|<jkc?|l<<ATh5n34&H3HlwT0RqUW
zN(%=Zg-oL<^@pF7jFNv7_uq5uc!87zEldf4*+*}5Xo6*6mk|=Y-=m1uvGycGl)(g&
zEdUsLogB*8eq!D4Pxm7zLJV070~2+Goyi9gf*og6rDJW1v2Pe?0zzqX@{%u^pGHk)
z-xL<X6Ey{NPQpS=iR*y=3mBm}N!sb;wTN*CtO|AbQ>(#8k1Btlb1WEzS@J+5NB6k%
zuB21G5eA~kJH;cH0J#M))}<VF^`lI*bBHd09=(Wl@BOYj_esw^7%>i&Asry`3)Mc0
zOb+&u_AvQiMXV1%nkz7^&U#Lb3B>k<{sAMm3Go%h$5-b?Noo+2S)n|X-yPXO>V6g1
z9uVt4EV7T1c^`jAV8JUC#IVSPZ+6i9RmWuo2eG!tbMWpaPkSFZq=x`TET^8k`Pv?I
zG+-qegm)A&n36mlRgY2&BiBDzl~KkxB|Y_!(osFaQAIrf=QKQcE46>}LbUS`Y*$03
zCTavn8X}b#<)`9I!D8QWJ%}bX!jnr+o&q$2Gimde>T73BbZmtP`g+ggdLm3@eNB5h
zbUtR%L8jGw(zq|hnH)m`!)KJyNhr1y@}cIxY?2>^QsB8V{)F3I!eR6I4@nBqDae|I
l!YH&YRkt}<x5!DSCkuV~AN{z~3l~%CzX3Btj9T3;004oc=1c$p

delta 1014
zcmV<S0}1?w2!#j-ABzYG0HqO;2Ooc+c3~?hq#7`g)<9$TN^r>$H8Gd7<wI-r(|0H<
zv4a9Vw1=S%dhD(^KXEya|4`Q-p9g<&fS8cN&1^Ma&MqJrvKZXW?D65YyPSPozq>Bc
zC&(9kaY#3_ea_=gi^Vie^Mn}&J*GKDw@7BsAJXD{aIu_U&DQTO_)!OVd3S$$TzfL>
zI!fYUESWUd(d=;quN}?9(JWT;5A)^XdQmTY`#qiVKz99p`Tj$<T6U{TzT|xD#n9rL
zrT=JiwOlDz&_YrkQx;+P`PaJx>d#{YZPOkgClQRz{q`4if2%h4zil5k4<ZbAKPK|=
zZWt+Mh$0BOry{csw0R0Nd9Z(OfFQI8^neP3g=aDAo~(*s#O$Y6KUBoA@_;aj!IaVJ
z3^L|JpsX`z3Za!)kM7XJSo2lNoTwn>$kXADazLk}ohL_KN!o!h33V1mPsbWCy2sH)
zZDAYBsQvaT({{CNiO<47h63No+!B_@12jK(Zy|3Z4J%9ehvons=81pe)6Ub>9NWOb
z{DD64+vd<}!Wr7bN>I<LT9HZC1ph*KN{=jFHXKzn22K<WB7XDP?d1b)uUaPjebZD-
zq14opYtKzfUqUV|5ltK@Xp(nB=P4x}WhzC(D<y-)G<tpHaY1H2Oi7uB3Hlwy0X)d5
zN(%=Xg-nwv^@pF7jFNv7_uq5uc!87%Elf!Uvkz#rZ-QlDmk|=Y-=m1uvGycGl%WWc
zEdUsL9UaNoerDZ|r~BbmCh|!NgJSM7btWA|2zH!Nl@7Hj#=c>o2?(Xl$xFT@ej0T)
z`=+omJm;o>&PiB^DRLdqe*q&jCs8{cQHvONz^YLDKeZY#ASr)O&aq$=W=TUHIl6}(
zx)M+MMi?kh=oF7!Jfs%DSeJ6x)lV|f&Jj5edPEWHh`8&{ebRFeMvOxxhzE%Ll53wu
zCI@54dzjE$5$gkF%@vqdXFaFJ2x5Ce|A3L(g!qbr;mvtbk{ZNhRwxgJyCXYD-LK->
z148|WMZzGN_i29w7Q8}1jEY?NW(Q4QbzD+#5Nmt91n+LbyvM*H-Fq-%IrZGl*Y==;
z0V~NMyrYo7l!*DLdXicgss6#Lj55Y^)Kd>J9@P^ZRnP-)PNM@_sr{2zq8);_T@9H!
z=SFa(Az}$Zek#rsEcPAOgJfbOJgM{<vxi1-CT;#weXVARj;#<uU+<Y*PlSmq*0iU6
zhaqL{Wm?TAjr&@h$uUGQd`1}^g<?y|=xhE<CgLcR0?(E4C)(~J4V%w@NK%MSLDDP~
kMxkw~y3N75MNT?BTj<OG=*OL2xR_f14K7+E9^Ebg070zZsQ>@~

diff --git a/1.4.0/sslsetup/index.html b/1.4.0/sslsetup/index.html
index e03bb1b0..c278c761 100644
--- a/1.4.0/sslsetup/index.html
+++ b/1.4.0/sslsetup/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1196,6 +1198,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1216,10 +1239,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1230,8 +1253,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/strawberry_key_name_providers/index.html b/1.4.0/strawberry_key_name_providers/index.html
index 9add96e7..e99aece3 100644
--- a/1.4.0/strawberry_key_name_providers/index.html
+++ b/1.4.0/strawberry_key_name_providers/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/strawberryfield-formatters/index.html b/1.4.0/strawberryfield-formatters/index.html
index d05586ab..f1d06f26 100644
--- a/1.4.0/strawberryfield-formatters/index.html
+++ b/1.4.0/strawberryfield-formatters/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/strawberryfields/index.html b/1.4.0/strawberryfields/index.html
index a6e83b12..9b1a7441 100644
--- a/1.4.0/strawberryfields/index.html
+++ b/1.4.0/strawberryfields/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -645,6 +645,8 @@
       
         
       
+        
+      
     
     
       
@@ -1210,6 +1212,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1230,10 +1253,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1244,8 +1267,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/strawberryrunners/index.html b/1.4.0/strawberryrunners/index.html
index 3d3f5592..37d90523 100644
--- a/1.4.0/strawberryrunners/index.html
+++ b/1.4.0/strawberryrunners/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/strawberryrunners_pager_ocr/index.html b/1.4.0/strawberryrunners_pager_ocr/index.html
index dab863e4..85926295 100644
--- a/1.4.0/strawberryrunners_pager_ocr/index.html
+++ b/1.4.0/strawberryrunners_pager_ocr/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/strawberryrunners_subtitle/index.html b/1.4.0/strawberryrunners_subtitle/index.html
index 0982635b..6f9cfce6 100644
--- a/1.4.0/strawberryrunners_subtitle/index.html
+++ b/1.4.0/strawberryrunners_subtitle/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/strawberryrunners_wacz_binary/index.html b/1.4.0/strawberryrunners_wacz_binary/index.html
index 30cfa911..34697164 100644
--- a/1.4.0/strawberryrunners_wacz_binary/index.html
+++ b/1.4.0/strawberryrunners_wacz_binary/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/strawberryrunners_webpage_text/index.html b/1.4.0/strawberryrunners_webpage_text/index.html
index 713cbaf2..989bfe90 100644
--- a/1.4.0/strawberryrunners_webpage_text/index.html
+++ b/1.4.0/strawberryrunners_webpage_text/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/traditional-install/index.html b/1.4.0/traditional-install/index.html
index 50340dcb..d29fec73 100644
--- a/1.4.0/traditional-install/index.html
+++ b/1.4.0/traditional-install/index.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -563,6 +563,8 @@
       
         
       
+        
+      
     
     
       
@@ -1128,6 +1130,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1148,10 +1171,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1162,8 +1185,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
@@ -3116,7 +3139,7 @@ <h2 id="traditional-installation-notes">Traditional Installation Notes</h2>
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 5, 2021</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 2, 2021</span>
   </span>
 
     
diff --git a/1.4.0/twig_extensions/index.html b/1.4.0/twig_extensions/index.html
index c9064104..253e450e 100644
--- a/1.4.0/twig_extensions/index.html
+++ b/1.4.0/twig_extensions/index.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -563,6 +563,8 @@
       
         
       
+        
+      
     
     
       
@@ -1128,6 +1130,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1148,10 +1171,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1162,8 +1185,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/twig_recipe_cards/index.html b/1.4.0/twig_recipe_cards/index.html
index 4774d50d..1a20785e 100644
--- a/1.4.0/twig_recipe_cards/index.html
+++ b/1.4.0/twig_recipe_cards/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/utility_scripts/index.html b/1.4.0/utility_scripts/index.html
index c6defaec..fd62143b 100644
--- a/1.4.0/utility_scripts/index.html
+++ b/1.4.0/utility_scripts/index.html
@@ -18,11 +18,11 @@
         <link rel="prev" href="../archipelago-deployment-live-updatingContainers/">
       
       
-        <link rel="next" href="../xdebug/">
+        <link rel="next" href="../security_bots/">
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1218,6 +1220,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1238,10 +1261,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1252,8 +1275,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/webformLoDfromCSV/index.html b/1.4.0/webformLoDfromCSV/index.html
index e5eb8672..aaf8a49e 100644
--- a/1.4.0/webformLoDfromCSV/index.html
+++ b/1.4.0/webformLoDfromCSV/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/webforms/index.html b/1.4.0/webforms/index.html
index 6d8d697e..6eb091fb 100644
--- a/1.4.0/webforms/index.html
+++ b/1.4.0/webforms/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/webformsasinput/index.html b/1.4.0/webformsasinput/index.html
index a8e663a3..281d6507 100644
--- a/1.4.0/webformsasinput/index.html
+++ b/1.4.0/webformsasinput/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
diff --git a/1.4.0/workingtwigs/index.html b/1.4.0/workingtwigs/index.html
index b926d405..c1a23d92 100644
--- a/1.4.0/workingtwigs/index.html
+++ b/1.4.0/workingtwigs/index.html
@@ -22,7 +22,7 @@
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
     
     
       
@@ -1134,6 +1136,27 @@
   
   
   
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
     
     
       
@@ -1154,10 +1177,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" >
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" >
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1168,8 +1191,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="false">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="false">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
@@ -3386,28 +3409,28 @@ <h1 id="working-with-twig-in-archipelago">Working with Twig in Archipelago</h1>
 </ul>
 <h2 id="prerequisites-with-food-analogy">Prerequisites (with food analogy)</h2>
 <ol>
-<li>Know your Data/Metadata. What do I have? <img alt="🤔" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f914.svg" title=":thinking:" /><ul>
-<li>What do I have in my Fridge? Do I have Tofu? Do I have Peppermint <img alt="🍬" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f36c.svg" title=":candy:" />? One Bunch?</li>
+<li>Know your Data/Metadata. What do I have? <img alt="🤔" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f914.svg" title=":thinking:" /><ul>
+<li>What do I have in my Fridge? Do I have Tofu? Do I have Peppermint <img alt="🍬" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f36c.svg" title=":candy:" />? One Bunch?</li>
 </ul>
 </li>
 <li>Know your final desired output Document: MODS, HTML, GEOJSON, etc. <ul>
-<li>What are you going to cook <img alt="🧑‍🍳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f9d1-200d-1f373.svg" title=":cook:" />? Do you have a picture of the Curry <img alt="🍛" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f35b.svg" title=":curry:" />? Have you ever had Curry <img alt="🍲" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f372.svg" title=":stew:" />?</li>
+<li>What are you going to cook <img alt="🧑‍🍳" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f9d1-200d-1f373.svg" title=":cook:" />? Do you have a picture of the Curry <img alt="🍛" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f35b.svg" title=":curry:" />? Have you ever had Curry <img alt="🍲" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f372.svg" title=":stew:" />?</li>
 </ul>
 </li>
 <li>Know your Twig Basics<ul>
-<li>How to cut and dice <img alt="🔪" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f52a.svg" title=":knife:" />, steam and sauté <img alt="🥣" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f963.svg" title=":bowl_with_spoon:" /></li>
+<li>How to cut and dice <img alt="🔪" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f52a.svg" title=":knife:" />, steam and sauté <img alt="🥣" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f963.svg" title=":bowl_with_spoon:" /></li>
 </ul>
 </li>
 <li>Do not be afraid<ul>
-<li>You can’t get burned <img alt="🔥" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f525.svg" title=":fire:" /> here and Ingredients <img alt="🍓" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f353.svg" title=":strawberry:" /> do not expire!</li>
+<li>You can’t get burned <img alt="🔥" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f525.svg" title=":fire:" /> here and Ingredients <img alt="🍓" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f353.svg" title=":strawberry:" /> do not expire!</li>
 </ul>
 </li>
-<li>Ask for help. Slack/Google Groups/Postcards <img alt="📧" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f4e7.svg" title=":email:" /> <img alt="📬" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f4ec.svg" title=":mailbox_with_mail:" /><ul>
-<li>Seeing others cook helps and also motivates. Others may share some spices. <img alt="🌶" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f336.svg" title=":hot_pepper:" /></li>
+<li>Ask for help. Slack/Google Groups/Postcards <img alt="📧" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f4e7.svg" title=":email:" /> <img alt="📬" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f4ec.svg" title=":mailbox_with_mail:" /><ul>
+<li>Seeing others cook helps and also motivates. Others may share some spices. <img alt="🌶" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f336.svg" title=":hot_pepper:" /></li>
 </ul>
 </li>
 <li>Use and Share your findings!<ul>
-<li>Eat what you cook. <img alt="🍪" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f36a.svg" title=":cookie:" /> Share with friends and family. <img alt="🍰" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f370.svg" title=":cake:" /></li>
+<li>Eat what you cook. <img alt="🍪" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f36a.svg" title=":cookie:" /> Share with friends and family. <img alt="🍰" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f370.svg" title=":cake:" /></li>
 </ul>
 </li>
 </ol>
@@ -3675,7 +3698,7 @@ <h3 id="a-recommended-workflow">A Recommended Workflow</h3>
 <li>Get yourself an example <strong>document (Frame)</strong>. If <strong>HTML</strong> copy the source. If <strong>XML</strong> copy the full XML. (<span class="keys"><kbd class="key-command">Cmd</kbd><span>+</span><kbd class="key-c">C</kbd></span> or <span class="keys"><kbd class="key-control">Ctrl</kbd><span>+</span><kbd class="key-c">C</kbd></span>)</li>
 <li>Create a new <strong>Metadata Display Entity</strong>. Copy the content (text) of your Frame into the Edit window. (<span class="keys"><kbd class="key-command">Cmd</kbd><span>+</span><kbd class="key-v">V</kbd></span> or <span class="keys"><kbd class="key-control">Ctrl</kbd><span>+</span><kbd class="key-v">V</kbd></span>)</li>
 <li>Select an existing (as complete as possible) ADO to use as preview, press Preview.</li>
-<li>Put your nice glasses <img alt="🤓" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.0.3/assets/svg/1f913.svg" title=":nerd_face:" /> on. What do you see? What data in your Frame do you have in your ADO (data)?</li>
+<li>Put your nice glasses <img alt="🤓" class="twemoji" src="https://cdn.jsdelivr.net/gh/jdecked/twemoji@15.1.0/assets/svg/1f913.svg" title=":nerd_face:" /> on. What do you see? What data in your Frame do you have in your ADO (data)?</li>
 <li>Start nimble. Select the <code>data.label</code> info and check where your <strong>Frame</strong> uses a <em>Title</em> or a <em>Label</em>. Remove that text (<span class="keys"><kbd class="key-command">Cmd</kbd><span>+</span><kbd class="key-x">X</kbd></span> or <span class="keys"><kbd class="key-control">Ctrl</kbd><span>+</span><kbd class="key-x">X</kbd></span>) and replace with a <code>{{ data.label }}</code>. Press Preview. Do you see your title?</li>
 <li>Keep doing <strong>5</strong>, over and over. Leave complex values for the end. (e.g <code>data.subject_loc</code>)</li>
 <li>Document your changes. <code>{# I added this because .. #}</code></li>
diff --git a/1.4.0/xdebug/index.html b/1.4.0/xdebug/index.html
index c3135ad2..2e808732 100644
--- a/1.4.0/xdebug/index.html
+++ b/1.4.0/xdebug/index.html
@@ -15,14 +15,14 @@
         <link rel="canonical" href="https://docs.archipelago.nyc/1.4.0/xdebug/">
       
       
-        <link rel="prev" href="../utility_scripts/">
+        <link rel="prev" href="../security_bots/">
       
       
         <link rel="next" href="../generalqa_minio_logging/">
       
       
       <link rel="icon" href="../images/ArchipelagoLogo-favicon-32x32.png">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.41">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
     
     
       
@@ -569,6 +569,8 @@
       
         
       
+        
+      
         
       
     
@@ -1133,6 +1135,27 @@
               
                 
   
+  
+  
+  
+    <li class="md-nav__item">
+      <a href="../security_bots/" class="md-nav__link">
+        
+  
+  <span class="md-ellipsis">
+    Managing Bots
+  </span>
+  
+
+      </a>
+    </li>
+  
+
+              
+            
+              
+                
+  
   
     
   
@@ -1158,10 +1181,10 @@
       
         
         
-        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_4" checked>
+        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_5" checked>
         
           
-          <label class="md-nav__link" for="__nav_3_1_4" id="__nav_3_1_4_label" tabindex="0">
+          <label class="md-nav__link" for="__nav_3_1_5" id="__nav_3_1_5_label" tabindex="0">
             
   
   <span class="md-ellipsis">
@@ -1172,8 +1195,8 @@
             <span class="md-nav__icon md-icon"></span>
           </label>
         
-        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_4_label" aria-expanded="true">
-          <label class="md-nav__title" for="__nav_3_1_4">
+        <nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_5_label" aria-expanded="true">
+          <label class="md-nav__title" for="__nav_3_1_5">
             <span class="md-nav__icon md-icon"></span>
             DevOps Q&A
           </label>
@@ -3309,7 +3332,7 @@ <h3 id="appendix-xdebug-on-a-linux-host">Appendix: XDebug on a linux host</h3>
     <span class="md-icon" title="Last update">
       <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1zM12.5 7v5.2l4 2.4-1 1L11 13V7zM11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2z"/></svg>
     </span>
-    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 5, 2021</span>
+    <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">October 2, 2021</span>
   </span>