[FEAT] Add browse by grant (#629)

--------- Co-authored-by: Quinn Hart <[email protected]>
ucd-library · Oct 25, 2024 · 151a04e · 151a04e
1 parent e9e5a6e
commit 151a04e
Show file tree

Hide file tree

Showing 37 changed files with 414 additions and 361 deletions.
diff --git a/docker-template.yaml b/docker-template.yaml
@@ -45,12 +45,11 @@ services:
     ports:
       - ${FUSEKI_PORT}:3030
     environment:
-      - JVM_ARGS=${JVM_ARGS:- -Xmx16g -Djena.scripting=true}
+      - JVM_ARGS=${JVM_ARGS:- -Xmx16g -Djena:scripting=true}
       - GOOGLE_APPLICATION_CREDENTIALS=/etc/fin/service-account.json
     restart: unless-stopped
-    command:
-      HDT=import server
-    # command: bash -c 'tail -f /dev/null'
+    command: server
+    # command: noop
 
 
   ###

diff --git a/harvest/Dockerfile b/harvest/Dockerfile
@@ -112,11 +112,8 @@ COPY experts-client ${AE_CLIENT}
 #RUN cd ${AE_CLIENT} && rm -rf package-lock.json node_modules && yarn global add simple-copy typescript  && yarn link @ucd-lib/experts-api && yarn install
 RUN cd ${AE_CLIENT} && rm -rf package-lock.json node_modules && npm link @ucd-lib/experts-api && npm install -g
 
-# Setup a place to hold the Makefile data
-RUN mkdir /usr/local/lib/harvest
-COPY harvest /usr/local/lib/harvest
-
 # Add startup file
+COPY /startup.mk /
 COPY /harvest-entrypoint.sh /
 
 # Add sqlcl and tarql to bin

diff --git a/harvest/experts-client/bin/experts-cdl.js b/harvest/experts-client/bin/experts-cdl.js
@@ -30,7 +30,7 @@ async function main(opt, cache) {
     await cache_expert.fetch();
     await cache_expert.load();
     await cache_expert.transform();
-    if (opt.fuseki.delete === true && cache_expert._db) {
+    if (opt["fuseki.delete"] === true && cache_expert._db) {
       log.info(`Deleting ${cache_expert._db.db}`);
       await cache_expert._db.drop();
     }

diff --git a/harvest/experts-client/lib/cache/query/grant/construct.rq b/harvest/experts-client/lib/cache/query/grant/construct.rq
@@ -3,6 +3,7 @@ PREFIX bibo: <http://purl.org/ontology/bibo/>
 PREFIX cite: <http://citationstyles.org/schema/>
 PREFIX expert: <http://experts.ucdavis.edu/expert/>
 PREFIX experts: <http://experts.ucdavis.edu/>
+PREFIX js: <http://jena.apache.org/ARQ/jsFunction#>
 PREFIX kfs: <ark:/87287/d7gt0q/schema#>
 PREFIX list: <http://jena.apache.org/ARQ/list#>
 PREFIX obo: <http://purl.obolibrary.org/obo/>
@@ -24,8 +25,7 @@ construct {
                 .
 
   ?grant a vivo:Grant,?grant_type;
-         rdfs:label ?grant_label;
-         schema:name ?grant_title;
+         schema:name ?grant_label;
          schema:identifier ?grant,?g;
          vivo:totalAwardAmount ?total_award_amount;
          vivo:sponsorAwardId ?sponsor_award_id;
@@ -34,8 +34,6 @@ construct {
          vivo:relatedBy ?RELATIONSHIP__,?labeled_only_role;
          .
 
-#  ?EXPERT__ schema:name ?user_name.
-
   ?labeled_only_role a vivo:CoPrincipalInvestigatorRole;
                            vivo:relates ?labeled_only_person,?grant;
                            .
@@ -130,36 +128,48 @@ WHERE {
                 .
       }
     }
-    ?record :native/:field ?field.
 
     OPTIONAL {
-      ?field :name "title";
-             :type "text";
-             :text ?grant_label;
-             .
-      bind(replace(?grant_label,"(?:SEE\\s+)?(?:(?:[ABCKKXYZ][0-9CF]{6})*(?:\\s*-)?\\s*)*\\s*(?:SP0A\\d{6})?(..*?)(\\s+K.[0-9]{2}.[0-9]{1,2})?$","$1") as ?grant_title)
+      ?record :native/:field [ :name "title";
+                               :type "text";
+                               :text ?grant_raw_title ];
+                                     .
+      bind(replace(?grant_raw_title,"(?:SEE\\s+)?(?:(?:[ABCKKXYZ][0-9CF]{6})*(?:\\s*-)?\\s*)*\\s*(?:SP0A\\d{6})?(..*?)(\\s+K.[0-9]{2}.[0-9]{1,2})?$","$1") as ?grant_title)
     }
 
     OPTIONAL {
-      ?field :name "funder-reference";
-             :type "text";
-             :text ?sponsor_award_id;
-             .
+      ?record :native/:field [ :name "funder-name";
+                               :type "text";
+                               :text ?funder_label_cap ].
+
+      ?record :native/:field [ :name "funder-reference";
+                               :type "text";
+                               :text ?sponsor_award_id ].
     }
 
     OPTIONAL {
-      ?field :name "amount";
-             :type "money";
-             :money/:field-value ?total_award_amount;
-             .
+      ?record :native/:field [ :name "amount";
+                               :type "money";
+                               :money/:field-value ?total_award_amount ];
+                                                   .
+    }
+
+    OPTIONAL {
+      ?record :native/:field [ :name "c-pi";
+                               :type "text";
+                               :text ?pi_name_cap ];
+                                     .
     }
 
     OPTIONAL {
-      ?field :name "c-co-pis";
-             :people/:person [ list:index(?pos ?elem) ] .
-      ?elem :last-name ?labeled_only_name_last .
+      ?record :native/:field [ :name "c-co-pis";
+                               :people/:person [ list:index(?pos ?elem) ]
+                             ].
+      ?elem :last-name ?labeled_only_name_last_cap .
+      bind(js:capitalizeName(?labeled_only_name_last_cap) as ?labeled_only_name_last)
       OPTIONAL {
-        ?elem :first-names ?labeled_only_name_first .
+        ?elem :first-names ?labeled_only_name_first_cap .
+        bind(js:capitalizeName(?labeled_only_name_first_cap) as ?labeled_only_name_first)
       }
       OPTIONAL {
         ?elem :initials ?labeled_only_name_initials .
@@ -182,15 +192,9 @@ WHERE {
              .
     }
 
-    OPTIONAL {
-      ?field :name "funder-name";
-             :type "text";
-             :text ?funder_label;
-             .
-    }
 
     {
-      select ?g ?start_date ?start_date_precision ?end_date ?end_date_precision
+      select ?g ?start_date ?start_year ?start_date_precision ?end_year ?end_date ?end_date_precision
       WHERE {
         {
           select ?relationship ?grant_id ?record ?source
@@ -224,7 +228,7 @@ WHERE {
            :type "date";
            :date ?s_date;
            .
-        ?s_date :year ?syear.
+        ?s_date :year ?start_year.
 
         BIND(vivo:yearPrecision AS ?syear_p)
         OPTIONAL {
@@ -239,14 +243,14 @@ WHERE {
             BIND(vivo:yearMonthDayPrecision AS ?sday_p)
           }
         }
-        BIND(CONCAT(?syear, COALESCE(?smonth, ""), COALESCE(?sday, "")) AS ?start_date)
+        BIND(CONCAT(?start_year, COALESCE(?smonth, ""), COALESCE(?sday, "")) AS ?start_date)
         BIND(coalesce(?sday_p,?smonth_p,?syear_p) as ?start_date_precision)
 
         ?e :name "end-date";
            :type "date";
            :date ?e_date;
            .
-        ?e_date :year ?eyear.
+        ?e_date :year ?end_year.
 
         BIND(vivo:yearPrecision AS ?eyear_p)
         OPTIONAL {
@@ -261,7 +265,7 @@ WHERE {
             BIND(vivo:yearMonthDayPrecision AS ?eday_p)
           }
         }
-        BIND(CONCAT(?eyear, COALESCE(?emonth, ""), COALESCE(?eday, "")) AS ?end_date)
+        BIND(CONCAT(?end_year, COALESCE(?emonth, ""), COALESCE(?eday, "")) AS ?end_date)
         BIND(coalesce(?eday_p,?emonth_p,?eyear_p) as ?end_date_precision)
 
       }
@@ -309,4 +313,15 @@ WHERE {
           uri(concat(str(?grant),"#roleof_",?labeled_only_name_match)),
           ?undefined)
        as ?labeled_only_role)
+
+  bind(js:capitalizeName(?funder_label_cap) as ?funder_label)
+  bind(js:capitalizeName(?pi_name_cap,"\\s+"," ") as ?pi_name)
+
+  bind(concat(?grant_title," § ",
+              coalesce(if(xsd:date(?end_date) < xsd:date(now()), "Completed", "Active"),""),
+              coalesce(concat(" • ",?start_year," - ",?end_year),""),
+              coalesce(concat(" • ",?pi_name),""),
+              " § ",
+              coalesce(?funder_label,""),
+              coalesce(concat(" • ",?sponsor_award_id),"")) as ?grant_label)
 }
diff --git a/harvest/harvest-entrypoint.sh b/harvest/harvest-entrypoint.sh
@@ -32,8 +32,8 @@ if [[ ${uid} = 0 ]]; then
   else
     echo \"no /etc/fin/service-account.json and GOOGLE_APPLICATION_CREDENTIALS_JSON is not set\";
   fi;
-  exec setpriv --reuid=ucd.process --init-groups -- make --file=/usr/local/lib/harvest/Makefile "$@"
+  exec setpriv --reuid=ucd.process --init-groups -- make --file=/startup.mk "$@"
 
 else
-  exec make --file=/usr/local/lib/harvest/Makefile "$@"
+  exec make --file=/startup.mk "$@"
 fi
diff --git a/harvest/harvest/Makefile → harvest/startup.mk b/harvest/harvest/Makefile → harvest/startup.mk
diff --git a/services/base-service/models/base/model.js b/services/base-service/models/base/model.js
@@ -187,6 +187,9 @@ class BaseModel extends FinEsDataModel {
       name: node['name'],
       "@graph": [node]
     };
+    ["@type","is-visible","updated","identifier"].forEach(key => {
+      if (node[key]) doc[key] = node[key];
+    });
     return doc;
   }
 
@@ -307,23 +310,20 @@ class BaseModel extends FinEsDataModel {
    * @returns {Object} ES results
    */
   async search(opts) {
-
+    opts.index || (opts.index = this.readIndexAlias);
     const params = this.common_parms(opts.params);
 
-    console.log(`searching ${this.readIndexAlias} with ${JSON.stringify(params)}`);
     const options = {
       id: (opts.id)?opts.id:"default",
-      index: this.readIndexAlias,
+      index: opts.index,
       params
     }
     const res=await this.client.searchTemplate(options);
     return this.compact_search_results(res,params);
   }
 
   async msearch(opts) {
-
-    opts.index = "expert-read";
-
+    opts.index || (opts.index = this.readIndexAlias);
     // Fix-up parms
     for(let i=0;i<opts.search_templates.length;i++) {
       let template=opts.search_templates[i];

diff --git a/services/base-service/models/base/schema/minimal.json b/services/base-service/models/base/schema/minimal.json
@@ -14,16 +14,19 @@
     },
     "name" : {
       "type" : "text",
-      "analyzer" : "title_sort",
+      "analyzer" : "title",
       "index_prefixes" : {
         "min_chars" : 1,
         "max_chars" : 7
       },
       "fields" : {
         "kw" : {
           "type" : "keyword",
-          "normalizer" : "title_prefix_analyzer",
-          "ignore_above" : 256
+          "normalizer" : "title_start"
+        },
+        "first" : {
+          "type" : "keyword",
+          "normalizer" : "starts_with"
         }
       }
     },
@@ -43,6 +46,7 @@
         "is-visible" : { "type" : "boolean" },
         "identifier" : {
           "type" : "keyword",
+
           "normalizer" : "identifier",
           "ignore_above": 128
         },
@@ -186,9 +190,6 @@
           "type" : "text",
           "analyzer" : "default"
         },
-        "issued" : {
-          "type" : "date"
-        },
         "title" : {
           "type" : "text",
           "analyzer" : "default"

diff --git a/services/base-service/models/base/schema/settings.json b/services/base-service/models/base/schema/settings.json
@@ -55,6 +55,26 @@
         "type": "pattern_replace",
         "pattern": "^(The|A|An)\\s+",
         "replacement": ""
+      },
+      "leading_space_filter": {
+        "type": "pattern_replace",
+        "pattern": "^\\s+",
+        "replacement": ""
+      },
+      "truncate_filter": {
+        "type": "pattern_replace",
+        "pattern": "^(.{1,128}).*",
+        "replacement": "$1"
+      },
+      "first_letter_filter": {
+        "type": "pattern_replace",
+        "pattern": "^(.).*",
+        "replacement": "$1"
+      },
+      "non_letter_filter": {
+        "type": "pattern_replace",
+        "pattern": "^(?s)(?i)([^a-z])",
+        "replacement": "other"
       }
     },
     "filter": {
@@ -77,11 +97,6 @@
         "type": "pattern_replace",
         "pattern": "(/|doi:|https://doi.org/)?(10\\.\\d{4,})",
         "replacement": "doi:$2"
-      },
-      "article_filter": {
-        "type": "pattern_replace",
-        "pattern": "^(The|A|An)\\s+",
-        "replacement": ""
       }
     },
     "normalizer" : {
@@ -101,10 +116,25 @@
           "asciifolding"
         ]
       },
-      "title_prefix_analyzer": {
+      "starts_with": {
         "type": "custom",
         "char_filter": [
-          "article_filter"
+          "leading_space_filter",
+          "article_filter",
+          "first_letter_filter",
+          "non_letter_filter"
+        ],
+        "filter": [
+          "lowercase",
+          "asciifolding"
+        ]
+      },
+      "title_start": {
+        "type": "custom",
+        "char_filter": [
+          "leading_space_filter",
+          "article_filter",
+          "truncate_filter"
         ],
         "filter": [
           "lowercase",
@@ -117,8 +147,9 @@
         "type": "custom",
         "tokenizer": "uax_url_email",
         "char_filter": [
-          "text_mapper","doi_to_url","orcid_to_url",
-          "mailto_to_email","ark_to_n2t","urn_to_n2t"
+          "leading_space_filter","text_mapper","doi_to_url","orcid_to_url",
+          "mailto_to_email","ark_to_n2t","urn_to_n2t",
+          "article_filter"
         ],
         "filter": [
           "email_to_mailto","xxx_to_doi","xxx_to_orcid", "n2t_to_id",
@@ -127,19 +158,18 @@
           "asciifolding"
         ]
       },
-      "title_sort": {
+      "title": {
         "type": "custom",
         "tokenizer": "uax_url_email",
         "char_filter": [
-          "text_mapper","doi_to_url","orcid_to_url",
+          "leading_space_filter","text_mapper","doi_to_url","orcid_to_url",
           "mailto_to_email","ark_to_n2t","urn_to_n2t"
         ],
         "filter": [
           "email_to_mailto","xxx_to_doi","xxx_to_orcid", "n2t_to_id",
           "lowercase",
           "stop",
-          "asciifolding",
-          "article_filter"
+          "asciifolding"
         ]
       },
       "autocomplete": {

diff --git a/...models/browse/template/family_prefix.json → ...e/models/base/template/family_prefix.json b/...models/browse/template/family_prefix.json → ...e/models/base/template/family_prefix.json