From b7c973f54dd75041cd381a8df8b6fc21515c01af Mon Sep 17 00:00:00 2001
From: Bob Coret <bob.coret+github@gmail.com>
Date: Fri, 22 Dec 2023 20:01:40 +0100
Subject: [PATCH] make loading of lists (SPARQL query) asynchronous

Not a real solution for #20 but at least is looks faster in the GUI (because loading isn't blocking)
---
 web/get-list.php |  26 +++
 web/search.php   | 432 ++++++++++++++++++++++++-----------------------
 2 files changed, 251 insertions(+), 207 deletions(-)
 create mode 100644 web/get-list.php

diff --git a/web/get-list.php b/web/get-list.php
new file mode 100644
index 0000000..f9edfdd
--- /dev/null
+++ b/web/get-list.php
@@ -0,0 +1,26 @@
+<?php
+
+include("includes/search.php");
+
+if (isset($_GET["list"])) {
+	if ($_GET["list"]=="publishers") {
+		$list=getPublishers();
+	} elseif ($_GET["list"]=="creators") {
+		$list=getCreators();
+	} elseif ($_GET["list"]=="formats") {
+		$list=getFormats();
+	} else {
+		exit;
+	}
+	
+	# client cache 2 hours
+	$now = time();
+	$generatedAt = gmdate('D, d M Y H:i:s T', $now);
+	$lastModified = gmdate('D, d M Y 00:00:00 T', $now);
+	$expiresAt = gmdate('D, d M Y H:i:s T', strtotime($lastModified) + 2*3600); # 2 hours
+	header('Last-modified: ' . $lastModified);
+	header('Cache-control: max-age=' . strtotime($expiresAt) - strtotime($generatedAt));
+
+	header('Content-Type: application/json; charset=utf-8');
+	echo json_encode($list);
+}
\ No newline at end of file
diff --git a/web/search.php b/web/search.php
index 0ee65ae..45e0ef5 100644
--- a/web/search.php
+++ b/web/search.php
@@ -29,53 +29,26 @@
           <label id="publisher_listLabel"><?= t('Uitgever')?></label>
           <select aria-labelledby="publisher_listLabel" class="form-control" id="publisher_list" name="publisher">
             <option value=""><?= t('Alle organisaties')?></option>
-<?php
-                     $publishers=getPublishers();
-                     foreach ($publishers as $publisher_uri => $publisher_name) {
-                     	echo '            <option ';
-						if (isset($o) && $publisher_uri==$o) { echo 'selected '; }
-                     	echo 'value="'.htmlentities($publisher_uri).'">'.str_replace(" ","&nbsp;",htmlentities($publisher_name))."</option>";
-                     }
-                     ?>
           </select>
           <br>- <?= t('of') ?> -<br>
           <label id="creator_listLabel"><?= t('Maker')?></label>
           <select aria-labelledby="creator_listLabel" class="form-control" id="creator_list"  name="creator">
             <option value=""><?= t('Alle organisaties')?></option>
-<?php
-                     $creators=getCreators();
-                     foreach ($creators as $creator_uri => $creator_name) {
-                     	echo '            <option ';
-						if (isset($o) && $creator_uri==$o) { echo 'selected '; }
-						echo 'value="'.htmlentities($creator_uri).'">'.str_replace(" ","&nbsp;",htmlentities($creator_name))."</option>";
-                     }
-                     ?>
           </select>
         </div>
         <div class="column2">
           <label><?= t('Formaat')?></label>
-          <div class="formatcheckboxes">
-                  <?php
-                     $formats=getFormats();
-                     foreach ($formats as $format_name) {
-                     	echo '<label>';
-                     	echo '<input class="choice" type="checkbox" name="format[]" value="'.htmlentities($format_name).'">';
-                     	echo '&nbsp;&nbsp;'.htmlentities($format_name);
-                     	echo '</label>';
-                     }
-                     ?>
-               </div>
-               <p class="choices"><a href="#" onclick="return set_lod_choices()"><?= t('Selecteer Linked Data formaten')?></a> | <a href="#" onclick="return set_sparql_choices()"><?= t('Selecteer SPARQL formaten')?></a> | <a href="#" onclick="return clear_formats()"><?= t('Verwijder selectie(s)')?></a></p>
-               <span class="btn btn--arrow m-t-half-space btn--api" style="display:block" onclick="searchDatasets()">
-               <?= t('Zoek datasets')?><div id="wait" class="lds-ellipsis"><div></div><div></div><div></div><div></div></div>
-                  <svg class="rect">
-                     <rect class="svgrect" width="100%" height="100%" style="stroke-width: 3; fill: transparent; stroke-dasharray: 586; stroke-dashoffset: 586;"></rect>
-                  </svg>
-                  <svg class="icon icon-arrow-right">
-                     <use xlink:href="#icon-arrow-right"></use>
-                  </svg>
-               </span>
-            </div>
+            <div id="format_list" class="formatcheckboxes"></div>
+            <p class="choices"><a href="#" onclick="return set_lod_choices()"><?= t('Selecteer Linked Data formaten')?></a> | <a href="#" onclick="return set_sparql_choices()"><?= t('Selecteer SPARQL formaten')?></a> | <a href="#" onclick="return clear_formats()"><?= t('Verwijder selectie(s)')?></a></p>
+            <span class="btn btn--arrow m-t-half-space btn--api" style="display:block" onclick="searchDatasets()">
+              <?= t('Zoek datasets')?><div id="wait" class="lds-ellipsis"><div></div><div></div><div></div><div></div></div>
+			  <svg class="rect">
+			    <rect class="svgrect" width="100%" height="100%" style="stroke-width: 3; fill: transparent; stroke-dasharray: 586; troke-dashoffset: 586;"></rect>
+			  </svg>
+			  <svg class="icon icon-arrow-right">
+				<use xlink:href="#icon-arrow-right"></use>
+			  </svg>
+            </span>
          </div>
       </div>
    </section>
@@ -97,9 +70,12 @@
 <script>
 const sparqlUrl = 'https://triplestore.netwerkdigitaalerfgoed.nl/sparql?query=';
 var sparqlQuery;
+var organisationFacet;
+var bSearch = 0;
+var creator = '';
+var publisher = '';
 
 function updateSparql() {
-
   searchTerm = document.getElementById("searchTerm").value.trim().replace(/"/g, '\\"');
 
   sparqlQuery = `PREFIX dcat: <http://www.w3.org/ns/dcat#>
@@ -128,25 +104,29 @@ function updateSparql() {
   if (formats.size > 0) {
     sparqlQuery += "  ?dataset dcat:distribution ?distribution .\n";
     sparqlQuery += "  ?distribution dct:format ?format .\n";
-    sparqlQuery += "  FILTER( ?format=\"" + Array.from(formats).join('" || ?format="') + "\")\n";
+    sparqlQuery += "  FILTER( ?format=\"" + Array.from(formats)
+      .join('" || ?format="') + "\")\n";
   }
 
   sparqlQuery += "} ORDER BY ?title";
 
-  document.getElementById('sparql-query').innerHTML = sparqlQuery;
-  console.log(sparqlQuery);
+  document.getElementById('sparql-query')
+    .innerHTML = sparqlQuery;
+  console.info(sparqlQuery);
   toHash();
 }
 
 function set_creator(value) {
   creator = value;
   publisher = "";
-  document.getElementById('publisher_list').options[0].selected = true;
+  document.getElementById('publisher_list')
+    .options[0].selected = true;
 }
 
 function set_publisher(value) {
   creator = "";
-  document.getElementById('creator_list').options[0].selected = true;
+  document.getElementById('creator_list')
+    .options[0].selected = true;
   publisher = value;
 }
 
@@ -159,13 +139,6 @@ function set_choice(name, value, checked) {
         formats.delete(value);
       }
       break;
-    case 'searchIn[]':
-      if (checked) {
-        searchIn.add(value);
-      } else {
-        searchIn.delete(value);
-      }
-      break;
   }
   updateSparql();
 }
@@ -173,23 +146,20 @@ function set_choice(name, value, checked) {
 function searchTriplestore() {
   updateSparql();
   var url = sparqlUrl + encodeURIComponent(sparqlQuery);
-  window.open(url, 'triplestore').focus();
+  window.open(url, 'triplestore')
+    .focus();
   return false;
 }
 
 function searchDatasets() {
-	
-  document.getElementById("wait").style.display="inline-block";
-	
+  document.getElementById("wait").style.display = "inline-block";
   updateSparql();
-
   document.getElementById("searchresults").style.display = "none";
 
   var url = 'https://triplestore.netwerkdigitaalerfgoed.nl/repositories/registry?query=' + encodeURIComponent(sparqlQuery);
   var xhr = new XMLHttpRequest();
   xhr.open("GET", url);
   xhr.setRequestHeader("Accept", "application/json");
-
   xhr.onreadystatechange = function() {
     if (xhr.readyState === 4) {
       if (xhr.status === 200) {
@@ -198,52 +168,48 @@ function searchDatasets() {
         console.log("Call to triplestore got HTTP code " + xhr.status);
       }
     }
-    document.getElementById("wait").style.display="none";
+    document.getElementById("wait").style.display = "none";
   };
-
   xhr.send();
 }
 
-var organisationFacet;
-
 function showDatasets(sparqlresult) {
-
   document.getElementById("countdatasets").innerHTML = sparqlresult.results.bindings.length;
   document.getElementById("searchresults").style.display = "block";
   document.getElementById("sparql").scrollIntoView();
 
-  organisationFacet=new Object();
-  
+  organisationFacet = new Object();
+
   var ul = document.getElementById("datasets");
   ul.innerHTML = "";
 
-  uriCount=0;
+  uriCount = 0;
   for (var prop in sparqlresult.results.bindings) {
     uriCount++;
     dataset = sparqlresult.results.bindings[prop].dataset.value;
     title = sparqlresult.results.bindings[prop].title.value;
     publisherName = sparqlresult.results.bindings[prop].publisherName.value;
-	if (organisationFacet[publisherName]) {
-		organisationFacet[publisherName]++;
-	} else {
-		organisationFacet[publisherName]=1;
-	}
-	
+    if (organisationFacet[publisherName]) {
+      organisationFacet[publisherName]++;
+    } else {
+      organisationFacet[publisherName] = 1;
+    }
+
     var li = document.createElement("li");
     li.setAttribute("class", "linkprop");
-	li.setAttribute("data-organisation", publisherName);
+    li.setAttribute("data-organisation", publisherName);
 
     var link = document.createElement("a");
     var linkText = document.createTextNode(title);
-    link.setAttribute("href", "show.php?lang=<?= $lang ?>&uri="+encodeURIComponent(dataset));
+    link.setAttribute("href", "show.php?lang=<?= $lang ?>&uri=" + encodeURIComponent(dataset));
     link.appendChild(linkText);
     li.appendChild(link);
-    li.appendChild(document.createTextNode(" ("+publisherName+")"));
+    li.appendChild(document.createTextNode(" (" + publisherName + ")"));
 
     var div = document.createElement("div");
     div.setAttribute("class", "scroll");
-	  li.appendChild(div);
- 
+    li.appendChild(div);
+
     var eul = document.createElement("table");
     eul.setAttribute("id", "props-" + dataset);
     div.appendChild(eul);
@@ -254,53 +220,58 @@ function showDatasets(sparqlresult) {
 }
 
 function showFacets() {
-	if (Object.keys(organisationFacet).length>1) {
-		  document.getElementById("facet_block").style.display = "block";
-
-		  facet_options="";
-		  // sort by number of occurences
-		  var organisations = Object.keys(organisationFacet);
-		  organisations.sort(function(a, b) { return organisationFacet[b] - organisationFacet[a] });
-
-		  organisations.forEach(function(org) {
-			facet_options+='<label><input type="checkbox" checked class="org_facet" onclick="showhideListFacets(this)" value="'+org+'"> '+org+' ('+organisationFacet[org]+')</label> ';			
-		  });
-		
-		  document.getElementById("facets").innerHTML=facet_options;
-	} else {
-		  document.getElementById("facet_block").style.display = "none";
-	}	
+  if (Object.keys(organisationFacet).length > 1) {
+    document.getElementById("facet_block").style.display = "block";
+
+    facet_options = "";
+    // sort by number of occurences
+    var organisations = Object.keys(organisationFacet);
+    organisations.sort(function(a, b) {
+      return organisationFacet[b] - organisationFacet[a]
+    });
+
+    organisations.forEach(function(org) {
+      facet_options += '<label><input type="checkbox" checked class="org_facet" onclick="showhideListFacets(this)" value="' + org + '"> ' + org + ' (' + organisationFacet[org] + ')</label> ';
+    });
+
+    document.getElementById("facets").innerHTML = facet_options;
+  } else {
+    document.getElementById("facet_block").style.display = "none";
+  }
 }
 
 function showhideListFacets(organisation) {
-	var ul = document.getElementById("datasets");
-	var items = ul.getElementsByTagName("li");
-	var nrFacetResults=0;
-	for (var i = 0; i < items.length; ++i) {
-		if(items[i].getAttribute('data-organisation')==organisation.value) {
-			if (organisation.checked) {
-				items[i].style.display="list-item";
-			} else {
-				items[i].style.display="none";
-			}
-		}
-        if (items[i].style.display!="none") {
-            nrFacetResults++;
-		}
-	}
-	document.getElementById("countdatasets").innerHTML=nrFacetResults;
+  var ul = document.getElementById("datasets");
+  var items = ul.getElementsByTagName("li");
+  var nrFacetResults = 0;
+  for (var i = 0; i < items.length; ++i) {
+    if (items[i].getAttribute('data-organisation') == organisation.value) {
+      if (organisation.checked) {
+        items[i].style.display = "list-item";
+      } else {
+        items[i].style.display = "none";
+      }
+    }
+    if (items[i].style.display != "none") {
+      nrFacetResults++;
+    }
+  }
+  document.getElementById("countdatasets")
+    .innerHTML = nrFacetResults;
 }
 
 function getSortedKeys(obj) {
-    var keys = Object.keys(obj);
-    return keys.sort(function(a,b){return obj[b]-obj[a]});
+  var keys = Object.keys(obj);
+  return keys.sort(function(a, b) {
+    return obj[b] - obj[a]
+  });
 }
 
 function set_lod_choices() {
   choices = document.getElementsByClassName('choice');
   for (i = 0; i < choices.length; i++) {
     if (choices[i].name == "format[]") {
-      if (["application/ld+json", "application/n-quads", "application/n-triples", "application/rdf+xml", "application/sparql-query", "application/trig", "application/turtle", "application/vnd.hdt", "text/turtle", "text/n3", "application/x-sparqlstar-results+json", "application/sparql-results+xml", "application/sparql-results+json", "application/x-sparqlstar-results", "application/sparql-results","application/ld+json+gzip","application/n-quads+gzip","application/n-triples+gzip","application/rdf+xml+gzip","text/n3+gzip","text/turtle+gzip"].includes(choices[i].value)) {
+      if (["application/ld+json", "application/n-quads", "application/n-triples", "application/rdf+xml", "application/sparql-query", "application/trig", "application/turtle", "application/vnd.hdt", "text/turtle", "text/n3", "application/x-sparqlstar-results+json", "application/sparql-results+xml", "application/sparql-results+json", "application/x-sparqlstar-results", "application/sparql-results", "application/ld+json+gzip", "application/n-quads+gzip", "application/n-triples+gzip", "application/rdf+xml+gzip", "text/n3+gzip", "text/turtle+gzip"].includes(choices[i].value)) {
         choices[i].checked = true;
         formats.add(choices[i].value);
       } else {
@@ -342,122 +313,169 @@ function clear_formats() {
 }
 
 function toHash() {
-	var searchData={};
-
-	if (creator) {
-		searchData.c=creator;
-	}
-	if (publisher) {
-		searchData.p=publisher;
-	}
-	if (formats.size > 0) {
-		searchData.f=Array.from(formats);
-	}
-	searchData.t=document.getElementById("searchTerm").value.trim();
-	history.pushState({}, "", "#"+btoa(JSON.stringify(searchData)));
+  var searchData = {};
+
+  if (creator) {
+    searchData.c = creator;
+  }
+  if (publisher) {
+    searchData.p = publisher;
+  }
+  if (formats.size > 0) {
+    searchData.f = Array.from(formats);
+  }
+  searchData.t = document.getElementById("searchTerm").value.trim();
+  history.pushState({}, "", "#" + btoa(JSON.stringify(searchData)));
 }
 
 function fromHash() {
+  var hash = new URL(document.URL).hash.substring(1);
+
+  if (hash.length > 0) {
+    try {
+      var searchData = JSON.parse(atob(hash));
+      if (searchData.c) {
+        creator = searchData.c;
+      }
+      if (searchData.p) {
+        publisher = searchData.p;
+      }
+      document.getElementById("searchTerm").value = searchData.t;
+      if (searchData.f) {
+        formats = new Set(searchData.f);
+      }
+      searchDatasets();
+    } catch (e) {
+      console.log(e);
+      return false;
+    }
+  } else {
+    updateSparql();
+  }
+}
 
-	var hash = new URL(document.URL).hash.substring(1);
-
-	if (hash.length>0) {
-		try {
-			var searchData=JSON.parse(atob(hash));
-			if (searchData.c) {
-				creator=searchData.c;
-				document.getElementById("creator_list").value=creator;
-			}
-			if (searchData.p) {
-				publisher=searchData.p;
-				document.getElementById("publisher_list").value=publisher;
-			}
-			
-			searchIn=new Set(searchData.i);	  
-			document.getElementById("searchTerm").value=searchData.t;
-
-			if (searchData.f) {
-				formats=new Set(searchData.f);		
-				formatChoices=document.getElementsByClassName("choice");
-				Array.prototype.forEach.call(formatChoices, function(el) {
-					el.checked=searchData.f.includes(el.value);
-				});
-			}
-			searchDatasets();
-		} catch (e) {
-			console.log(e);
-			return false;
-		}
-	} else {
-		updateSparql();		
-	}
+async function fetchData(url) {
+  try {
+    const response = await fetch(url);
+    const jsonData = await response.json();
+    return jsonData;
+  } catch (error) {
+    console.error('Error fetching data:', error);
+  }
 }
 
-var bSearch=0;
-var creator = '';
-if (document.getElementById('creator_list').selectedIndex>0) { 
-	creator=document.getElementById('creator_list').options[document.getElementById('creator_list').selectedIndex].value; 
-	bSearch=1;
+if (document.getElementById('creator_list')
+  .selectedIndex > 0) {
+  creator = document.getElementById('creator_list')
+    .options[document.getElementById('creator_list')
+      .selectedIndex].value;
+  bSearch = 1;
 }
-var publisher = '';
-if (document.getElementById('publisher_list').selectedIndex>0) { 
-	publisher=document.getElementById('publisher_list').options[document.getElementById('publisher_list').selectedIndex].value; 
-	bSearch=1;
+if (document.getElementById('publisher_list')
+  .selectedIndex > 0) {
+  publisher = document.getElementById('publisher_list')
+    .options[document.getElementById('publisher_list')
+      .selectedIndex].value;
+  bSearch = 1;
 }
 var formats = new Set();
-var searchIn = new Set(["dct:title"]);
 
 if (bSearch) {
-	 searchDatasets();
+  searchDatasets();
 } else {
-	fromHash();
+  fromHash();
 }
 
-document.getElementById('creator_list').addEventListener('change', function() {
-  set_creator(this.value);
-  updateSparql();
-});
 
-document.getElementById('publisher_list').addEventListener('change', function() {
-  set_publisher(this.value);
-  updateSparql();
-});
+document.getElementById('creator_list')
+  .addEventListener('change', function() {
+    set_creator(this.value);
+    updateSparql();
+  });
 
-document.getElementById('sparql-query').addEventListener(
-  "click",
-  function(event) {
+document.getElementById('publisher_list')
+  .addEventListener('change', function() {
+    set_publisher(this.value);
+    updateSparql();
+  });
 
-    if (!navigator.clipboard) {
-      // Clipboard API not available
-      return;
+document.getElementById('sparql-query').addEventListener("click", function(event) {
+      if (!navigator.clipboard) {
+        // Clipboard API not available
+        return;
+      }
+      const text = document.getElementById('sparql-query')
+        .innerHTML;
+      try {
+        navigator.clipboard.writeText(text);
+        document.getElementById("copy-status")
+          .innerText = "<?= t('De SPARQL is gekopieerd.') ?>";
+        setTimeout(function() {
+          document.getElementById("copy-status")
+            .innerText = "<?= t('Klik de SPARQL om deze te kopieren.') ?>";
+        }, 1200);
+      } catch (err) {
+        console.error("Failed to copy!", err);
+      }
+    },
+    false
+  );
+
+document.getElementById("searchTerm")
+  .addEventListener("keypress", function(event) {
+    if (event.key === "Enter") {
+      event.preventDefault();
+      searchDatasets();
     }
-    const text = document.getElementById('sparql-query').innerHTML;
-    try {
-      navigator.clipboard.writeText(text);
-      document.getElementById("copy-status").innerText = "<?= t('De SPARQL is gekopieerd.') ?>";
-      setTimeout(function() {
-        document.getElementById("copy-status").innerText = "<?= t('Klik de SPARQL om deze te kopieren.') ?>";
-      }, 1200);
-    } catch (err) {
-      console.error("Failed to copy!", err);
+  });
+
+fetchData('https://datasetregister.coret.org/get-list.php?list=formats').then(data => {
+    const listformatsDiv = document.getElementById("format_list");
+    for (const item of data) {
+      const newFormat = document.createElement('label');
+      const inputElement = document.createElement('input');
+      inputElement.type = 'checkbox';
+      inputElement.class = 'choice';
+      inputElement.value = item;
+      inputElement.name = 'format[]';
+      inputElement.checked = formats.has(item);
+      inputElement.addEventListener('click', function(a) {
+        set_choice(this.name, this.value, this.checked);
+        updateSparql();
+      });
+      newFormat.appendChild(inputElement);
+      const labelText = document.createTextNode('  ' + item);
+      newFormat.appendChild(labelText);
+      listformatsDiv.append(newFormat);
     }
-  },
-  false
-);
-
-document.getElementById("searchTerm").addEventListener("keypress", function(event) {
-  if (event.key === "Enter") {
-    event.preventDefault();
-	searchDatasets();
-  }
-});
+    console.info("Loaded " + data.length + " formats in checklist");
+  });
 
-choices = document.getElementsByClassName('choice');
-for (i = 0; i < choices.length; i++) {
-  choices[i].addEventListener('click', function(a) {
-    set_choice(this.name, this.value, this.checked);
-    updateSparql();
+fetchData('https://datasetregister.coret.org/get-list.php?list=publishers').then(data => {
+    const listpublishersDiv = document.getElementById("publisher_list");
+    for (const key in data) {
+      const value = data[key];
+      const newPublisherItem = document.createElement('option');
+      newPublisherItem.value = key;
+      newPublisherItem.textContent = value;
+      newPublisherItem.selected = (publisher == key);
+      listpublishersDiv.appendChild(newPublisherItem);
+    }
+    console.info("Loaded " + Object.keys(data).length + " publishers in dropdown");
+  });
+
+fetchData('https://datasetregister.coret.org/get-list.php?list=creators').then(data => {
+    const listcreatorsDiv = document.getElementById("creator_list");
+    for (const key in data) {
+      const value = data[key];
+      const newCreatorItem = document.createElement('option');
+      newCreatorItem.class = 'choice';
+      newCreatorItem.textContent = value;
+      newCreatorItem.selected = (creator == key);
+      listcreatorsDiv.appendChild(newCreatorItem);
+    }
+    console.info("Loaded " + Object.keys(data)
+      .length + " creators in dropdown");
   });
-}
 </script>
 <?php include("includes/footer.php") ?>
\ No newline at end of file