Skip to content

Commit

Permalink
Merge pull request #305 from RDFLib/lawson/fts
Browse files Browse the repository at this point in the history 
Lawson/fts
  • Loading branch information
@lalewis1
lalewis1 authored Nov 21, 2024
2 parents d2ef2d9 + 16c0e11 commit d5f5a2a
Show file tree
Hide file tree
Showing 42 changed files with 1,287 additions and 331 deletions.
83 changes: 78 additions & 5 deletions dev/dev-config.ttl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,12 @@
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix tdb2: <http://jena.apache.org/2016/tdb#> .
@prefix text: <http://jena.apache.org/text#> .
@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
@prefix dcterms: <http://purl.org/dc/terms/> .
@prefix sdo: <https://schema.org/> .
@prefix ex: <http://www.example.org/resources#> .


tdb2:GraphTDB rdfs:subClassOf ja:Model .

Expand Down Expand Up @@ -42,7 +48,7 @@ ja:DatasetRDFS rdfs:subClassOf ja:RDFDataset .

:service rdf:type fuseki:Service ;
rdfs:label "myds" ;
fuseki:dataset :dataset_readwrite ;
fuseki:dataset <#textds> ;
fuseki:endpoint [ fuseki:operation fuseki:update ] ;
fuseki:endpoint [ fuseki:name "update" ;
fuseki:operation fuseki:update
Expand All @@ -63,7 +69,74 @@ ja:DatasetRDFS rdfs:subClassOf ja:RDFDataset .
fuseki:endpoint [ fuseki:operation fuseki:query ] ;
fuseki:name "myds" .

:dataset_readwrite
rdf:type tdb2:DatasetTDB2 ;
tdb2:unionDefaultGraph true ;
tdb2:location "/fuseki/databases/myds" .
<#textds> rdf:type text:TextDataset ;
text:dataset <#myds> ;
text:index <#indexLucene> ;
.

# Text index description
<#indexLucene> a text:TextIndexLucene ;
text:analyzer: [ a text:StandardAnalyzer ];
text:directory "databases/text-index-with-rdfs" ;
text:storeValues true ;
text:entityMap <#entMap> ;
text:propLists (
[ text:propListProp ex:NameProps ;
text:props ( rdfs:label
sdo:name
sdo:alternateName
) ;
]
[ text:propListProp ex:DescriptiveProps ;
text:props ( sdo:description
rdfs:comment
) ;
]
) ;
.

<#entMap> a text:EntityMap ;
text:entityField "uri" ;
text:graphField "graph" ;
text:defaultField "preflabel" ; ## Should be defined in the text:map.
text:map (
[ text:field "definition" ;
text:predicate skos:definition ;
]
[ text:field "preflabel" ;
text:predicate skos:prefLabel ;
]
[ text:field "altlabel" ;
text:predicate skos:altLabel ;
]
[ text:field "rdfslabel" ;
text:predicate rdfs:label ;
]
[ text:field "identifier" ;
text:predicate dcterms:identifier ;
]
[ text:field "description" ;
text:predicate dcterms:description ;
]
[ text:field "comment" ;
text:predicate rdfs:comment ;
]
[ text:field "type" ;
text:predicate rdf:type ;
]
[ text:field "historyNote" ;
text:predicate skos:historyNote ;
]
[ text:field "sdodescription" ;
text:predicate sdo:description ;
]
[ text:field "headline" ;
text:predicate sdo:headline ;
]
) .

# TDB2 dataset
<#myds> rdf:type tdb2:DatasetTDB2 ;
tdb2:unionDefaultGraph true ;
tdb2:location "fuseki/databases/myds" ;
.
191 changes: 37 additions & 154 deletions dev/dev-setup.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import os
import time

import requests
import glob

os.system("docker context use default")
response = os.system(
Expand All @@ -13,158 +13,41 @@

def setup():
url = "http://localhost:3030/myds"

payload = {}
files = [
(
"myfile",
(
"geofabric_small.ttl",
open("tests/data/spaceprez/input/geofabric_small.ttl", "rb"),
"application/octet-stream",
),
),
(
"myfile2",
(
"gnaf_small.ttl",
open("tests/data/spaceprez/input/gnaf_small.ttl", "rb"),
"application/octet-stream",
),
),
(
"myfile3",
(
"labels.ttl",
open("tests/data/spaceprez/input/labels.ttl", "rb"),
"application/octet-stream",
),
),
(
"myfile4",
(
"sandgate.ttl",
open("tests/data/spaceprez/input/sandgate.ttl", "rb"),
"application/octet-stream",
),
),
(
"myfile5",
(
"multiple_object.ttl",
open("tests/data/spaceprez/input/multiple_object.ttl", "rb"),
"application/octet-stream",
),
),
(
"myfile6",
(
"dublin_core_terms.ttl",
open("tests/data/vocprez/input/dublin_core_terms.ttl", "rb"),
"application/octet-stream",
),
),
(
"myfile7",
(
"remote_profile.ttl",
open("tests/data/profiles/remote_profile.ttl", "rb"),
"application/octet-stream",
),
),
(
"myfile8",
(
"alteration-types.ttl",
open("tests/data/vocprez/input/alteration-types.ttl", "rb"),
"application/octet-stream",
),
),
(
"myfile9",
(
"contacttype.ttl",
open("tests/data/vocprez/input/contacttype.ttl", "rb"),
"application/octet-stream",
),
),
(
"myfile10",
(
"dublin_core_terms.ttl",
open("tests/data/vocprez/input/dublin_core_terms.ttl", "rb"),
"application/octet-stream",
),
),
(
"myfile11",
(
"_idn-ac.ttl",
open("tests/data/catprez/input/_idn-ac.ttl", "rb"),
"application/octet-stream",
),
),
(
"myfile12",
(
"_idn-dc.ttl",
open("tests/data/catprez/input/_idn-dc.ttl", "rb"),
"application/octet-stream",
),
),
(
"myfile13",
(
"_system-catalog.ttl",
open("tests/data/catprez/input/_system-catalog.ttl", "rb"),
"application/octet-stream",
),
),
(
"myfile14",
(
"AAC-SA.ttl",
open("tests/data/catprez/input/AAC-SA.ttl", "rb"),
"application/octet-stream",
),
),
(
"myfile15",
(
"agents.ttl",
open("tests/data/catprez/input/agents.ttl", "rb"),
"application/octet-stream",
),
),
(
"myfile17",
(
"agents.ttl",
open("tests/data/catprez/input/pd_democat.ttl", "rb"),
"application/octet-stream",
),
),
(
"myfile16",
(
"dublin_core_terms.ttl",
open("tests/data/vocprez/input/dublin_core_terms.ttl", "rb"),
"application/octet-stream",
),
),
]
headers = {}

response = requests.request(
"POST",
url,
headers=headers,
data=payload,
files=files,
params={"graph": "http://exampledatagraph"},
)

print(response.text)


setup()
# Get all TTL files from test_data directory
ttl_files = glob.glob("test_data/*.ttl")

# Process each file sequentially
for i, file_path in enumerate(ttl_files, 1):
file_name = os.path.basename(file_path)
print(f"Loading file {i}/{len(ttl_files)}: {file_name}")

files = [
(
"file",
(
file_name,
open(file_path, "rb"),
"application/octet-stream",
),
)
]

response = requests.request(
"POST",
url,
headers=headers,
data={},
files=files,
params={"graph": "http://exampledatagraph"},
)

if response.status_code != 200:
print(f"Error loading {file_name}: {response.status_code}")
print(response.text)
else:
print(f"Successfully loaded {file_name}")


setup()
2 changes: 2 additions & 0 deletions prez/app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
healthcheck_sparql_endpoints,
populate_api_info,
prefix_initialisation,
retrieve_remote_jena_fts_shapes,
retrieve_remote_queryable_definitions,
retrieve_remote_template_queries,
)
Expand Down Expand Up @@ -119,6 +120,7 @@ async def lifespan(app: FastAPI):

await prefix_initialisation(app.state.repo)
await retrieve_remote_template_queries(app.state.repo)
await retrieve_remote_jena_fts_shapes(app.state.repo)
await create_profiles_graph(app.state.repo)
await create_endpoints_graph(app.state)
await count_objects(app.state.repo)
Expand Down
2 changes: 2 additions & 0 deletions prez/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from rdflib import DCTERMS, RDFS, SDO, URIRef
from rdflib.namespace import SKOS

from prez.enums import SearchMethod
from prez.reference_data.prez_ns import EP, REG


Expand Down Expand Up @@ -89,6 +90,7 @@ class Settings(BaseSettings):
temporal_predicate: Optional[URIRef] = SDO.temporal
endpoint_to_template_query_filename: Optional[Dict[str, str]] = {}
prez_ui_url: Optional[str] = None
search_method: SearchMethod = SearchMethod.DEFAULT
required_header: dict[str, str] | None = None
proxy_headers: bool = False
forwarded_allow_ips: str = "127.0.0.1"
Expand Down
Loading

0 comments on commit d5f5a2a

Please sign in to comment.