Skip to content

Commit

Permalink
Plone upgrade, conversion from alpine, and resolve #18
Browse files Browse the repository at this point in the history
  • Loading branch information
nutjob4life committed Dec 6, 2022
1 parent 8f80467 commit 105dd8c
Show file tree
Hide file tree
Showing 18 changed files with 237 additions and 51 deletions.
9 changes: 2 additions & 7 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,16 @@
#
# Defines the image for the CancerDataExpo

FROM plone:5.2.2-alpine
FROM plone:5.2.9

ENV numpy=1.19.3
ENV numpy=1.23.4

COPY site.cfg /plone/instance/
COPY src/ /plone/instance/src/
COPY etc/ /plone/instance/etc/

RUN : &&\
build_deps="gcc musl-dev" &&\
apk update --quiet &&\
apk add --quiet --virtual /build $build_deps &&\
pip --quiet install numpy==${numpy} &&\
buildout -c site.cfg &&\
find /plone -not -user plone -exec chown plone:plone {} \+ &&\
apk del --quiet /build &&\
rm -rf /var/cache/apk/* &&\
:
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,12 @@ The answer will then be in the result element of the response element of the env

## 🔧 Developing

**👉 Note:** Plone 5.2.2 (used here) is not compatible with any Python newer than 3.8. Stick with 3.8.
**👉 Note:** Plone 5.2.9 (used here) is not compatible with any Python newer than 3.8. Stick with 3.8.

Do the following:
```console
python3.8 -m venv venv
venv/bin/pip install --upgrade pip build wheel zc.buildout setuptools==42.0.2 numpy==1.19.3
venv/bin/pip install --upgrade pip build wheel zc.buildout setuptools==42.0.2 numpy==1.23.4
venv/bin/buildout -c dev.cfg
```

Expand Down
2 changes: 1 addition & 1 deletion etc/versions/dists.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
#
# That's it (for now).
[buildout]
extends = https://dist.plone.org/release/5.2.2/versions.cfg
extends = https://dist.plone.org/release/5.2.9/versions.cfg


# Software Distributions to Compile/Make/Install
Expand Down
189 changes: 189 additions & 0 deletions etc/versions/known-good-versions.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -223,3 +223,192 @@ suds2 = 0.7.1
# Required by:
# unittest2==1.1.0
traceback2 = 1.4.0

# Added by buildout at 2022-12-05 16:58:47.176461
biothings-client = 0.2.6
collective.recipe.backup = 4.2.0
isort = 5.10.1
modernize = 0.8.0
mr.scripty = 1.0

# Required by:
# edrn.summarizer==0.0.8
biopython = 1.80

# Required by:
# edrn.rdf==1.3.8
# edrn.summarizer==0.0.8
collective.autopermission = 1.0b2

# Required by:
# modernize==0.8.0
fissix = 21.11.13

# Required by:
# rdflib==4.2.2
isodate = 0.6.1

# Required by:
# edrn.summarizer==0.0.8
mygene = 3.2.2

# Required by:
# edrn.rdf==1.3.8
pysolr = 3.9.0

# Required by:
# edrn.rdf==1.3.8
# edrn.summarizer==0.0.8
suds2 = 0.7.1

# Required by:
# python-dotenv==0.15.0
typing = 3.10.0.0

# Added by buildout at 2022-12-05 17:12:03.918426
biothings-client = 0.2.6
collective.recipe.backup = 4.2.0
isort = 5.10.1
modernize = 0.8.0
mr.scripty = 1.0

# Required by:
# edrn.summarizer==0.0.8
biopython = 1.80

# Required by:
# edrn.rdf==1.3.8
# edrn.summarizer==0.0.8
collective.autopermission = 1.0b2

# Required by:
# modernize==0.8.0
fissix = 21.11.13

# Required by:
# rdflib==4.2.2
isodate = 0.6.1

# Required by:
# edrn.summarizer==0.0.8
mygene = 3.2.2

# Required by:
# edrn.rdf==1.3.8
pysolr = 3.9.0

# Required by:
# edrn.rdf==1.3.8
# edrn.summarizer==0.0.8
suds2 = 0.7.1

# Added by buildout at 2022-12-06 09:35:44.943510
biothings-client = 0.2.6
collective.recipe.backup = 4.2.0
isort = 5.10.1
modernize = 0.8.0
mr.scripty = 1.0

# Required by:
# edrn.summarizer==0.0.8
biopython = 1.80

# Required by:
# edrn.rdf==1.3.8
# edrn.summarizer==0.0.8
collective.autopermission = 1.0b2

# Required by:
# modernize==0.8.0
fissix = 21.11.13

# Required by:
# rdflib==6.2.0
isodate = 0.6.1

# Required by:
# edrn.summarizer==0.0.8
mygene = 3.2.2

# Required by:
# edrn.rdf==1.3.8
pysolr = 3.9.0

# Required by:
# edrn.rdf==1.3.8
# edrn.summarizer==0.0.8
suds2 = 0.7.1

# Added by buildout at 2022-12-06 09:44:08.666120
biothings-client = 0.2.6
collective.recipe.backup = 4.2.0
isort = 5.10.1
modernize = 0.8.0
mr.scripty = 1.0

# Required by:
# edrn.summarizer==0.0.8
biopython = 1.80

# Required by:
# edrn.rdf==1.3.8
# edrn.summarizer==0.0.8
collective.autopermission = 1.0b2

# Required by:
# modernize==0.8.0
fissix = 21.11.13

# Required by:
# rdflib==5.0.0
isodate = 0.6.1

# Required by:
# edrn.summarizer==0.0.8
mygene = 3.2.2

# Required by:
# edrn.rdf==1.3.8
pysolr = 3.9.0

# Required by:
# edrn.rdf==1.3.8
# edrn.summarizer==0.0.8
suds2 = 0.7.1

# Added by buildout at 2022-12-06 10:22:10.706859
biothings-client = 0.2.6
collective.recipe.backup = 4.2.0
isort = 5.10.1
modernize = 0.8.0
mr.scripty = 1.0

# Required by:
# edrn.summarizer==0.0.8
biopython = 1.80

# Required by:
# edrn.rdf==1.3.8
# edrn.summarizer==0.0.8
collective.autopermission = 1.0b2

# Required by:
# modernize==0.8.0
fissix = 21.11.13

# Required by:
# rdflib==6.2.0
isodate = 0.6.1

# Required by:
# edrn.summarizer==0.0.8
mygene = 3.2.2

# Required by:
# edrn.rdf==1.3.8
pysolr = 3.9.0

# Required by:
# edrn.rdf==1.3.8
# edrn.summarizer==0.0.8
suds2 = 0.7.1
1 change: 0 additions & 1 deletion etc/zope.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ eggs =
plone.testing
plone.app.testing
plone.app.robotframework
unittest2
plone.app.upgrade
defaults = ['--auto-color', '--auto-progress', '--verbose']

Expand Down
12 changes: 6 additions & 6 deletions src/edrn.rdf/edrn/rdf/README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,7 @@ And now::
'application/rdf+xml'
>>> import rdflib
>>> graph = rdflib.Graph()
>>> graph.parse(data=browser.contents)
>>> graph.parse(data=browser.contents, format='xml')
<Graph identifier=...(<class 'rdflib.graph.Graph'>)>
>>> len(graph)
68
Expand Down Expand Up @@ -431,7 +431,7 @@ And now::

>>> browser.open(portalURL + '/a-simple-source/@@rdf')
>>> graph = rdflib.Graph()
>>> graph.parse(data=browser.contents)
>>> graph.parse(data=browser.contents, format='xml')
<Graph identifier=...(<class 'rdflib.graph.Graph'>)>
>>> len(graph)
124
Expand Down Expand Up @@ -541,7 +541,7 @@ And now for the RDF::

>>> browser.open(portalURL + '/a-simple-source/@@rdf')
>>> graph = rdflib.Graph()
>>> graph.parse(data=browser.contents)
>>> graph.parse(data=browser.contents, format='xml')
<Graph identifier=...(<class 'rdflib.graph.Graph'>)>
>>> len(graph)
31809
Expand Down Expand Up @@ -672,7 +672,7 @@ And now for the RDF::

>>> browser.open(portalURL + '/a-committee-source/@@rdf')
>>> graph = rdflib.Graph()
>>> graph.parse(data=browser.contents)
>>> graph.parse(data=browser.contents, format='xml')
<Graph identifier=...(<class 'rdflib.graph.Graph'>)>
>>> len(graph)
298
Expand Down Expand Up @@ -804,7 +804,7 @@ And now for the RDF::

>>> browser.open(portalURL + '/a-protocol-source/@@rdf')
>>> graph = rdflib.Graph()
>>> graph.parse(data=browser.contents)
>>> graph.parse(data=browser.contents, format='xml')
<Graph identifier=...(<class 'rdflib.graph.Graph'>)>
>>> len(graph) > 9422
True
Expand Down Expand Up @@ -969,7 +969,7 @@ And now for the RDF::

>>> browser.open(portalURL + '/a-biomuta-source/@@rdf')
>>> graph = rdflib.Graph()
>>> graph.parse(data=browser.contents)
>>> graph.parse(data=browser.contents, format='xml')
<Graph identifier=...(<class 'rdflib.graph.Graph'>)>
>>> len(graph)
131187
Expand Down
6 changes: 3 additions & 3 deletions src/edrn.rdf/edrn/rdf/labcascollectionrdfgenerator.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,11 @@ def generateGraph(self):
context = aq_inner(self.context)
graph = rdflib.Graph()
solr = Solr(context.labcasSolrURL + '/datasets', auth=(context.username, context.password))
numDatasets = solr.search(q='*:*', rows=0).hits
numDatasets = solr.search(q='Consortium:EDRN', rows=0).hits
solr = Solr(context.labcasSolrURL + '/files', auth=(context.username, context.password))
numFiles = solr.search(q='*:*', rows=0).hits
numFiles = solr.search(q='Consortium:EDRN', rows=0).hits
solr = Solr(context.labcasSolrURL + '/collections', auth=(context.username, context.password))
results = solr.search(q='*:*', rows=999999) # 😮 TODO This'll fail once we get to a million collections
results = solr.search(q='Consortium:EDRN', rows=999999) # 😮 TODO This'll fail once we get to a million collections
numCollections = results.hits
for i in results:
collectionID, name, consortia = i.get('id'), i.get('CollectionName', '«unknown»'), i.get('Consortium', [])
Expand Down
10 changes: 7 additions & 3 deletions src/edrn.rdf/edrn/rdf/rdfupdater.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,13 @@ def updateRDF(self):
# Is there an active file?
if context.approvedFile:
# Is it identical to what we just generated?
current = Graph().parse(data=context.approvedFile.to_object.file.data)
if isomorphic(graph, current):
raise NoUpdateRequired(context)
try:
current = Graph().parse(data=context.approvedFile.to_object.file.data, format='xml')
if isomorphic(graph, current):
raise NoUpdateRequired(context)
except AttributeError:
# File not found
pass

# Create a new file and set it active
# TODO: Add validation steps here
Expand Down
2 changes: 1 addition & 1 deletion src/edrn.rdf/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
'plone.app.relationfield',
'plone.behavior',
'Products.CMFPlone',
'rdflib==4.2.2',
'rdflib==6.2.0',
'setuptools',
'z3c.relationfield',
'suds2',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from zope import schema
from zope.component import queryUtility
from plone.i18n.normalizer.interfaces import IIDNormalizer
import jsonlib
import json

_typeURI = URIRef('http://www.w3.org/1999/02/22-rdf-syntax-ns#type')
_bmOrganDataTypeURI = URIRef('http://edrn.nci.nih.gov/rdf/rdfs/bmdb-1.0.0#BiomarkerOrganData')
Expand Down Expand Up @@ -108,7 +108,7 @@ def generateJson(self):
raise RDFIngestException(_('This generator folder lacks one or both of its RDF source URLs.'))
normalizerFunction = queryUtility(IIDNormalizer).normalize
graph = ConjunctiveGraph()
graph.parse(URLInputSource(rdfDataSource))
graph.parse(URLInputSource(rdfDataSource), format='xml')
statements = self._parseRDF(graph)

biomarkerTypeFreq = {}
Expand Down Expand Up @@ -141,11 +141,11 @@ def generateJson(self):

# Add organ-specific information
graph = ConjunctiveGraph()
graph.parse(URLInputSource(bmoDataSource))
graph.parse(URLInputSource(bmoDataSource), format='xml')
organStatements = self._parseRDF(graph)
self.addOrganSpecificInformation(allBiomarkers, organStatements, biomarkerOrganFreq)

jsondata = self.generateOrganTypeStats(biomarkerTypeFreq,biomarkerOrganFreq)

# C'est tout.
return jsonlib.write(jsondata)
return json.dumps(jsondata)
Loading

0 comments on commit 105dd8c

Please sign in to comment.