Invenio pins revert

cap/alembic/f8fbe488976d_add_config_field_for_schemas.py

@@ -0,0 +1,36 @@
+#
+# This file is part of Invenio.
+# Copyright (C) 2016-2018 CERN.
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Add config field for schemas"""
+
+from alembic import op
+import sqlalchemy as sa
+
+from cap.types import json_type
+
+# revision identifiers, used by Alembic.
+revision = 'f8fbe488976d'
+down_revision = '637a2a528700'
+branch_labels = ()
+depends_on = None
+
+
+def upgrade():
+    """Upgrade database."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('schema', sa.Column('config', json_type,
+                                      default=lambda: dict(),
+                                      nullable=True,
+                                      server_default='{}'))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    """Downgrade database."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('schema', 'config')
+    # ### end Alembic commands ###

cap/config.py

@@ -616,12 +616,13 @@ def _(x):
 CMS_CONVENERS_EGROUP = 'cms-phys-conveners-{wg}@cern.ch'
 
 #: CADI database
-CADI_AUTH_URL = 'https://icms.cern.ch/tools/api/cadiLine/BPH-13-009'
-CADI_GET_CHANGES_URL = 'https://icms.cern.ch/tools/api/updatedCadiLines/'
-CADI_GET_ALL_URL = 'https://icms.cern.ch/tools/restplus/relay/piggyback/' + \
-    'cadi/history/capInfo'
-CADI_GET_RECORD_URL = 'https://icms.cern.ch/tools/restplus/relay/' + \
-    'piggyback/cadi/history/capInfo/{id}'
+CADI_AUTH_URL = 'https://icms.cern.ch/tools-api/restplus/' + \
+                'relay/piggyback/cadi/history/capInfo/BPH-13-009'
+CADI_GET_CHANGES_URL = 'https://icms.cern.ch/tools-api/api/updatedCadiLines/'
+CADI_GET_ALL_URL = 'https://icms.cern.ch/tools-api/restplus/' + \
+                   'relay/piggyback/cadi/history/capInfo'
+CADI_GET_RECORD_URL = 'https://icms.cern.ch/tools-api/restplus' + \
+                      '/relay/piggyback/cadi/history/capInfo/{id}'
 CADI_REGEX = "^[A-Z]{3}-[0-9]{2}-[0-9]{3}$"
 
 # ATLAS

cap/modules/deposit/api.py

@@ -32,6 +32,7 @@
 from invenio_access.models import ActionRoles, ActionUsers
 from invenio_db import db
 from invenio_deposit.api import Deposit, has_status, index, preserve
+from invenio_deposit.signals import post_action
 from invenio_deposit.utils import mark_as_action
 from invenio_files_rest.errors import MultipartMissingParts
 from invenio_files_rest.models import (Bucket, FileInstance, ObjectVersion,
@@ -52,7 +53,6 @@
 from cap.modules.deposit.errors import DisconnectWebhookError, FileUploadError
 from cap.modules.deposit.validators import NoRequiredValidator
 from cap.modules.experiments.permissions import exp_need_factory
-from cap.modules.mail.utils import post_action_notifications
 from cap.modules.records.api import CAPRecord
 from cap.modules.records.errors import get_error_path
 from cap.modules.repos.errors import GitError
@@ -231,12 +231,7 @@ def publish(self, *args, **kwargs):
                     raise MultipartMissingParts()
 
             try:
-                deposit = super(CAPDeposit, self).publish(*args, **kwargs)
-                post_action_notifications("publish",
-                                          deposit,
-                                          host_url=request.host_url)
-
-                return deposit
+                return super(CAPDeposit, self).publish(*args, **kwargs)
             except ValidationError as e:
                 raise DepositValidationError(e.message)
 
@@ -367,9 +362,6 @@ def review(self, pid, *args, **kwargs):
                     self.update_review(data)
                 else:
                     self.create_review(data)
-
-                post_action_notifications(
-                    "review", self, host_url=request.host_url)
             else:
                 raise ReviewError(None)
 

cap/modules/deposit/review.py

@@ -67,12 +67,11 @@ class ReviewUpdatePayload(Schema):
 
 class Reviewable(object):
     def schema_is_reviewable(self):
-        schema = self.get("$schema", None)
+        config = self.schema.config
 
-        if "questionnaire" in schema:
-            return True
-        else:
-            return False
+        if config:
+            return config.get('reviewable', False)
+        return False
 
     def create_review(self, data):
         new_review, errors = ReviewCreatePayload().load(data=data)

cap/modules/experiments/utils/cadi.py

@@ -28,7 +28,7 @@
 
 import requests
 from elasticsearch_dsl import Q
-from flask import current_app
+from flask import current_app, abort
 from invenio_db import db
 from invenio_search import RecordsSearch
 
@@ -174,7 +174,7 @@ def _get_admin_egroups(wg):
     return roles
 
 
-def get_from_cadi_by_id(cadi_id):
+def get_from_cadi_by_id(cadi_id, from_validator=False):
     """Retrieve entry with given id from CADI database.
 
     :params str cadi_id: CADI identifier
@@ -185,9 +185,14 @@ def get_from_cadi_by_id(cadi_id):
         id=cadi_id.upper())
 
     cookie = get_sso_cookie_for_cadi()
-    response = requests.get(url, cookies=cookie)
+    response = requests.get(url, cookies=cookie, verify=False)
 
     if not response.ok:
+        if from_validator:
+            return False
+        if response.status_code == 404:
+            abort(400, 'No CADI entry found')
+
         raise ExternalAPIException(response)
 
     entry = response.json()
@@ -206,7 +211,7 @@ def get_all_from_cadi():
     url = current_app.config.get('CADI_GET_ALL_URL')
 
     cookie = get_sso_cookie_for_cadi()
-    response = requests.get(url=url, cookies=cookie)
+    response = requests.get(url=url, cookies=cookie, verify=False)
 
     if not response.ok:
         raise ExternalAPIException(response)

cap/modules/experiments/utils/cern_sso.py

@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of CERN Analysis Preservation Framework.
+# Copyright (C) 2016 CERN.
+#
+# CERN Analysis Preservation Framework is free software; you can redistribute
+# it and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# CERN Analysis Preservation Framework is distributed in the hope that it will
+# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with CERN Analysis Preservation Framework; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+"""Experiments CERN SSO utils."""
+
+# Code reused from 'auth-get-sso-cookie' repo
+# https://gitlab.cern.ch/authzsvc/tools/auth-get-sso-cookie
+
+import logging
+import requests
+import time
+import uuid
+from requests_kerberos import HTTPKerberosAuth, OPTIONAL
+from bs4 import BeautifulSoup
+
+try:
+    from http.cookiejar import MozillaCookieJar, Cookie
+    from urllib.parse import parse_qs
+except ImportError:  # python 2.7 compatibility
+    from cookielib import MozillaCookieJar, Cookie
+    from urlparse import parse_qs
+
+
+def save_cookies_lwp(cookiejar):
+    """Return cookies from a requests.
+
+    Session cookies member in the Netscape format.
+    """
+
+    lwp_cookiejar = MozillaCookieJar()
+    for c in cookiejar:
+        args = dict(vars(c).items())
+        args["rest"] = args["_rest"]
+        del args["_rest"]
+        if args["expires"] is None:
+            args["expires"] = int(time.time()) + 86400
+        c = Cookie(**args)
+        lwp_cookiejar.set_cookie(c)
+    return lwp_cookiejar
+
+
+def post_session_saml(session, response):
+    """Performs the SAML POST.
+
+    Performs the SAML POST request given a session and a
+    successful Keycloak authentication response in SAML
+    """
+
+    soup_saml = BeautifulSoup(response.text, features="html.parser")
+    action = soup_saml.form.get("action")
+    post_key = soup_saml.form.input.get("name")
+    post_value = soup_saml.form.input.get("value")
+    session.post(action, data={post_key: post_value})
+
+
+def login_with_kerberos(login_page, verify_cert, auth_hostname, silent):
+    """Simulates a browser session to log in using SPNEGO protocol"""
+
+    session = requests.Session()
+    if not silent:
+        logging.info("Fetching target URL and its redirects")
+    r_login_page = session.get(login_page, verify=verify_cert)
+    if not silent:
+        logging.debug("Landing page: {}".format(r_login_page.url))
+        logging.info("Parsing landing page to get the Kerberos login URL")
+    soup = BeautifulSoup(r_login_page.text, features="html.parser")
+    kerberos_button = soup.find(id="zocial-kerberos")
+    if not kerberos_button:
+        error_message = get_error_message(r_login_page.text)
+        if error_message:
+            raise Exception("Login failed: {}".format(error_message))
+        else:
+            raise Exception(
+                "Login failed: Landing page not recognized.")
+    kerberos_path = kerberos_button.get("href")
+    if not silent:
+        logging.info("Fetching Kerberos login URL")
+    r_kerberos_redirect = session.get(
+        "https://{}{}".format(auth_hostname, kerberos_path)
+    )
+    if not silent:
+        logging.info("Logging in using Kerberos Auth")
+    r_kerberos_auth = session.get(
+        r_kerberos_redirect.url,
+        auth=HTTPKerberosAuth(mutual_authentication=OPTIONAL),
+        allow_redirects=False,
+    )
+    while (
+        r_kerberos_auth.status_code == 302 and
+        auth_hostname in r_kerberos_auth.headers["Location"]
+    ):
+        r_kerberos_auth = session.get(
+            r_kerberos_auth.headers["Location"], allow_redirects=False
+        )
+    if r_kerberos_auth.status_code != 302:
+        error_message = get_error_message(r_kerberos_auth.text)
+        if not error_message:
+            logging.debug(
+                "Not automatically redirected: trying SAML authentication")
+            post_session_saml(session, r_kerberos_auth)
+        else:
+            raise Exception("Login failed: {}".format(error_message))
+    return session, r_kerberos_auth
+
+
+def get_error_message(response_html):
+    soup_err_page = BeautifulSoup(response_html, features="html.parser")
+    error_message = soup_err_page.find(id="kc-error-message")
+    if not error_message:
+        return None
+    else:
+        return error_message.find("p").text
+
+
+def save_sso_cookie(url, verify_cert, auth_hostname, silent=True):
+    """Log in into a URL that redirects to the SSO.
+
+    Log in into a URL that redirects to the SSO and
+    save the session cookies
+    """
+
+    try:
+        session, response = login_with_kerberos(
+            url, verify_cert, auth_hostname, silent=silent)
+        if response.status_code == 302:
+            redirect_uri = response.headers["Location"]
+            if not silent:
+                logging.info(
+                    "Logged in. Fetching redirect URL to get cookies")
+            session.get(redirect_uri, verify=verify_cert)
+
+        return save_cookies_lwp(session.cookies)
+
+    except Exception as e:
+        logging.error(
+            "An error occurred while trying to log in and save cookies.")
+        raise e

cap/modules/experiments/utils/common.py

@@ -28,7 +28,8 @@
 from os.path import join
 from subprocess import CalledProcessError, check_output
 
-import cern_sso
+import cern_sso as cern_sso_old
+import cap.modules.experiments.utils.cern_sso as cern_sso_current
 from cachetools.func import ttl_cache
 from elasticsearch import helpers
 from flask import current_app
@@ -72,6 +73,26 @@ def wrapped_function(*args, **kwargs):
     return decorator
 
 
+@ttl_cache(ttl=86000)  # cookie expires after 24 hours
+def generate_krb_cookie_cern_sso_old(principal, kt, url):
+    """Generate a HTTP cookie with given kerberos credentials.
+
+    :param str principal: Kerberos principal, e.g. [email protected]
+    :param str kt: Keytab filename e.g user.keytab
+    :param str url: URL
+
+    :returns: Generated HTTP Cookie
+    :rtype `requests.cookies.RequestsCookieJar`
+    """
+    @kinit(principal, kt)
+    def generate(url):
+        cookie = cern_sso_old.krb_sign_on(url)
+
+        return cookie
+
+    return generate(url)
+
+
 @ttl_cache(ttl=86000)  # cookie expires after 24 hours
 def generate_krb_cookie(principal, kt, url):
     """Generate a HTTP cookie with given kerberos credentials.
@@ -85,7 +106,8 @@ def generate_krb_cookie(principal, kt, url):
     """
     @kinit(principal, kt)
     def generate(url):
-        cookie = cern_sso.krb_sign_on(url)
+        cookie = cern_sso_current.save_sso_cookie(url, False, "auth.cern.ch")
+
         return cookie
 
     return generate(url)

cap/modules/experiments/validators.py

@@ -57,5 +57,5 @@ def validate_das_path(validator, value, instance, schema):
 
 def validate_cadi_id(validator, value, instance, schema):
     from .utils.cadi import get_from_cadi_by_id
-    if not get_from_cadi_by_id(instance):
+    if not get_from_cadi_by_id(instance, from_validator=True):
         yield ValidationError("{} not found in CADI.".format(instance))

cap/modules/fixtures/schemas/cms-questionnaire.json

@@ -5,6 +5,9 @@
   "experiment":"CMS",
   "is_indexed":true,
   "use_deposit_as_record":true,
+  "config": {
+    "reviewable": true
+  },
   "deposit_schema":{
     "additionalProperties":false,
     "$schema":"http://json-schema.org/draft-04/schema#",

cap/modules/fixtures/schemas/cms-questionnaire_v2.json

@@ -5,6 +5,9 @@
   "experiment":"CMS",
   "is_indexed":true,
   "use_deposit_as_record":true,
+  "config": {
+    "reviewable": true
+  },
   "deposit_schema":{
     "additionalProperties":false,
     "$schema":"http://json-schema.org/draft-04/schema#",