From cca855fd3205273bc7f646beaa8077c69456fa34 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 16 Jul 2024 15:21:03 +0200 Subject: [PATCH 1/2] Add myproxy to dirac image --- dirac/Dockerfile | 11 ++- dirac/README.md | 2 +- dirac/entrypoint.sh | 1 + dirac/myproxy-server.config | 157 ++++++++++++++++++++++++++++++++++++ 4 files changed, 167 insertions(+), 4 deletions(-) create mode 100644 dirac/myproxy-server.config diff --git a/dirac/Dockerfile b/dirac/Dockerfile index ab147b1..fbb7390 100644 --- a/dirac/Dockerfile +++ b/dirac/Dockerfile @@ -4,9 +4,9 @@ LABEL org.opencontainers.image.source=https://github.com/xenon-middleware/xenon- LABEL org.opencontainers.image.documentation=https://github.com/xenon-middleware/xenon-docker-images/blob/dirac/dirac/README.md LABEL org.opencontainers.image.licenses=Apache-2.0 -ARG dirac_version=8.0.39 -ARG dirac_pilot_version=v8r0p39 -ARG diracos_version=2.38 +ARG dirac_version=8.0.49 +ARG dirac_pilot_version=v8r0p49 +ARG diracos_version=2.42 # Use BUILDKIT_SANDBOX_HOSTNAME to force hostname # see https://docs.docker.com/engine/reference/builder/#buildkit-built-in-build-args @@ -120,4 +120,9 @@ RUN mkdir -p /cvmfs/dirac.egi.eu/dirac/${dirac_pilot_version} && \ COPY --chown=diracuser:diracuser dirac.client.cfg /home/diracuser/dirac.cfg +RUN yum install -y myproxy myproxy-server myproxy-admin +RUN chown dirac:dirac /var/lib/myproxy/ + +COPY myproxy-server.config /etc/myproxy-server.config + CMD ["/bin/entrypoint.sh"] diff --git a/dirac/README.md b/dirac/README.md index 4f02bcb..2b36419 100644 --- a/dirac/README.md +++ b/dirac/README.md @@ -56,7 +56,7 @@ This can be done with `docker-compose` see [../diracclient](diracclient/README.m ## Build ```shell -docker build -t ghcr.io/xenon-middleware/dirac:8.0.39 --progress plain \ +docker build -t ghcr.io/xenon-middleware/dirac:8.0.49 --progress plain \ --build-arg BUILDKIT_SANDBOX_HOSTNAME=dirac-tuto . ``` During build need to interact with services which require host certificates. diff --git a/dirac/entrypoint.sh b/dirac/entrypoint.sh index 523aec3..5bce1ab 100644 --- a/dirac/entrypoint.sh +++ b/dirac/entrypoint.sh @@ -2,4 +2,5 @@ mariadbd-safe & /usr/sbin/sshd -De & +su -c "/usr/sbin/myproxy-server" dirac /opt/dirac/sbin/runsvdir-start diff --git a/dirac/myproxy-server.config b/dirac/myproxy-server.config new file mode 100644 index 0000000..170145d --- /dev/null +++ b/dirac/myproxy-server.config @@ -0,0 +1,157 @@ +##################################################################### +accepted_credentials "*" +authorized_retrievers "*" +default_retrievers "*" +authorized_renewers "*" +default_renewers "none" +authorized_key_retrievers "*" +default_key_retrievers "none" +trusted_retrievers "*" +default_trusted_retrievers "none" +cert_dir /opt/dirac/etc/grid-security/certificates + +#authorized_retrievers "*" +#pam "sufficient" +#sasl "sufficient" +certificate_issuer_cert /opt/dirac/etc/grid-security/ca/ca.cert.pem +certificate_issuer_key /opt/dirac/etc/grid-security/ca/ca.key.pem +#certificate_issuer_key_passphrase "myproxy" +#certificate_serialfile /home/globus/.globus/simpleCA/serial +#certificate_out_dir /home/globus/.globus/simpleCA/newcerts +#certificate_mapfile /etc/grid-security/grid-mapfile +#cert_dir /etc/grid-security/certificates + +#accepted_credentials "/C=US/O=National Computational Science Alliance/CN=*" +#accepted_credentials "/C=US/O=Globus/*" +#accepted_credentials "/O=Grid/O=Globus/*" +#accepted_credentials "*" + +#authorized_retrievers "/C=US/O=National Computational Science Alliance/CN=portal/*" +#authorized_retrievers "*" + +#default_retrievers "/C=US/O=National Computational Science Alliance/CN=portal/*" + +#authorized_renewers "/C=US/O=National Computational Science Alliance/CN=scheduler/*" +#authorized_renewers "*" + +#default_renewers "none" +#default_renewers "/C=US/O=National Computational Science Alliance/CN=condorg/modi4.ncsa.uiuc.edu" + +#authorized_key_retrievers "*" + +#default_key_retrievers "none" + +#trusted_retrievers "*" + +#default_trusted_retrievers "none" + + +#allow_self_authorization true + +#passphrase_policy_program /usr/local/sbin/myproxy-passphrase-policy + +#cert_dir /etc/grid-security/certificates + +#max_proxy_lifetime 12 + +#max_cred_lifetime 12 + +#ignore_globus_limited_proxy_flag true + +#pam "disabled" + +#pam_id "myproxy" + +#sasl "disabled" + +#sasl_mech GSSAPI + +#sasl_serverFQDN myproxy.teragrid.org + +#sasl_user_realm TERAGRID.ORG + +#certificate_issuer_cert /home/globus/.globus/simpleCA/cacert.pem + +#certificate_issuer_key /home/globus/.globus/simpleCA/private/cakey.pem + +#certificate_issuer_key_passphrase "myproxy" + +#certificate_issuer_subca_certfile "/etc/grid-security/subca_certificates" + +#certificate_issuer_hashalg "sha256" + +#certificate_issuer_program /usr/local/sbin/myproxy-ca + +#certificate_openssl_engine_id "dynamic" + +#certificate_openssl_engine_lockfile /var/lib/myproxy/enginelock + + + +#certificate_serialfile /home/globus/.globus/simpleCA/serial + +#certificate_serial_skip 1 + +#certificate_out_dir /home/globus/.globus/simpleCA/newcerts + +#certificate_issuer_email_domain "ncsa.uiuc.edu" + +#max_cert_lifetime 12 + +#min_keylen 1024 + +#certificate_extfile /etc/myproxy-ca-extfile.txt + +#certificate_extapp /usr/local/sbin/myproxy-extapp + +#certificate_mapfile /etc/grid-security/grid-mapfile + +#certificate_mapapp /usr/local/sbin/myproxy-mapapp + +#certificate_request_checker /usr/local/bin/certreq-checker + +#certificate_issuer_checker /usr/local/bin/cert-checker + +#ca_ldap_server "ldap://localhost:389/" + +#ca_ldap_uid_attribute "uid" + +#ca_ldap_searchbase "ou=people,dc=bullwinkle,dc=lbl,dc=gov" + +#ca_ldap_dn_attribute "subjectDN" + +#ca_ldap_connect_dn "cn=Monte Goode,ou=ldapusers,dc=bullwinkle,dc=lbl,dc=gov" +#ca_ldap_connect_passphrase "passphrase" + +#ca_ldap_start_tls true + +#slave_servers + +#accepted_credentials_mapfile /etc/grid-security/store-mapfile + +#accepted_credentials_mapapp /usr/local/sbin/myproxy-accepted-mapapp + +#check_multiple_credentials true + +#ocsp_policy "aia" + +#ocsp_responder_url "http://ca.ncsa.uiuc.edu:8888/" + +#ocsp_responder_cert /etc/grid-security/trustedocspresponder.pem + +#syslog_ident myproxy-server + + +#syslog_facility user + +#request_timeout 120 + +#request_size_limit 1048576 + +#proxy_extfile /etc/myproxy-proxy-extfile.txt + +#proxy_extapp /usr/local/sbin/myproxy-extapp + +#allow_voms_attribute_requests true + +voms_userconf /opt/dirac/etc/grid-security/vomses From c8b61bc9556700efafdbae64e499fcc973736f88 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 17 Jul 2024 10:00:24 +0200 Subject: [PATCH 2/2] Bump to dirac==8.0.49 + Try to use myproxy-init and failing --- dirac/README.md | 36 +++++++++++++++++++++++++++++++--- diracclient/Dockerfile | 12 ++++++------ diracclient/README.md | 6 +++--- diracclient/docker-compose.yml | 4 ++-- diracclient/test_submit.py | 3 ++- 5 files changed, 46 insertions(+), 15 deletions(-) diff --git a/dirac/README.md b/dirac/README.md index 2b36419..a98f567 100644 --- a/dirac/README.md +++ b/dirac/README.md @@ -13,7 +13,7 @@ and integration test scripts. Run image from https://github.com/xenon-middleware/xenon-docker-images/pkgs/container/dirac with: ```shell -docker run --privileged --hostname dirac-tuto ghcr.io/xenon-middleware/dirac:8.0.39 +docker run --privileged --hostname dirac-tuto ghcr.io/xenon-middleware/dirac:8.0.49 ``` The `--privileged` flag is required to run apptainer containers inside Docker container. @@ -68,8 +68,8 @@ The `--progress plain` makes it possible to see all the output logs. Make sure to [configure Docker to be able to push to GitHub container registry](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#authenticating-to-the-container-registry). ```shell -docker push ghcr.io/xenon-middleware/dirac:8.0.39 -docker tag ghcr.io/xenon-middleware/dirac:8.0.39 ghcr.io/xenon-middleware/dirac:latest +docker push ghcr.io/xenon-middleware/dirac:8.0.49 +docker tag ghcr.io/xenon-middleware/dirac:8.0.49 ghcr.io/xenon-middleware/dirac:latest docker push ghcr.io/xenon-middleware/dirac:latest ``` @@ -105,6 +105,36 @@ cat 1/StdOut # -rw-r--r-- 1 diracpilot diracpilot 604 Apr 21 12:08 job.info ``` +Using myproxy + +``` +export MYPROXY_SERVER=dirac-tuto +myproxy-init -d -n -v +MyProxy v6.2 Jan 2024 PAM SASL KRB5 LDAP VOMS OCSP +Attempting to connect to 172.17.0.2:7512 +Successfully connected to dirac-tuto:7512 + +User Cert File: /home/diracuser/.globus/usercert.pem +User Key File: /home/diracuser/.globus/userkey.pem + +Trusted CA Cert Dir: /opt/dirac/etc/grid-security/certificates + +Output File: /tmp/myproxy-proxy.1002.2393 +Your identity: /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser +Creating proxy .......+.........+............+..+..........+...+.........+.....+...+...+....+......+.....+...+......+....+..+.......+..+...+..........+.....+......+.+...........+....+...+.....+......+................+..............+....+..+.............+..+...+.+..+....+.....+..........+.........+..+.......+..+++++++++++++++++++++++++++++++++++++++*.....+.+..+...+++++++++++++++++++++++++++++++++++++++*...+......++++++ +.+..................+.+...+..+.............+..+....+++++++++++++++++++++++++++++++++++++++*.+...+.+..+...+.........+...+...+.......+++++++++++++++++++++++++++++++++++++++*....+..+..........+.................+......+.......+..+......+....+...............+......+.....+.........+.+......+.....++++++ + Done +Error: Couldn't verify the authenticity of the user's credential to generate a proxy from. + grid_proxy_init.c:957: globus_credential: Error verifying credential: Failed to verify credential +globus_gsi_callback_module: Could not verify credential +globus_gsi_callback_module: Could not verify credential +globus_gsi_callback_module: Error with signing policy +globus_gsi_callback_module: Error with signing policy +globus_sysconfig: Error getting signing policy file +globus_sysconfig: File does not exist: /opt/dirac/etc/grid-security/certificates/855f710d.signing_policy is not a valid file +grid-proxy-init failed +``` + ## DIRAC web portal The [DIRAC web portal](https://dirac.readthedocs.io/en/latest/UserGuide/WebPortalReference/Overview/index.html) can be accessed with: diff --git a/diracclient/Dockerfile b/diracclient/Dockerfile index 51889b1..b7d3845 100644 --- a/diracclient/Dockerfile +++ b/diracclient/Dockerfile @@ -12,7 +12,7 @@ RUN useradd diracuser -m -s /bin/bash && \ USER diracuser WORKDIR /home/diracuser -# TODO reuse /cvmfs/dirac.egi.eu/dirac/v8r0p39/Linux-x86_64/ +# TODO reuse /cvmfs/dirac.egi.eu/dirac/v8r0p49/Linux-x86_64/ # from dirac image RUN curl -LO https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/DIRACOS-Linux-$(uname -m).sh && \ bash DIRACOS-Linux-$(uname -m).sh && \ @@ -20,16 +20,16 @@ RUN curl -LO https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/DIRA RUN echo '. /home/diracuser/diracos/diracosrc' >> /home/diracuser/.profile SHELL ["/bin/bash", "-l", "-c"] -# TODO silence `#0 0.390 realpath: '': No such file or directory` warnings from diracosrc script +# TODO silence `#0 0.490 realpath: '': No such file or directory` warnings from diracosrc script -RUN pip install DIRAC==8.0.39 +RUN pip install DIRAC==8.0.49 # Copy host certs, so server is trusted by dirac clients -COPY --chown=diracuser:diracuser --from=ghcr.io/xenon-middleware/dirac:8.0.39 /opt/dirac/etc/grid-security/certificates /etc/grid-security/certificates +COPY --chown=diracuser:diracuser --from=ghcr.io/xenon-middleware/dirac:8.0.49 /opt/dirac/etc/grid-security/certificates /etc/grid-security/certificates # Copy diracuser certs from dirac image to here -COPY --chown=diracuser:diracuser --from=ghcr.io/xenon-middleware/dirac:8.0.39 /home/diracuser/.globus /home/diracuser/.globus -COPY --chown=diracuser:diracuser --from=ghcr.io/xenon-middleware/dirac:8.0.39 /home/diracuser/dirac.cfg /home/diracuser/diracos/etc/dirac.cfg +COPY --chown=diracuser:diracuser --from=ghcr.io/xenon-middleware/dirac:8.0.49 /home/diracuser/.globus /home/diracuser/.globus +COPY --chown=diracuser:diracuser --from=ghcr.io/xenon-middleware/dirac:8.0.49 /home/diracuser/dirac.cfg /home/diracuser/diracos/etc/dirac.cfg VOLUME /src WORKDIR /src diff --git a/diracclient/README.md b/diracclient/README.md index bcc3c2d..6f8aaea 100644 --- a/diracclient/README.md +++ b/diracclient/README.md @@ -24,11 +24,11 @@ docker compose run -ti test 'dirac-proxy-init -g dirac_user && pytest test_submi ## Build & push ```shell -docker build -t ghcr.io/xenon-middleware/diracclient:8.0.39 . +docker build -t ghcr.io/xenon-middleware/diracclient:8.0.49 . ``` ```shell -docker push ghcr.io/xenon-middleware/diracclient:8.0.39 -docker tag ghcr.io/xenon-middleware/diracclient:8.0.39 ghcr.io/xenon-middleware/diracclient:latest +docker push ghcr.io/xenon-middleware/diracclient:8.0.49 +docker tag ghcr.io/xenon-middleware/diracclient:8.0.49 ghcr.io/xenon-middleware/diracclient:latest docker push ghcr.io/xenon-middleware/diracclient:latest ``` diff --git a/diracclient/docker-compose.yml b/diracclient/docker-compose.yml index e949eb0..f45dd16 100644 --- a/diracclient/docker-compose.yml +++ b/diracclient/docker-compose.yml @@ -2,11 +2,11 @@ version: '3.9' services: dirac-tuto: - image: ghcr.io/xenon-middleware/dirac:8.0.18 + image: ghcr.io/xenon-middleware/dirac:8.0.49 privileged: true hostname: dirac-tuto test: - image: ghcr.io/xenon-middleware/diracclient:8.0.18 + image: ghcr.io/xenon-middleware/diracclient:8.0.49 build: . volumes: - .:/src diff --git a/diracclient/test_submit.py b/diracclient/test_submit.py index 65b4a81..07d1e0e 100644 --- a/diracclient/test_submit.py +++ b/diracclient/test_submit.py @@ -25,8 +25,9 @@ def test_submit(): for i in range(max_checks): print('Checking status') result = monitoring.getJobsStatus(job_id) + print(result) if result['Value'][job_id]['Status'] == 'Done': - break; + break time.sleep(sleep_time) else: raise Exception("Failed to finish job") \ No newline at end of file