diff --git a/Cargo.toml b/Cargo.toml index 9fcf300..bb214af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ mockall = "0.12" [workspace.package] version = "0.1.0" edition = "2021" -authors = ["Ivan Kudriavtsev ", "Ksenia Vazhdaeva ", "Ksenia Vazhdaeva "] description = "Media Gateway (client and server)" homepage = "https://github.com/insight-platform/MediaGateway" repository = "https://github.com/insight-platform/MediaGateway" diff --git a/docs/diagrams/media-gateway.drawio b/docs/diagrams/media-gateway.drawio new file mode 100644 index 0000000..0424a42 --- /dev/null +++ b/docs/diagrams/media-gateway.drawio @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/source/_static/media-gateway.png b/docs/source/_static/media-gateway.png new file mode 100644 index 0000000..feb4d54 Binary files /dev/null and b/docs/source/_static/media-gateway.png differ diff --git a/docs/source/cookbook/0_https.rst b/docs/source/cookbook/0_https.rst new file mode 100644 index 0000000..0539488 --- /dev/null +++ b/docs/source/cookbook/0_https.rst @@ -0,0 +1,287 @@ +HTTPS +===== + +This guide shows how to enable HTTPS protocol in Media Gateway. Both self-signed and signed by CA server certificates are supported. Certificates should be in PEM format. + +.. important:: + + The protocol in ``url`` field in the client configuration must be updated to ``https``. + +Prerequisites +------------- + +* Docker +* Docker Compose +* openssl +* curl + +Using a self-signed certificate +------------------------------- + +Configuring Media Gateway +^^^^^^^^^^^^^^^^^^^^^^^^^ + +To use a self-signed certificate update server and client configurations. + +.. code-block:: json + :caption: server + + "tls": { + "identity": { + "certificate": "server.crt", + "key": "server.key" + } + } + +.. code-block:: json + :caption: client + + "tls": { + "root_certificate": "server.crt" + } + +where + +* ``server.crt`` is a file with the server certificate in PEM format. + +* ``server.key`` is a file with the server key in PEM format. + +Generating certificates +^^^^^^^^^^^^^^^^^^^^^^^ + +Generate a private key and certificate signing request + +.. code-block:: bash + + mkdir certs + + openssl genpkey -algorithm RSA -out certs/server.key + + openssl req -new -key certs/server.key -out certs/server.csr -subj "/CN=media-gateway-server" + +If the client connects to the server by IP generate a certificate with IP subject alternative name. Otherwise generate a certificate with DNS subject alternative name. + +In commands below replace `192.168.0.108` and `media-gateway-server` with your values. + +.. code-block:: bash + :caption: IP SAN + + export HOST_IP="192.168.0.108" + + openssl x509 -req -days 365 -key certs/server.key -in certs/server.csr -out certs/server.crt -extfile <(echo "subjectAltName=IP:${HOST_IP}") + +.. code-block:: bash + :caption: DNS SAN + + export MEDIA_GATEWAY_SERVER_DNS="media-gateway-server" + + openssl x509 -req -days 365 -key certs/server.key -in certs/server.csr -out certs/server.crt -extfile <(echo "subjectAltName=DNS:${MEDIA_GATEWAY_SERVER_DNS}") + +Testing +^^^^^^^ + +Server +"""""" + +To test the server only a certificate with IP SAN is used. + +Prepare the configuration file + +.. code-block:: bash + + cat << EOF > media-gateway-server.json + { + "ip": "0.0.0.0", + "port": 8080, + "tls": { + "identity": { + "certificate": "/etc/certs/server.crt", + "key": "/etc/certs/server.key" + } + }, + "out_stream": { + "url": "pub+bind:ipc:///tmp/server", + "send_timeout": { + "secs": 1, + "nanos": 0 + }, + "send_retries": 3, + "receive_timeout": { + "secs": 1, + "nanos": 0 + }, + "receive_retries": 3, + "send_hwm": 1000, + "receive_hwm": 1000, + "fix_ipc_permissions": 511 + } + } + EOF + +Launch the server (change the value of `MEDIA_GATEWAY_PORT` in the command below if required) + +.. code-block:: bash + :caption: x86_64 + + export MEDIA_GATEWAY_PORT=8080 + + docker run -d \ + -v $(pwd)/media-gateway-server.json:/opt/etc/custom_config.json \ + -v $(pwd)/certs:/etc/certs \ + -p $MEDIA_GATEWAY_PORT:8080 \ + --name media-gateway-server \ + ghcr.io/insight-platform/media-gateway-server-x86:latest \ + /opt/etc/custom_config.json + +.. code-block:: bash + :caption: ARM64 + + export MEDIA_GATEWAY_PORT=8080 + + docker run -d \ + -v $(pwd)/media-gateway-server.json:/opt/etc/custom_config.json \ + -v $(pwd)/certs:/etc/certs \ + -p $MEDIA_GATEWAY_PORT:8080 \ + --name media-gateway-server \ + ghcr.io/insight-platform/media-gateway-server-arm64:latest \ + /opt/etc/custom_config.json + +Send the request to the server + +.. code-block:: bash + + curl --cacert certs/server.crt -v https://$HOST_IP:$MEDIA_GATEWAY_PORT/health + +HTTP response with ``200 OK`` status code and the body as below should be returned. + +.. code-block:: json + + {"status": "healthy"} + +Clean up after testing + +.. code-block:: bash + + docker stop media-gateway-server + + docker rm media-gateway-server + + rm -rf certs media-gateway-server.json + +e2e +""" + +To test both server and client based on :doc:`3_usage_example` + +* generate a certificate with DNS SAN +* update ``server_config.json`` and ``client_config.json`` in the downloaded archive as described above +* add volumes for ``media-gateway-client``` (a certificate file) and ``media-gateway-server`` (key and certificate files) in ``docker-compose-x86.yaml`` and ``docker-compose-arm64.yaml`` in the downloaded archive + +Clean up after testing + +.. code-block:: bash + + rm -rf certs + +.. _private ca https: + +Using a certificate signed by a private CA +------------------------------------------ + +Configuring Media Gateway +^^^^^^^^^^^^^^^^^^^^^^^^^ + +To use a certificate signed by a private CA update server and client configurations. + +.. code-block:: json + :caption: server + + "tls": { + "identity": { + "certificate": "server.crt", + "key": "server.key" + } + } + +.. code-block:: json + :caption: client + + "tls": { + "root_certificate": "ca.crt" + } + +where + +* ``server.crt`` is a file with the server certificate in PEM format. + +* ``server.key`` is a file with the server key in PEM format. + +* ``ca.crt`` is a file with the CA certificate in PEM format. + +Generating certificates +^^^^^^^^^^^^^^^^^^^^^^^ + +Generate a private key and a certificate for CA and a private key and certificate signing request for the server + +.. code-block:: bash + + mkdir certs + + openssl genpkey -algorithm RSA -out certs/ca.key + + openssl req -new -x509 -days 365 -key certs/ca.key -out certs/ca.crt -subj "/CN=media-gateway-ca" + + openssl genpkey -algorithm RSA -out certs/server.key + + openssl req -new -key certs/server.key -out certs/server.csr -subj "/CN=media-gateway-server" + +If the client connects to the server by IP generate a certificate with IP subject alternative name. Otherwise generate a certificate with DNS subject alternative name. + +In commands below replace `192.168.0.108` and `media-gateway-server` with your values. + +.. code-block:: bash + :caption: IP SAN + + export HOST_IP="192.168.0.108" + + openssl x509 -req -days 365 -in certs/server.csr -CA certs/ca.crt -CAkey certs/ca.key -CAcreateserial -out certs/server.crt -extfile <(echo "subjectAltName=IP:${HOST_IP}") + +.. code-block:: bash + :caption: DNS SAN + + export MEDIA_GATEWAY_SERVER_DNS="media-gateway-server" + + openssl x509 -req -days 365 -in certs/server.csr -CA certs/ca.crt -CAkey certs/ca.key -CAcreateserial -out certs/server.crt -extfile <(echo "subjectAltName=DNS:${MEDIA_GATEWAY_SERVER_DNS}") + +Testing +^^^^^^^ + +Testing is the same as for a self-signed certificate except the request to the server for checking + +.. code-block:: bash + + curl --cacert certs/ca.crt -v https://$HOST_IP:$MEDIA_GATEWAY_PORT/health + +Using a certificate signed by a public CA +----------------------------------------- + +Configuring Media Gateway +^^^^^^^^^^^^^^^^^^^^^^^^^ + +To use a certificate signed by a public CA update the server configuration. + +.. code-block:: json + :caption: server + + "tls": { + "identity": { + "certificate": "server.crt", + "key": "server.key" + } + } + +where + +* ``server.crt`` is a file with a sequence of certificates, the first being the leaf certificate, and the remainder forming the chain of certificates up to and including the trusted root certificate. + +* ``server.key`` is a file with the server key in PEM format. diff --git a/docs/source/cookbook/1_certificate_auth.rst b/docs/source/cookbook/1_certificate_auth.rst new file mode 100644 index 0000000..8f89ee1 --- /dev/null +++ b/docs/source/cookbook/1_certificate_auth.rst @@ -0,0 +1,426 @@ +Client certificate authentication +================================= + +Client certificate authentication is an optional feature in Media Gateway and can be used only if HTTPS is enabled (see :doc:`0_https`). Only signed by CA client certificates can be used. Certificates should be in PEM format. CRLs are supported but they are not mandatory. + +The server uses a store with trusted X509 certificates and CRLs to verify peer certificates. The store automatically (without a server restart) loads certificates and CRLs from the specified directory. Certificates and CRLs should be added to the directory in accordance with `X509_LOOKUP_hash_dir method `__ requirements. If CRLs are enabled for each certificate at least one CRL must be in the directory. The CRL may contain no revoked certificates. A new CRL must be added when the previous CRL is expired. + +Prerequisites +------------- + +* Docker +* Docker Compose +* openssl +* curl + +Configuring Media Gateway +------------------------- + +To enable client certificate authentication in Media Gateway both server and client configuration should be updated. + +.. code-block:: json + :caption: server with enabled CRLs + + "tls": { + // see HTTPS + "peers": { + "lookup_hash_directory" : "/etc/certs/lookup-hash-dir", + "crl_enabled": true + } + } + +.. code-block:: json + :caption: server with disabled CRLs + + "tls": { + // see HTTPS + "peers": { + "lookup_hash_directory" : "/etc/certs/lookup-hash-dir", + "crl_enabled": false + } + } + +.. code-block:: json + :caption: client + + "tls": { + // see HTTPS + "identity": { + "certificate": "client.crt", + "key": "client.key" + } + } + +where + +* ``/etc/certs/lookup-hash-dir`` is a directory with CA certificates and CRLs. + +* ``client.crt`` is a file with a client certificate in PEM format. + +* ``client.key`` is a file with a PEM encoded PKCS #8 formatted client key. + +Generating certificates and CRLs +-------------------------------- + +This section describes how to generate certificates and CLRs signed by a private CA using `OpenSSL `_. Provided instructions specifies the minimum required information only. For production usage see OpenSSL documentation. + +Creating a private CA +^^^^^^^^^^^^^^^^^^^^^ + +Create directories + +.. code-block:: bash + + CA_DIR="$(pwd)/ca" + + mkdir "${CA_DIR}" "${CA_DIR}/certs" "${CA_DIR}/crl" + +Prepare a CA database + +.. code-block:: bash + + touch "${CA_DIR}/index.txt" + + echo 01 > "${CA_DIR}/serial" + + echo 1000 > "${CA_DIR}/crlnumber" + +Prepare a CA configuration file + +.. code-block:: bash + + echo "[ ca ] + + default_ca = CA_default + + [ CA_default ] + + dir = ${CA_DIR} + certificate = \$dir/ca.crt + private_key = \$dir/ca.key + database = \$dir/index.txt + new_certs_dir = \$dir/certs + serial = \$dir/serial + crl_dir = \$dir/crl + crl = \$dir/crl/ca.crl + crlnumber = \$dir/crlnumber + + x509_extensions = v3_ca + crl_extensions = crl_ext + + name_opt = ca_default + cert_opt = ca_default + + default_days = 365 + default_crl_days = 30 + default_md = default + preserve = no + policy = policy_any + + [ policy_any ] + countryName = optional + stateOrProvinceName = optional + organizationName = optional + organizationalUnitName = optional + commonName = supplied + emailAddress = optional + + #################################################################### + + [ req ] + default_bits = 2048 + default_keyfile = privkey.pem + distinguished_name = req_distinguished_name + attributes = req_attributes + x509_extensions = v3_ca + + [ req_distinguished_name ] + countryName = Country Name (2 letter code) + countryName_default = US + countryName_min = 2 + countryName_max = 2 + stateOrProvinceName = State or Province Name (full name) + stateOrProvinceName_default = + localityName = Locality Name (eg, city) + localityName+default = + 0.organizationName = Organization Name (eg, company) + 0.organizationName_default = + organizationalUnitName = Organizational Unit Name (eg, section) + organizationalUnitName_default = + commonName = Common Name (e.g. server FQDN or YOUR name) + commonName_max = 64 + emailAddress = Email Address + emailAddress_max = 64 + + [ req_attributes ] + challengePassword = A challenge password + challengePassword_min = 4 + challengePassword_max = 20 + unstructuredName = An optional company name + + [ v3_req ] + basicConstraints = CA:FALSE + keyUsage = nonRepudiation, digitalSignature, keyEncipherment + + [ v3_ca ] + subjectKeyIdentifier=hash + authorityKeyIdentifier=keyid:always,issuer + basicConstraints = critical,CA:true + keyUsage = critical, digitalSignature, cRLSign, keyCertSign + + [ crl_ext ] + authorityKeyIdentifier=keyid:always + " > "${CA_DIR}/ca.conf" + +Generate a CA private key and certificate + +.. code-block:: bash + + openssl genpkey -algorithm RSA -out "${CA_DIR}/ca.key" + + openssl req -new -x509 -days 365 -config "${CA_DIR}/ca.conf" -key "${CA_DIR}/ca.key" -out "${CA_DIR}/ca.crt" -subj "/CN=media-gateway-ca" + +Generating a server certificate +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Generate a private key and certificate signing request + +.. code-block:: bash + + openssl genpkey -algorithm RSA -out "${CA_DIR}/certs/server.key" + + openssl req -new -key "${CA_DIR}/certs/server.key" -out "${CA_DIR}/certs/server.csr" -subj "/CN=media-gateway-server" + +If the client connects to the server by IP generate a certificate with IP subject alternative name. Otherwise generate a certificate with DNS subject alternative name. + +In commands below replace `192.168.0.108` and `media-gateway-server` with your values. + +.. code-block:: bash + :caption: IP SAN + + export HOST_IP="192.168.0.108" + + openssl ca -config "${CA_DIR}/ca.conf" -in "${CA_DIR}/certs/server.csr" -out "${CA_DIR}/certs/server.crt" -extfile <(echo "basicConstraints=CA:FALSE + nsComment=\"OpenSSL Generated Certificate\" + subjectKeyIdentifier=hash + authorityKeyIdentifier=keyid,issuer + keyUsage=critical,digitalSignature,keyEncipherment + extendedKeyUsage=serverAuth + subjectAltName=IP:${HOST_IP}") + +.. code-block:: bash + :caption: DNS SAN + + export MEDIA_GATEWAY_SERVER_DNS="media-gateway-server" + + openssl ca -config "${CA_DIR}/ca.conf" -in "${CA_DIR}/certs/server.csr" -out "${CA_DIR}/certs/server.crt" -extfile <(echo "basicConstraints=CA:FALSE + nsComment=\"OpenSSL Generated Certificate\" + subjectKeyIdentifier=hash + authorityKeyIdentifier=keyid,issuer + keyUsage=critical,digitalSignature,keyEncipherment + extendedKeyUsage=serverAuth + subjectAltName=DNS:${MEDIA_GATEWAY_SERVER_DNS}") + +Generating a client certificate +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Generate a private key, certificate signing request and a certificate + +.. code-block:: bash + + openssl genpkey -algorithm RSA -out "${CA_DIR}/certs/client.key" + + openssl req -new -key "${CA_DIR}/certs/client.key" -out "${CA_DIR}/certs/client.csr" -subj "/CN=media-gateway-client" + + openssl ca -config "${CA_DIR}/ca.conf" -in "${CA_DIR}/certs/client.csr" -out "${CA_DIR}/certs/client.crt" -extfile <(echo 'basicConstraints=CA:FALSE + nsComment="OpenSSL Generated Certificate" + subjectKeyIdentifier=hash + keyUsage=critical,nonRepudiation,digitalSignature,keyEncipherment + extendedKeyUsage=clientAuth + authorityKeyIdentifier=keyid,issuer') + +Preparing X509 lookup hash directory +------------------------------------ + +.. admonition:: OpenSSL documentation + + X509_LOOKUP_hash_dir is a more advanced method, which loads certificates and CRLs on demand, and caches them in memory once they are loaded. As of OpenSSL 1.0.0, it also checks for newer CRLs upon each lookup, so that newer CRLs are as soon as they appear in the directory. + + The directory should contain one certificate or CRL per file in PEM format, with a filename of the form hash.N for a certificate, or hash.rN for a CRL. The hash is the value returned by the X509_NAME_hash(3) function applied to the subject name for certificates or issuer name for CRLs. The hash can also be obtained via the -hash option of the x509(1) or crl(1) commands. + + The .N or .rN suffix is a sequence number that starts at zero, and is incremented consecutively for each certificate or CRL with the same hash value. Gaps in the sequence numbers are not supported, it is assumed that there are no more objects with the same hash beyond the first missing number in the sequence. + + Sequence numbers make it possible for the directory to contain multiple certificates with same subject name hash value. For example, it is possible to have in the store several certificates with same subject or several CRLs with same issuer (and, for example, different validity period). + + When checking for new CRLs once one CRL for given hash value is loaded, hash_dir lookup method checks only for certificates with sequence number greater than that of the already cached CRL. + +Create a directory + +.. code-block:: bash + + mkdir "${CA_DIR}/lookup-hash-dir" + +Add the CA certificate to the directory + +.. code-block:: bash + + CA_HASH=$(openssl x509 -in "${CA_DIR}/ca.crt" -subject_hash -noout) + + cp "${CA_DIR}/ca.crt" "${CA_DIR}/lookup-hash-dir/$CA_HASH.0" + +If CRLs are used generate and add an empty CRL + +.. code-block:: bash + + openssl ca -config "${CA_DIR}/ca.conf" -gencrl -out "${CA_DIR}/crl/ca.crl" + + CRL_HASH=$(openssl crl -in "${CA_DIR}/crl/ca.crl" -hash -noout) + + cp "${CA_DIR}/crl/ca.crl" "${CA_DIR}/lookup-hash-dir/$CRL_HASH.r0" + +.. _certificate revocation: + +Revoking certificates +--------------------- + +Omit this section if CRLs are not used or Media Gateway has not been launched. + +Revoke a client certificate + +.. code-block:: bash + + openssl ca -config "${CA_DIR}/ca.conf" -revoke "${CA_DIR}/certs/client.crt" + +Generate a new CRL and update X509 lookup hash directory. + +.. warning:: + + The sequence number N in the filename of the form ``hash.rN`` must be increased each time. + +.. code-block:: bash + + openssl ca -config "${CA_DIR}/ca.conf" -gencrl -out "${CA_DIR}/crl/ca.crl" + + CRL_HASH=$(openssl crl -in "${CA_DIR}/crl/ca.crl" -hash -noout) + + cp "${CA_DIR}/crl/ca.crl" "${CA_DIR}/lookup-hash-dir/$CRL_HASH.r1" + +Testing +------- + +Server +^^^^^^ + +To test the server only a certificate with IP SAN is used. + +Prepare the configuration file with enabled CRLs + +.. code-block:: bash + + cat << EOF > media-gateway-server.json + { + "ip": "0.0.0.0", + "port": 8080, + "tls": { + "identity": { + "certificate": "/etc/certs/server.crt", + "key": "/etc/certs/server.key" + }, + "peers": { + "lookup_hash_directory": "/etc/certs/lookup-hash-dir", + "crl_enabled": true + } + }, + "out_stream": { + "url": "pub+bind:ipc:///tmp/server", + "send_timeout": { + "secs": 1, + "nanos": 0 + }, + "send_retries": 3, + "receive_timeout": { + "secs": 1, + "nanos": 0 + }, + "receive_retries": 3, + "send_hwm": 1000, + "receive_hwm": 1000, + "inflight_ops": 100 + } + } + EOF + +Launch the server (change the value of ``MEDIA_GATEWAY_PORT`` in the command below if required) + +.. code-block:: bash + :caption: x86_64 + + export MEDIA_GATEWAY_PORT=8080 + + docker run -d \ + -v $(pwd)/media-gateway-server.json:/opt/etc/custom_config.json \ + -v ${CA_DIR}/certs/server.key:/etc/certs/server.key \ + -v ${CA_DIR}/certs/server.crt:/etc/certs/server.crt \ + -v ${CA_DIR}/lookup-hash-dir:/etc/certs/lookup-hash-dir \ + -p ${MEDIA_GATEWAY_PORT}:8080 \ + --name media-gateway-server \ + ghcr.io/insight-platform/media-gateway-server-x86:latest \ + /opt/etc/custom_config.json + +.. code-block:: bash + :caption: ARM64 + + export MEDIA_GATEWAY_PORT=8080 + + docker run -d \ + -v $(pwd)/media-gateway-server.json:/opt/etc/custom_config.json \ + -v ${CA_DIR}/certs/server.key:/etc/certs/server.key \ + -v ${CA_DIR}/certs/server.crt:/etc/certs/server.crt \ + -v ${CA_DIR}/lookup-hash-dir:/etc/certs/lookup-hash-dir \ + -p ${MEDIA_GATEWAY_PORT}:8080 \ + --name media-gateway-server \ + ghcr.io/insight-platform/media-gateway-server-arm64:latest \ + /opt/etc/custom_config.json + +Send the request to the server + +.. code-block:: bash + + curl --cacert "${CA_DIR}/ca.crt" --cert "${CA_DIR}/certs/client.crt" --key "${CA_DIR}/certs/client.key" -v https://$HOST_IP:$MEDIA_GATEWAY_PORT/health + +HTTP response with ``200 OK`` status code and the body as below should be returned. + +.. code-block:: json + + {"status": "healthy"} + +Revoke the client certificate using :ref:`the section ` and send the request to the server again. An error response with the message as below should be returned. + +.. code-block:: + + error:0A000414:SSL routines::sslv3 alert certificate revoked + +Clean up after testing + +.. code-block:: bash + + docker stop media-gateway-server + + docker rm media-gateway-server + + rm -rf ca media-gateway-server.json + +e2e +^^^ + +To test both server and client based on :doc:`3_usage_example` + +* generate a certificate with DNS SAN +* update ``server_config.json`` and ``client_config.json`` in the downloaded archive as described above and in :ref:`HTTPS ` guide +* add volumes for ``media-gateway-client``` (key and certificate files) and ``media-gateway-server`` (key and certificate files) in ``docker-compose-x86.yaml`` and ``docker-compose-arm64.yaml`` in the downloaded archive + +Clean up after testing + +.. code-block:: bash + + rm -rf ca diff --git a/docs/source/cookbook/1_tls.rst b/docs/source/cookbook/1_tls.rst deleted file mode 100644 index 6e966af..0000000 --- a/docs/source/cookbook/1_tls.rst +++ /dev/null @@ -1,354 +0,0 @@ -TLS -=== - -Media Gateway TLS features: - -* HTTPS -* client certificate authentication - -TLS features are implemented using `OpenSSL `__. - -HTTPS ------ - -Media Gateway supports both self-signed and signed by CA server certificates. Certificates should be in PEM format. To enable HTTPS in Media Gateway update both server and client configuration. - -The protocol in ``url`` field in the client configuration must be updated to ``https``. - -Self-signed server certificates -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``server.crt`` is a file with the server certificate in PEM format. - -``server.key`` is a file with the server key in PEM format. - -.. code-block:: json - :caption: server - - "tls": { - "identity": { - "certificate": "server.crt", - "key": "server.key" - } - } - -.. code-block:: json - :caption: client - - "tls": { - "root_certificate": "server.crt" - } - -Signed by a private CA server certificates -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``server.crt`` is a file with the server certificate in PEM format. - -``server.key`` is a file with the server key in PEM format. - -``ca.crt`` is a file with the CA certificate in PEM format. - -.. code-block:: json - :caption: server - - "tls": { - "identity": { - "certificate": "server.crt", - "key": "server.key" - } - } - -.. code-block:: json - :caption: client - - "tls": { - "root_certificate": "ca.crt" - } - -Signed by a public CA server certificates -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``server.crt`` is a file with a sequence of certificates, the first being the leaf certificate, and the remainder forming the chain of certificates up to and including the trusted root certificate. - -``server.key`` is a file with the server key in PEM format. - -.. code-block:: json - :caption: server - - "tls": { - "identity": { - "certificate": "server.crt", - "key": "server.key" - } - } - -Client certificate authentication ---------------------------------- - -Client certificate authentication is an optional feature in Media Gateway. Only signed by CA client certificates can be used. Certificates should be in PEM format. - -The server uses a store with trusted X509 certificates to verify peer certificates. The store automatically (without a server restart) loads certificates and CRLs from the specified directory. Certificates and CRLs should be added to the directory in accordance with `X509_LOOKUP_hash_dir method `__ requirements. For each certificate at least one CRL must be in the directory. CRL might be without revoked certificates. A new CRL must be loaded when the previous CRL is expired. - -``ca.crt`` is a file with the CA certificate in PEM format. - -``ca.crl`` is a file with CRL in PEM format. - -``/opt/etc/certs/lookup-hash-dir`` is a directory with CA certificates and CRLs. - -To add a new certificate and corresponding CRL - -.. code-block:: bash - - CA_HASH=$(openssl x509 -in ca.crt -subject_hash -noout) - - cp ca.crt "/opt/etc/certs/lookup-hash-dir/$CA_HASH.0" - - CRL_HASH=$(openssl crl -in ca.crl -hash -noout) - - cp ca.crl "/opt/etc/certs/lookup-hash-dir/$CRL_HASH.r0" - -To enable client certificate authentication in Media Gateway update both server and client configuration. - -``/opt/etc/certs/lookup-hash-dir`` is a directory with CA certificates and CRLs. - -``client.crt`` is a file with a client certificate in PEM format. - -``client.key`` is a file with a PEM encoded PKCS #8 formatted client key. - -.. code-block:: json - :caption: server - - "tls": { - // see HTTPS section - "peers": { - "lookup_hash_directory" : "/opt/etc/certs/lookup-hash-dir", - "crl_enabled": true - } - } - -.. code-block:: json - :caption: client - - "tls": { - // see HTTPS section - "identity": { - "certificate": "client.crt", - "key": "client.key" - } - } - -Certificate generation with a private CA ----------------------------------------- - -This section describes how to generate certificates and CLRs signed by a private CA using `OpenSSL `_. Provided instructions specifies the minimum required information only. For production usage see OpenSSL documentation. - -CA -^^ - -To set up a private CA and generate a certificate - -.. code-block:: bash - - CA_DIR="$(pwd)/ca" - - mkdir "${CA_DIR}" - - cd "${CA_DIR}" - - mkdir certs crl - - touch index.txt - - echo 01 > serial - - echo 1000 > crlnumber - - echo "[ ca ] - - default_ca = CA_default - - [ CA_default ] - - dir = ${CA_DIR} - certificate = \$dir/ca.crt - private_key = \$dir/ca.key - database = \$dir/index.txt - new_certs_dir = \$dir/certs - serial = \$dir/serial - crl_dir = \$dir/crl - crl = \$dir/crl/ca.crl - crlnumber = \$dir/crlnumber - - x509_extensions = v3_ca - crl_extensions = crl_ext - - name_opt = ca_default - cert_opt = ca_default - - default_days = 365 - default_crl_days = 30 - default_md = default - preserve = no - policy = policy_any - - [ policy_any ] - countryName = optional - stateOrProvinceName = optional - organizationName = optional - organizationalUnitName = optional - commonName = supplied - emailAddress = optional - - #################################################################### - - [ req ] - default_bits = 2048 - default_keyfile = privkey.pem - distinguished_name = req_distinguished_name - attributes = req_attributes - x509_extensions = v3_ca - - [ req_distinguished_name ] - countryName = Country Name (2 letter code) - countryName_default = US - countryName_min = 2 - countryName_max = 2 - stateOrProvinceName = State or Province Name (full name) - stateOrProvinceName_default = - localityName = Locality Name (eg, city) - localityName+default = - 0.organizationName = Organization Name (eg, company) - 0.organizationName_default = - organizationalUnitName = Organizational Unit Name (eg, section) - organizationalUnitName_default = - commonName = Common Name (e.g. server FQDN or YOUR name) - commonName_max = 64 - emailAddress = Email Address - emailAddress_max = 64 - - [ req_attributes ] - challengePassword = A challenge password - challengePassword_min = 4 - challengePassword_max = 20 - unstructuredName = An optional company name - - [ v3_req ] - basicConstraints = CA:FALSE - keyUsage = nonRepudiation, digitalSignature, keyEncipherment - - [ v3_ca ] - subjectKeyIdentifier=hash - authorityKeyIdentifier=keyid:always,issuer - basicConstraints = critical,CA:true - keyUsage = critical, digitalSignature, cRLSign, keyCertSign - - [ crl_ext ] - authorityKeyIdentifier=keyid:always - " > ca.conf - - openssl genpkey -algorithm RSA -out ca.key - - openssl req -new -x509 -days 365 -config ca.conf -key ca.key -out ca.crt -subj "/CN=ca.example.com" - -``ca.crt`` is a file with CA certificate in PEM format. - -``ca.key`` is a file with CA key in PEM format. - -Server -^^^^^^ - -To generate a server certificate signed by the CA with a simple subject name and IP (both ``127.0.0.1`` and ``192.168.0.100``) subject alternative name - -.. code-block:: bash - - openssl genpkey -algorithm RSA -out certs/server.key - - openssl req -new -key certs/server.key -out certs/server.csr -subj "/CN=server.example.com" - - openssl ca -config ca.conf -in certs/server.csr -out certs/server.crt -extfile <(echo 'basicConstraints=CA:FALSE - nsComment="OpenSSL Generated Certificate" - subjectKeyIdentifier=hash - authorityKeyIdentifier=keyid,issuer - keyUsage=critical,digitalSignature,keyEncipherment - extendedKeyUsage=serverAuth - subjectAltName=IP:127.0.0.1,IP:192.168.0.100') - -To generate a server certificate signed by CA with a simple subject name and DNS (``server.example.com``) subject alternative name - -.. code-block:: bash - - openssl genpkey -algorithm RSA -out certs/server.key - - openssl req -new -key certs/server.key -out certs/server.csr -subj "/CN=server.example.com" - - openssl ca -config ca.conf -in certs/server.csr -out certs/server.crt -extfile <(echo 'basicConstraints=CA:FALSE - nsComment="OpenSSL Generated Certificate" - subjectKeyIdentifier=hash - authorityKeyIdentifier=keyid,issuer - keyUsage=critical,digitalSignature,keyEncipherment - extendedKeyUsage=serverAuth - subjectAltName=DNS:server.example.com') - -``certs/server.crt`` is a file with a server certificate in PEM format. - -``certs/server.key`` is a file with a server key in PEM format. - -Client ------- - -To generate a client certificate signed by the CA with a simple subject name - -.. code-block:: bash - - openssl genpkey -algorithm RSA -out certs/client.key - - openssl req -new -key certs/client.key -out certs/client.csr -subj "/CN=client.example.com" - - openssl ca -config ca.conf -in certs/client.csr -out certs/client.crt -extfile <(echo 'basicConstraints=CA:FALSE - nsComment="OpenSSL Generated Certificate" - subjectKeyIdentifier=hash - keyUsage=critical,nonRepudiation,digitalSignature,keyEncipherment - extendedKeyUsage=clientAuth - authorityKeyIdentifier=keyid,issuer') - -``certs/client.crt`` is a file with a client certificate in PEM format. - -``certs/client.key`` is a file with a client key in PEM format. - -X509 lookup hash dir --------------------- - -To prepare certificates signed by the CA for `X509_LOOKUP_hash_dir method `__ in ``certs/client`` directory - -.. code-block:: bash - - mkdir lookup-hash-dir - - CA_HASH=$(openssl x509 -in ca.crt -subject_hash -noout) - - cp ca.crt "lookup-hash-dir/$CA_HASH.0" - - openssl ca -config ca.conf -gencrl -out crl/ca.crl - - CRL_HASH=$(openssl crl -in crl/ca.crl -hash -noout) - - cp crl/ca.crl "lookup-hash-dir/$CRL_HASH.r0" - -A filename has the form ``hash.N`` for a certificate and the form ``hash.rN`` for a CRL where N is a sequence number that starts at zero, and is incremented consecutively for each certificate or CRL with the same hash value. - -CRL ---- - -To revoke a client certificate signed by the CA - -.. code-block:: bash - - openssl ca -config ca.conf -revoke certs/client.crt - - openssl ca -config ca.conf -gencrl -out crl/ca.crl - - CRL_HASH=$(openssl crl -in crl/ca.crl -hash -noout) - - cp crl/ca.crl "lookup-hash-dir/$CRL_HASH.r1" - -⚠️ The sequence number N in the filename of the form ``hash.rN`` must be increased each time. diff --git a/docs/source/cookbook/2_basic_auth.rst b/docs/source/cookbook/2_basic_auth.rst index 3f52809..3d90568 100644 --- a/docs/source/cookbook/2_basic_auth.rst +++ b/docs/source/cookbook/2_basic_auth.rst @@ -1,16 +1,95 @@ HTTP Basic authentication ========================= -Media Gateway can control access to the server via a username/password authentication (HTTP Basic Authentication). HTTP Basic authentication should be used with HTTPS (see :doc:`1_tls`) to provide confidentiality. The only endpoint that is available to anyone is :ref:`a health endpoint `. Usernames and passwords are taken from `etcd `__ and cached to decrease deserialization costs. Data in the cache is automatically reloaded from `etcd` when its checksum is changed. An additional cache is used to decrease cryptographic costs. It stores results of authentication checks which are used for subsequent requests if provided credentials are the same and the user password is not changed in `etcd`. +Media Gateway can control access to the server via a username/password authentication (HTTP Basic Authentication). HTTP Basic authentication should be used with HTTPS (see :doc:`0_https`) to provide confidentiality. The only endpoint that is available to anyone is :ref:`a health endpoint `. Usernames and passwords are taken from `etcd `__. `Savant messages `__ contain routing labels. Media Gateway server can be configured to accept messages from a user only if routing labels are allowed. Allowed labels in the form of `a label filter rule `__ are taken from `etcd`. Prerequisites ------------- -* `Docker` -* `openssl` -* `curl` +* Docker +* Docker Compose +* openssl +* curl + +Configuring Media Gateway +------------------------- + +To use HTTP Basic authentication update server and client configurations. Examples below show the full possible configuration. See :doc:`/reference/0_configuration` and :doc:`/miscellaneous/2_caching` for more details. + +.. code-block:: json + :caption: server + + "auth": { + "basic": { + "etcd": { + "urls": [ + "https://etcd:2379" + ], + "tls": { + "root_certificate": "etcd-ca.crt", + "identity": { + "certificate": "etcd-client.crt", + "key": "etcd-client.key" + } + }, + "credentials": { + "username": "etcd-user", + "password": "etcd-password" + }, + "path": "/users", + "data_format": "json", + "lease_timeout": { + "secs": 60, + "nanos": 0 + }, + "connect_timeout": { + "secs": 30, + "nanos": 0 + }, + "cache": { + "size": 10, + "usage": { + "period": { + "secs": 60, + "nanos": 0 + }, + "evicted_threshold": 10 + } + } + }, + "cache": { + "size": 10, + "usage": { + "period": { + "secs": 60, + "nanos": 0 + }, + "evicted_threshold": 10 + } + } + } + } + +.. code-block:: json + :caption: client + + "auth": { + "basic": { + "username": "user", + "password": "password" + } + } + +where + +* ``etcd-ca.crt`` is a file with the CA certificate in PEM format. + +* ``etcd-client.crt`` is a file with the client in PEM format. + +* ``etcd-client.key`` is a file with the client key in PEM format. + Running etcd with TLS authentication ------------------------------------ @@ -24,7 +103,7 @@ In order to expose the etcd API to clients outside of the Docker host use the ho Generating certificates ^^^^^^^^^^^^^^^^^^^^^^^ -To generate certificates signed by a private CA +Generate certificates signed by a private CA .. code-block:: bash @@ -39,37 +118,32 @@ To generate certificates signed by a private CA # Generate server private key openssl genpkey -algorithm RSA -out certs/server.key - # Generate server CSR with SAN + # Generate server CSR openssl req -new -key certs/server.key -out certs/server.csr -subj "/CN=etcd-server" # Generate server certificate signed by the CA with IP address subject alternative name - openssl x509 -req -days 365 -in certs/server.csr -CA certs/ca.crt -CAkey certs/ca.key -CAcreateserial -out certs/server.crt -extfile <(echo "subjectAltName=IP:127.0.0.1,IP:${HOST_IP}") + openssl x509 -req -days 365 -in certs/server.csr -CA certs/ca.crt -CAkey certs/ca.key -CAcreateserial -out certs/server.crt -extfile <(echo "subjectAltName=IP:${HOST_IP}") # Generate client private key openssl genpkey -algorithm RSA -out certs/client.key - # Generate client CSR with SAN + # Generate client CSR openssl req -new -key certs/client.key -out certs/client.csr -subj "/CN=etcd-client" # Generate client certificate signed by the CA openssl x509 -req -days 365 -in certs/client.csr -CA certs/ca.crt -CAkey certs/ca.key -CAcreateserial -out certs/client.crt - chmod 644 certs/ca.key certs/server.key certs/client.key - -``certs/*.crt`` is a file with a certificate in PEM format and ``certs/*.key`` is a file with a key in PEM format where ``*`` is a replacement for `ca`, `server` and `client`. - Launching etcd ^^^^^^^^^^^^^^ -Environment variables below declare the docker image and the port on the host for etcd. Note that the password is passed in a non-secure way (it will be seen in the command history) and `root` etcd user is used for simplicity. +Environment variables below declare the docker image and the port on the host for etcd. .. code-block:: bash ETCD_IMAGE="bitnami/etcd:3.5" ETCD_PORT=42379 - ETCD_ROOT_PASSWORD="etcd-password" -To start etcd +Launch etcd .. code-block:: bash @@ -81,7 +155,7 @@ To start etcd -e ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379 \ -e ETCD_ADVERTISE_CLIENT_URLS=https://0.0.0.0:$ETCD_PORT \ -e ETCD_CLIENT_CERT_AUTH=true \ - -e ETCD_ROOT_PASSWORD=$ETCD_ROOT_PASSWORD \ + -e ALLOW_NONE_AUTHENTICATION=yes \ -v $(pwd)/certs:/etc/certs \ --name etcd \ $ETCD_IMAGE @@ -205,7 +279,7 @@ User data in `etcd` are stored as an object in JSON/YAML format with the followi Saving user data ^^^^^^^^^^^^^^^^ -To store data with a password `password1` for a user with the name `user1` in etcd under the path `/users` +Save data with a password `password1` for a user with the name `user1` in etcd under the path `/users` .. code-block:: bash @@ -216,110 +290,97 @@ To store data with a password `password1` for a user with the name `user1` in et --cacert /etc/certs/ca.crt \ --cert /etc/certs/client.crt \ --key /etc/certs/client.key \ - --user=root:$ETCD_ROOT_PASSWORD \ --endpoints https://$HOST_IP:$ETCD_PORT \ put \ /users/user1 \ '{"password_hash": "$argon2i$v=19$m=12,t=3,p=1$YXkzZmx1eTFwVW5hZ0R2S1dXazA$VxVMw2Omh1CeVqry8Cay+4OZ69OGvn4fma2M5rURZhI"}' -Running Media Gateway server with HTTP Basic authentication ------------------------------------------------------------ +Testing +------- -Preparing a configuration file -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Server +^^^^^^ -The configuration below does not contain TLS settings for simplicity. For production HTTP Basic authentication should be used with HTTPS (see :doc:`1_tls`). +To test the server only prepare a configuration file. The configuration below does not contain TLS settings for simplicity. For production HTTP Basic authentication should be used with HTTPS (see :doc:`0_https`). .. code-block:: bash - echo "{ - \"ip\": \"0.0.0.0\", - \"port\": 8080, - \"auth\": { - \"basic\": { - \"etcd\": { - \"urls\": [ - \"https://$HOST_IP:$ETCD_PORT\" + cat << EOF > media-gateway-server.json + { + "ip": "0.0.0.0", + "port": 8080, + "auth": { + "basic": { + "etcd": { + "urls": [ + "https://$HOST_IP:$ETCD_PORT" ], - \"credentials\": { - \"username\": \"root\", - \"password\": \"etcd-password\" - }, - \"tls\": { - \"root_certificate\": \"/etc/certs/ca.crt\", - \"identity\": { - \"certificate\": \"/etc/certs/client.crt\", - \"key\": \"/etc/certs/client.key\" + "tls": { + "root_certificate": "/etc/certs/ca.crt", + "identity": { + "certificate": "/etc/certs/client.crt", + "key": "/etc/certs/client.key" } }, - \"path\": \"/users\", - \"data_format\": \"json\", - \"lease_timeout\": { - \"secs\": 60, - \"nanos\": 0 + "path": "/users", + "data_format": "json", + "lease_timeout": { + "secs": 60, + "nanos": 0 }, - \"connect_timeout\": { - \"secs\": 30, - \"nanos\": 0 + "connect_timeout": { + "secs": 30, + "nanos": 0 }, - \"cache\": { - \"size\": 10, - \"usage\": { - \"period\": { - \"secs\": 60, - \"nanos\": 0 + "cache": { + "size": 10, + "usage": { + "period": { + "secs": 60, + "nanos": 0 }, - \"evicted_threshold\": 10 + "evicted_threshold": 10 } } }, - \"cache\": { - \"size\": 10, - \"usage\": { - \"period\": { - \"secs\": 60, - \"nanos\": 0 + "cache": { + "size": 10, + "usage": { + "period": { + "secs": 60, + "nanos": 0 }, - \"evicted_threshold\": 10 + "evicted_threshold": 10 } } } }, - \"out_stream\": { - \"url\": \"pub+bind:ipc:///etc/media-gateway/server\", - \"send_timeout\": { - \"secs\": 1, - \"nanos\": 0 + "out_stream": { + "url": "pub+bind:ipc:///tmp/server", + "send_timeout": { + "secs": 1, + "nanos": 0 }, - \"send_retries\": 3, - \"receive_timeout\": { - \"secs\": 1, - \"nanos\": 0 + "send_retries": 3, + "receive_timeout": { + "secs": 1, + "nanos": 0 }, - \"receive_retries\": 3, - \"send_hwm\": 1000, - \"receive_hwm\": 1000, - \"fix_ipc_permissions\": 511 + "receive_retries": 3, + "send_hwm": 1000, + "receive_hwm": 1000, + "fix_ipc_permissions": 511 } } - " > media-gateway-server.json + EOF -Launching Media Gateway server -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Environment variables below declare and create the data directory and the port on the host for Media Gateway server. - -.. code-block:: bash - - MEDIA_GATEWAY_PORT=8080 - MEDIA_GATEWAY_DATA_DIR=media-gateway - mkdir $MEDIA_GATEWAY_DATA_DIR - -To start Media Gateway server with the prepared configuration +Launch the server (change the value of `MEDIA_GATEWAY_PORT` in the command below if required) .. code-block:: bash :caption: x86_64 + export MEDIA_GATEWAY_PORT=8080 + docker run -d \ -v $(pwd)/media-gateway-server.json:/opt/etc/custom_config.json \ -v $(pwd)/$MEDIA_GATEWAY_DATA_DIR:/etc/media-gateway \ @@ -332,6 +393,8 @@ To start Media Gateway server with the prepared configuration .. code-block:: bash :caption: ARM64 + export MEDIA_GATEWAY_PORT=8080 + docker run -d \ -v $(pwd)/media-gateway-server.json:/opt/etc/custom_config.json \ -v $(pwd)/$MEDIA_GATEWAY_DATA_DIR:/etc/media-gateway \ @@ -341,26 +404,21 @@ To start Media Gateway server with the prepared configuration ghcr.io/insight-platform/media-gateway-server-arm64:latest \ /opt/etc/custom_config.json -Testing HTTP Basic authentication ---------------------------------- - -For simplicity an invalid request is used for testing. - -Send a valid user name and password. +For simplicity an invalid request is used for testing. Send a request with a valid user name and password. .. code-block:: bash curl -v -u user1:password1 http://$HOST_IP:$MEDIA_GATEWAY_PORT/ -X POST -``400 Bad Request`` response should be returned. It means that authentication is successful. +HTTP response with ``400 Bad Request`` status code should be returned. It means that authentication is successful. -Send an invalid user name and password. +Send a request with an invalid user name and password. .. code-block:: bash curl -v -u user1:password http://$HOST_IP:$MEDIA_GATEWAY_PORT/ -X POST -``401 Unauthorized`` response should be returned. It means that authentication fails. +HTTP response with ``401 Unauthorized`` status code should be returned. It means that authentication fails. Add a new user `user2` with a password `password2` and send a request using it to test that new users are loaded. @@ -373,7 +431,6 @@ Add a new user `user2` with a password `password2` and send a request using it t --cacert /etc/certs/ca.crt \ --cert /etc/certs/client.crt \ --key /etc/certs/client.key \ - --user=root:$ETCD_ROOT_PASSWORD \ --endpoints https://$HOST_IP:$ETCD_PORT \ put \ /users/user2 \ @@ -392,7 +449,6 @@ Change the password for the user `user2` to `password` and send a request using --cacert /etc/certs/ca.crt \ --cert /etc/certs/client.crt \ --key /etc/certs/client.key \ - --user=root:$ETCD_ROOT_PASSWORD \ --endpoints https://$HOST_IP:$ETCD_PORT \ put \ /users/user2 \ @@ -402,10 +458,7 @@ Change the password for the user `user2` to `password` and send a request using curl -v -u user2:password http://$HOST_IP:$MEDIA_GATEWAY_PORT/ -X POST -Cleaning up ------------ - -Stop and remove Docker containers +Clean up after testing .. code-block:: bash @@ -413,8 +466,22 @@ Stop and remove Docker containers docker rm media-gateway-server etcd -Remove certificates, a configuration file and the data directory + rm -rf certs media-gateway-server.json $MEDIA_GATEWAY_DATA_DIR + +e2e +^^^ + +To test both server and client based on :doc:`3_usage_example` + +* update ``server_config.json`` and ``client_config.json`` in the downloaded archive as described above +* add volumes for ``media-gateway-server`` (key and certificate files) in ``docker-compose-x86.yaml`` and ``docker-compose-arm64.yaml`` in the downloaded archive + +Clean up after testing .. code-block:: bash - rm -rf certs media-gateway-server.json $MEDIA_GATEWAY_DATA_DIR + docker stop etcd + + docker rm etcd + + rm -rf certs diff --git a/docs/source/cookbook/0_usage_example.rst b/docs/source/cookbook/3_usage_example.rst similarity index 87% rename from docs/source/cookbook/0_usage_example.rst rename to docs/source/cookbook/3_usage_example.rst index 42e1fed..a728ff3 100644 --- a/docs/source/cookbook/0_usage_example.rst +++ b/docs/source/cookbook/3_usage_example.rst @@ -6,7 +6,8 @@ This example shows how to ingest `the video file `__ and `Docker Compose `__ are installed and `NVIDIA GPU `__ is configured. +* Docker with configured `NVIDIA GPU `__ +* Docker Compose Launch ------ diff --git a/docs/source/getting_started/0_configuration.rst b/docs/source/getting_started/0_configuration.rst deleted file mode 100644 index 6f8159f..0000000 --- a/docs/source/getting_started/0_configuration.rst +++ /dev/null @@ -1,534 +0,0 @@ -Configuration -============= - -Both server and client applications are configured via JSON files. - -Server ------- - -The server configuration consists of following fields: - -.. list-table:: - :header-rows: 1 - - * - Field - - Description - - Mandatory - * - ip - - A string representation of an IP address or a host name to bind to. Both IPv4 or IPv6 are supported. If the host name is specified the server is bound to both the IPv4 and IPv6 addresses that result from a DNS lookup. - - yes - * - port - - A port to bind to. - - yes - * - out_stream - - A configuration how to write to the target ZeroMQ. - - yes - * - auth - - Authentication settings. - - no - * - tls - - TLS settings. - - no - * - statistics - - Statistics settings. - - no - -out_stream -^^^^^^^^^^ - -.. list-table:: - :header-rows: 1 - - * - Field - - Description - - Mandatory - - Default value - * - url - - The URL in Savant ZMQ format. - - yes - - - * - send_timeout - - The timeout for sending data to the egress stream. The default value is . - - no - - ``{"secs": 1, "nanos": 0}`` - * - send_retries - - The number of retries for sending data to the egress stream. - - no - - ``3`` - * - receive_timeout - - The timeout for receiving data from the egress stream. Valid only for dealer and req socket types. - - no - - ``{"secs": 1, "nanos": 0}`` - * - receive_retries - - The number of retries for receiving data from the egress stream (crucial for req/rep communication). - - yes - - ``3`` - * - send_hwm - - The high-water mark for the egress stream. This parameter is used to control backpressure. Please consult with ZeroMQ documentation for more details. - - no - - ``1000`` - * - receive_hwm - - The high-water mark for the egress stream. This parameter is used to control backpressure. Please consult with ZeroMQ documentation for more details. - - no - - ``1000`` - * - fix_ipc_permissions - - If set, Media Gateway will fix the UNIX file permissions for IPC sockets. - - no - - - -auth -^^^^ - -The only supported authentication type is ``basic``. Provided user names and passwords are loaded from `etcd `__. The key in `etcd` is a user name. The value in `etcd` is an object in JSON/YAML format with the following schema - -.. code-block:: json - - { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Media Gateway user data schema", - "type": "object", - "properties": { - "password_hash": { - "description": "Argon2 password hash in PHC string format.", - "type": "string" - }, - "allowed_routing_labels": { - "type": "object", - "anyOf": [ - {"$ref": "#/$defs/set"}, - {"$ref": "#/$defs/unset"}, - {"$ref": "#/$defs/and"}, - {"$ref": "#/$defs/or"}, - {"$ref": "#/$defs/not"} - ] - } - }, - "required": [ "password_hash" ], - "$defs": { - "set": { - "description": "Set label rule: routing labels must contain a specified label.", - "type": "string" - }, - "unset": { - "description": "Unset label rule: routing labels must not contain a specified label.", - "type": "string" - }, - "and" : { - "description": "And label rule: labels rules combined with and logic.", - "type": "array", - "items": { - "anyOf": [ - {"$ref": "#/$defs/set"}, - {"$ref": "#/$defs/unset"}, - {"$ref": "#/$defs/and"}, - {"$ref": "#/$defs/or"}, - {"$ref": "#/$defs/not"} - ] - } - }, - "or" : { - "description": "Or label rule: labels rules combined with or logic.", - "type": "array", - "items": { - "anyOf": [ - {"$ref": "#/$defs/set"}, - {"$ref": "#/$defs/unset"}, - {"$ref": "#/$defs/and"}, - {"$ref": "#/$defs/or"}, - {"$ref": "#/$defs/not"} - ] - } - }, - "not" : { - "description": "Not label rule: a negation of the specified label rule.", - "type": "object", - "items": { - "anyOf": [ - {"$ref": "#/$defs/set"}, - {"$ref": "#/$defs/unset"}, - {"$ref": "#/$defs/and"}, - {"$ref": "#/$defs/or"}, - {"$ref": "#/$defs/not"} - ] - } - } - } - } - -.. list-table:: - :header-rows: 1 - - * - Field - - Description - - Mandatory - * - etcd - - etcd configuration. - - true - * - etcd.urls - - A list of etcd server endpoints to connect to. - - true - * - etcd.tls - - TLS options to use while connecting to etcd servers. - - false - * - etcd.tls.root_certificate - - CA certificate against which to verify the etcd server's TLS certificate. - - false - * - etcd.tls.identity - - The client identity to present to the etcd server. - - false - * - etcd.tls.identity.certificate - - A path to a chain of PEM encoded certificates, with the leaf certificate first. - - true - * - etcd.tls.identity.key - - A path to a PEM encoded private key - - true - * - etcd.credentials - - Credentials for basic authentication in etcd. - - false - * - etcd.credentials.username - - A user name for basic authentication in etcd. - - true - * - etcd.credentials.password - - A password for basic authentication in etcd. - - true - * - etcd.path - - The path of the hierarchically organized directories (as in a standard filesystem) for the stored key/value(-s). - - true - * - etcd.data_format - - The format of the data stored in etcd. Possible values are `json`, `yaml`. - - true - * - etcd.connect_timeout - - A timeout for each request to etcd in the duration format, e.g. ``{"secs": 1, "nanos": 0}``. - - true - * - etcd.lease_timeout - - A timeout to hold keys if the etcd server does not receive a keepAlive, in the duration format, e.g. ``{"secs": 60, "nanos": 0}`` - - true - * - etcd.cache - - Settings for user data cache. See :ref:`cache configuration section `. - - true - * - cache - - Settings for authentication cache. See :ref:`cache configuration section `. - - true - -.. _cache configuration: - -cache -^^^^^ - -.. list-table:: - :header-rows: 1 - - * - Field - - Description - - Mandatory - * - size - - A size of LRU cache for credentials to exclude running encryption functions when the same credentials are passed. - - true - * - usage - - Settings to monitor LRU cache usage and to produce a warning when more than X keys per a period are evicted. - - false - * - usage.period - - A period to track evicted keys in the duration format, e.g. ``{"secs": 1, "nanos": 0}``. - - true - * - usage.evicted_threshold - - The positive integer number of keys allowed for eviction for the period. - - true - -tls -^^^ -.. list-table:: - :header-rows: 1 - - * - Field - - Description - - Mandatory - * - identity - - HTTPS settings. - - yes - * - identity.certificate - - A path to a PEM encoded certificate (can be a self-signed certificate). - - yes - * - identity.key - - A path to a private key for the certificate. - - yes - * - peers - - Settings to verify peer certificates. - - no - * - peers.lookup_hash_directory - - A directory with certificates and CRLs to verify client certificates. See `X509_LOOKUP_hash_dir method `_ for more details. - - yes - * - peers.crl_enabled - - ``true`` if CRLs must be checked during client certificate verification, ``false`` otherwise. - - yes - -statistics -^^^^^^^^^^ - -Exactly one of ``frame_period`` and ``timestamp_period`` must be specified. - -.. list-table:: - :header-rows: 1 - - * - Field - - Description - - Mandatory - * - frame_period - - A frame period - - no - * - timestamp_period - - A timestamp period in the duration format with millisecond precision, e.g. ``{"secs": 1, "nanos": 0}`` - - no - -Client ------- - -The client configuration consist of following fields: - -.. list-table:: - :header-rows: 1 - - * - Field - - Description - - Mandatory - * - ip - - A string representation of an IP address or a host name to bind to. Both IPv4 or IPv6 are supported. If the host name is specified the server is bound to both the IPv4 and IPv6 addresses that result from a DNS lookup. - - yes - * - port - - A port to bind to. - - yes - * - url - - An endpoint of the media gateway service to accept messages. - - yes - * - retry_strategy - - A strategy how to retry to send a message to the media gateway service. The default value is an exponential strategy with the initial delay 1 ms, the maximum delay 1 sec and the multiplier 2. - - no - * - in_stream - - A configuration how to read from the source ZeroMQ. - - yes - * - wait_strategy - - A strategy how to wait for data from the source ZeroMQ. The default value is 1 ms sleep strategy. - - no - * - auth - - Authentication settings. - - no - * - tls - - TLS settings. - - no - * - statistics - - Statistics settings. - - no - -retry_strategy -^^^^^^^^^^^^^^ - -There is only one retry strategy - exponential. The strategy executes next attempt after the delay which is calculated for each attempt. The delay for the first attempt is the initial delay. The delay for subsequent attempts is calculated as maximum between multiplication of last attempt delay by the specified multiplier and the maximum delay. - -Retry strategy is an object with the following schema - -.. code-block:: json - - { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Media Gateway Client retry strategy schema", - "anyOf": [ - { - "description": "A strategy that executes next attempt after a delay that starts from the initial delay and increases (multiplying by the specified multiplier) with each attempt up to the maximum.", - "type": "object", - "properties": { - "exponential": { - "type": "object", - "properties": { - "initial_delay": { - "description": "The delay for the first attempt.", - "$ref": "#/$defs/duration" - }, - "maximum_delay": { - "description": "The maximum delay", - "$ref": "#/$defs/duration" - }, - "multiplier": { - "description": "A multiplier to calculate the delay for next attempt by multiplying last attempt delay.", - "type": "integer", - "minimum": 2 - } - }, - "required": [ - "initial_delay", - "maximum_delay", - "multiplier" - ] - } - }, - "required": [ - "exponential" - ] - } - ], - "$defs": { - "duration": { - "description": "A duration composed of a whole number of seconds and a fractional part represented in nanoseconds.", - "type": "object", - "properties": { - "secs": { - "description": "Duration seconds.", - "type": "integer", - "minimum": 0 - }, - "nanos": { - "description": "Duration nanoseconds.", - "type": "integer", - "minimum": 0, - "maximum": 999999999 - } - }, - "required": ["secs", "nanos"] - } - } - } - -in_stream -^^^^^^^^^ - -.. list-table:: - :header-rows: 1 - - * - Field - - Description - - Mandatory - * - url - - The URL for the data ingress in Savant ZMQ format. - - yes - * - receive_timeout - - The timeout for receiving data from the ingress stream. - - yes - * - receive_hwm - - The high-water mark for the ingress stream. This parameter is used to control backpressure. Please consult with ZeroMQ documentation for more details. - - yes - * - topic_prefix_spec - - The topic prefix specification for the ingress stream. Possible values are ``{"none": null}``, ``{"source_id": "topic"}`` or ``{"prefix": "prefix"}`` - - yes - * - source_cache_size - - The size of the whitelist cache used only when prefix-based filtering is in use. This parameter is used to quickly check if the source ID is in the whitelist or must be checked. - - yes - * - fix_ipc_permissions - - If set, Media Gateway will fix the UNIX file permissions for IPC sockets. - - no - * - inflight_ops - - The maximum number of read messages for non-blocking mode. - - yes - -.. _wait strategy: - -wait_strategy -^^^^^^^^^^^^^ - -There are two wait strategies: - -* yield - -A strategy that pauses execution using `Tokio yield_now `__. - -* sleep - -A strategy that pauses execution using `tokio_timerfd sleep `__ for the specified duration with nanosecond precision. - -Wait strategy is an object with the following schema - -.. code-block:: json - - { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Media Gateway Client wait strategy schema", - "anyOf": [ - { - "description": "A strategy that pauses execution using https://docs.rs/tokio/1.39.2/tokio/task/fn.yield_now.html.", - "type": "string", - "pattern": "^yield$" - }, - { - "description": "A strategy that pauses execution using https://docs.rs/tokio-timerfd/0.2.0/tokio_timerfd/fn.sleep.html for the specified duration with nanosecond precision.", - "type": "object", - "properties": { - "sleep": { - "description": "A duration to sleep composed of a whole number of seconds and a fractional part represented in nanoseconds.", - "type": "object", - "properties": { - "secs": { - "description": "Duration seconds.", - "type": "integer", - "minimum": 0 - }, - "nanos": { - "description": "Duration nanoseconds.", - "type": "integer", - "minimum": 0 - } - } - } - } - } - ] - } - -auth -^^^^ - -The only supported authentication type is ``basic``. - -.. code-block:: json - - "auth": { - "basic": { - "username": "user", - "password": "password" - } - } - -tls -^^^ -.. list-table:: - :header-rows: 1 - - * - Field - - Description - - Mandatory - * - root_certificate - - A path to a self-signed PEM encoded server certificate or PEM encoded CA certificate - - yes - * - identity - - Client certificate authentication settings. - - no - * - identity.certificate - - A path to a chain of PEM encoded X509 certificates, with the leaf certificate first. - - yes - * - identity.key - - A path to a PEM encoded PKCS #8 formatted private key - - yes - -statistics -^^^^^^^^^^ - -Exactly one of ``frame_period`` and ``timestamp_period`` must be specified. - -.. list-table:: - :header-rows: 1 - - * - Field - - Description - - Mandatory - * - frame_period - - A frame period - - no - * - timestamp_period - - A timestamp period in the duration format with millisecond precision, e.g. ``{"secs": 1, "nanos": 0}`` - - no - -Environment variables in configuration files --------------------------------------------- - -You can use environment variables in the configuration file. The syntax is ``${VAR_NAME:-default_value}``. If the environment variable is not set, the default value will be used. - -Examples --------- -Examples of configuration files can be found `here `_. - diff --git a/docs/source/getting_started/1_deployment.rst b/docs/source/getting_started/0_deployment.rst similarity index 57% rename from docs/source/getting_started/1_deployment.rst rename to docs/source/getting_started/0_deployment.rst index b072b71..18035b8 100644 --- a/docs/source/getting_started/1_deployment.rst +++ b/docs/source/getting_started/0_deployment.rst @@ -1,7 +1,9 @@ Deployment ========== -Both server and client are deployed as Docker containers. Docker images are built for x86_64 and ARM64 architectures. +Media Gateway server and client are deployed as Docker containers. Docker images are built for x86_64 and ARM64 architectures and available on GitHub Container Registry. + +If you do not want to run Media Gateway in Docker, you can build it from source or contact us for building binaries for your platform. Common environment variables ---------------------------- @@ -21,7 +23,18 @@ Common environment variables Server ------ -To run the server with `the default configuration `__ and to mount ``/tmp`` directory and publish the port from the default configuration +Docker images: + +* `media-gateway-server-x86 `__ for x86_64 architecture + +* `media-gateway-server-arm64 `__ for ARM64 architecture + +The server is configured via a file in JSON format (see :ref:`server configuration `). + +Default configuration +^^^^^^^^^^^^^^^^^^^^^ + +To run the server with `the default configuration `__, to mount ``/tmp`` directory and publish the port from the default configuration .. code-block:: bash :caption: x86_64 @@ -39,14 +52,17 @@ To run the server with `the default configuration : \ ghcr.io/insight-platform/media-gateway-server-x86:latest \ /opt/etc/custom_config.json @@ -55,13 +71,26 @@ To run the server with another configuration (``/home/user/server_config.json``) docker run \ -v /home/user/server_config.json:/opt/etc/custom_config.json \ - -p HOST_PORT:CONFIG_PORT \ + -p : \ ghcr.io/insight-platform/media-gateway-server-arm64:latest \ /opt/etc/custom_config.json +where ```` is the port specified in the configuration file and ```` is the port on the host machine. + Client ------ +Docker images: + +* `media-gateway-client-x86 `__ for x86_64 architecture + +* `media-gateway-client-arm64 `__ for ARM64 architecture + +The client is configured via a file in JSON format (see :ref:`client configuration `). + +Default configuration +^^^^^^^^^^^^^^^^^^^^^ + To run the client with `the default configuration `__, to mount ``/tmp`` directory and publish the port from the default configuration .. code-block:: bash @@ -82,9 +111,12 @@ To run the client with `the default configuration " \ ghcr.io/insight-platform/media-gateway-client-arm64:latest -where ```` is the server URL, e.g. ``http://192.168.0.100:8080`` +where ```` is Media Gateway server URL, e.g. ``http://192.168.0.100:8080`` + +Custom configuration +^^^^^^^^^^^^^^^^^^^^ -To run the server with another configuration (``/home/user/client_config.json``) +To run the client with a custom configuration file ``/home/user/client_config.json`` and publish the port specified in it .. code-block:: bash :caption: x86_64 diff --git a/docs/source/getting_started/1_secure_communication.rst b/docs/source/getting_started/1_secure_communication.rst new file mode 100644 index 0000000..192971a --- /dev/null +++ b/docs/source/getting_started/1_secure_communication.rst @@ -0,0 +1,22 @@ +Secure communication +==================== + +Media Gateway supports secure communication between the server and client via following features: + +* HTTPS protocol +* authentication + + * client certificate authentication (including client revocation lists) + * HTTP Basic authentication + +Features can be used separately or in combinations. The recommended combinations are + +* HTTPS protocol + HTTP Basic authentication +* HTTPS protocol + client certificate authentication +* HTTPS protocol + client certificate authentication + HTTP Basic authentication + +Guides how to enable security features in Media Gateway: + +* :doc:`/cookbook/0_https` +* :doc:`/cookbook/1_certificate_auth` +* :doc:`/cookbook/2_basic_auth` diff --git a/docs/source/index.rst b/docs/source/index.rst index 31cc980..3973f1c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,7 +3,11 @@ Media Gateway's documentation ============================= -`Media Gateway `_ is an application that provides a functionality to forward `Savant `_ messages from one `ZeroMQ `_ instance to another. The media gateway consists of two parts - a server and client. The client reads messages from the source ZeroMQ instance and sends them to the server via HTTP/HTTPS. The server writes received messages to the target ZeroMQ instance. +Media Gateway is a service that provides a secure bridge (with encryption and authentication) between `Savant `_ edge and cloud components by forwarding messages from one `ZeroMQ `_ socket to another. The media gateway consists of two parts - a server and client. The client reads messages from the source ZeroMQ socket and sends them to the server via HTTP/HTTPS. The server writes received messages to the target ZeroMQ socket. + +.. image:: _static/media-gateway.png + :width: 1281 + :align: center * **Repository**: https://github.com/insight-platform/MediaGateway * **License**: Business Source License 1.1 @@ -11,33 +15,40 @@ Media Gateway's documentation Features -------- * HTTPS -* basic authentication -* client certificate authentication -* FPS statistics logging (by frame or timestamp period) +* HTTP Basic authentication with `etcd `__ as a credentials storage +* X509 client certificate authentication .. toctree:: :maxdepth: 1 :caption: Getting Started - getting_started/0_configuration - getting_started/1_deployment + getting_started/0_deployment + getting_started/1_secure_communication .. toctree:: :maxdepth: 1 :caption: Reference - reference/0_api + reference/0_configuration + reference/1_api + +.. _cookbook: .. toctree:: :maxdepth: 1 :caption: Cookbook - cookbook/0_usage_example - cookbook/1_tls + cookbook/0_https + cookbook/1_certificate_auth cookbook/2_basic_auth + cookbook/3_usage_example + +.. _miscellaneous: .. toctree:: :maxdepth: 1 :caption: Miscellaneous miscellaneous/0_troubleshooting + miscellaneous/1_benchmarking + miscellaneous/2_caching diff --git a/docs/source/miscellaneous/0_troubleshooting.rst b/docs/source/miscellaneous/0_troubleshooting.rst index 172fe2b..5df3d57 100644 --- a/docs/source/miscellaneous/0_troubleshooting.rst +++ b/docs/source/miscellaneous/0_troubleshooting.rst @@ -6,4 +6,4 @@ This guide explains how to fix issues you might encounter when using Media Gatew High CPU usage by Media Gateway client -------------------------------------- -Media Gateway client uses one of strategies to wait for data while reading from ZeroMQ socket. If the chosen strategy does not fit, the client checks for new data too often causing high CPU usage. Try to use another strategy or increase a timeout (see :ref:`wait strategy configuration `). +Media Gateway client uses one of strategies to wait for data while reading from ZeroMQ socket. If the chosen strategy does not fit, the client checks for new data too often causing high CPU usage. Try to use another strategy or increase a timeout (see :ref:`wait strategy configuration `). diff --git a/docs/source/miscellaneous/1_benchmarking.rst b/docs/source/miscellaneous/1_benchmarking.rst new file mode 100644 index 0000000..c42d2d8 --- /dev/null +++ b/docs/source/miscellaneous/1_benchmarking.rst @@ -0,0 +1,50 @@ +Benchmarking +============ + +Media Gateway can collect statistics and report it to logs. Statistics includes FPS (frames per second) metric which can be used for benchmarking. FPS metric can be calculated by a timestamp period or frame period. + +Statistics by timestamp period +------------------------------ + +Statics calculation by a timestamp period means that data is collected during the timeout and by the end of the timeout metrics are calculated from the collected data. + +.. code-block:: + :caption: statistics logs + + [2024-08-02T08:38:52Z INFO savant_core::pipeline::stats] Time-based FPS counter triggered: FPS = 2536.46, OPS = 0.00, frame_delta = 2539, time_delta = 1.001 sec , period=[1722587931826, 1722587932827] ms + [2024-08-02T08:38:53Z INFO savant_core::pipeline::stats] Time-based FPS counter triggered: FPS = 2501.00, OPS = 0.00, frame_delta = 2501, time_delta = 1 sec , period=[1722587932827, 1722587933827] ms + [2024-08-02T08:38:54Z INFO savant_core::pipeline::stats] Time-based FPS counter triggered: FPS = 2530.00, OPS = 0.00, frame_delta = 2530, time_delta = 1 sec , period=[1722587933827, 1722587934827] ms + +Statistics by frame period +-------------------------- + +Statics calculation by a frame period means that data is collected until the number of collected frames reaches the specified number and then metrics are calculated from the collected data. + +.. code-block:: + :caption: statistics logs + + [2024-08-02T08:37:56Z INFO savant_core::pipeline::stats] Frame-based FPS counter triggered: FPS = 2386.63, OPS = 0.00, frame_delta = 1000, time_delta = 0.419 sec, period=[1722587875889, 1722587876308] ms + [2024-08-02T08:37:56Z INFO savant_core::pipeline::stats] Frame-based FPS counter triggered: FPS = 2427.18, OPS = 0.00, frame_delta = 1000, time_delta = 0.412 sec, period=[1722587876308, 1722587876720] ms + [2024-08-02T08:37:57Z INFO savant_core::pipeline::stats] Frame-based FPS counter triggered: FPS = 2487.56, OPS = 0.00, frame_delta = 1000, time_delta = 0.402 sec, period=[1722587876720, 1722587877122] ms + +Configuring Media Gateway +------------------------- + +Both server and client can be configured to enable statistics. See :doc:`/reference/0_configuration`. + +.. code-block:: json + :caption: by a timestamp period + + "statistics": { + "timestamp_period": { + "secs": 1, + "nanos": 0 + } + } + +.. code-block:: json + :caption: by a frame period + + "statistics": { + "frame_period": 1000 + } diff --git a/docs/source/miscellaneous/2_caching.rst b/docs/source/miscellaneous/2_caching.rst new file mode 100644 index 0000000..136edd9 --- /dev/null +++ b/docs/source/miscellaneous/2_caching.rst @@ -0,0 +1,52 @@ +Caching +======= + +Media Gateway server uses several caches to speed up processing. + +User data cache +--------------- + +User data such as usernames, passwords and allowed routing labels are required for HTTP Basic authentication and authorization and stored in `etcd` (see :doc:`/cookbook/2_basic_auth`). User data is cached and automatically reloaded from `etcd` when its checksum is changed. + +Authentication cache +-------------------- + +A separate cache is used to decrease cryptographic costs for HTTP Basic authentication (see :doc:`/cookbook/2_basic_auth`). It holds results of authentication checks which are used for subsequent requests if provided credentials are the same and the user's password is not changed. + +Cache configuration +------------------- + +Caches use LRU eviction policy. The maximum number of entries the cache may contain is specified in the configuration. The cache might be inefficient if its size is not suitable. To detect such cases cache usage tracking is supported. Cache usage statistics includes evicted entries per period metric. If the metric value exceeds the threshold a warning is reported to logs. + +.. code-block:: + :caption: logs for the exceeded evicted entries threshold in user data cache + + [2024-08-05T04:40:20Z WARN media_gateway_server::server::service::cache] Evicted entities threshold is exceeded for user: 7 per 60.001 seconds + +.. code-block:: + :caption: logs for the exceeded evicted entries threshold in authentication cache + + [2024-08-05T04:40:20Z WARN media_gateway_server::server::service::cache] Evicted entities threshold is exceeded for auth: 14 per 60.001 seconds + +The period and the threshold are specified in the configuration (see :ref:`cache configuration `). + +.. code-block:: json + :caption: cache configuration without cache usage + + { + "size": 10, + } + +.. code-block:: json + :caption: cache configuration with cache usage + + { + "size": 10, + "usage": { + "period": { + "secs": 60, + "nanos": 0 + }, + "evicted_threshold": 10 + } + } diff --git a/docs/source/reference/0_configuration.rst b/docs/source/reference/0_configuration.rst new file mode 100644 index 0000000..94464b1 --- /dev/null +++ b/docs/source/reference/0_configuration.rst @@ -0,0 +1,482 @@ +Configuration +============= + +.. _server configuration: + +Server +------ + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - ip + - A string representation of an IP address or a host name to bind to. Both IPv4 or IPv6 are supported. If the host name is specified the server is bound to both the IPv4 and IPv6 addresses that result from a DNS lookup. + - yes + * - port + - A port to bind to. + - yes + * - out_stream + - A configuration how to write to ZeroMQ socket. See :ref:`sink configuration `. + - yes + * - tls + - TLS settings. See :ref:`server TLS settings configuration `. + - no + * - auth + - Authentication settings. See :ref:`server authentication settings configuration `. + - no + * - statistics + - Statistics settings. See :ref:`statistics configuration `. + - no + +.. _client configuration: + +Client +------ + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - ip + - A string representation of an IP address or a host name to bind to. Both IPv4 or IPv6 are supported. If the host name is specified the server is bound to both the IPv4 and IPv6 addresses that result from a DNS lookup. + - yes + * - port + - A port to bind to. + - yes + * - url + - Media Gateway server URL. + - yes + * - retry_strategy + - A strategy how to retry to send a message to Media Gateway server. The default value is an exponential strategy with the initial delay 1 ms, the maximum delay 1 sec and the multiplier 2. See :ref:`retry strategy configuration `. + - no + * - in_stream + - A configuration how to read from ZeroMQ socket. See :ref:`source configuration `. + - yes + * - wait_strategy + - A strategy how to wait for data from ZeroMQ socket. The default value is 1 ms sleep strategy. See :ref:`wait strategy configuration `. + - no + * - tls + - TLS settings. See :ref:`client TLS settings configuration `. + - no + * - auth + - Authentication settings. See :ref:`client authentication settings configuration `. + - no + * - statistics + - Statistics settings. See :ref:`statistics configuration `. + - no + +Subconfigurations +----------------- + +.. _duration configuration: + +Duration +^^^^^^^^ + +A duration is specified as a composition of a whole number of seconds and a fractional part represented in nanoseconds. + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - secs + - Duration seconds + - yes + * - nanos + - Duration nanoseconds + - yes + +.. _sink configuration: + +Sink +^^^^ + +A configuration how to write to ZeroMQ socket. + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + - Default value + * - url + - The URL in Savant ZMQ format. + - yes + - + * - send_timeout + - The timeout for sending data. The default value is ok for most cases. See :ref:`duration configuration `. + - no + - ``{"secs": 1, "nanos": 0}`` + * - send_retries + - The number of retries for sending data. The default value is ok for most cases. For unstable or busy recipients the value might be increased. + - no + - ``3`` + * - receive_timeout + - The timeout for receiving data. Valid only for ``dealer`` and ``req`` socket types. The default value is ok for most cases. See :ref:`duration configuration `. + - no + - ``{"secs": 1, "nanos": 0}`` + * - receive_retries + - The number of retries for receiving data (crucial for req/rep communication). The default value is ok for most cases. For unstable or busy senders the value might be increased. + - yes + - ``3`` + * - send_hwm + - The high-water mark for sending data. This parameter is used to control backpressure. Consult with ZeroMQ documentation for more details. + - no + - ``1000`` + * - receive_hwm + - The high-water mark for receiving data. This parameter is used to control backpressure. Consult with ZeroMQ documentation for more details. Change only if you are using req/rep communication. + - no + - ``1000`` + * - fix_ipc_permissions + - UNIX file permissions for IPC sockets. + - no + - + +.. _source configuration: + +Source +^^^^^^ + +A configuration how to read from ZeroMQ socket. + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - url + - The URL in Savant ZMQ format. + - yes + * - receive_timeout + - The timeout for receiving data. The default value is ok for most cases. See :ref:`duration configuration `. + - yes + * - receive_hwm + - The high-water mark for receiving data. This parameter is used to control backpressure. Consult with ZeroMQ documentation for more details. + - yes + * - topic_prefix_spec + - The topic prefix specification for receiving data. Possible values are ``{"none": null}``, ``{"source_id": "topic"}`` or ``{"prefix": "prefix"}`` + - yes + * - source_cache_size + - The size of the whitelist cache used only when prefix-based filtering is in use. This parameter is used to quickly check if the source ID is in the whitelist or must be checked. + - yes + * - fix_ipc_permissions + - UNIX file permissions for IPC sockets. + - no + * - inflight_ops + - The maximum number of read messages for non-blocking mode. + - yes + +.. _retry strategy configuration: + +Retry strategy +^^^^^^^^^^^^^^ + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - exponential + - Settings for exponential retry strategy. + - true + +Exponential retry strategy +"""""""""""""""""""""""""" + +The strategy executes next attempt after the delay which is calculated for each attempt. The delay for the first attempt is the initial delay. The delay for subsequent attempts is calculated as maximum between multiplication of last attempt delay by the specified multiplier and the maximum delay. + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - initial_delay + - The delay with nanosecond precision for the first attempt. See :ref:`duration configuration `. + - true + * - maximum_delay + - The maximum delay with nanosecond precision. See :ref:`duration configuration `. + - true + * - multiplier + - A multiplier to calculate the delay for next attempt by multiplying last attempt delay. The minimum value is 2. + - true + +.. _wait strategy configuration: + +Wait strategy +^^^^^^^^^^^^^ + +Yield wait strategy +""""""""""""""""""" + +A strategy that pauses execution using `Tokio yield_now `__. The strategy does not have configuration parameters and is specified as the string ``yield``. + +Sleep wait strategy +""""""""""""""""""" + +A strategy that pauses execution using `tokio_timerfd sleep `__ for the specified duration with nanosecond precision. + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - sleep + - A duration with nanosecond precision to sleep. See :ref:`duration configuration `. + - true + +.. _cache configuration: + +Cache +^^^^^ + +Cache configuration. + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - size + - The maximum number of entries. + - yes + * - usage + - Cache usage settings. See below. + - no + +Cache usage +""""""""""" + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - period + - A period with millisecond precision to collect data before calculating usage metrics. See :ref:`duration configuration `. + - yes + * - evicted_threshold + - A number of cache entries allowed for eviction for the period. + - yes + +.. _identity configuration: + +Identity +^^^^^^^^ + +An identity represents a private key and X509 certificate. + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - certificate + - A path to the file with a chain of PEM encoded X509 certificates, the first being the leaf certificate, and the remainder forming the chain of certificates up to and including the trusted root certificate. + - yes + * - key + - A path to the file with a PEM encoded private key. + - yes + +.. _credentials configuration: + +Credentials +^^^^^^^^^^^ + +Credentials represent a username and password. + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - username + - A username. + - yes + * - password + - A password. + - yes + +.. _client tls settings configuration: + +Client TLS settings +^^^^^^^^^^^^^^^^^^^ + +TLS settings used by the client to connect to the server. + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - root_certificate + - A path to the file with a PEM encoded X509 certificate against which to verify the server's TLS certificate. The file might contain a chain of CA certificates, the first being the leaf certificate, and the remainder forming the chain of certificates up to and including the trusted root certificate. For the server with the self-signed certificate the file contains the certificate itself. + - no + * - identity + - An identity to be presented to the server for client certificate authentication. See :ref:`identity configuration `. + - no + +.. _server tls settings configuration: + +Server TLS settings +^^^^^^^^^^^^^^^^^^^ + +TLS settings for the server. + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - identity + - An identity to be presented to peers. See :ref:`identity configuration `. + - yes + * - peers + - Settings for peer certificate verification. See below. + - no + +**Peer certificate verification settings** + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - lookup_hash_directory + - A directory with allowed certificates and CRLs. See `X509_LOOKUP_hash_dir method `_ for more details. + - yes + * - crl_enabled + - ``true`` if CRLs must be checked during certificate verification, ``false`` otherwise. + - yes + +.. _client authentication settings configuration: + +Client authentication settings +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Authentication settings for the client. + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - basic + - Credentials used to connect to Media Gateway server. See :ref:`credentials configuration `. + - true + +.. _server authentication settings configuration: + +Server authentication settings +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Authentication settings for the server. + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - basic + - Settings for HTTP Basic authentication. See below. + - true + +**HTTP Basic authentication settings** + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - etcd + - etcd configuration. See below. + - true + * - cache + - Settings for authentication cache. See :ref:`cache configuration section `. + - true + +**etcd** + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - urls + - A list of etcd server endpoints to connect to. + - true + * - tls + - TLS options to use while connecting to etcd servers. See :ref:`client TLS settings configuration `. + - false + * - credentials + - Credentials for basic authentication in etcd. See :ref:`credentials configuration `. + - false + * - path + - The path of the hierarchically organized directories (as in a standard filesystem) for the stored key/value(-s). + - true + * - data_format + - The format of the data stored in etcd. Possible values are ``json``, ``yaml``. + - true + * - connect_timeout + - A timeout with millisecond precision for each request to etcd. See :ref:`duration configuration `. + - true + * - lease_timeout + - A timeout with millisecond precision to hold keys if the etcd server does not receive a keepAlive. See :ref:`duration configuration `. + - true + * - cache + - Settings for user data cache. See :ref:`cache configuration section `. + - true + +.. _statistics configuration: + +Statistics +^^^^^^^^^^ + +Statistics settings. + +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Mandatory + * - frame_period + - A number of frames to collect before calculating statistics metrics. + - no* + * - timestamp_period + - A period with millisecond precision to collect data before calculating statistics metrics. See :ref:`duration configuration `. + - no* + +\* exactly one of ``frame_period`` and ``timestamp_period`` must be specified. + +Environment variables in configuration files +-------------------------------------------- + +Environment variables can be used in the configuration file. The syntax is ``${VAR_NAME:-default_value}``. If the environment variable is not set, the default value will be used. + +Examples +-------- +Examples of configuration files can be found `here `_. + diff --git a/docs/source/reference/0_api.rst b/docs/source/reference/1_api.rst similarity index 100% rename from docs/source/reference/0_api.rst rename to docs/source/reference/1_api.rst diff --git a/samples/configuration/server/basic_auth_config.json b/samples/configuration/server/basic_auth_config.json index 4125685..3692d07 100644 --- a/samples/configuration/server/basic_auth_config.json +++ b/samples/configuration/server/basic_auth_config.json @@ -4,7 +4,9 @@ "auth": { "basic": { "etcd": { - "urls": "http://etcd:2379", + "urls": [ + "https://etcd:2379" + ], "tls": { "root_certificate": "etcd-ca.crt", "identity": { diff --git a/samples/configuration/server/certificate_auth_config.json b/samples/configuration/server/certificate_auth_config.json index 0f84fd9..bb35f70 100644 --- a/samples/configuration/server/certificate_auth_config.json +++ b/samples/configuration/server/certificate_auth_config.json @@ -6,7 +6,7 @@ "certificate": "server.crt", "key": "server.key" }, - "peer": { + "peers": { "lookup_hash_directory": "lookup-hash-dir", "crl_enabled": true }