Merge branch 'feat/tls-on-gateway'

.env.example

@@ -3,9 +3,7 @@
 IC_URL=https://icp0.io
 # gateway to canister polling interval in milliseconds
 POLLING_INTERVAL=200
-
-#### NGINX reverse proxy
-# the public port where NGINX reverse proxy will listen
+# the public port where the gateway will listen
 LISTEN_PORT=443
 # the public domain name of the server
 DOMAIN_NAME=example.com

Dockerfile

@@ -10,7 +10,11 @@ COPY . .
 RUN cargo chef prepare --recipe-path recipe.json
 
 ### build the IC WS Gateway
-FROM deps AS builder 
+FROM deps AS builder
+
+RUN apt update
+RUN apt install -y pkg-config libssl-dev
+
 COPY --from=planner /ic-ws-gateway/recipe.json recipe.json
 # Build dependencies - this is the caching Docker layer!
 RUN cargo chef cook --release --recipe-path recipe.json

README.md

@@ -37,39 +37,28 @@ A [Dockerfile](./Dockerfile) is provided, together with a [docker-compose.yml](.
     ```
     cp .env.example .env
     ```
-    You can ignore the `NGINX reverse proxy` variables section for now.
-2. Run the gateway:
+2. The docker-compose.yml file is configured to make the gateway run with TLS enabled. For this, you need a public domain (that you will put in the `DOMAIN_NAME` environment variable) and a TLS certificate for that domain. See [Obtain a TLS certificate](#obtain-a-tls-certificate) for more details.
+3. Open the `443` port (or the port that you set in the `LISTEN_PORT` environment variable) on your server and make it reachable from the Internet.
+4. Run the gateway:
     ```
     docker compose up
     ```
-3. The Gateway will print its principal in the container logs, just as explained above.
-4. Whenever you want to rebuild the gateway image, run:
+5. The Gateway will print its principal in the container logs, just as explained above.
+6. Whenever you want to rebuild the gateway image, run:
     ```
     docker compose up --build
     ```
 
-### Adding NGINX as a reverse proxy (with TLS)
-
-It's possible to run the gateway behind an [NGINX reverse proxy](https://docs.nginx.com/nginx/admin-guide/web-server/reverse-proxy/), which also enables to run the WebSocket server over TLS.
+### Obtain a TLS certificate
 
-1. Obtain a domain name and point it to the server where you are running the gateway.
-2. Set the environment variables:
-
-    ```
-    cp .env.example .env
-    ```
-    In this case you have to set the `NGINX reverse proxy` variables.
+1. Buy a domain name and point it to the server where you are running the gateway.
+2. Make sure the `.env` file is configured with the correct domain name, see above.
 3. Obtain an SSL certificate for your domain:
     ```
     ./scripts/certbot_certonly.sh
     ```
     This will guide you in obtaining a certificate using [Certbot](https://certbot.eff.org/) in [Standalone mode](https://eff-certbot.readthedocs.io/en/stable/using.html#standalone).
     > Make sure you have port `80` open on your server and reachable from the Internet, otherwise certbot will not be able to verify your domain. Port `80` is used only for the certificate generation and can be closed afterwards.
-4. Open the `443` port (or the port that you set in the `LISTEN_PORT` environment variable) on your server and make it reachable from the Internet.
-5. Run the gateway:
-    ```
-    docker compose --profile nginx up
-    ```
 
 To renew the SSL certificate, you can run the same command as above:
 ```

docker-compose.yml

@@ -6,39 +6,26 @@ services:
         image: ic-websocket-gateway
         container_name: ic-websocket-gateway
         restart: unless-stopped
+        ports:
+            - ${LISTEN_PORT}:443
         command:
             [
                 "--gateway-address",
-                "0.0.0.0:8080",
+                "0.0.0.0:443",
                 "--subnet-url",
                 "${IC_URL}",
                 "--polling-interval",
                 "${POLLING_INTERVAL}",
+                "--tls-certificate-pem-path",
+                "/ic-ws-gateway/data/certs/live/${DOMAIN_NAME}/fullchain.pem",
+                "--tls-certificate-key-pem-path",
+                "/ic-ws-gateway/data/certs/live/${DOMAIN_NAME}/privkey.pem",
             ]
         volumes:
             - ./volumes/ic-ws-gateway/data:/ic-ws-gateway/data
         networks:
             - ic-ws-gateway-network
 
-    nginx_reverse_proxy:
-        image: nginx:latest
-        container_name: nginx-reverse-proxy
-        profiles:
-            - nginx
-        restart: always
-        ports:
-            - ${LISTEN_PORT}:443
-        environment:
-            - GATEWAY_ADDRESS=ic-websocket-gateway:8080
-            - DOMAIN_NAME=${DOMAIN_NAME}
-        volumes:
-            - ./nginx/nginx.conf.template:/etc/nginx/templates/default.conf.template
-            - ./volumes/nginx/certs:/etc/letsencrypt
-        depends_on:
-            - ic_websocket_gateway
-        networks:
-            - ic-ws-gateway-network
-
 networks:
     ic-ws-gateway-network:
         name: ic-ws-gateway-network

src/ic-websocket-gateway/Cargo.toml

@@ -12,6 +12,8 @@ ic-cdk-macros = "0.7.0"
 serde = "1.0.176"
 serde_cbor = "0.11.2"
 tokio = { version = "1.29.1", features = ["full"] }
+tokio-native-tls = "0.3.1"
+native-tls = "0.2.11"
 serde_bytes = "0.11.12"
 tokio-tungstenite = "0.20.0"
 ed25519-compact = "2.0.4"

src/ic-websocket-gateway/src/client_connection_handler.rs

@@ -1,19 +1,22 @@
-use std::sync::Arc;
+use std::{fs, sync::Arc};
 
 use futures_util::{stream::SplitSink, SinkExt, StreamExt, TryStreamExt};
 use ic_agent::Agent;
+use native_tls::Identity;
 use serde_cbor::to_vec;
 use tokio::{
+    io::{AsyncRead, AsyncWrite},
     net::{TcpListener, TcpStream},
     select,
     sync::mpsc::{self, Sender},
 };
+use tokio_native_tls::{TlsAcceptor, TlsStream};
 use tokio_tungstenite::{
     accept_async,
     tungstenite::{Error, Message},
     WebSocketStream,
 };
-use tracing::{error, info, span, warn, Instrument, Level};
+use tracing::{debug, error, info, span, warn, Instrument, Level};
 
 use crate::{
     canister_methods::{self, CanisterWsOpenResultValue},
@@ -44,9 +47,21 @@ pub enum IcWsError {
     WsError(String),
 }
 
+/// Possible TCP streams.
+enum CustomTcpStream {
+    Tcp(TcpStream),
+    TcpWithTls(TlsStream<TcpStream>),
+}
+
+pub struct TlsConfig {
+    pub certificate_pem_path: String,
+    pub certificate_key_pem_path: String,
+}
+
 pub struct WsConnectionsHandler {
     // listener of incoming TCP connections
     listener: TcpListener,
+    tls_acceptor: Option<TlsAcceptor>,
     agent: Arc<Agent>,
     client_connection_handler_tx: Sender<WsConnectionState>,
     // needed to know which gateway_session to delete in case of error or WS closed
@@ -58,12 +73,31 @@ impl WsConnectionsHandler {
         gateway_address: &str,
         agent: Arc<Agent>,
         client_connection_handler_tx: Sender<WsConnectionState>,
+        tls_config: Option<TlsConfig>,
     ) -> Self {
         let listener = TcpListener::bind(&gateway_address)
             .await
             .expect("Can't listen");
+        let mut tls_acceptor = None;
+        if let Some(tls_config) = tls_config {
+            let chain = fs::read(tls_config.certificate_pem_path).expect("Can't read certificate");
+            let privkey =
+                fs::read(tls_config.certificate_key_pem_path).expect("Can't read private key");
+            let tls_identity =
+                Identity::from_pkcs8(&chain, &privkey).expect("Can't create a TLS identity");
+            let acceptor = TlsAcceptor::from(
+                native_tls::TlsAcceptor::builder(tls_identity)
+                    .build()
+                    .expect("Can't create a TLS acceptor from the TLS identity"),
+            );
+            tls_acceptor = Some(acceptor);
+            info!("TLS enabled");
+        } else {
+            info!("TLS disabled");
+        }
         Self {
             listener,
+            tls_acceptor,
             agent,
             client_connection_handler_tx,
             next_client_id: 0,
@@ -72,6 +106,22 @@ impl WsConnectionsHandler {
 
     pub async fn listen_for_incoming_requests(&mut self) {
         while let Ok((stream, client_addr)) = self.listener.accept().await {
+            let stream = match self.tls_acceptor {
+                Some(ref acceptor) => {
+                    let tls_stream = acceptor.accept(stream).await;
+                    match tls_stream {
+                        Ok(tls_stream) => {
+                            debug!("TLS handshake successful");
+                            CustomTcpStream::TcpWithTls(tls_stream)
+                        },
+                        Err(e) => {
+                            error!("TLS handshake failed: {:?}", e);
+                            continue;
+                        },
+                    }
+                },
+                None => CustomTcpStream::Tcp(stream),
+            };
             let agent_cl = Arc::clone(&self.agent);
             let client_connection_handler_tx_cl = self.client_connection_handler_tx.clone();
             // spawn a connection handler task for each incoming client connection
@@ -90,7 +140,14 @@ impl WsConnectionsHandler {
                         client_connection_handler_tx_cl,
                     );
                     info!("Spawned new connection handler");
-                    client_connection_handler.handle_stream(stream).await;
+                    match stream {
+                        CustomTcpStream::Tcp(stream) => {
+                            client_connection_handler.handle_stream(stream).await
+                        },
+                        CustomTcpStream::TcpWithTls(stream) => {
+                            client_connection_handler.handle_stream(stream).await
+                        },
+                    }
                 }
                 .instrument(span),
             );
@@ -116,7 +173,7 @@ impl ClientConnectionHandler {
             client_connection_handler_tx,
         }
     }
-    pub async fn handle_stream(&self, stream: TcpStream) {
+    pub async fn handle_stream<S: AsyncRead + AsyncWrite + Unpin>(&self, stream: S) {
         match accept_async(stream).await {
             Ok(ws_stream) => {
                 info!("Accepted WebSocket connection");
@@ -256,8 +313,8 @@ impl ClientConnectionHandler {
     }
 }
 
-async fn send_ws_message_to_client(
-    ws_write: &mut SplitSink<WebSocketStream<TcpStream>, Message>,
+async fn send_ws_message_to_client<S: AsyncRead + AsyncWrite + Unpin>(
+    ws_write: &mut SplitSink<WebSocketStream<S>, Message>,
     message: Message,
 ) {
     if let Err(e) = ws_write.send(message).await {

src/ic-websocket-gateway/src/gateway_server.rs

@@ -11,7 +11,7 @@ use crate::{
     canister_poller::{
         CanisterPoller, CertifiedMessage, PollerChannelsPollerEnds, PollerToClientChannelData,
     },
-    client_connection_handler::{WsConnectionState, WsConnectionsHandler},
+    client_connection_handler::{TlsConfig, WsConnectionState, WsConnectionsHandler},
 };
 
 /// contains the information needed by the WS Gateway to maintain the state of the WebSocket connection
@@ -101,16 +101,20 @@ impl GatewayServer {
         panic!("TODO: graceful shutdown");
     }
 
-    pub fn start_accepting_incoming_connections(&self) {
+    pub fn start_accepting_incoming_connections(&self, tls_config: Option<TlsConfig>) {
         // spawn a task which keeps listening for incoming client connections
         let gateway_address = self.address.clone();
         let agent = Arc::clone(&self.agent);
         let client_connection_handler_tx = self.client_connection_handler_tx.clone();
         info!("Start accepting incoming connections");
         tokio::spawn(async move {
-            let mut ws_connections_hanlders =
-                WsConnectionsHandler::new(&gateway_address, agent, client_connection_handler_tx)
-                    .await;
+            let mut ws_connections_hanlders = WsConnectionsHandler::new(
+                &gateway_address,
+                agent,
+                client_connection_handler_tx,
+                tls_config,
+            )
+            .await;
             ws_connections_hanlders.listen_for_incoming_requests().await;
         });
     }