Add support for env vars (#42)

* Add support for env vars

.github/workflows/test.yml

@@ -14,9 +14,15 @@ jobs:
         run: ls -al
       - name: Change permission for test keys
         run: make permissions
+      - name: Install docker compose
+        run: |
+          curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          chmod +x /usr/local/bin/docker-compose
       - name: Build docker-compose stack
         run: docker-compose -f docker-compose.yml up -d
       - name: Check running containers
         run: docker ps -a
+      - name: Install lib required to run erlang/beam on ssh server
+        run: docker exec openssh-server apk add libstdc++
       - name: Run test suite
         run: docker exec control-node make test

.tool-versions

@@ -1,2 +1 @@
-elixir 1.10.4-otp-23
-erlang 23.0
+elixir 1.14.4

config/config.exs

@@ -0,0 +1,7 @@
+import Config
+
+# Configures Elixir's Logger
+config :logger, :console,
+  level: :info,
+  format: "$time $metadata[$level] $message\n",
+  metadata: :all

docker-compose.yml

@@ -6,15 +6,19 @@ services:
     environment:
       - PUID=1000
       - PGID=1000
+      - HOME=/app
       - TZ=Europe/Berlin
       - PUBLIC_KEY=ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDg+KMD7QAU+qtH3duwTHmBaJE/WUdiOwC87cqP5cL21 [email protected]
+      - SUDO_ACCESS=true
+      - PASSWORD_ACCESS=false
+      - USER_PASSWORD=password
     volumes:
       - ./test/fixture/config:/config
       - /tmp:/tmp
     ports:
       - 2222:2222
   control-node:
-    image: beamx/elixir:1.10.4-otp-23
+    image: elixir:1.14.5-otp-24
     container_name: control-node
     depends_on:
       - openssh-server

example/Dockerfile

@@ -1,96 +1,23 @@
-FROM linuxserver/openssh-server
+# Deprecated, not used anymore but was an attempt to
+# - install nix in docker
+# - install elixir using nix
+# - build elixir service
+# The issue is that the build fails to run because the during build the executables generated source their sh path from nix store i.e.
+#
+# for eg.  #!/nix/store/7mbf4p3z7pyvpha1fwv29n1cw0ms65wg-nix-something/sh
+#
+# so the release tar cannot be deployed to a target host since these paths do not exist there
 
-# -------------------------------------
-# Erlang (23.0) alpine Dockerfile
-# -------------------------------------
-ENV OTP_VERSION="23.1" \
-    REBAR3_VERSION="3.14.1"
+FROM  elixir:1.14.5-otp-24-alpine
 
-LABEL org.opencontainers.image.version=$OTP_VERSION
+RUN curl -L https://nixos.org/nix/install > /tmp/nix.install
 
-RUN set -xe \
-	&& OTP_DOWNLOAD_URL="https://github.com/erlang/otp/archive/OTP-${OTP_VERSION}.tar.gz" \
-	&& OTP_DOWNLOAD_SHA256="3591903503ea70be3ef1e42abc7a3e1f8af90f2c8989506bf9832175f091e6e5" \
-	&& REBAR3_DOWNLOAD_SHA256="b01275b6cbdb354dcf9ed686fce2b5f9dfdd58972ded9e970e31b9215a8521f2" \
-	&& apk add --no-cache --virtual .fetch-deps \
-		curl \
-		ca-certificates \
-	&& curl -fSL -o otp-src.tar.gz "$OTP_DOWNLOAD_URL" \
-	&& echo "$OTP_DOWNLOAD_SHA256  otp-src.tar.gz" | sha256sum -c - \
-	&& apk add --no-cache --virtual .build-deps \
-		dpkg-dev dpkg \
-		gcc \
-		g++ \
-		libc-dev \
-		linux-headers \
-		make \
-		autoconf \
-		ncurses-dev \
-		openssl-dev \
-		unixodbc-dev \
-		lksctp-tools-dev \
-		tar \
-	&& export ERL_TOP="/usr/src/otp_src_${OTP_VERSION%%@*}" \
-	&& mkdir -vp $ERL_TOP \
-	&& tar -xzf otp-src.tar.gz -C $ERL_TOP --strip-components=1 \
-	&& rm otp-src.tar.gz \
-	&& ( cd $ERL_TOP \
-	  && ./otp_build autoconf \
-	  && gnuArch="$(dpkg-architecture --query DEB_HOST_GNU_TYPE)" \
-	  && ./configure --build="$gnuArch" \
-	  && make -j$(getconf _NPROCESSORS_ONLN) \
-	  && make install ) \
-	&& rm -rf $ERL_TOP \
-	&& find /usr/local -regex '/usr/local/lib/erlang/\(lib/\|erts-\).*/\(man\|doc\|obj\|c_src\|emacs\|info\|examples\)' | xargs rm -rf \
-	&& find /usr/local -name src | xargs -r find | grep -v '\.hrl$' | xargs rm -v || true \
-	&& find /usr/local -name src | xargs -r find | xargs rmdir -vp || true \
-	&& scanelf --nobanner -E ET_EXEC -BF '%F' --recursive /usr/local | xargs -r strip --strip-all \
-	&& scanelf --nobanner -E ET_DYN -BF '%F' --recursive /usr/local | xargs -r strip --strip-unneeded \
-	&& runDeps="$( \
-		scanelf --needed --nobanner --format '%n#p' --recursive /usr/local \
-			| tr ',' '\n' \
-			| sort -u \
-			| awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \
-	)" \
-	&& REBAR3_DOWNLOAD_URL="https://github.com/erlang/rebar3/archive/${REBAR3_VERSION}.tar.gz" \
-	&& curl -fSL -o rebar3-src.tar.gz "$REBAR3_DOWNLOAD_URL" \
-	&& echo "${REBAR3_DOWNLOAD_SHA256}  rebar3-src.tar.gz" | sha256sum -c - \
-	&& mkdir -p /usr/src/rebar3-src \
-	&& tar -xzf rebar3-src.tar.gz -C /usr/src/rebar3-src --strip-components=1 \
-	&& rm rebar3-src.tar.gz \
-	&& cd /usr/src/rebar3-src \
-	&& HOME=$PWD ./bootstrap \
-	&& install -v ./rebar3 /usr/local/bin/ \
-	&& rm -rf /usr/src/rebar3-src \
-	&& apk add --virtual .erlang-rundeps \
-		$runDeps \
-		lksctp-tools \
-		ca-certificates \
-	&& apk del .fetch-deps .build-deps
+RUN sh /tmp/nix.install --daemon
 
-# -------------------------------------
-# ELixir (1.10.4) alpine Dockerfile
-# -------------------------------------
-# elixir expects utf8.
-ENV ELIXIR_VERSION="v1.10.4" \
-LANG=C.UTF-8
+RUN cp /etc/bashrc /root/.bashrc
 
-RUN set -xe \
-&& ELIXIR_DOWNLOAD_URL="https://github.com/elixir-lang/elixir/archive/${ELIXIR_VERSION}.tar.gz" \
-&& ELIXIR_DOWNLOAD_SHA256="8518c78f43fe36315dbe0d623823c2c1b7a025c114f3f4adbb48e04ef63f1d9f" \
-&& buildDeps=' \
-  ca-certificates \
-  curl \
-  make \
-  ' \
-&& apk add --no-cache --virtual .build-deps $buildDeps \
-&& curl -fSL -o elixir-src.tar.gz $ELIXIR_DOWNLOAD_URL \
-&& echo "$ELIXIR_DOWNLOAD_SHA256  elixir-src.tar.gz" | sha256sum -c - \
-&& mkdir -p /usr/local/src/elixir \
-&& tar -xzC /usr/local/src/elixir --strip-components=1 -f elixir-src.tar.gz \
-&& rm elixir-src.tar.gz \
-&& cd /usr/local/src/elixir \
-&& make install clean \
-&& apk del .build-deps
+ENV LANG=C.UTF-8
+USER root
+RUN /root/.nix-profile/bin/nix-env -iA nixpkgs.elixir_1_14
 
 CMD ["iex"]

example/README.md

@@ -0,0 +1,11 @@
+### How to rebuild service_app
+
+``` sh
+docker run -v ./service_app:/app -it elixir:1.14.5-otp-24-alpine sh
+
+$ cd /app && MIX_ENV=prod mix release --overwrite 
+$ exit
+
+cp service_app/_build/prod/service_app-0.1.0.tar.gz .
+sudo rm -rf service_app/_build
+```

example/service_app/mix.exs

@@ -5,7 +5,7 @@ defmodule ServiceApp.MixProject do
     [
       app: :service_app,
       version: "0.1.0",
-      elixir: "~> 1.10",
+      elixir: "~> 1.14",
       start_permanent: Mix.env() == :prod,
       deps: deps(),
       releases: releases()
@@ -23,6 +23,7 @@ defmodule ServiceApp.MixProject do
   def releases do
     [
       service_app: [
+        cookie: :"YFWZXAOJGTABHNGIT6KVAC2X6TEHA6WCIRDKSLFD6JZWRC4YHMMA====",
         include_erts: true,
         include_executables_for: [:unix],
         steps: [:assemble, :tar]

lib/control_node/host/ssh.ex

@@ -12,7 +12,8 @@ defmodule ControlNode.Host.SSH do
             private_key_dir: nil,
             conn: nil,
             hostname: nil,
-            via_ssh_agent: false
+            via_ssh_agent: false,
+            env_vars: nil
 
   @typedoc """
   SSH spec defines a host which shall be used to connect and deploy releases. Following fields should be
@@ -24,6 +25,7 @@ defmodule ControlNode.Host.SSH do
   * `:user` : SSH user name
   * `:private_key_dir` : Path to the `.ssh` folder (eg. `/home/user/.ssh`)
   * `via_ssh_agent`:  Use SSH Agent for authentication (default `false`)
+  * `env_vars`:  Define env vars (key, value) to be passed when running a command on the remote host
   """
   @type t :: %__MODULE__{
           host: binary,
@@ -127,14 +129,28 @@ defmodule ControlNode.Host.SSH do
   """
   @spec exec(t, list | binary) :: {:ok, ExecStatus.t()} | :failure | {:error, any}
   def exec(ssh_config, commands, skip_eof \\ false) do
-    do_exec(ssh_config, commands, skip_eof)
+    env_vars = to_shell_env_vars(ssh_config.env_vars, :inline)
+    Logger.debug("Processed env var", env_vars: env_vars)
+
+    script =
+      if env_vars != "" do
+        "#{env_vars} #{commands}"
+      else
+        commands
+      end
+
+    do_exec(ssh_config, script, skip_eof)
   end
 
   defp do_exec(ssh_config, commands, skip_eof) when is_list(commands) do
+    env_vars = to_shell_env_vars(ssh_config.env_vars, :export)
+    commands = env_vars <> Enum.join(commands, "; ")
     do_exec(ssh_config, Enum.join(commands, "; "), skip_eof)
   end
 
   defp do_exec(ssh_config, script, skip_eof) when is_binary(script) do
+    Logger.debug("Executing script on host", host: ssh_config.host, script: script)
+
     with {:ok, conn} <- connect_host(ssh_config),
          {:ok, channel_id} <- :ssh_connection.session_channel(conn, @timeout),
          :success <- :ssh_connection.exec(conn, channel_id, to_list(script), @timeout) do
@@ -147,6 +163,23 @@ defmodule ControlNode.Host.SSH do
     end
   end
 
+  @spec to_shell_env_vars(Map.t() | nil, :inline | :export) :: String.t()
+  defp to_shell_env_vars(nil, _), do: ""
+
+  defp to_shell_env_vars(env_vars, :inline) do
+    Enum.map(env_vars, fn {key, value} ->
+      "#{key}='#{value}'"
+    end)
+    |> Enum.join(" ")
+  end
+
+  defp to_shell_env_vars(env_vars, :export) do
+    Enum.map(env_vars, fn {key, value} ->
+      "export #{key}=#{value}"
+    end)
+    |> Enum.join("; ")
+  end
+
   defp get_exec_status(conn, status, skip_eof) do
     receive do
       {:ssh_cm, ^conn, {:closed, _channel_id}} ->

lib/control_node/release.ex

@@ -205,6 +205,8 @@ defmodule ControlNode.Release do
             # register node locally
             register_node(release_spec, host_spec, local_port)
 
+            # TODO: handle scenario where conneting to node fails because of
+            # incorrect cookie
             true = connect_and_monitor(release_spec, host_spec, cookie)
 
             release_state = %State{
@@ -215,6 +217,7 @@ defmodule ControlNode.Release do
               release_path: release_path(release_spec, host_spec)
             }
 
+            # Get the pid of the release service running on the remote host
             release_pid = rpc(release_state, release_spec, &:os.getpid/0, [])
 
             case get_version(release_spec, host_spec) do

mix.exs

@@ -7,8 +7,8 @@ defmodule ControlNode.MixProject do
     [
       name: "Control Node",
       app: :control_node,
-      version: "0.6.0",
-      elixir: "~> 1.10",
+      version: "0.7.0",
+      elixir: "~> 1.14",
       start_permanent: Mix.env() == :prod,
       elixirc_paths: elixirc_paths(Mix.env()),
       elixirc_options: [warnings_as_errors: true],

test/control_node/host/ssh_test.exs

@@ -1,5 +1,7 @@
 defmodule ControlNode.Host.SSHTest do
   use ExUnit.Case
+  require Logger
+  import ExUnit.CaptureLog
   import ControlNode.TestUtils
   alias ControlNode.Host.SSH
 
@@ -32,21 +34,21 @@ defmodule ControlNode.Host.SSHTest do
       on_exit(fn -> System.cmd("rm", ["-rf", "/tmp/config.txt"]) end)
     end
 
+    # NOTE: in order to have ENV vars the ssh server config has to be updated
+    # so we just assert that the correct command was sent to the server
     test "run list of commands on remote SSH server", %{ssh_config: ssh_config} do
-      assert {:ok, %SSH.ExecStatus{exit_status: :success}} =
-               SSH.exec(ssh_config, [
-                 "export ENV_TEST='hello world'",
-                 "echo $ENV_TEST > /tmp/config.txt"
-               ])
+      ssh_config = %SSH{ssh_config | env_vars: %{"ENV_TEST" => "hello world"}}
 
-      assert {:ok, "hello world\n"} = File.read("/tmp/config.txt")
+      assert capture_log(fn ->
+        assert {:ok, %SSH.ExecStatus{exit_status: :success}} = SSH.exec(ssh_config, ["echo $ENV_TEST > /tmp/config.txt"])
+      end) =~ "ENV_TEST='hello world' echo $ENV_TEST > /tmp/config.txt"
     end
 
     test "runs script on remote SSH server", %{ssh_config: ssh_config} do
       script = """
       #!/bin/sh
 
-      export ENV_TEST='hello world';
+      export ENV_TEST="hello world"
       echo $ENV_TEST > /tmp/config.txt
       """
 

test/control_node/namespace/deploy_test.exs

@@ -57,7 +57,7 @@ defmodule ControlNode.Namespace.DeployTest do
     test "transitions to [state: :initialize] after failing to terminate deployment" do
       data = build_workflow_data("0.1.0")
 
-      assert {:next_state, :initialize, data, next_actions} =
+      assert {:next_state, :initialize, _data, next_actions} =
                Namespace.Deploy.handle_event(:internal, {:ensure_running, "0.2.0"}, :ignore, data)
 
       assert next_actions == expected_actions("0.2.0")
@@ -66,7 +66,7 @@ defmodule ControlNode.Namespace.DeployTest do
     test "transitions to [state: :initialize] after failing to start deployment" do
       data = build_workflow_data("0.2.0")
 
-      assert {:next_state, :initialize, data, next_actions} =
+      assert {:next_state, :initialize, _data, next_actions} =
                Namespace.Deploy.handle_event(:internal, {:ensure_running, "0.3.0"}, :ignore, data)
 
       assert next_actions == expected_actions("0.3.0")