From d8e93aa4f7712c7389b0e03783be0b74c746e148 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Sun, 9 Oct 2022 23:33:48 -0400
Subject: [PATCH 001/109] TF 0.12+ and Cloudflare 2.10+ domain->zone_id

---
 terraform-modules/binderhub/main.tf      | 84 ++++++++++++------------
 terraform-modules/binderhub/variables.tf |  4 +-
 terraform-modules/binderhub/versions.tf  |  4 ++
 3 files changed, 49 insertions(+), 43 deletions(-)
 create mode 100644 terraform-modules/binderhub/versions.tf

diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index a96fcf4..9ba2618 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -4,109 +4,109 @@ resource "random_id" "token" {
 }
 
 data "template_file" "config" {
-  template = "${file("${path.module}/assets/config.yaml")}"
+  template = file("${path.module}/assets/config.yaml")
   vars = {
-    domain          = "${var.domain}"
-    TLS_name        = "${var.TLS_name}"
-    cpu_alloc       = "${var.cpu_alloc}"
-    mem_alloc       = "${var.mem_alloc_gb}"
-    docker_registry = "${var.docker_registry}"
-    docker_id       = "${var.docker_id}"
+    domain          = var.domain
+    TLS_name        = var.TLS_name
+    cpu_alloc       = var.cpu_alloc
+    mem_alloc       = var.mem_alloc_gb
+    docker_registry = var.docker_registry
+    docker_id       = var.docker_id
   }
 }
 
 data "template_file" "secrets" {
-  template = "${file("${path.module}/assets/secrets.yaml")}"
+  template = file("${path.module}/assets/secrets.yaml")
   vars = {
-    api_token       = "${random_id.token.0.hex}"
-    secret_token    = "${random_id.token.1.hex}"
-    docker_registry = "${var.docker_registry}"
-    docker_id       = "${var.docker_id}"
-    docker_password = "${var.docker_password}"
+    api_token       = random_id.token[0].hex
+    secret_token    = random_id.token[1].hex
+    docker_registry = var.docker_registry
+    docker_id       = var.docker_id
+    docker_password = var.docker_password
   }
 }
 
 data "template_file" "pv" {
-  template = "${file("${path.module}/assets/pv.yaml")}"
-  vars = { }
+  template = file("${path.module}/assets/pv.yaml")
+  vars     = {}
 }
 
 data "template_file" "nginx-ingress" {
-  template = "${file("${path.module}/assets/nginx-ingress.yaml")}"
-  vars = { }
+  template = file("${path.module}/assets/nginx-ingress.yaml")
+  vars     = {}
 }
 
 data "template_file" "production-binderhub-issuer" {
-  template = "${file("${path.module}/assets/production-binderhub-issuer.yaml")}"
-  vars = { 
-    domain    = "${var.domain}"
-    TLS_email = "${var.TLS_email}"
+  template = file("${path.module}/assets/production-binderhub-issuer.yaml")
+  vars = {
+    domain    = var.domain
+    TLS_email = var.TLS_email
   }
 }
 
 data "template_file" "staging-binderhub-issuer" {
-  template = "${file("${path.module}/assets/staging-binderhub-issuer.yaml")}"
-  vars = { 
-    domain    = "${var.domain}"
-    TLS_email = "${var.TLS_email}"
+  template = file("${path.module}/assets/staging-binderhub-issuer.yaml")
+  vars = {
+    domain    = var.domain
+    TLS_email = var.TLS_email
   }
 }
 
 data "template_file" "install-binderhub" {
-  template = "${file("${path.module}/assets/install-binderhub.sh")}"
+  template = file("${path.module}/assets/install-binderhub.sh")
   vars = {
-    binder_version  = "${var.binder_version}"
-    admin_user      = "${var.admin_user}"
-    docker_id       = "${var.docker_id}"
-    docker_password = "${var.docker_password}"
+    binder_version  = var.binder_version
+    admin_user      = var.admin_user
+    docker_id       = var.docker_id
+    docker_password = var.docker_password
   }
 }
 
 resource "null_resource" "remote_install" {
   connection {
-    user        = "${var.admin_user}"
-    host        = "${var.ip}"
+    user = var.admin_user
+    host = var.ip
   }
 
   provisioner "file" {
-    content     = "${data.template_file.config.rendered}"
+    content     = data.template_file.config.rendered
     destination = "/home/${var.admin_user}/config.yaml"
   }
 
   provisioner "file" {
-    content     = "${data.template_file.secrets.rendered}"
+    content     = data.template_file.secrets.rendered
     destination = "/home/${var.admin_user}/secrets.yaml"
   }
 
   provisioner "file" {
-    content     = "${data.template_file.pv.rendered}"
+    content     = data.template_file.pv.rendered
     destination = "/home/${var.admin_user}/pv.yaml"
   }
 
   provisioner "file" {
-    content     = "${data.template_file.nginx-ingress.rendered}"
+    content     = data.template_file.nginx-ingress.rendered
     destination = "/home/${var.admin_user}/nginx-ingress.yaml"
   }
 
-
   provisioner "file" {
-    content     = "${data.template_file.production-binderhub-issuer.rendered}"
+    content     = data.template_file.production-binderhub-issuer.rendered
     destination = "/home/${var.admin_user}/production-binderhub-issuer.yaml"
   }
-  
+
   provisioner "file" {
-    content     = "${data.template_file.staging-binderhub-issuer.rendered}"
+    content     = data.template_file.staging-binderhub-issuer.rendered
     destination = "/home/${var.admin_user}/staging-binderhub-issuer.yaml"
   }
 
   provisioner "file" {
-    content     = "${data.template_file.install-binderhub.rendered}"
+    content     = data.template_file.install-binderhub.rendered
     destination = "/home/${var.admin_user}/install-binderhub.sh"
   }
 
   provisioner "remote-exec" {
     inline = [
-      "bash /home/${var.admin_user}/install-binderhub.sh"
+      "bash /home/${var.admin_user}/install-binderhub.sh",
     ]
   }
 }
+
diff --git a/terraform-modules/binderhub/variables.tf b/terraform-modules/binderhub/variables.tf
index 948321b..880bbf9 100644
--- a/terraform-modules/binderhub/variables.tf
+++ b/terraform-modules/binderhub/variables.tf
@@ -1,6 +1,7 @@
 variable "ip" {
   description = "ip address of the master"
 }
+
 variable "domain" {
   description = "Domain name"
 }
@@ -31,7 +32,7 @@ variable "binder_version" {
 
 variable "docker_registry" {
   description = "Docker registry url"
-  default = "docker.io"
+  default     = "docker.io"
 }
 
 variable "docker_id" {
@@ -41,3 +42,4 @@ variable "docker_id" {
 variable "docker_password" {
   description = "Docker hub password"
 }
+
diff --git a/terraform-modules/binderhub/versions.tf b/terraform-modules/binderhub/versions.tf
new file mode 100644
index 0000000..ac97c6a
--- /dev/null
+++ b/terraform-modules/binderhub/versions.tf
@@ -0,0 +1,4 @@
+
+terraform {
+  required_version = ">= 0.12"
+}

From d798329a397124786aab2c82f77866eb3a9d0ba3 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Sun, 9 Oct 2022 23:35:48 -0400
Subject: [PATCH 002/109] TF 0.12+ and Cloudflare 2.10+ domain->zone_id

---
 terraform-modules/dns/cloudflare/main.tf      |  11 +-
 terraform-modules/dns/cloudflare/outputs.tf   |   5 +-
 terraform-modules/dns/cloudflare/variables.tf |   1 +
 terraform-modules/dns/cloudflare/versions.tf  |   4 +
 terraform-modules/providers/openstack/main.tf | 118 +++++++++---------
 .../providers/openstack/outputs.tf            |  11 +-
 .../providers/openstack/variables.tf          |   5 +-
 .../providers/openstack/versions.tf           |   4 +
 8 files changed, 88 insertions(+), 71 deletions(-)
 create mode 100644 terraform-modules/dns/cloudflare/versions.tf
 create mode 100644 terraform-modules/providers/openstack/versions.tf

diff --git a/terraform-modules/dns/cloudflare/main.tf b/terraform-modules/dns/cloudflare/main.tf
index d5fecd2..c9dc500 100644
--- a/terraform-modules/dns/cloudflare/main.tf
+++ b/terraform-modules/dns/cloudflare/main.tf
@@ -1,10 +1,11 @@
 provider "cloudflare" {
-  version = "<= 1.13.0"
+  version = "<= 2.10.1"
 }
 
 resource "cloudflare_record" "domain" {
-  domain = "${join(".", slice(split(".", var.domain), 1, length(split(".", var.domain))))}"
-  name   = "${element(split(".", var.domain), 0)}"
-  value  = "${var.public_ip}"
-  type   = "A"
+  zone_id = "ae42bc72343b3e27ab10ad833086b679"
+  name    = element(split(".", var.domain), 0)
+  value   = var.public_ip
+  type    = "A"
 }
+
diff --git a/terraform-modules/dns/cloudflare/outputs.tf b/terraform-modules/dns/cloudflare/outputs.tf
index f31f872..a59f138 100644
--- a/terraform-modules/dns/cloudflare/outputs.tf
+++ b/terraform-modules/dns/cloudflare/outputs.tf
@@ -1,3 +1,4 @@
-output domain {
-  value = "${var.domain}"
+output "domain" {
+  value = var.domain
 }
+
diff --git a/terraform-modules/dns/cloudflare/variables.tf b/terraform-modules/dns/cloudflare/variables.tf
index 79d30be..06d45a9 100644
--- a/terraform-modules/dns/cloudflare/variables.tf
+++ b/terraform-modules/dns/cloudflare/variables.tf
@@ -5,3 +5,4 @@ variable "domain" {
 variable "public_ip" {
   description = "Public address IP to associate"
 }
+
diff --git a/terraform-modules/dns/cloudflare/versions.tf b/terraform-modules/dns/cloudflare/versions.tf
new file mode 100644
index 0000000..ac97c6a
--- /dev/null
+++ b/terraform-modules/dns/cloudflare/versions.tf
@@ -0,0 +1,4 @@
+
+terraform {
+  required_version = ">= 0.12"
+}
diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index fa72b52..8d8e5fd 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -1,8 +1,9 @@
 provider "openstack" {
   version = "<= 1.24.0"
 }
+
 data "openstack_images_image_v2" "ubuntu" {
-  name = "${var.image_name}"
+  name        = var.image_name
   most_recent = true
 }
 
@@ -30,8 +31,8 @@ resource "openstack_compute_secgroup_v2" "secgroup_1" {
     ip_protocol = "udp"
     self        = true
   }
-	
-    rule {
+
+  rule {
     from_port   = -1
     to_port     = -1
     ip_protocol = "icmp"
@@ -79,32 +80,31 @@ locals {
 }
 
 resource "openstack_networking_subnet_v2" "subnet" {
-  count = "${1 - var.is_computecanada}"
+  count = 1 - var.is_computecanada
 
   name        = "subnet"
-  network_id  = "${openstack_networking_network_v2.network_1.id}"
+  network_id  = openstack_networking_network_v2.network_1[0].id
   ip_version  = 4
   cidr        = "10.0.1.0/24"
   enable_dhcp = true
 }
 
-
 resource "openstack_networking_network_v2" "network_1" {
-  count = "${1 - var.is_computecanada}"
+  count = 1 - var.is_computecanada
 
-  name = "${local.network_name}"
+  name = local.network_name
 }
 
 data "template_file" "kubeadm_master" {
-  template = "${file("${path.module}/../../../cloud-init/kubeadm/master.yaml")}"
+  template = file("${path.module}/../../../cloud-init/kubeadm/master.yaml")
 
-  vars {
-    admin_user      = "${var.admin_user}"
-    project_name    = "${var.project_name}"
-    nb_nodes 	    = "${var.nb_nodes}"
-    docker_registry = "${var.docker_registry}"
-    docker_id 	    = "${var.docker_id}"
-    docker_password = "${var.docker_password}"
+  vars = {
+    admin_user      = var.admin_user
+    project_name    = var.project_name
+    nb_nodes        = var.nb_nodes
+    docker_registry = var.docker_registry
+    docker_id       = var.docker_id
+    docker_password = var.docker_password
   }
 }
 
@@ -117,36 +117,36 @@ data "openstack_networking_network_v2" "int_network" {
 }
 
 resource "openstack_networking_router_v2" "router_1" {
-  count = "${1 - var.is_computecanada}"
+  count = 1 - var.is_computecanada
 
   name                = "${var.project_name}-router"
-  external_network_id = "${data.openstack_networking_network_v2.ext_network.id}"
+  external_network_id = data.openstack_networking_network_v2.ext_network.id
 }
 
 resource "openstack_networking_router_interface_v2" "router_interface_1" {
-  count = "${1 - var.is_computecanada}"
+  count = 1 - var.is_computecanada
 
-  router_id = "${openstack_networking_router_v2.router_1.id}"
-  subnet_id = "${openstack_networking_subnet_v2.subnet.id}"
+  router_id = openstack_networking_router_v2.router_1[0].id
+  subnet_id = openstack_networking_subnet_v2.subnet[0].id
 }
 
 data "template_file" "kubeadm_node" {
-  template = "${file("${path.module}/../../../cloud-init/kubeadm/node.yaml")}"
+  template = file("${path.module}/../../../cloud-init/kubeadm/node.yaml")
 
-  vars {
-    master_ip  = "${openstack_compute_instance_v2.master.network.0.fixed_ip_v4}"
-    admin_user = "${var.admin_user}"
-    docker_registry = "${var.docker_registry}"
-    docker_id 	    = "${var.docker_id}"
-    docker_password = "${var.docker_password}"
+  vars = {
+    master_ip       = openstack_compute_instance_v2.master.network[0].fixed_ip_v4
+    admin_user      = var.admin_user
+    docker_registry = var.docker_registry
+    docker_id       = var.docker_id
+    docker_password = var.docker_password
   }
 }
 
 data "template_file" "kubeadm_common" {
-  template = "${file("${path.module}/../../../cloud-init/kubeadm/common.yaml")}"
+  template = file("${path.module}/../../../cloud-init/kubeadm/common.yaml")
 
-  vars {
-    ssh_authorized_keys = "${indent(2, join("\n", formatlist("- %s", var.ssh_authorized_keys)))}"
+  vars = {
+    ssh_authorized_keys = indent(2, join("\n", formatlist("- %s", var.ssh_authorized_keys)))
   }
 }
 
@@ -155,14 +155,14 @@ data "template_cloudinit_config" "node_config" {
     filename     = "common.yaml"
     merge_type   = "list(append)+dict(recurse_array)+str()"
     content_type = "text/cloud-config"
-    content      = "${data.template_file.kubeadm_common.rendered}"
+    content      = data.template_file.kubeadm_common.rendered
   }
 
   part {
     filename     = "node.yaml"
     merge_type   = "list(append)+dict(recurse_array)+str()"
     content_type = "text/cloud-config"
-    content      = "${data.template_file.kubeadm_node.rendered}"
+    content      = data.template_file.kubeadm_node.rendered
   }
 }
 
@@ -171,71 +171,75 @@ data "template_cloudinit_config" "master_config" {
     filename     = "common.yaml"
     merge_type   = "list(append)+dict(recurse_array)+str()"
     content_type = "text/cloud-config"
-    content      = "${data.template_file.kubeadm_common.rendered}"
+    content      = data.template_file.kubeadm_common.rendered
   }
 
   part {
     filename     = "master.yaml"
     merge_type   = "list(append)+dict(recurse_array)+str()"
     content_type = "text/cloud-config"
-    content      = "${data.template_file.kubeadm_master.rendered}"
+    content      = data.template_file.kubeadm_master.rendered
   }
 }
 
 resource "openstack_compute_keypair_v2" "keypair" {
   name       = "${var.project_name}-keypair"
-  public_key = "${element(var.ssh_authorized_keys, 0)}"
+  public_key = element(var.ssh_authorized_keys, 0)
 }
 
 resource "openstack_compute_instance_v2" "master" {
   name            = "${var.project_name}-master"
-  flavor_name     = "${var.os_flavor_master}"
-  key_pair        = "${openstack_compute_keypair_v2.keypair.name}"
-  security_groups = ["${openstack_compute_secgroup_v2.secgroup_1.name}"]
-  user_data       = "${data.template_cloudinit_config.master_config.rendered}"
+  flavor_name     = var.os_flavor_master
+  key_pair        = openstack_compute_keypair_v2.keypair.name
+  security_groups = [openstack_compute_secgroup_v2.secgroup_1.name]
+  user_data       = data.template_cloudinit_config.master_config.rendered
 
   block_device {
-    uuid                  = "${data.openstack_images_image_v2.ubuntu.id}"
+    uuid                  = data.openstack_images_image_v2.ubuntu.id
     source_type           = "image"
-    volume_size           = "${var.instance_volume_size}"
+    volume_size           = var.instance_volume_size
     boot_index            = 0
     destination_type      = "volume"
     delete_on_termination = true
   }
 
-  network = {
-    name = "${var.is_computecanada ? data.openstack_networking_network_v2.int_network.name : local.network_name}"
+  network {
+    name = var.is_computecanada ? data.openstack_networking_network_v2.int_network.name : local.network_name
   }
 }
 
 resource "openstack_compute_instance_v2" "node" {
-  count    = "${var.nb_nodes}"
-  name     = "${var.project_name}-node${count.index + 1}"
+  count = var.nb_nodes
+  name  = "${var.project_name}-node${count.index + 1}"
 
-  flavor_name     = "${var.os_flavor_node}"
-  key_pair        = "${openstack_compute_keypair_v2.keypair.name}"
-  security_groups = ["${openstack_compute_secgroup_v2.secgroup_1.name}"]
-  user_data       = "${element(data.template_cloudinit_config.node_config.*.rendered, count.index)}"
+  flavor_name     = var.os_flavor_node
+  key_pair        = openstack_compute_keypair_v2.keypair.name
+  security_groups = [openstack_compute_secgroup_v2.secgroup_1.name]
+  user_data = element(
+    data.template_cloudinit_config.node_config.*.rendered,
+    count.index,
+  )
 
   block_device {
-    uuid                  = "${data.openstack_images_image_v2.ubuntu.id}"
+    uuid                  = data.openstack_images_image_v2.ubuntu.id
     source_type           = "image"
-    volume_size           = "${var.instance_volume_size}"
+    volume_size           = var.instance_volume_size
     boot_index            = 0
     destination_type      = "volume"
     delete_on_termination = true
   }
 
-  network = {
-    name = "${var.is_computecanada ? data.openstack_networking_network_v2.int_network.name : local.network_name}"
+  network {
+    name = var.is_computecanada ? data.openstack_networking_network_v2.int_network.name : local.network_name
   }
 }
 
 resource "openstack_networking_floatingip_v2" "fip_1" {
-	pool = "${data.openstack_networking_network_v2.ext_network.name}"
+  pool = data.openstack_networking_network_v2.ext_network.name
 }
 
 resource "openstack_compute_floatingip_associate_v2" "fip_1" {
-	floating_ip = "${openstack_networking_floatingip_v2.fip_1.address}"
-	instance_id = "${openstack_compute_instance_v2.master.id}"
+  floating_ip = openstack_networking_floatingip_v2.fip_1.address
+  instance_id = openstack_compute_instance_v2.master.id
 }
+
diff --git a/terraform-modules/providers/openstack/outputs.tf b/terraform-modules/providers/openstack/outputs.tf
index f428f8b..c01d24e 100644
--- a/terraform-modules/providers/openstack/outputs.tf
+++ b/terraform-modules/providers/openstack/outputs.tf
@@ -1,19 +1,20 @@
 output "public_ip" {
-	value = "${openstack_networking_floatingip_v2.fip_1.address}"
+  value = openstack_networking_floatingip_v2.fip_1.address
 }
 
 output "admin_user" {
-	value = "${var.admin_user}"
+  value = var.admin_user
 }
 
 output "docker_registry" {
-	value = "${var.docker_registry}"
+  value = var.docker_registry
 }
 
 output "docker_id" {
-	value = "${var.docker_id}"
+  value = var.docker_id
 }
 
 output "docker_password" {
-	value = "${var.docker_password}"
+  value = var.docker_password
 }
+
diff --git a/terraform-modules/providers/openstack/variables.tf b/terraform-modules/providers/openstack/variables.tf
index 5386455..73203a2 100644
--- a/terraform-modules/providers/openstack/variables.tf
+++ b/terraform-modules/providers/openstack/variables.tf
@@ -8,7 +8,7 @@ variable "instance_volume_size" {
 
 variable "ssh_authorized_keys" {
   description = "List of public SSH keys that can connect to the cluster"
-  type    = "list"
+  type        = list(string)
 }
 
 variable "os_flavor_node" {
@@ -44,7 +44,7 @@ variable "cc_private_network" {
 
 variable "docker_registry" {
   description = "Docker registry url"
-  default = "docker.io"
+  default     = "docker.io"
 }
 
 variable "docker_id" {
@@ -54,3 +54,4 @@ variable "docker_id" {
 variable "docker_password" {
   description = "Docker hub password"
 }
+
diff --git a/terraform-modules/providers/openstack/versions.tf b/terraform-modules/providers/openstack/versions.tf
new file mode 100644
index 0000000..ac97c6a
--- /dev/null
+++ b/terraform-modules/providers/openstack/versions.tf
@@ -0,0 +1,4 @@
+
+terraform {
+  required_version = ">= 0.12"
+}

From e2a52b831a404dedb2ff6c1b9ce3d11d706445e6 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Sun, 9 Oct 2022 23:42:21 -0400
Subject: [PATCH 003/109] Update DNS vars

---
 terraform-modules/dns/cloudflare/main.tf      | 4 ++--
 terraform-modules/dns/cloudflare/variables.tf | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/terraform-modules/dns/cloudflare/main.tf b/terraform-modules/dns/cloudflare/main.tf
index c9dc500..863bca2 100644
--- a/terraform-modules/dns/cloudflare/main.tf
+++ b/terraform-modules/dns/cloudflare/main.tf
@@ -3,8 +3,8 @@ provider "cloudflare" {
 }
 
 resource "cloudflare_record" "domain" {
-  zone_id = "ae42bc72343b3e27ab10ad833086b679"
-  name    = element(split(".", var.domain), 0)
+  zone_id = var.zone_id
+  name    = var.domain
   value   = var.public_ip
   type    = "A"
 }
diff --git a/terraform-modules/dns/cloudflare/variables.tf b/terraform-modules/dns/cloudflare/variables.tf
index 06d45a9..cd8d95f 100644
--- a/terraform-modules/dns/cloudflare/variables.tf
+++ b/terraform-modules/dns/cloudflare/variables.tf
@@ -2,6 +2,10 @@ variable "domain" {
   description = "Domain name"
 }
 
+variable "zone_id" {
+  description = "Cloudflare zone ID"
+}
+
 variable "public_ip" {
   description = "Public address IP to associate"
 }

From daca1a19b1b5514517ecbd459b297d3b4f91b66d Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Sun, 9 Oct 2022 23:51:46 -0400
Subject: [PATCH 004/109] Change var type proper bool

---
 terraform-modules/providers/openstack/main.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 8d8e5fd..178d1f8 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -109,11 +109,11 @@ data "template_file" "kubeadm_master" {
 }
 
 data "openstack_networking_network_v2" "ext_network" {
-  external = "True"
+  external = true
 }
 
 data "openstack_networking_network_v2" "int_network" {
-  external = "False"
+  external = false
 }
 
 resource "openstack_networking_router_v2" "router_1" {

From 0191b1d612dc5398634a71bd8d6bf88b3143aea6 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Mon, 10 Oct 2022 00:05:02 -0400
Subject: [PATCH 005/109] rm non-cc case from network

---
 terraform-modules/providers/openstack/main.tf | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 178d1f8..bed84f3 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -108,10 +108,6 @@ data "template_file" "kubeadm_master" {
   }
 }
 
-data "openstack_networking_network_v2" "ext_network" {
-  external = true
-}
-
 data "openstack_networking_network_v2" "int_network" {
   external = false
 }

From 7c0ac29e19f3cf4a91672dbacd0d1c6745a8fb0f Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Mon, 10 Oct 2022 00:07:09 -0400
Subject: [PATCH 006/109] rm non-cc bypass resources

---
 terraform-modules/providers/openstack/main.tf | 30 -------------------
 1 file changed, 30 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index bed84f3..af3d9ee 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -79,22 +79,6 @@ locals {
   network_name = "${var.project_name}-network"
 }
 
-resource "openstack_networking_subnet_v2" "subnet" {
-  count = 1 - var.is_computecanada
-
-  name        = "subnet"
-  network_id  = openstack_networking_network_v2.network_1[0].id
-  ip_version  = 4
-  cidr        = "10.0.1.0/24"
-  enable_dhcp = true
-}
-
-resource "openstack_networking_network_v2" "network_1" {
-  count = 1 - var.is_computecanada
-
-  name = local.network_name
-}
-
 data "template_file" "kubeadm_master" {
   template = file("${path.module}/../../../cloud-init/kubeadm/master.yaml")
 
@@ -112,20 +96,6 @@ data "openstack_networking_network_v2" "int_network" {
   external = false
 }
 
-resource "openstack_networking_router_v2" "router_1" {
-  count = 1 - var.is_computecanada
-
-  name                = "${var.project_name}-router"
-  external_network_id = data.openstack_networking_network_v2.ext_network.id
-}
-
-resource "openstack_networking_router_interface_v2" "router_interface_1" {
-  count = 1 - var.is_computecanada
-
-  router_id = openstack_networking_router_v2.router_1[0].id
-  subnet_id = openstack_networking_subnet_v2.subnet[0].id
-}
-
 data "template_file" "kubeadm_node" {
   template = file("${path.module}/../../../cloud-init/kubeadm/node.yaml")
 

From 275c0a503092c2483b577366182169cefe9c96db Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Mon, 10 Oct 2022 00:08:52 -0400
Subject: [PATCH 007/109] rm fip ext network rsc

---
 terraform-modules/providers/openstack/main.tf | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index af3d9ee..01791ee 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -200,9 +200,6 @@ resource "openstack_compute_instance_v2" "node" {
   }
 }
 
-resource "openstack_networking_floatingip_v2" "fip_1" {
-  pool = data.openstack_networking_network_v2.ext_network.name
-}
 
 resource "openstack_compute_floatingip_associate_v2" "fip_1" {
   floating_ip = openstack_networking_floatingip_v2.fip_1.address

From 6578575b839fa603c854028bbaa35f022cec7150 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Mon, 10 Oct 2022 00:23:44 -0400
Subject: [PATCH 008/109] change public network resources

---
 terraform-modules/providers/openstack/main.tf | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 01791ee..35ac794 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -79,6 +79,22 @@ locals {
   network_name = "${var.project_name}-network"
 }
 
+resource "openstack_networking_subnet_v2" "subnet" {
+  count = 1 - var.is_computecanada
+
+  name        = "subnet"
+  network_id  = openstack_networking_network_v2.network_1[0].id
+  ip_version  = 4
+  cidr        = "10.0.1.0/24"
+  enable_dhcp = true
+}
+
+resource "openstack_networking_network_v2" "network_1" {
+  count = 1 - var.is_computecanada
+
+  name = local.network_name
+}
+
 data "template_file" "kubeadm_master" {
   template = file("${path.module}/../../../cloud-init/kubeadm/master.yaml")
 
@@ -92,10 +108,32 @@ data "template_file" "kubeadm_master" {
   }
 }
 
+data "openstack_networking_network_v2" "ext_network" {
+  name = "public"
+}
+
+data "openstack_networking_subnet_ids_v2" "ext_subnets" {
+  network_id = data.openstack_networking_network_v2.ext_network.id
+}
+
 data "openstack_networking_network_v2" "int_network" {
   external = false
 }
 
+resource "openstack_networking_router_v2" "router_1" {
+  count = 1 - var.is_computecanada
+
+  name                = "${var.project_name}-router"
+  external_network_id = data.openstack_networking_network_v2.ext_network.id
+}
+
+resource "openstack_networking_router_interface_v2" "router_interface_1" {
+  count = 1 - var.is_computecanada
+
+  router_id = openstack_networking_router_v2.router_1[0].id
+  subnet_id = openstack_networking_subnet_v2.subnet[0].id
+}
+
 data "template_file" "kubeadm_node" {
   template = file("${path.module}/../../../cloud-init/kubeadm/node.yaml")
 
@@ -200,6 +238,10 @@ resource "openstack_compute_instance_v2" "node" {
   }
 }
 
+resource "openstack_networking_floatingip_v2" "fip_1" {
+  pool       = data.openstack_networking_network_v2.ext_network.name
+  subnet_ids = data.openstack_networking_subnet_ids_v2.ext_subnets.ids
+}
 
 resource "openstack_compute_floatingip_associate_v2" "fip_1" {
   floating_ip = openstack_networking_floatingip_v2.fip_1.address

From a398ca94bb16bc9ba33b94cb099190f84474c6fd Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Mon, 10 Oct 2022 00:25:08 -0400
Subject: [PATCH 009/109] remove unsupported data source

---
 terraform-modules/providers/openstack/main.tf | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 35ac794..75dbb21 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -112,10 +112,6 @@ data "openstack_networking_network_v2" "ext_network" {
   name = "public"
 }
 
-data "openstack_networking_subnet_ids_v2" "ext_subnets" {
-  network_id = data.openstack_networking_network_v2.ext_network.id
-}
-
 data "openstack_networking_network_v2" "int_network" {
   external = false
 }
@@ -240,7 +236,6 @@ resource "openstack_compute_instance_v2" "node" {
 
 resource "openstack_networking_floatingip_v2" "fip_1" {
   pool       = data.openstack_networking_network_v2.ext_network.name
-  subnet_ids = data.openstack_networking_subnet_ids_v2.ext_subnets.ids
 }
 
 resource "openstack_compute_floatingip_associate_v2" "fip_1" {

From c521d0a69afbe9a0b9054837cbcf2ac3e50aa843 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Mon, 10 Oct 2022 00:35:30 -0400
Subject: [PATCH 010/109] Sort out existing network names

---
 terraform-modules/providers/openstack/main.tf      | 4 ++--
 terraform-modules/providers/openstack/variables.tf | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 75dbb21..ae3ff7e 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -109,11 +109,11 @@ data "template_file" "kubeadm_master" {
 }
 
 data "openstack_networking_network_v2" "ext_network" {
-  name = "public"
+  name = var.public_network
 }
 
 data "openstack_networking_network_v2" "int_network" {
-  external = false
+  name = var.cc_private_network
 }
 
 resource "openstack_networking_router_v2" "router_1" {
diff --git a/terraform-modules/providers/openstack/variables.tf b/terraform-modules/providers/openstack/variables.tf
index 73203a2..5a21fee 100644
--- a/terraform-modules/providers/openstack/variables.tf
+++ b/terraform-modules/providers/openstack/variables.tf
@@ -55,3 +55,6 @@ variable "docker_password" {
   description = "Docker hub password"
 }
 
+variable "public_network" {
+  default  = "Public-Network"
+}
\ No newline at end of file

From 65b635850ad6655b37d7bbe99201978e9d449c4e Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Mon, 10 Oct 2022 00:45:14 -0400
Subject: [PATCH 011/109] Fix is_computecanada count conditions

---
 terraform-modules/providers/openstack/main.tf | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index ae3ff7e..d7c9d45 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -80,7 +80,7 @@ locals {
 }
 
 resource "openstack_networking_subnet_v2" "subnet" {
-  count = 1 - var.is_computecanada
+  count = var.is_computecanada ? 0 : 1
 
   name        = "subnet"
   network_id  = openstack_networking_network_v2.network_1[0].id
@@ -90,7 +90,7 @@ resource "openstack_networking_subnet_v2" "subnet" {
 }
 
 resource "openstack_networking_network_v2" "network_1" {
-  count = 1 - var.is_computecanada
+  count = var.is_computecanada ? 0 : 1
 
   name = local.network_name
 }
@@ -117,14 +117,14 @@ data "openstack_networking_network_v2" "int_network" {
 }
 
 resource "openstack_networking_router_v2" "router_1" {
-  count = 1 - var.is_computecanada
+  count = var.is_computecanada ? 0 : 1
 
   name                = "${var.project_name}-router"
   external_network_id = data.openstack_networking_network_v2.ext_network.id
 }
 
 resource "openstack_networking_router_interface_v2" "router_interface_1" {
-  count = 1 - var.is_computecanada
+  count = var.is_computecanada ? 0 : 1
 
   router_id = openstack_networking_router_v2.router_1[0].id
   subnet_id = openstack_networking_subnet_v2.subnet[0].id

From 863dd713f113d9484f7911aab51833429112797b Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Mon, 10 Oct 2022 01:21:07 -0400
Subject: [PATCH 012/109] Create new security group

---
 terraform-modules/providers/openstack/main.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index d7c9d45..dc13867 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -8,8 +8,8 @@ data "openstack_images_image_v2" "ubuntu" {
 }
 
 resource "openstack_compute_secgroup_v2" "secgroup_1" {
-  name        = "${var.project_name}-secgroup"
-  description = "BinderHub security group"
+  name        = "${var.project_name}-secgroup-new"
+  description = "BinderHub security group - Agah"
 
   rule {
     from_port   = -1

From 537d2ec8000d3e1c5fa82e8edfa5d793cce6bae2 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Mon, 10 Oct 2022 03:24:12 -0400
Subject: [PATCH 013/109] specify external

---
 terraform-modules/providers/openstack/main.tf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index dc13867..95941dd 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -110,6 +110,7 @@ data "template_file" "kubeadm_master" {
 
 data "openstack_networking_network_v2" "ext_network" {
   name = var.public_network
+  external = true
 }
 
 data "openstack_networking_network_v2" "int_network" {

From a8d2128f80b25d34ffbea45d73b4f67f458024ea Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Mon, 10 Oct 2022 04:01:42 -0400
Subject: [PATCH 014/109] add private key

---
 terraform-modules/binderhub/main.tf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index 9ba2618..1205379 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -66,6 +66,7 @@ resource "null_resource" "remote_install" {
   connection {
     user = var.admin_user
     host = var.ip
+    private_key = file("${var.key_path}")
   }
 
   provisioner "file" {

From e7c1137abb8c4b8c7df065164fdbde3c87dce370 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Mon, 10 Oct 2022 04:02:29 -0400
Subject: [PATCH 015/109] key path

---
 terraform-modules/binderhub/variables.tf | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/terraform-modules/binderhub/variables.tf b/terraform-modules/binderhub/variables.tf
index 880bbf9..23613a5 100644
--- a/terraform-modules/binderhub/variables.tf
+++ b/terraform-modules/binderhub/variables.tf
@@ -43,3 +43,6 @@ variable "docker_password" {
   description = "Docker hub password"
 }
 
+variable "key_path" {
+  description = "Private key path"
+}

From a23b7a443ac4908820949be8849f2be09ed381f3 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Mon, 10 Oct 2022 04:18:44 -0400
Subject: [PATCH 016/109] try this one

---
 terraform-modules/providers/openstack/main.tf | 1 -
 1 file changed, 1 deletion(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 95941dd..4ba7fd3 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -185,7 +185,6 @@ data "template_cloudinit_config" "master_config" {
 
 resource "openstack_compute_keypair_v2" "keypair" {
   name       = "${var.project_name}-keypair"
-  public_key = element(var.ssh_authorized_keys, 0)
 }
 
 resource "openstack_compute_instance_v2" "master" {

From b541ff71e40aede068fe49bf4b849868381a079a Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Mon, 10 Oct 2022 11:49:32 -0400
Subject: [PATCH 017/109] revert to 537d2ec8000d3e1c5fa82e8edfa5d793cce6bae2

---
 terraform-modules/binderhub/main.tf           | 1 -
 terraform-modules/binderhub/variables.tf      | 3 ---
 terraform-modules/providers/openstack/main.tf | 1 +
 3 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index 1205379..9ba2618 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -66,7 +66,6 @@ resource "null_resource" "remote_install" {
   connection {
     user = var.admin_user
     host = var.ip
-    private_key = file("${var.key_path}")
   }
 
   provisioner "file" {
diff --git a/terraform-modules/binderhub/variables.tf b/terraform-modules/binderhub/variables.tf
index 23613a5..880bbf9 100644
--- a/terraform-modules/binderhub/variables.tf
+++ b/terraform-modules/binderhub/variables.tf
@@ -43,6 +43,3 @@ variable "docker_password" {
   description = "Docker hub password"
 }
 
-variable "key_path" {
-  description = "Private key path"
-}
diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 4ba7fd3..95941dd 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -185,6 +185,7 @@ data "template_cloudinit_config" "master_config" {
 
 resource "openstack_compute_keypair_v2" "keypair" {
   name       = "${var.project_name}-keypair"
+  public_key = element(var.ssh_authorized_keys, 0)
 }
 
 resource "openstack_compute_instance_v2" "master" {

From c9907b607d212be0f22323e0bc5832516c81732f Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Mon, 10 Oct 2022 13:36:00 -0400
Subject: [PATCH 018/109] do not specify internal net name

---
 terraform-modules/providers/openstack/main.tf | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 95941dd..d7f64e6 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -113,8 +113,7 @@ data "openstack_networking_network_v2" "ext_network" {
   external = true
 }
 
-data "openstack_networking_network_v2" "int_network" {
-  name = var.cc_private_network
+data "openstack_networking_network_v2" "int_network" { 
 }
 
 resource "openstack_networking_router_v2" "router_1" {

From b817d2cefcff30a3e24320b46ce291708bcfe503 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Mon, 10 Oct 2022 13:48:17 -0400
Subject: [PATCH 019/109] debug install binderhub sh

---
 terraform-modules/binderhub/assets/install-binderhub.sh | 7 +++++++
 terraform-modules/providers/openstack/main.tf           | 3 ++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/terraform-modules/binderhub/assets/install-binderhub.sh b/terraform-modules/binderhub/assets/install-binderhub.sh
index 945548b..86cf0a0 100755
--- a/terraform-modules/binderhub/assets/install-binderhub.sh
+++ b/terraform-modules/binderhub/assets/install-binderhub.sh
@@ -5,6 +5,13 @@ while [ ! -f /shared/k8s-initialized ]; do sleep 1; done
 
 cd /home/${admin_user}
 
+# See if vars are in scope
+echo ${admin_user}
+echo ${binder_version}
+
+# Try resolving : Kubernetes cluster unreachable: Get "http://localhost:8080/version" problem
+export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+
 # node helath monitoring
 sudo helm repo add deliveryhero https://charts.deliveryhero.io/
 sudo helm install deliveryhero/node-problem-detector --generate-name
diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index d7f64e6..95941dd 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -113,7 +113,8 @@ data "openstack_networking_network_v2" "ext_network" {
   external = true
 }
 
-data "openstack_networking_network_v2" "int_network" { 
+data "openstack_networking_network_v2" "int_network" {
+  name = var.cc_private_network
 }
 
 resource "openstack_networking_router_v2" "router_1" {

From e755b89b73a253413e14001f4daec5c2f451f7d9 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Mon, 10 Oct 2022 15:47:18 -0400
Subject: [PATCH 020/109] move var elsewhere

---
 terraform-modules/binderhub/assets/install-binderhub.sh | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/terraform-modules/binderhub/assets/install-binderhub.sh b/terraform-modules/binderhub/assets/install-binderhub.sh
index 86cf0a0..945548b 100755
--- a/terraform-modules/binderhub/assets/install-binderhub.sh
+++ b/terraform-modules/binderhub/assets/install-binderhub.sh
@@ -5,13 +5,6 @@ while [ ! -f /shared/k8s-initialized ]; do sleep 1; done
 
 cd /home/${admin_user}
 
-# See if vars are in scope
-echo ${admin_user}
-echo ${binder_version}
-
-# Try resolving : Kubernetes cluster unreachable: Get "http://localhost:8080/version" problem
-export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
-
 # node helath monitoring
 sudo helm repo add deliveryhero https://charts.deliveryhero.io/
 sudo helm install deliveryhero/node-problem-detector --generate-name

From 6266ba1699d3b0c5bf2bc6313e60610ee4789a2c Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Wed, 12 Oct 2022 00:03:13 -0400
Subject: [PATCH 021/109] do not spec ext

---
 terraform-modules/providers/openstack/main.tf | 1 -
 1 file changed, 1 deletion(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 95941dd..dc13867 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -110,7 +110,6 @@ data "template_file" "kubeadm_master" {
 
 data "openstack_networking_network_v2" "ext_network" {
   name = var.public_network
-  external = true
 }
 
 data "openstack_networking_network_v2" "int_network" {

From c6ff9241c6573386060902248ae9a37b24877828 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Wed, 12 Oct 2022 03:05:55 -0400
Subject: [PATCH 022/109] force helm to use right kubeconfig

---
 .../binderhub/assets/install-binderhub.sh        | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/terraform-modules/binderhub/assets/install-binderhub.sh b/terraform-modules/binderhub/assets/install-binderhub.sh
index 945548b..a4d11ab 100755
--- a/terraform-modules/binderhub/assets/install-binderhub.sh
+++ b/terraform-modules/binderhub/assets/install-binderhub.sh
@@ -7,7 +7,7 @@ cd /home/${admin_user}
 
 # node helath monitoring
 sudo helm repo add deliveryhero https://charts.deliveryhero.io/
-sudo helm install deliveryhero/node-problem-detector --generate-name
+sudo helm install deliveryhero/node-problem-detector --generate-name --kubeconfig ~/.kube/config
 
 #Persistent volume
 kubectl create -f pv.yaml
@@ -21,20 +21,21 @@ sudo helm repo update
 sudo helm install cert-manager --namespace cert-manager --version v1.0.3 jetstack/cert-manager --set installCRDs=true \
   --set nodeSelector."node-role\.kubernetes\.io/master=" \
   --set cainjector.nodeSelector."node-role\.kubernetes\.io/master=" \
-  --set webhook.nodeSelector."node-role\.kubernetes\.io/master="
+  --set webhook.nodeSelector."node-role\.kubernetes\.io/master=" \
+  --kubeconfig ~/.kube/config
 #wait until cert-manager is ready
 kubectl wait --namespace cert-manager \
   --for=condition=ready pod \
   --selector=app.kubernetes.io/instance=cert-manager \
   --timeout=300s
 # apply the issuer(s)
-# kubectl create namespace binderhub
+kubectl create namespace binderhub
 # kubectl apply -f staging-binderhub-issuer.yaml
 kubectl apply -f production-binderhub-issuer.yaml
 
 # Binderhub proxy
 sudo helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx/
-sudo helm install binderhub-proxy ingress-nginx/ingress-nginx --namespace=binderhub -f nginx-ingress.yaml
+sudo helm install binderhub-proxy ingress-nginx/ingress-nginx --namespace=binderhub -f nginx-ingress.yaml --kubeconfig ~/.kube/config
 # wait until nginx is ready (https://kubernetes.github.io/ingress-nginx/deploy/)
 kubectl wait --namespace binderhub \
   --for=condition=ready pod \
@@ -52,7 +53,8 @@ sudo helm install binderhub jupyterhub/binderhub --version=${binder_version} \
   --namespace=binderhub -f config.yaml -f secrets.yaml \
   --set-file jupyterhub.singleuser.extraFiles.repo2data.stringData=repo2data.bash \
   --set-file jupyterhub.singleuser.extraFiles.fill_submission_metadata.stringData=fill_submission_metadata.bash \
-  --set-file jupyterhub.singleuser.extraFiles.jb_build.stringData=jb_build.bash
+  --set-file jupyterhub.singleuser.extraFiles.jb_build.stringData=jb_build.bash \
+  --kubeconfig ~/.kube/config
 kubectl wait --namespace binderhub \
   --for=condition=ready pod \
   --selector=release=binderhub \
@@ -64,5 +66,5 @@ sudo helm repo add grafana https://grafana.github.io/helm-charts
 sudo helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
 sudo helm repo add kube-state-metrics https://kubernetes.github.io/kube-state-metrics
 sudo helm repo update
-sudo helm install grafana-prod grafana/grafana
-sudo helm install prometheus-prod prometheus-community/prometheus
+sudo helm install grafana-prod grafana/grafana --kubeconfig ~/.kube/config
+sudo helm install prometheus-prod prometheus-community/prometheus --kubeconfig ~/.kube/config
\ No newline at end of file

From 1822663e4d783624e139174e36925775fb1b6b40 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Wed, 12 Oct 2022 12:02:29 -0400
Subject: [PATCH 023/109] try 22.04 ssh-rsa enable

---
 cloud-init/kubeadm/common.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cloud-init/kubeadm/common.yaml b/cloud-init/kubeadm/common.yaml
index 0fc45f5..1bcbc52 100644
--- a/cloud-init/kubeadm/common.yaml
+++ b/cloud-init/kubeadm/common.yaml
@@ -7,6 +7,7 @@ manage_resolv_conf: true
 
 runcmd:
   - echo "127.0.0.1 $(hostname)" | sudo tee -a /etc/hosts
+  - echo "PubkeyAcceptedKeyTypes +ssh-rsa" >> /etc/ssh/ssh_config
 
 ssh_authorized_keys:
   ${ssh_authorized_keys}

From dcecddd9b3a46ae0d40c722c4f8c5fe3ef53dc85 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Wed, 12 Oct 2022 12:11:32 -0400
Subject: [PATCH 024/109] gave up on 22.04

---
 cloud-init/kubeadm/common.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cloud-init/kubeadm/common.yaml b/cloud-init/kubeadm/common.yaml
index 1bcbc52..0fc45f5 100644
--- a/cloud-init/kubeadm/common.yaml
+++ b/cloud-init/kubeadm/common.yaml
@@ -7,7 +7,6 @@ manage_resolv_conf: true
 
 runcmd:
   - echo "127.0.0.1 $(hostname)" | sudo tee -a /etc/hosts
-  - echo "PubkeyAcceptedKeyTypes +ssh-rsa" >> /etc/ssh/ssh_config
 
 ssh_authorized_keys:
   ${ssh_authorized_keys}

From 0538bcb07dbaec2bb6a1f712d158bf5be6e6c1db Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Thu, 13 Oct 2022 21:56:05 -0400
Subject: [PATCH 025/109] specify ngx-ingress version and add files

---
 .../binderhub/assets/install-binderhub.sh     |  2 +-
 terraform-modules/binderhub/main.tf           | 30 +++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/terraform-modules/binderhub/assets/install-binderhub.sh b/terraform-modules/binderhub/assets/install-binderhub.sh
index a4d11ab..4c97fe6 100755
--- a/terraform-modules/binderhub/assets/install-binderhub.sh
+++ b/terraform-modules/binderhub/assets/install-binderhub.sh
@@ -35,7 +35,7 @@ kubectl apply -f production-binderhub-issuer.yaml
 
 # Binderhub proxy
 sudo helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx/
-sudo helm install binderhub-proxy ingress-nginx/ingress-nginx --namespace=binderhub -f nginx-ingress.yaml --kubeconfig ~/.kube/config
+sudo helm install binderhub-proxy ingress-nginx/ingress-nginx --namespace=binderhub -f nginx-ingress.yaml --kubeconfig ~/.kube/config --version 4.1.4
 # wait until nginx is ready (https://kubernetes.github.io/ingress-nginx/deploy/)
 kubectl wait --namespace binderhub \
   --for=condition=ready pod \
diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index 9ba2618..6533612 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -62,12 +62,42 @@ data "template_file" "install-binderhub" {
   }
 }
 
+data "template_file" "fill_submission_metadata" {
+  template = file("${path.module}/assets/fill_submission_metadata.sh")
+  vars     = {}
+}
+
+data "template_file" "repo2data" {
+  template = file("${path.module}/assets/repo2data.sh")
+  vars     = {}
+}
+
+data "template_file" "jb_build" {
+  template = file("${path.module}/assets/jb_build.sh")
+  vars     = {}
+}
+
 resource "null_resource" "remote_install" {
   connection {
     user = var.admin_user
     host = var.ip
   }
 
+  provisioner "file" {
+    content     = data.template_file.fill_submission_metadata.rendered
+    destination = "/home/${var.admin_user}/fill_submission_metadata.sh"
+  }
+
+  provisioner "file" {
+    content     = data.template_file.repo2data.rendered
+    destination = "/home/${var.admin_user}/repo2data.sh"
+  }
+
+  provisioner "file" {
+    content     = data.template_file.jb_build.rendered
+    destination = "/home/${var.admin_user}/jb_build.sh"
+  }
+
   provisioner "file" {
     content     = data.template_file.config.rendered
     destination = "/home/${var.admin_user}/config.yaml"

From 1e736438cf97dc027da39d54cc41369d1df752fc Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Thu, 13 Oct 2022 21:59:52 -0400
Subject: [PATCH 026/109] fix format

---
 terraform-modules/binderhub/main.tf | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index 6533612..c335643 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -63,17 +63,17 @@ data "template_file" "install-binderhub" {
 }
 
 data "template_file" "fill_submission_metadata" {
-  template = file("${path.module}/assets/fill_submission_metadata.sh")
+  template = file("${path.module}/assets/fill_submission_metadata.bash")
   vars     = {}
 }
 
 data "template_file" "repo2data" {
-  template = file("${path.module}/assets/repo2data.sh")
+  template = file("${path.module}/assets/repo2data.bash")
   vars     = {}
 }
 
 data "template_file" "jb_build" {
-  template = file("${path.module}/assets/jb_build.sh")
+  template = file("${path.module}/assets/jb_build.bash")
   vars     = {}
 }
 
@@ -85,17 +85,17 @@ resource "null_resource" "remote_install" {
 
   provisioner "file" {
     content     = data.template_file.fill_submission_metadata.rendered
-    destination = "/home/${var.admin_user}/fill_submission_metadata.sh"
+    destination = "/home/${var.admin_user}/fill_submission_metadata.bash"
   }
 
   provisioner "file" {
     content     = data.template_file.repo2data.rendered
-    destination = "/home/${var.admin_user}/repo2data.sh"
+    destination = "/home/${var.admin_user}/repo2data.bash"
   }
 
   provisioner "file" {
     content     = data.template_file.jb_build.rendered
-    destination = "/home/${var.admin_user}/jb_build.sh"
+    destination = "/home/${var.admin_user}/jb_build.bash"
   }
 
   provisioner "file" {

From 7a32bd14a216eacdd297392cbd79b3f3f41d9e20 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Thu, 13 Oct 2022 22:09:53 -0400
Subject: [PATCH 027/109] switch to source/dest for bash

---
 terraform-modules/binderhub/main.tf | 21 +++------------------
 1 file changed, 3 insertions(+), 18 deletions(-)

diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index c335643..358a621 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -62,21 +62,6 @@ data "template_file" "install-binderhub" {
   }
 }
 
-data "template_file" "fill_submission_metadata" {
-  template = file("${path.module}/assets/fill_submission_metadata.bash")
-  vars     = {}
-}
-
-data "template_file" "repo2data" {
-  template = file("${path.module}/assets/repo2data.bash")
-  vars     = {}
-}
-
-data "template_file" "jb_build" {
-  template = file("${path.module}/assets/jb_build.bash")
-  vars     = {}
-}
-
 resource "null_resource" "remote_install" {
   connection {
     user = var.admin_user
@@ -84,17 +69,17 @@ resource "null_resource" "remote_install" {
   }
 
   provisioner "file" {
-    content     = data.template_file.fill_submission_metadata.rendered
+    source     = "${path.module}/assets/fill_submission_metadata.bash"
     destination = "/home/${var.admin_user}/fill_submission_metadata.bash"
   }
 
   provisioner "file" {
-    content     = data.template_file.repo2data.rendered
+    source     = "${path.module}/assets/repo2data.bash"
     destination = "/home/${var.admin_user}/repo2data.bash"
   }
 
   provisioner "file" {
-    content     = data.template_file.jb_build.rendered
+    source     = "${path.module}/assets/jb_build.bash"
     destination = "/home/${var.admin_user}/jb_build.bash"
   }
 

From e10b48992f6fb22ede09950e12f6235b4583d324 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Wed, 21 Dec 2022 17:16:42 +0300
Subject: [PATCH 028/109] add sftp secgroup

---
 terraform-modules/binderhub/.DS_Store         | Bin 0 -> 6148 bytes
 terraform-modules/providers/openstack/main.tf |   4 ++--
 2 files changed, 2 insertions(+), 2 deletions(-)
 create mode 100644 terraform-modules/binderhub/.DS_Store

diff --git a/terraform-modules/binderhub/.DS_Store b/terraform-modules/binderhub/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..a1e5eba800aa83362ec5047eeadc300ea442ca8a
GIT binary patch
literal 6148
zcmeHK%}T>S5Z<k|CKe=u2L%rS4_<m`)hgaX{W%GW5k2@*6B2A7B$A}<p%j7--~;H@
zpZ8E7!WZ%G%<fhw)uW;^Q)a)(?96QPrR-!GW4zJ!YK$3-F%1;4pu_Nq;5_Px6qE;%
ziy9eA-EMm6hOs*mO^)Bl0KU5tb65&LpAEmiH}^6O(yUs2<%O~0cxgh@C(C+ys;r;d
zt+;DvPS#EvPSP~y?ZES0Cpikdtaac=TcO|aqupf3$^7IPgAd|3gaBzf?qT3}o5svW
z9A!=r`H83}Yn&Vfkr#IxaXa!-5wD61d(X*QP)A#^&Ly3?rZ<h*Ua!7hub3-KwSL9y
zt*uqTUS93@bxoUF+}=NHUtQna-rYYuK0V7c4u4o7(+UUh0>%&_9XpA_IJyA;AZHLu
zNDL4I#K0eAKy7DQ`H$W|-8L~m416C0xIdVnh?c@!qS!iMz|TjVFCwCVjb{l&Q=z3W
zmk2W;T&4obRIZK~T&9EHRB@KVT%t^8T&xUr)XH3)P`Fqf{H6$J+>%HmF+dCqGcYDE
z4ebA0U!VVnljw>VAO`*w1H8CtZ#LkORBs)5IPA41=s73~#^n+}Qos<c7-F#%&w>g8
YzljE*r7)KW9uWEv5H!$04E!hq?{A7?CjbBd

literal 0
HcmV?d00001

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index dc13867..070b2e6 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -191,7 +191,7 @@ resource "openstack_compute_instance_v2" "master" {
   name            = "${var.project_name}-master"
   flavor_name     = var.os_flavor_master
   key_pair        = openstack_compute_keypair_v2.keypair.name
-  security_groups = [openstack_compute_secgroup_v2.secgroup_1.name]
+  security_groups = [openstack_compute_secgroup_v2.secgroup_1.name,"neurolibre-sftp"]
   user_data       = data.template_cloudinit_config.master_config.rendered
 
   block_device {
@@ -214,7 +214,7 @@ resource "openstack_compute_instance_v2" "node" {
 
   flavor_name     = var.os_flavor_node
   key_pair        = openstack_compute_keypair_v2.keypair.name
-  security_groups = [openstack_compute_secgroup_v2.secgroup_1.name]
+  security_groups = [openstack_compute_secgroup_v2.secgroup_1.name,"neurolibre-sftp"]
   user_data = element(
     data.template_cloudinit_config.node_config.*.rendered,
     count.index,

From 489d5b414b1fd2afb6fbcf2723e5b3ae49f309b6 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Wed, 21 Dec 2022 20:22:29 +0300
Subject: [PATCH 029/109] network and secgroup rules

---
 terraform-modules/providers/openstack/main.tf | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 070b2e6..2b58498 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -110,6 +110,7 @@ data "template_file" "kubeadm_master" {
 
 data "openstack_networking_network_v2" "ext_network" {
   name = var.public_network
+  external = true
 }
 
 data "openstack_networking_network_v2" "int_network" {
@@ -191,7 +192,7 @@ resource "openstack_compute_instance_v2" "master" {
   name            = "${var.project_name}-master"
   flavor_name     = var.os_flavor_master
   key_pair        = openstack_compute_keypair_v2.keypair.name
-  security_groups = [openstack_compute_secgroup_v2.secgroup_1.name,"neurolibre-sftp"]
+  security_groups = [openstack_compute_secgroup_v2.secgroup_1.name,"neurolibre-secgroup"]
   user_data       = data.template_cloudinit_config.master_config.rendered
 
   block_device {
@@ -214,7 +215,7 @@ resource "openstack_compute_instance_v2" "node" {
 
   flavor_name     = var.os_flavor_node
   key_pair        = openstack_compute_keypair_v2.keypair.name
-  security_groups = [openstack_compute_secgroup_v2.secgroup_1.name,"neurolibre-sftp"]
+  security_groups = [openstack_compute_secgroup_v2.secgroup_1.name,"neurolibre-secgroup"]
   user_data = element(
     data.template_cloudinit_config.node_config.*.rendered,
     count.index,

From d54183bc494398c5f4289c41085f61b6d00a85fb Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Wed, 21 Dec 2022 23:56:01 +0300
Subject: [PATCH 030/109] use consistent dirs

---
 terraform-modules/binderhub/assets/install-binderhub.sh | 6 +++---
 terraform-modules/binderhub/main.tf                     | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/terraform-modules/binderhub/assets/install-binderhub.sh b/terraform-modules/binderhub/assets/install-binderhub.sh
index 4c97fe6..302a6d7 100755
--- a/terraform-modules/binderhub/assets/install-binderhub.sh
+++ b/terraform-modules/binderhub/assets/install-binderhub.sh
@@ -51,9 +51,9 @@ sudo helm repo add jupyterhub https://jupyterhub.github.io/helm-chart
 sudo helm repo update
 sudo helm install binderhub jupyterhub/binderhub --version=${binder_version} \
   --namespace=binderhub -f config.yaml -f secrets.yaml \
-  --set-file jupyterhub.singleuser.extraFiles.repo2data.stringData=repo2data.bash \
-  --set-file jupyterhub.singleuser.extraFiles.fill_submission_metadata.stringData=fill_submission_metadata.bash \
-  --set-file jupyterhub.singleuser.extraFiles.jb_build.stringData=jb_build.bash \
+  --set-file jupyterhub.singleuser.extraFiles.repo2data.stringData=/usr/local/share/repo2data.bash \
+  --set-file jupyterhub.singleuser.extraFiles.fill_submission_metadata.stringData=/usr/local/share/fill_submission_metadata.bash \
+  --set-file jupyterhub.singleuser.extraFiles.jb_build.stringData=/usr/local/share/jb_build.bash \
   --kubeconfig ~/.kube/config
 kubectl wait --namespace binderhub \
   --for=condition=ready pod \
diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index 358a621..05c04e8 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -70,17 +70,17 @@ resource "null_resource" "remote_install" {
 
   provisioner "file" {
     source     = "${path.module}/assets/fill_submission_metadata.bash"
-    destination = "/home/${var.admin_user}/fill_submission_metadata.bash"
+    destination = "/usr/local/share/fill_submission_metadata.bash"
   }
 
   provisioner "file" {
     source     = "${path.module}/assets/repo2data.bash"
-    destination = "/home/${var.admin_user}/repo2data.bash"
+    destination = "/usr/local/share/repo2data.bash"
   }
 
   provisioner "file" {
     source     = "${path.module}/assets/jb_build.bash"
-    destination = "/home/${var.admin_user}/jb_build.bash"
+    destination = "/usr/local/share/jb_build.bash"
   }
 
   provisioner "file" {

From 65c9f01ec6276d46932f5d4dc2ab7bfb6c626790 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Thu, 22 Dec 2022 00:30:54 +0300
Subject: [PATCH 031/109] Revert shared dir

---
 terraform-modules/binderhub/assets/install-binderhub.sh | 6 +++---
 terraform-modules/binderhub/main.tf                     | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/terraform-modules/binderhub/assets/install-binderhub.sh b/terraform-modules/binderhub/assets/install-binderhub.sh
index 302a6d7..4c97fe6 100755
--- a/terraform-modules/binderhub/assets/install-binderhub.sh
+++ b/terraform-modules/binderhub/assets/install-binderhub.sh
@@ -51,9 +51,9 @@ sudo helm repo add jupyterhub https://jupyterhub.github.io/helm-chart
 sudo helm repo update
 sudo helm install binderhub jupyterhub/binderhub --version=${binder_version} \
   --namespace=binderhub -f config.yaml -f secrets.yaml \
-  --set-file jupyterhub.singleuser.extraFiles.repo2data.stringData=/usr/local/share/repo2data.bash \
-  --set-file jupyterhub.singleuser.extraFiles.fill_submission_metadata.stringData=/usr/local/share/fill_submission_metadata.bash \
-  --set-file jupyterhub.singleuser.extraFiles.jb_build.stringData=/usr/local/share/jb_build.bash \
+  --set-file jupyterhub.singleuser.extraFiles.repo2data.stringData=repo2data.bash \
+  --set-file jupyterhub.singleuser.extraFiles.fill_submission_metadata.stringData=fill_submission_metadata.bash \
+  --set-file jupyterhub.singleuser.extraFiles.jb_build.stringData=jb_build.bash \
   --kubeconfig ~/.kube/config
 kubectl wait --namespace binderhub \
   --for=condition=ready pod \
diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index 05c04e8..358a621 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -70,17 +70,17 @@ resource "null_resource" "remote_install" {
 
   provisioner "file" {
     source     = "${path.module}/assets/fill_submission_metadata.bash"
-    destination = "/usr/local/share/fill_submission_metadata.bash"
+    destination = "/home/${var.admin_user}/fill_submission_metadata.bash"
   }
 
   provisioner "file" {
     source     = "${path.module}/assets/repo2data.bash"
-    destination = "/usr/local/share/repo2data.bash"
+    destination = "/home/${var.admin_user}/repo2data.bash"
   }
 
   provisioner "file" {
     source     = "${path.module}/assets/jb_build.bash"
-    destination = "/usr/local/share/jb_build.bash"
+    destination = "/home/${var.admin_user}/jb_build.bash"
   }
 
   provisioner "file" {

From 6d8d02a15e4de40beb70ff40942bfd3014d28b2f Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Thu, 22 Dec 2022 01:48:35 +0300
Subject: [PATCH 032/109] docker group

---
 cloud-init/kubeadm/master.yaml | 5 ++++-
 cloud-init/kubeadm/node.yaml   | 5 ++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/cloud-init/kubeadm/master.yaml b/cloud-init/kubeadm/master.yaml
index 923896b..3766a2f 100644
--- a/cloud-init/kubeadm/master.yaml
+++ b/cloud-init/kubeadm/master.yaml
@@ -31,7 +31,10 @@ runcmd:
   - echo "Host registry" >> /home/${admin_user}/.ssh/config;
   - echo "        HostName "${docker_registry} >> /home/${admin_user}/.ssh/config;
   - echo "        User "${admin_user} >> /home/${admin_user}/.ssh/config; 
-  - su ${admin_user} -c "sudo docker login ${docker_registry} --username ${docker_id} --password ${docker_password}"
+  #- su ${admin_user} -c "sudo docker login ${docker_registry} --username ${docker_id} --password ${docker_password}"
+  - sudo groupadd docker
+  - sudo usermod -aG docker $USER
+  - docker login ${docker_registry} --username ${docker_id} --password ${docker_password}
   - while [ ! -d /var/lib/kubelet/ ]; do sleep 1; done; 
   - cp /home/${admin_user}/.docker/config.json /var/lib/kubelet/
   # creating flag to say that master configuration finished
diff --git a/cloud-init/kubeadm/node.yaml b/cloud-init/kubeadm/node.yaml
index 4fab01f..1c4e925 100644
--- a/cloud-init/kubeadm/node.yaml
+++ b/cloud-init/kubeadm/node.yaml
@@ -12,6 +12,9 @@ runcmd:
   - echo "while [ ! -f /shared/kubeadm-join.bash ]; do sleep 1; done; sudo bash /shared/kubeadm-join.bash" >> /tmp/install.sh
   - su ${admin_user} -c "/tmp/install.sh"
   # authorizing docker
-  - su ${admin_user} -c "sudo docker login ${docker_registry} --username ${docker_id} --password ${docker_password}"
+  #- su ${admin_user} -c "sudo docker login ${docker_registry} --username ${docker_id} --password ${docker_password}"
+  - sudo groupadd docker
+  - sudo usermod -aG docker $USER
+  - docker login ${docker_registry} --username ${docker_id} --password ${docker_password}
   - while [ ! -d /var/lib/kubelet/ ]; do sleep 1; done; 
   - cp /home/${admin_user}/.docker/config.json /var/lib/kubelet/

From 6a8fcbfc3cd0e6e1ef596740b3ca51a25f50c4cc Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Thu, 22 Dec 2022 02:17:43 +0300
Subject: [PATCH 033/109] pass proper arg

---
 cloud-init/kubeadm/master.yaml | 2 +-
 cloud-init/kubeadm/node.yaml   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cloud-init/kubeadm/master.yaml b/cloud-init/kubeadm/master.yaml
index 3766a2f..0f694ea 100644
--- a/cloud-init/kubeadm/master.yaml
+++ b/cloud-init/kubeadm/master.yaml
@@ -33,7 +33,7 @@ runcmd:
   - echo "        User "${admin_user} >> /home/${admin_user}/.ssh/config; 
   #- su ${admin_user} -c "sudo docker login ${docker_registry} --username ${docker_id} --password ${docker_password}"
   - sudo groupadd docker
-  - sudo usermod -aG docker $USER
+  - sudo usermod -aG docker ${admin_user}
   - docker login ${docker_registry} --username ${docker_id} --password ${docker_password}
   - while [ ! -d /var/lib/kubelet/ ]; do sleep 1; done; 
   - cp /home/${admin_user}/.docker/config.json /var/lib/kubelet/
diff --git a/cloud-init/kubeadm/node.yaml b/cloud-init/kubeadm/node.yaml
index 1c4e925..f2a8815 100644
--- a/cloud-init/kubeadm/node.yaml
+++ b/cloud-init/kubeadm/node.yaml
@@ -14,7 +14,7 @@ runcmd:
   # authorizing docker
   #- su ${admin_user} -c "sudo docker login ${docker_registry} --username ${docker_id} --password ${docker_password}"
   - sudo groupadd docker
-  - sudo usermod -aG docker $USER
+  - sudo usermod -aG docker ${admin_user}
   - docker login ${docker_registry} --username ${docker_id} --password ${docker_password}
   - while [ ! -d /var/lib/kubelet/ ]; do sleep 1; done; 
   - cp /home/${admin_user}/.docker/config.json /var/lib/kubelet/

From bde0b0c49824175fd69a4b6c5291840e545f0511 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Thu, 22 Dec 2022 03:17:58 +0300
Subject: [PATCH 034/109] Fix docker conf and helm add file

---
 cloud-init/kubeadm/master.yaml                          | 2 +-
 cloud-init/kubeadm/node.yaml                            | 2 +-
 terraform-modules/binderhub/assets/config.yaml          | 2 +-
 terraform-modules/binderhub/assets/install-binderhub.sh | 8 +++++---
 4 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/cloud-init/kubeadm/master.yaml b/cloud-init/kubeadm/master.yaml
index 0f694ea..c772790 100644
--- a/cloud-init/kubeadm/master.yaml
+++ b/cloud-init/kubeadm/master.yaml
@@ -34,7 +34,7 @@ runcmd:
   #- su ${admin_user} -c "sudo docker login ${docker_registry} --username ${docker_id} --password ${docker_password}"
   - sudo groupadd docker
   - sudo usermod -aG docker ${admin_user}
-  - docker login ${docker_registry} --username ${docker_id} --password ${docker_password}
+  - su ${admin_user} -c "docker login ${docker_registry} --username ${docker_id} --password ${docker_password}"
   - while [ ! -d /var/lib/kubelet/ ]; do sleep 1; done; 
   - cp /home/${admin_user}/.docker/config.json /var/lib/kubelet/
   # creating flag to say that master configuration finished
diff --git a/cloud-init/kubeadm/node.yaml b/cloud-init/kubeadm/node.yaml
index f2a8815..b061904 100644
--- a/cloud-init/kubeadm/node.yaml
+++ b/cloud-init/kubeadm/node.yaml
@@ -15,6 +15,6 @@ runcmd:
   #- su ${admin_user} -c "sudo docker login ${docker_registry} --username ${docker_id} --password ${docker_password}"
   - sudo groupadd docker
   - sudo usermod -aG docker ${admin_user}
-  - docker login ${docker_registry} --username ${docker_id} --password ${docker_password}
+  - su ${admin_user} -c "docker login ${docker_registry} --username ${docker_id} --password ${docker_password}"
   - while [ ! -d /var/lib/kubelet/ ]; do sleep 1; done; 
   - cp /home/${admin_user}/.docker/config.json /var/lib/kubelet/
diff --git a/terraform-modules/binderhub/assets/config.yaml b/terraform-modules/binderhub/assets/config.yaml
index 32abfd0..99cd726 100644
--- a/terraform-modules/binderhub/assets/config.yaml
+++ b/terraform-modules/binderhub/assets/config.yaml
@@ -76,7 +76,7 @@ jupyterhub:
     lifecycleHooks:
       postStart:
         exec:
-          command: ["/bin/sh", "-c", "/usr/local/share/repo2data.bash; /usr/local/share/jb_build.bash"]   
+          command: ["/bin/sh", "-c", "/usr/local/share/repo2data.bash; /usr/local/share/jb_build.bash"]
 
 # BinderHub config
 config:
diff --git a/terraform-modules/binderhub/assets/install-binderhub.sh b/terraform-modules/binderhub/assets/install-binderhub.sh
index 4c97fe6..b453ba2 100755
--- a/terraform-modules/binderhub/assets/install-binderhub.sh
+++ b/terraform-modules/binderhub/assets/install-binderhub.sh
@@ -51,9 +51,11 @@ sudo helm repo add jupyterhub https://jupyterhub.github.io/helm-chart
 sudo helm repo update
 sudo helm install binderhub jupyterhub/binderhub --version=${binder_version} \
   --namespace=binderhub -f config.yaml -f secrets.yaml \
-  --set-file jupyterhub.singleuser.extraFiles.repo2data.stringData=repo2data.bash \
-  --set-file jupyterhub.singleuser.extraFiles.fill_submission_metadata.stringData=fill_submission_metadata.bash \
-  --set-file jupyterhub.singleuser.extraFiles.jb_build.stringData=jb_build.bash \
+  --kubeconfig ~/.kube/config
+sudo helm upgrade binderhub jupyterhub/binderhub --version=${binder_version} \
+  --set-file jupyterhub.singleuser.extraFiles.repo2data.stringData=./repo2data.bash \
+  --set-file jupyterhub.singleuser.extraFiles.fill_submission_metadata.stringData=./fill_submission_metadata.bash \
+  --set-file jupyterhub.singleuser.extraFiles.jb_build.stringData=./jb_build.bash \
   --kubeconfig ~/.kube/config
 kubectl wait --namespace binderhub \
   --for=condition=ready pod \

From 8788fdfba6a428c9d46b347665111bfa1312a6d9 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Thu, 22 Dec 2022 03:56:57 +0300
Subject: [PATCH 035/109] use abs path

---
 .../binderhub/assets/install-binderhub.sh           | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/terraform-modules/binderhub/assets/install-binderhub.sh b/terraform-modules/binderhub/assets/install-binderhub.sh
index b453ba2..eb4171c 100755
--- a/terraform-modules/binderhub/assets/install-binderhub.sh
+++ b/terraform-modules/binderhub/assets/install-binderhub.sh
@@ -51,12 +51,15 @@ sudo helm repo add jupyterhub https://jupyterhub.github.io/helm-chart
 sudo helm repo update
 sudo helm install binderhub jupyterhub/binderhub --version=${binder_version} \
   --namespace=binderhub -f config.yaml -f secrets.yaml \
+  --set-file jupyterhub.singleuser.extraFiles.repo2data.stringData=/home/${admin_user}/repo2data.bash \
+  --set-file jupyterhub.singleuser.extraFiles.fill_submission_metadata.stringData=/home/${admin_user}/fill_submission_metadata.bash \
+  --set-file jupyterhub.singleuser.extraFiles.jb_build.stringData=/home/${admin_user}/jb_build.bash \
   --kubeconfig ~/.kube/config
-sudo helm upgrade binderhub jupyterhub/binderhub --version=${binder_version} \
-  --set-file jupyterhub.singleuser.extraFiles.repo2data.stringData=./repo2data.bash \
-  --set-file jupyterhub.singleuser.extraFiles.fill_submission_metadata.stringData=./fill_submission_metadata.bash \
-  --set-file jupyterhub.singleuser.extraFiles.jb_build.stringData=./jb_build.bash \
-  --kubeconfig ~/.kube/config
+# sudo helm upgrade binderhub jupyterhub/binderhub --version=${binder_version} \
+#   --set-file jupyterhub.singleuser.extraFiles.repo2data.stringData=/home/${admin_user}/repo2data.bash \
+#   --set-file jupyterhub.singleuser.extraFiles.fill_submission_metadata.stringData=/home/${admin_user}/fill_submission_metadata.bash \
+#   --set-file jupyterhub.singleuser.extraFiles.jb_build.stringData=/home/${admin_user}/jb_build.bash \
+#   --kubeconfig ~/.kube/config
 kubectl wait --namespace binderhub \
   --for=condition=ready pod \
   --selector=release=binderhub \

From bec40d0216b762416950d139ec63ded7ae8d986c Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Thu, 22 Dec 2022 16:19:53 +0300
Subject: [PATCH 036/109] Updade BinderHub version and helm command

---
 .../binderhub/assets/install-binderhub.sh         | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/terraform-modules/binderhub/assets/install-binderhub.sh b/terraform-modules/binderhub/assets/install-binderhub.sh
index eb4171c..cf5a2a4 100755
--- a/terraform-modules/binderhub/assets/install-binderhub.sh
+++ b/terraform-modules/binderhub/assets/install-binderhub.sh
@@ -51,14 +51,15 @@ sudo helm repo add jupyterhub https://jupyterhub.github.io/helm-chart
 sudo helm repo update
 sudo helm install binderhub jupyterhub/binderhub --version=${binder_version} \
   --namespace=binderhub -f config.yaml -f secrets.yaml \
-  --set-file jupyterhub.singleuser.extraFiles.repo2data.stringData=/home/${admin_user}/repo2data.bash \
-  --set-file jupyterhub.singleuser.extraFiles.fill_submission_metadata.stringData=/home/${admin_user}/fill_submission_metadata.bash \
-  --set-file jupyterhub.singleuser.extraFiles.jb_build.stringData=/home/${admin_user}/jb_build.bash \
+  --set-file jupyterhub.singleuser.extraFiles.repo2data.stringData=./repo2data.bash \
+  --set-file jupyterhub.singleuser.extraFiles.fill_submission_metadata.stringData=./fill_submission_metadata.bash \
+  --set-file jupyterhub.singleuser.extraFiles.jb_build.stringData=./jb_build.bash \
   --kubeconfig ~/.kube/config
-# sudo helm upgrade binderhub jupyterhub/binderhub --version=${binder_version} \
-#   --set-file jupyterhub.singleuser.extraFiles.repo2data.stringData=/home/${admin_user}/repo2data.bash \
-#   --set-file jupyterhub.singleuser.extraFiles.fill_submission_metadata.stringData=/home/${admin_user}/fill_submission_metadata.bash \
-#   --set-file jupyterhub.singleuser.extraFiles.jb_build.stringData=/home/${admin_user}/jb_build.bash \
+# sudo helm upgrade binderhub jupyterhub/binderhub -n binderhub --version=${binder_version} \
+#   -f confgi.yaml -f secrets.yaml \
+#   --set-file jupyterhub.singleuser.extraFiles.repo2data.stringData=./repo2data.bash \
+#   --set-file jupyterhub.singleuser.extraFiles.fill_submission_metadata.stringData=./fill_submission_metadata.bash \
+#   --set-file jupyterhub.singleuser.extraFiles.jb_build.stringData=./jb_build.bash \
 #   --kubeconfig ~/.kube/config
 kubectl wait --namespace binderhub \
   --for=condition=ready pod \

From a4b7e13b26f014ec9b6c8fa8d0601fdc5e813963 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Fri, 23 Dec 2022 18:43:20 +0300
Subject: [PATCH 037/109] workaround

---
 terraform-modules/binderhub/assets/config.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/terraform-modules/binderhub/assets/config.yaml b/terraform-modules/binderhub/assets/config.yaml
index 99cd726..670e8b9 100644
--- a/terraform-modules/binderhub/assets/config.yaml
+++ b/terraform-modules/binderhub/assets/config.yaml
@@ -17,6 +17,7 @@ jupyterhub:
       - secretName: binder-conp-cloud-tls
         hosts:
           - binder.conp.cloud
+          - tmp.binder.conp.cloud
   #https://discourse.jupyter.org/t/pre-building-images-on-binderhub/4325/4
   prePuller:
     continuous:
@@ -115,3 +116,4 @@ ingress:
     - secretName: binder-conp-cloud-tls
       hosts: 
         - binder.conp.cloud
+        - tmp.binder.conp.cloud

From 961ab31ed1a9f060af64dc39f7af72069c5db0ef Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Fri, 23 Dec 2022 19:51:03 +0300
Subject: [PATCH 038/109] change host name binder --> test

---
 terraform-modules/binderhub/assets/config.yaml   | 14 +++++++-------
 terraform-modules/binderhub/assets/jb_build.bash |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/terraform-modules/binderhub/assets/config.yaml b/terraform-modules/binderhub/assets/config.yaml
index 670e8b9..196934b 100644
--- a/terraform-modules/binderhub/assets/config.yaml
+++ b/terraform-modules/binderhub/assets/config.yaml
@@ -8,7 +8,7 @@ jupyterhub:
   ingress:
     enabled: true
     hosts:
-      - binder.conp.cloud
+      - test.conp.cloud
     annotations:
       kubernetes.io/ingress.class: nginx
       kubernetes.io/tls-acme: "true"
@@ -16,8 +16,8 @@ jupyterhub:
     tls:
       - secretName: binder-conp-cloud-tls
         hosts:
-          - binder.conp.cloud
-          - tmp.binder.conp.cloud
+          - test.conp.cloud
+          - tmp.test.conp.cloud
   #https://discourse.jupyter.org/t/pre-building-images-on-binderhub/4325/4
   prePuller:
     continuous:
@@ -94,7 +94,7 @@ config:
       - ^shishirchoudharygic/mltraining.*
       - ^hmharshit/mltraining.*
   BinderHub:
-    hub_url: https://binder.conp.cloud/jupyter
+    hub_url: https://test.conp.cloud/jupyter
     use_registry: true
     image_prefix: binder-registry.conp.cloud/binder-registry.conp.cloud/binder-
 
@@ -104,7 +104,7 @@ service:
 ingress:
   enabled: true
   hosts:
-    - binder.conp.cloud
+    - test.conp.cloud
   annotations:
     kubernetes.io/ingress.class: nginx
     kubernetes.io/tls-acme: "true"
@@ -115,5 +115,5 @@ ingress:
   tls:
     - secretName: binder-conp-cloud-tls
       hosts: 
-        - binder.conp.cloud
-        - tmp.binder.conp.cloud
+        - test.conp.cloud
+        - tmp.test.conp.cloud
diff --git a/terraform-modules/binderhub/assets/jb_build.bash b/terraform-modules/binderhub/assets/jb_build.bash
index 6b91fad..4bb2d32 100755
--- a/terraform-modules/binderhub/assets/jb_build.bash
+++ b/terraform-modules/binderhub/assets/jb_build.bash
@@ -11,7 +11,7 @@ CONFIG_FILE="content/_config.yml"
 BOOK_DST_PATH="/mnt/books/${USER_NAME}/${PROVIDER_NAME}/${REPO_NAME}/${COMMIT_REF}"
 BOOK_BUILT_FLAG="${BOOK_DST_PATH}/successfully_built"
 BOOK_BUILD_LOG="${BOOK_DST_PATH}/book-build.log"
-BINDERHUB_URL="https://binder.conp.cloud"
+BINDERHUB_URL="https://test.conp.cloud"
 BOOK_CACHE_PATH=${BOOK_DST_PATH}"/_build/.jupyter_cache"
 
 # checking if book build is necessary

From 544f51e8c39e912ef5622ce734490a7293148065 Mon Sep 17 00:00:00 2001
From: Agah Karakuzu <agahkarakuzu@gmail.com>
Date: Fri, 23 Dec 2022 19:51:52 +0300
Subject: [PATCH 039/109] rm tmp

---
 terraform-modules/binderhub/assets/config.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/terraform-modules/binderhub/assets/config.yaml b/terraform-modules/binderhub/assets/config.yaml
index 196934b..d8bdb80 100644
--- a/terraform-modules/binderhub/assets/config.yaml
+++ b/terraform-modules/binderhub/assets/config.yaml
@@ -17,7 +17,6 @@ jupyterhub:
       - secretName: binder-conp-cloud-tls
         hosts:
           - test.conp.cloud
-          - tmp.test.conp.cloud
   #https://discourse.jupyter.org/t/pre-building-images-on-binderhub/4325/4
   prePuller:
     continuous:
@@ -116,4 +115,3 @@ ingress:
     - secretName: binder-conp-cloud-tls
       hosts: 
         - test.conp.cloud
-        - tmp.test.conp.cloud

From ec586c89c8caf7c057660f762d0881f4e4b4e09c Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Mon, 15 May 2023 06:27:44 +0400
Subject: [PATCH 040/109] Update cert-manager and apis

---
 .../binderhub/assets/install-binderhub.sh          | 14 +++++++-------
 .../assets/production-binderhub-issuer.yaml        |  2 +-
 .../binderhub/assets/staging-binderhub-issuer.yaml |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/terraform-modules/binderhub/assets/install-binderhub.sh b/terraform-modules/binderhub/assets/install-binderhub.sh
index cf5a2a4..61025d5 100755
--- a/terraform-modules/binderhub/assets/install-binderhub.sh
+++ b/terraform-modules/binderhub/assets/install-binderhub.sh
@@ -18,7 +18,7 @@ kubectl create namespace cert-manager
 sudo helm repo add jetstack https://charts.jetstack.io
 sudo helm repo update
 # running on master node to avoid issues with webhook not in the k8s network
-sudo helm install cert-manager --namespace cert-manager --version v1.0.3 jetstack/cert-manager --set installCRDs=true \
+sudo helm install cert-manager --namespace cert-manager --version v1.9.0 jetstack/cert-manager --set installCRDs=true \
   --set nodeSelector."node-role\.kubernetes\.io/master=" \
   --set cainjector.nodeSelector."node-role\.kubernetes\.io/master=" \
   --set webhook.nodeSelector."node-role\.kubernetes\.io/master=" \
@@ -68,9 +68,9 @@ kubectl wait --namespace binderhub \
 
 # Grafana and prometheus
 # https://github.com/pangeo-data/pangeo-binder#binder-monitoring
-sudo helm repo add grafana https://grafana.github.io/helm-charts
-sudo helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
-sudo helm repo add kube-state-metrics https://kubernetes.github.io/kube-state-metrics
-sudo helm repo update
-sudo helm install grafana-prod grafana/grafana --kubeconfig ~/.kube/config
-sudo helm install prometheus-prod prometheus-community/prometheus --kubeconfig ~/.kube/config
\ No newline at end of file
+# sudo helm repo add grafana https://grafana.github.io/helm-charts
+# sudo helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+# sudo helm repo add kube-state-metrics https://kubernetes.github.io/kube-state-metrics
+# sudo helm repo update
+# sudo helm install grafana-prod grafana/grafana --kubeconfig ~/.kube/config
+# sudo helm install prometheus-prod prometheus-community/prometheus --kubeconfig ~/.kube/config
\ No newline at end of file
diff --git a/terraform-modules/binderhub/assets/production-binderhub-issuer.yaml b/terraform-modules/binderhub/assets/production-binderhub-issuer.yaml
index 4c8e2c7..e4a84fc 100644
--- a/terraform-modules/binderhub/assets/production-binderhub-issuer.yaml
+++ b/terraform-modules/binderhub/assets/production-binderhub-issuer.yaml
@@ -1,4 +1,4 @@
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Issuer
 metadata:
   name: letsencrypt-production
diff --git a/terraform-modules/binderhub/assets/staging-binderhub-issuer.yaml b/terraform-modules/binderhub/assets/staging-binderhub-issuer.yaml
index 000ea29..3a9e754 100644
--- a/terraform-modules/binderhub/assets/staging-binderhub-issuer.yaml
+++ b/terraform-modules/binderhub/assets/staging-binderhub-issuer.yaml
@@ -1,4 +1,4 @@
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Issuer
 metadata:
   name: letsencrypt-staging

From 9bdc6d0235eab4632660620d6ea3ccf79e6ea052 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Mon, 15 May 2023 10:18:12 +0400
Subject: [PATCH 041/109] Hacky solution

---
 terraform-modules/providers/openstack/main.tf      |  6 ++++--
 terraform-modules/providers/openstack/variables.tf | 12 +++++++++++-
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 2b58498..a677f55 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -196,12 +196,14 @@ resource "openstack_compute_instance_v2" "master" {
   user_data       = data.template_cloudinit_config.master_config.rendered
 
   block_device {
-    uuid                  = data.openstack_images_image_v2.ubuntu.id
+    uuid                  = var.volume_uuid_1
     source_type           = "image"
     volume_size           = var.instance_volume_size
     boot_index            = 0
     destination_type      = "volume"
     delete_on_termination = true
+    allocate_retries = 300
+    allocate_retries_interval = 3
   }
 
   network {
@@ -222,7 +224,7 @@ resource "openstack_compute_instance_v2" "node" {
   )
 
   block_device {
-    uuid                  = data.openstack_images_image_v2.ubuntu.id
+    uuid                  = var.volume_uuid_2
     source_type           = "image"
     volume_size           = var.instance_volume_size
     boot_index            = 0
diff --git a/terraform-modules/providers/openstack/variables.tf b/terraform-modules/providers/openstack/variables.tf
index 5a21fee..32afb59 100644
--- a/terraform-modules/providers/openstack/variables.tf
+++ b/terraform-modules/providers/openstack/variables.tf
@@ -57,4 +57,14 @@ variable "docker_password" {
 
 variable "public_network" {
   default  = "Public-Network"
-}
\ No newline at end of file
+}
+
+variable "volume_uuid_1" {
+  description  = "Manual volume master"
+}
+
+variable "volume_uuid_2" {
+  description  = "Manual volume node"
+}
+
+

From da0458b71be2455df6e48b08923b3442ec5c4788 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Mon, 15 May 2023 10:21:02 +0400
Subject: [PATCH 042/109] Trying manual boot volume

---
 terraform-modules/providers/openstack/main.tf | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index a677f55..081ee9b 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -197,13 +197,10 @@ resource "openstack_compute_instance_v2" "master" {
 
   block_device {
     uuid                  = var.volume_uuid_1
-    source_type           = "image"
-    volume_size           = var.instance_volume_size
+    source_type           = "volume"
     boot_index            = 0
     destination_type      = "volume"
     delete_on_termination = true
-    allocate_retries = 300
-    allocate_retries_interval = 3
   }
 
   network {
@@ -225,8 +222,7 @@ resource "openstack_compute_instance_v2" "node" {
 
   block_device {
     uuid                  = var.volume_uuid_2
-    source_type           = "image"
-    volume_size           = var.instance_volume_size
+    source_type           = "volume"
     boot_index            = 0
     destination_type      = "volume"
     delete_on_termination = true

From 33c8638c3676d1cada5e57723221042eb454a840 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Mon, 15 May 2023 12:24:16 +0400
Subject: [PATCH 043/109] feed image ID manualy

---
 terraform-modules/providers/openstack/main.tf | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 081ee9b..3af9bc3 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -196,8 +196,9 @@ resource "openstack_compute_instance_v2" "master" {
   user_data       = data.template_cloudinit_config.master_config.rendered
 
   block_device {
-    uuid                  = var.volume_uuid_1
-    source_type           = "volume"
+    uuid                  = var.image_id
+    source_type           = "image"
+    volume_size           = var.instance_volume_size
     boot_index            = 0
     destination_type      = "volume"
     delete_on_termination = true
@@ -221,8 +222,9 @@ resource "openstack_compute_instance_v2" "node" {
   )
 
   block_device {
-    uuid                  = var.volume_uuid_2
-    source_type           = "volume"
+    uuid                  = var.image_id
+    source_type           = "image"
+    volume_size           = var.instance_volume_size
     boot_index            = 0
     destination_type      = "volume"
     delete_on_termination = true

From b1a2143fd57e1d7dcabf5ee09cb2a36ff9893293 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Mon, 15 May 2023 12:25:33 +0400
Subject: [PATCH 044/109] add variabels

---
 terraform-modules/providers/openstack/variables.tf | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/terraform-modules/providers/openstack/variables.tf b/terraform-modules/providers/openstack/variables.tf
index 32afb59..dc04ed8 100644
--- a/terraform-modules/providers/openstack/variables.tf
+++ b/terraform-modules/providers/openstack/variables.tf
@@ -67,4 +67,7 @@ variable "volume_uuid_2" {
   description  = "Manual volume node"
 }
 
+variable "image_id" {
+  description  = "Manual image id"
+}
 

From 8508a1cb73f652e9020d88e51abb53d9fc0ad58b Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Mon, 15 May 2023 13:57:51 +0400
Subject: [PATCH 045/109] Manual again

---
 terraform-modules/providers/openstack/main.tf | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 3af9bc3..081ee9b 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -196,9 +196,8 @@ resource "openstack_compute_instance_v2" "master" {
   user_data       = data.template_cloudinit_config.master_config.rendered
 
   block_device {
-    uuid                  = var.image_id
-    source_type           = "image"
-    volume_size           = var.instance_volume_size
+    uuid                  = var.volume_uuid_1
+    source_type           = "volume"
     boot_index            = 0
     destination_type      = "volume"
     delete_on_termination = true
@@ -222,9 +221,8 @@ resource "openstack_compute_instance_v2" "node" {
   )
 
   block_device {
-    uuid                  = var.image_id
-    source_type           = "image"
-    volume_size           = var.instance_volume_size
+    uuid                  = var.volume_uuid_2
+    source_type           = "volume"
     boot_index            = 0
     destination_type      = "volume"
     delete_on_termination = true

From bbcf81777093dc51718dd5886793676834b2ce0f Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Wed, 5 Jul 2023 17:33:02 -0400
Subject: [PATCH 046/109] Add neurolibre customization

---
 .../binderhub/assets/jb_build.bash            | 49 ++++++++++++++++---
 1 file changed, 41 insertions(+), 8 deletions(-)

diff --git a/terraform-modules/binderhub/assets/jb_build.bash b/terraform-modules/binderhub/assets/jb_build.bash
index 4bb2d32..150ce26 100755
--- a/terraform-modules/binderhub/assets/jb_build.bash
+++ b/terraform-modules/binderhub/assets/jb_build.bash
@@ -8,14 +8,28 @@ REPO_NAME=${BINDER_PARAMS[-3]}
 COMMIT_REF=${BINDER_PARAMS[-1]}
 # paths
 CONFIG_FILE="content/_config.yml"
+NEUROLIBRE_CUSTOM="content/_neurolibre.yaml"
 BOOK_DST_PATH="/mnt/books/${USER_NAME}/${PROVIDER_NAME}/${REPO_NAME}/${COMMIT_REF}"
 BOOK_BUILT_FLAG="${BOOK_DST_PATH}/successfully_built"
 BOOK_BUILD_LOG="${BOOK_DST_PATH}/book-build.log"
 BINDERHUB_URL="https://test.conp.cloud"
 BOOK_CACHE_PATH=${BOOK_DST_PATH}"/_build/.jupyter_cache"
 
+extract_yaml_field() {
+  local yaml_file="$1"
+  local field_name="$2"
+  if [ -f "$yaml_file" ]; then
+    local field_value=$(sed -n "s/^[[:blank:]]*${field_name}:[[:blank:]]*\(.*\)/\1/p" "$yaml_file")
+    field_value=$(sed 's/^"\(.*\)"$/\1/' <<< "$field_value")
+    field_value=$(sed 's/[[:space:]]//g' <<< "$field_value")
+    echo "$field_value"
+  else
+    echo "YAML file not found: $yaml_file"
+  fi
+}
+
 # checking if book build is necessary
-echo "Checking if jupyter book build will be done..." 2>&1 | tee ${BOOK_BUILD_LOG}
+echo "Checking if the book will be built..." 2>&1 | tee ${BOOK_BUILD_LOG}
 if [ -f "${CONFIG_FILE}" ]; then
   echo -e "\t ${CONFIG_FILE} exists." 2>&1 | tee -a ${BOOK_BUILD_LOG}
 else
@@ -31,7 +45,7 @@ else
   echo -e "\t ${BOOK_BUILT_FLAG} not found." 2>&1 | tee -a ${BOOK_BUILD_LOG}
 fi
 if git log -1 | grep "neurolibre-debug"; then
-    echo "Bypassing jupyter-book build from user request." 2>&1 | tee -a ${BOOK_BUILD_LOG}
+    echo "Bypassing jupyter-book build as requested by the user (neurolibre-debug)" 2>&1 | tee -a ${BOOK_BUILD_LOG}
     exit 0
 fi
 # changing config if test submission
@@ -59,17 +73,36 @@ execute:
 EOF
 fi
 
+
+if [ -f "$NEUROLIBRE_CUSTOM" ]; then
+    BOOK_LAYOUT=$(extract_yaml_field "$NEUROLIBRE_CUSTOM" "book_layout")
+    SINGLE_PAGE=$(extract_yaml_field "$NEUROLIBRE_CUSTOM" "single_page")
+  else
+    echo "YAML file not found: $yaml_file"
+fi
+
+
 # building jupyter book
 echo "" 2>&1 | tee -a ${BOOK_BUILD_LOG}
-echo "Building jupyter-book for ${USER_NAME}/${PROVIDER_NAME}/${REPO_NAME}/${COMMIT_REF}" 2>&1 | tee -a ${BOOK_BUILD_LOG}
+echo "Build source: ${USER_NAME}/${PROVIDER_NAME}/${REPO_NAME}/${COMMIT_REF}" 2>&1 | tee -a ${BOOK_BUILD_LOG}
 echo "" 2>&1 | tee -a ${BOOK_BUILD_LOG}
 mkdir -p ${BOOK_DST_PATH}
 mkdir -p ${BOOK_CACHE_PATH}
 touch ${BOOK_BUILD_LOG}
-jupyter-book build --all --verbose --path-output ${BOOK_DST_PATH} content 2>&1 | tee -a ${BOOK_BUILD_LOG}
+# Write the first line to the log
+echo "" 2>&1 | tee -a ${BOOK_BUILD_LOG}
+
+if [ "$BOOK_LAYOUT" = "traditional" ]; then
+    # SINGLE_PAGE exists when BOOK_LAYOUT is traditional (documentation)
+    echo -e "Customized book build: traditional paper layout based on ${SINGLE_PAGE}" 2>&1 | tee -a ${BOOK_BUILD_LOG}
+    jupyter-book build --all --verbose --path-output ${BOOK_DST_PATH} --builder singlehtml content/${SINGLE_PAGE} 2>&1 | tee -a ${BOOK_BUILD_LOG}
+  else
+    # Use default build otherwise
+    jupyter-book build --all --verbose --path-output ${BOOK_DST_PATH} content 2>&1 | tee -a ${BOOK_BUILD_LOG}
+fi
+
 # https://stackoverflow.com/a/1221870
 JB_EXIT_CODE=${PIPESTATUS[0]}
-echo "" 2>&1 | tee -a ${BOOK_BUILD_LOG}
 # checking execution
 if grep ${BOOK_BUILD_LOG} -e "Execution Failed"; then
   echo -e "Jupyter-book execution failed!" 2>&1 | tee -a ${BOOK_BUILD_LOG}
@@ -80,9 +113,9 @@ if [ ${JB_EXIT_CODE} -ne 0 ] ; then
   echo -e "Jupyter-book build failed!" 2>&1 | tee -a ${BOOK_BUILD_LOG}
   exit 0
 else
-  echo "Taring book build artifacts..." 2>&1 | tee -a ${BOOK_BUILD_LOG}
+  echo "Compressing book build artifacts..." 2>&1 | tee -a ${BOOK_BUILD_LOG}
   tar -zcvf ${BOOK_DST_PATH}".tar.gz" ${BOOK_DST_PATH} 2>&1 | tee -a ${BOOK_BUILD_LOG}
   touch ${BOOK_BUILT_FLAG}
-  echo "Filling metadata for current submission..." 2>&1 | tee -a ${BOOK_BUILD_LOG}
+  echo "Saving metadata for current submission..." 2>&1 | tee -a ${BOOK_BUILD_LOG}
   /bin/bash /usr/local/share/fill_submission_metadata.bash
-fi
+fi
\ No newline at end of file

From 4b748a6e778d0891dfe3284f771b67984041945b Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Wed, 5 Jul 2023 17:38:09 -0400
Subject: [PATCH 047/109] Use yml extension

---
 terraform-modules/binderhub/assets/jb_build.bash | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-modules/binderhub/assets/jb_build.bash b/terraform-modules/binderhub/assets/jb_build.bash
index 150ce26..e0c1c19 100755
--- a/terraform-modules/binderhub/assets/jb_build.bash
+++ b/terraform-modules/binderhub/assets/jb_build.bash
@@ -8,7 +8,7 @@ REPO_NAME=${BINDER_PARAMS[-3]}
 COMMIT_REF=${BINDER_PARAMS[-1]}
 # paths
 CONFIG_FILE="content/_config.yml"
-NEUROLIBRE_CUSTOM="content/_neurolibre.yaml"
+NEUROLIBRE_CUSTOM="content/_neurolibre.yml"
 BOOK_DST_PATH="/mnt/books/${USER_NAME}/${PROVIDER_NAME}/${REPO_NAME}/${COMMIT_REF}"
 BOOK_BUILT_FLAG="${BOOK_DST_PATH}/successfully_built"
 BOOK_BUILD_LOG="${BOOK_DST_PATH}/book-build.log"

From 822b56afa7d15cd45a368fd4f33a31f76d5614cb Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Fri, 17 May 2024 18:02:11 +0100
Subject: [PATCH 048/109] volume management

---
 terraform-modules/providers/openstack/main.tf | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 081ee9b..2b58498 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -196,8 +196,9 @@ resource "openstack_compute_instance_v2" "master" {
   user_data       = data.template_cloudinit_config.master_config.rendered
 
   block_device {
-    uuid                  = var.volume_uuid_1
-    source_type           = "volume"
+    uuid                  = data.openstack_images_image_v2.ubuntu.id
+    source_type           = "image"
+    volume_size           = var.instance_volume_size
     boot_index            = 0
     destination_type      = "volume"
     delete_on_termination = true
@@ -221,8 +222,9 @@ resource "openstack_compute_instance_v2" "node" {
   )
 
   block_device {
-    uuid                  = var.volume_uuid_2
-    source_type           = "volume"
+    uuid                  = data.openstack_images_image_v2.ubuntu.id
+    source_type           = "image"
+    volume_size           = var.instance_volume_size
     boot_index            = 0
     destination_type      = "volume"
     delete_on_termination = true

From ac816ad5eb4bb74be12de0534d1cbb2487600ef0 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Fri, 17 May 2024 18:06:21 +0100
Subject: [PATCH 049/109] rm var

---
 .../providers/openstack/variables.tf              | 15 +--------------
 1 file changed, 1 insertion(+), 14 deletions(-)

diff --git a/terraform-modules/providers/openstack/variables.tf b/terraform-modules/providers/openstack/variables.tf
index dc04ed8..5a21fee 100644
--- a/terraform-modules/providers/openstack/variables.tf
+++ b/terraform-modules/providers/openstack/variables.tf
@@ -57,17 +57,4 @@ variable "docker_password" {
 
 variable "public_network" {
   default  = "Public-Network"
-}
-
-variable "volume_uuid_1" {
-  description  = "Manual volume master"
-}
-
-variable "volume_uuid_2" {
-  description  = "Manual volume node"
-}
-
-variable "image_id" {
-  description  = "Manual image id"
-}
-
+}
\ No newline at end of file

From 85241da64dba40dd615ff07a94e0016d49b4343c Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Fri, 17 May 2024 18:42:25 +0100
Subject: [PATCH 050/109] make it make sense

---
 terraform-modules/providers/openstack/main.tf | 26 +++++++++++++------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 2b58498..4e6dac1 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -188,6 +188,12 @@ resource "openstack_compute_keypair_v2" "keypair" {
   public_key = element(var.ssh_authorized_keys, 0)
 }
 
+resource "openstack_blockstorage_volume_v1" "mastervolume" {
+  name        = "master-volume"
+  size        = var.instance_volume_size
+  image_id    = data.openstack_images_image_v2.ubuntu.id
+}
+
 resource "openstack_compute_instance_v2" "master" {
   name            = "${var.project_name}-master"
   flavor_name     = var.os_flavor_master
@@ -196,11 +202,10 @@ resource "openstack_compute_instance_v2" "master" {
   user_data       = data.template_cloudinit_config.master_config.rendered
 
   block_device {
-    uuid                  = data.openstack_images_image_v2.ubuntu.id
-    source_type           = "image"
-    volume_size           = var.instance_volume_size
-    boot_index            = 0
+    uuid                  = openstack_blockstorage_mastervolume_v1.volume.id
+    source_type           = "volume"
     destination_type      = "volume"
+    boot_index            = 0
     delete_on_termination = true
   }
 
@@ -209,6 +214,12 @@ resource "openstack_compute_instance_v2" "master" {
   }
 }
 
+resource "openstack_blockstorage_volume_v1" "nodevolume" {
+  name        = "node${count.index + 1}-volume"
+  size        = var.instance_volume_size
+  image_id    = data.openstack_images_image_v2.ubuntu.id
+}
+
 resource "openstack_compute_instance_v2" "node" {
   count = var.nb_nodes
   name  = "${var.project_name}-node${count.index + 1}"
@@ -222,11 +233,10 @@ resource "openstack_compute_instance_v2" "node" {
   )
 
   block_device {
-    uuid                  = data.openstack_images_image_v2.ubuntu.id
-    source_type           = "image"
-    volume_size           = var.instance_volume_size
-    boot_index            = 0
+    uuid                  = openstack_blockstorage_mastervolume_v1.nodevolume.id
+    source_type           = "volume"
     destination_type      = "volume"
+    boot_index            = 0
     delete_on_termination = true
   }
 

From e60c49cee3d6f80fa9f74ce96017b6a7d25e2d94 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Fri, 17 May 2024 18:49:00 +0100
Subject: [PATCH 051/109] wrong nspace

---
 terraform-modules/providers/openstack/main.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 4e6dac1..6507ed8 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -202,7 +202,7 @@ resource "openstack_compute_instance_v2" "master" {
   user_data       = data.template_cloudinit_config.master_config.rendered
 
   block_device {
-    uuid                  = openstack_blockstorage_mastervolume_v1.volume.id
+    uuid                  = openstack_blockstorage_volume_v1.mastervolume.id
     source_type           = "volume"
     destination_type      = "volume"
     boot_index            = 0
@@ -233,7 +233,7 @@ resource "openstack_compute_instance_v2" "node" {
   )
 
   block_device {
-    uuid                  = openstack_blockstorage_mastervolume_v1.nodevolume.id
+    uuid                  = openstack_blockstorage_volume_v1.nodevolume.id
     source_type           = "volume"
     destination_type      = "volume"
     boot_index            = 0

From 111b74e030f7537a93e47caf070c82599ed9a047 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Fri, 17 May 2024 18:52:59 +0100
Subject: [PATCH 052/109] Not allowed

---
 terraform-modules/providers/openstack/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 6507ed8..b1bae27 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -215,7 +215,7 @@ resource "openstack_compute_instance_v2" "master" {
 }
 
 resource "openstack_blockstorage_volume_v1" "nodevolume" {
-  name        = "node${count.index + 1}-volume"
+  name        = "node-volume"
   size        = var.instance_volume_size
   image_id    = data.openstack_images_image_v2.ubuntu.id
 }

From a8a379aa4e718f9e06c5e231b859580bb267cc25 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Fri, 17 May 2024 19:08:08 +0100
Subject: [PATCH 053/109] v2

---
 terraform-modules/providers/openstack/main.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index b1bae27..a6cc87f 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -214,7 +214,7 @@ resource "openstack_compute_instance_v2" "master" {
   }
 }
 
-resource "openstack_blockstorage_volume_v1" "nodevolume" {
+resource "openstack_blockstorage_volume_v2" "nodevolume" {
   name        = "node-volume"
   size        = var.instance_volume_size
   image_id    = data.openstack_images_image_v2.ubuntu.id
@@ -233,7 +233,7 @@ resource "openstack_compute_instance_v2" "node" {
   )
 
   block_device {
-    uuid                  = openstack_blockstorage_volume_v1.nodevolume.id
+    uuid                  = openstack_blockstorage_volume_v2.nodevolume.id
     source_type           = "volume"
     destination_type      = "volume"
     boot_index            = 0

From 54a8a416bf70a9ad04c9216794b2573edb9d1d57 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Fri, 17 May 2024 19:09:43 +0100
Subject: [PATCH 054/109] fix

---
 terraform-modules/providers/openstack/main.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index a6cc87f..17cb846 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -188,7 +188,7 @@ resource "openstack_compute_keypair_v2" "keypair" {
   public_key = element(var.ssh_authorized_keys, 0)
 }
 
-resource "openstack_blockstorage_volume_v1" "mastervolume" {
+resource "openstack_blockstorage_volume_v2" "mastervolume" {
   name        = "master-volume"
   size        = var.instance_volume_size
   image_id    = data.openstack_images_image_v2.ubuntu.id
@@ -202,7 +202,7 @@ resource "openstack_compute_instance_v2" "master" {
   user_data       = data.template_cloudinit_config.master_config.rendered
 
   block_device {
-    uuid                  = openstack_blockstorage_volume_v1.mastervolume.id
+    uuid                  = openstack_blockstorage_volume_v2.mastervolume.id
     source_type           = "volume"
     destination_type      = "volume"
     boot_index            = 0

From dfed415df768e951956cafcb92609131b2a98bd5 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sat, 18 May 2024 13:24:13 +0100
Subject: [PATCH 055/109] Change kubeadm source

---
 cloud-init/kubeadm/common.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloud-init/kubeadm/common.yaml b/cloud-init/kubeadm/common.yaml
index 0fc45f5..ab73087 100644
--- a/cloud-init/kubeadm/common.yaml
+++ b/cloud-init/kubeadm/common.yaml
@@ -16,7 +16,7 @@ write_files:
     permissions: 0755
     content: |
       #!/bin/bash
-      git clone https://github.com/neurolibre/kubeadm-bootstrap.git /tmp/kubeadm-bootstrap
+      git clone https://github.com/agahkarakuzu/kubeadm-bootstrap.git /tmp/kubeadm-bootstrap
       cd /tmp/kubeadm-bootstrap
       sudo ./install-kubeadm.bash
 

From e519a42456c5c15fd3f25be7863b0d1e75ba1706 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sat, 18 May 2024 17:49:07 +0100
Subject: [PATCH 056/109] Now this fails

---
 terraform-modules/binderhub/main.tf      | 3 +++
 terraform-modules/binderhub/variables.tf | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index 358a621..309fba3 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -66,6 +66,9 @@ resource "null_resource" "remote_install" {
   connection {
     user = var.admin_user
     host = var.ip
+    type     = "ssh"
+    agent    = "false"
+    private_key = var.admin_key
   }
 
   provisioner "file" {
diff --git a/terraform-modules/binderhub/variables.tf b/terraform-modules/binderhub/variables.tf
index 880bbf9..b910bb2 100644
--- a/terraform-modules/binderhub/variables.tf
+++ b/terraform-modules/binderhub/variables.tf
@@ -26,6 +26,10 @@ variable "admin_user" {
   description = "User with root access"
 }
 
+variable "admin_key" {
+  description = "You know what it is."
+}
+
 variable "binder_version" {
   description = "binderhub helm chart version - https://jupyterhub.github.io/helm-chart/#development-releases-binderhub"
 }

From 3879a72af8d996ba6a647ec7a606235d1a3498ff Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sat, 18 May 2024 17:52:58 +0100
Subject: [PATCH 057/109] fix

---
 terraform-modules/binderhub/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index 309fba3..444cec9 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -68,7 +68,7 @@ resource "null_resource" "remote_install" {
     host = var.ip
     type     = "ssh"
     agent    = "false"
-    private_key = var.admin_key
+    private_key = file(var.admin_key)
   }
 
   provisioner "file" {

From e8f970c0f004bf3a3ff33d3bbcc290e7b3eaca1d Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sat, 18 May 2024 18:10:54 +0100
Subject: [PATCH 058/109] another method

---
 terraform-modules/binderhub/main.tf      | 3 ---
 terraform-modules/binderhub/variables.tf | 4 ----
 2 files changed, 7 deletions(-)

diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index 444cec9..358a621 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -66,9 +66,6 @@ resource "null_resource" "remote_install" {
   connection {
     user = var.admin_user
     host = var.ip
-    type     = "ssh"
-    agent    = "false"
-    private_key = file(var.admin_key)
   }
 
   provisioner "file" {
diff --git a/terraform-modules/binderhub/variables.tf b/terraform-modules/binderhub/variables.tf
index b910bb2..880bbf9 100644
--- a/terraform-modules/binderhub/variables.tf
+++ b/terraform-modules/binderhub/variables.tf
@@ -26,10 +26,6 @@ variable "admin_user" {
   description = "User with root access"
 }
 
-variable "admin_key" {
-  description = "You know what it is."
-}
-
 variable "binder_version" {
   description = "binderhub helm chart version - https://jupyterhub.github.io/helm-chart/#development-releases-binderhub"
 }

From 7695f68725e619c29aeff9687a628da41871962c Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sat, 18 May 2024 18:29:10 +0100
Subject: [PATCH 059/109] remove ducking files

---
 terraform-modules/.DS_Store           | Bin 0 -> 6148 bytes
 terraform-modules/binderhub/.DS_Store | Bin 6148 -> 6148 bytes
 terraform-modules/binderhub/main.tf   |  66 +-------------------------
 3 files changed, 1 insertion(+), 65 deletions(-)
 create mode 100644 terraform-modules/.DS_Store

diff --git a/terraform-modules/.DS_Store b/terraform-modules/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..1ddfc1d53ecd42054d2727ea9e9b7864c7377388
GIT binary patch
literal 6148
zcmeHK%}T>S5T0$TO(;SS3VK`cTClC9DqcdZFJMFuDzzcS24lAL#~ex_cYPsW#OHBl
zcPo}w@FZeqVD_7xpDg<&>|_Bzv?gH<pbP*GDq+sWVTDkfbV+j7Ln!n!dN71OOdtr-
zrEGTmMF!~Ixe!1E354+F{f!1etOAU_1%o&mjjPpnk;@l0Hj7TtDLHq3U#0%IJD!YM
z-Qf+*E|rRdx!nz}qF&l5Z=b1T+>MgnP!~jD4^wWhqa;*mOHGn+q-%W>a9qdjl=r67
zTC;jok@e%otRkl;^=d^n4x6)?>+I|woL+Pu<EKQu7(NAldRn$D&fyh}6<d4uhe@oG
z2aL`u=M{|105iZ0tO*15_;X5YvR~dNGr$b|oB=u?Br2h2F*B%-4jkzBk>UkH613?q
zK`1SH7BhnwK@p}D(Ub~%#So?({nF-n7BhpU9E4sOpJP`R_J$(#>gbm`9fW6)M`nN-
zSY#k?re*5?=ilG|7mIkt3@`)#iUCn*`|TEP$@bQz%~7wFsQ0KOl$RO&EI~uJV$7vh
dyo;&@{gMns&thf}Jt+JmplRTN8TeHOJ^+fnO@#me

literal 0
HcmV?d00001

diff --git a/terraform-modules/binderhub/.DS_Store b/terraform-modules/binderhub/.DS_Store
index a1e5eba800aa83362ec5047eeadc300ea442ca8a..ba47e95ed67723e1bff85d7cb2b5461f403d13fe 100644
GIT binary patch
delta 176
zcmZoMXfc@J&&a+pU^gQp`(_>{MaKG+;^ds9{QMk71_lN;hD3&9AWUT_VJOZ?Hw;eB
z&n*Co1A#j`kU){o&3AE0%E?axigPGb*e`fcdBhP@O$q@uLJY+P8ORnLVAw3n?8&m3
Io#QV*0OFr3EdT%j

delta 34
qcmZoMXfc@J&&aefU^nAr0}+<Z{7iC;o6j*@vP^7PxS5^fFFydha|;Im

diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index 358a621..96c15bf 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -60,68 +60,4 @@ data "template_file" "install-binderhub" {
     docker_id       = var.docker_id
     docker_password = var.docker_password
   }
-}
-
-resource "null_resource" "remote_install" {
-  connection {
-    user = var.admin_user
-    host = var.ip
-  }
-
-  provisioner "file" {
-    source     = "${path.module}/assets/fill_submission_metadata.bash"
-    destination = "/home/${var.admin_user}/fill_submission_metadata.bash"
-  }
-
-  provisioner "file" {
-    source     = "${path.module}/assets/repo2data.bash"
-    destination = "/home/${var.admin_user}/repo2data.bash"
-  }
-
-  provisioner "file" {
-    source     = "${path.module}/assets/jb_build.bash"
-    destination = "/home/${var.admin_user}/jb_build.bash"
-  }
-
-  provisioner "file" {
-    content     = data.template_file.config.rendered
-    destination = "/home/${var.admin_user}/config.yaml"
-  }
-
-  provisioner "file" {
-    content     = data.template_file.secrets.rendered
-    destination = "/home/${var.admin_user}/secrets.yaml"
-  }
-
-  provisioner "file" {
-    content     = data.template_file.pv.rendered
-    destination = "/home/${var.admin_user}/pv.yaml"
-  }
-
-  provisioner "file" {
-    content     = data.template_file.nginx-ingress.rendered
-    destination = "/home/${var.admin_user}/nginx-ingress.yaml"
-  }
-
-  provisioner "file" {
-    content     = data.template_file.production-binderhub-issuer.rendered
-    destination = "/home/${var.admin_user}/production-binderhub-issuer.yaml"
-  }
-
-  provisioner "file" {
-    content     = data.template_file.staging-binderhub-issuer.rendered
-    destination = "/home/${var.admin_user}/staging-binderhub-issuer.yaml"
-  }
-
-  provisioner "file" {
-    content     = data.template_file.install-binderhub.rendered
-    destination = "/home/${var.admin_user}/install-binderhub.sh"
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "bash /home/${var.admin_user}/install-binderhub.sh",
-    ]
-  }
-}
-
+}
\ No newline at end of file

From c175c083eef8a803d2fecf5b2bb5b50fbdc4aee1 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sat, 18 May 2024 19:11:35 +0100
Subject: [PATCH 060/109] use agent

---
 terraform-modules/binderhub/main.tf | 64 +++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)

diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index 96c15bf..b9704bb 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -60,4 +60,68 @@ data "template_file" "install-binderhub" {
     docker_id       = var.docker_id
     docker_password = var.docker_password
   }
+}
+
+resource "null_resource" "remote_install" {
+  connection {
+    user = var.admin_user
+    host = var.ip
+    agent= true
+  }
+
+  provisioner "file" {
+    source     = "${path.module}/assets/fill_submission_metadata.bash"
+    destination = "/home/${var.admin_user}/fill_submission_metadata.bash"
+  }
+
+  provisioner "file" {
+    source     = "${path.module}/assets/repo2data.bash"
+    destination = "/home/${var.admin_user}/repo2data.bash"
+  }
+
+  provisioner "file" {
+    source     = "${path.module}/assets/jb_build.bash"
+    destination = "/home/${var.admin_user}/jb_build.bash"
+  }
+
+  provisioner "file" {
+    content     = data.template_file.config.rendered
+    destination = "/home/${var.admin_user}/config.yaml"
+  }
+
+  provisioner "file" {
+    content     = data.template_file.secrets.rendered
+    destination = "/home/${var.admin_user}/secrets.yaml"
+  }
+
+  provisioner "file" {
+    content     = data.template_file.pv.rendered
+    destination = "/home/${var.admin_user}/pv.yaml"
+  }
+
+  provisioner "file" {
+    content     = data.template_file.nginx-ingress.rendered
+    destination = "/home/${var.admin_user}/nginx-ingress.yaml"
+  }
+
+  provisioner "file" {
+    content     = data.template_file.production-binderhub-issuer.rendered
+    destination = "/home/${var.admin_user}/production-binderhub-issuer.yaml"
+  }
+
+  provisioner "file" {
+    content     = data.template_file.staging-binderhub-issuer.rendered
+    destination = "/home/${var.admin_user}/staging-binderhub-issuer.yaml"
+  }
+
+  provisioner "file" {
+    content     = data.template_file.install-binderhub.rendered
+    destination = "/home/${var.admin_user}/install-binderhub.sh"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "bash /home/${var.admin_user}/install-binderhub.sh",
+    ]
+  }
 }
\ No newline at end of file

From e8b3269ee4245f7fd3cd12e79d796679172d9c32 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sat, 18 May 2024 22:26:35 +0100
Subject: [PATCH 061/109] v0.13

---
 terraform-modules/binderhub/versions.tf | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/terraform-modules/binderhub/versions.tf b/terraform-modules/binderhub/versions.tf
index ac97c6a..9bc021f 100644
--- a/terraform-modules/binderhub/versions.tf
+++ b/terraform-modules/binderhub/versions.tf
@@ -1,4 +1,15 @@
 
 terraform {
-  required_version = ">= 0.12"
+  required_version = ">= 0.13"
+  required_providers {
+    null = {
+      source = "hashicorp/null"
+    }
+    random = {
+      source = "hashicorp/random"
+    }
+    template = {
+      source = "hashicorp/template"
+    }
+  }
 }

From ec031714ea316e53f3d49d4c6609a1f680d7336f Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sat, 18 May 2024 22:27:09 +0100
Subject: [PATCH 062/109] 0.13

---
 terraform-modules/dns/cloudflare/versions.tf      |  7 ++++++-
 terraform-modules/providers/openstack/versions.tf | 10 +++++++++-
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/terraform-modules/dns/cloudflare/versions.tf b/terraform-modules/dns/cloudflare/versions.tf
index ac97c6a..34893cc 100644
--- a/terraform-modules/dns/cloudflare/versions.tf
+++ b/terraform-modules/dns/cloudflare/versions.tf
@@ -1,4 +1,9 @@
 
 terraform {
-  required_version = ">= 0.12"
+  required_version = ">= 0.13"
+  required_providers {
+    cloudflare = {
+      source = "terraform-providers/cloudflare"
+    }
+  }
 }
diff --git a/terraform-modules/providers/openstack/versions.tf b/terraform-modules/providers/openstack/versions.tf
index ac97c6a..c3d0e65 100644
--- a/terraform-modules/providers/openstack/versions.tf
+++ b/terraform-modules/providers/openstack/versions.tf
@@ -1,4 +1,12 @@
 
 terraform {
-  required_version = ">= 0.12"
+  required_version = ">= 0.13"
+  required_providers {
+    openstack = {
+      source = "terraform-providers/openstack"
+    }
+    template = {
+      source = "hashicorp/template"
+    }
+  }
 }

From 5ff42bd3624d00d1ce9c6b4a72dd93500b4a6ef1 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sat, 18 May 2024 22:40:11 +0100
Subject: [PATCH 063/109] try

---
 terraform-modules/providers/openstack/main.tf     | 3 +--
 terraform-modules/providers/openstack/versions.tf | 1 +
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 17cb846..8f28e21 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -1,6 +1,4 @@
 provider "openstack" {
-  version = "<= 1.24.0"
-}
 
 data "openstack_images_image_v2" "ubuntu" {
   name        = var.image_name
@@ -254,3 +252,4 @@ resource "openstack_compute_floatingip_associate_v2" "fip_1" {
   instance_id = openstack_compute_instance_v2.master.id
 }
 
+}
\ No newline at end of file
diff --git a/terraform-modules/providers/openstack/versions.tf b/terraform-modules/providers/openstack/versions.tf
index c3d0e65..2bd38ae 100644
--- a/terraform-modules/providers/openstack/versions.tf
+++ b/terraform-modules/providers/openstack/versions.tf
@@ -4,6 +4,7 @@ terraform {
   required_providers {
     openstack = {
       source = "terraform-providers/openstack"
+      version = "<= 1.24.0"
     }
     template = {
       source = "hashicorp/template"

From 0274e86cf116e02692f086a1aeb60a0b811eea07 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sat, 18 May 2024 22:45:41 +0100
Subject: [PATCH 064/109] rm locals

---
 terraform-modules/providers/openstack/main.tf | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 8f28e21..c7f4fbd 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -73,9 +73,6 @@ resource "openstack_compute_secgroup_v2" "secgroup_1" {
   }
 }
 
-locals {
-  network_name = "${var.project_name}-network"
-}
 
 resource "openstack_networking_subnet_v2" "subnet" {
   count = var.is_computecanada ? 0 : 1
@@ -90,7 +87,7 @@ resource "openstack_networking_subnet_v2" "subnet" {
 resource "openstack_networking_network_v2" "network_1" {
   count = var.is_computecanada ? 0 : 1
 
-  name = local.network_name
+  name = "${var.project_name}-network"
 }
 
 data "template_file" "kubeadm_master" {
@@ -208,7 +205,7 @@ resource "openstack_compute_instance_v2" "master" {
   }
 
   network {
-    name = var.is_computecanada ? data.openstack_networking_network_v2.int_network.name : local.network_name
+    name = var.is_computecanada ? data.openstack_networking_network_v2.int_network.name : "${var.project_name}-network"
   }
 }
 
@@ -239,7 +236,7 @@ resource "openstack_compute_instance_v2" "node" {
   }
 
   network {
-    name = var.is_computecanada ? data.openstack_networking_network_v2.int_network.name : local.network_name
+    name = var.is_computecanada ? data.openstack_networking_network_v2.int_network.name : "${var.project_name}-network"
   }
 }
 

From 68f440a69725e6fbab56185554bae9a44cc7ebb3 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sat, 18 May 2024 22:49:19 +0100
Subject: [PATCH 065/109] compatibility issues

---
 terraform-modules/dns/cloudflare/main.tf          | 3 +--
 terraform-modules/dns/cloudflare/versions.tf      | 3 ++-
 terraform-modules/providers/openstack/versions.tf | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/terraform-modules/dns/cloudflare/main.tf b/terraform-modules/dns/cloudflare/main.tf
index 863bca2..7543439 100644
--- a/terraform-modules/dns/cloudflare/main.tf
+++ b/terraform-modules/dns/cloudflare/main.tf
@@ -1,6 +1,4 @@
 provider "cloudflare" {
-  version = "<= 2.10.1"
-}
 
 resource "cloudflare_record" "domain" {
   zone_id = var.zone_id
@@ -9,3 +7,4 @@ resource "cloudflare_record" "domain" {
   type    = "A"
 }
 
+}
\ No newline at end of file
diff --git a/terraform-modules/dns/cloudflare/versions.tf b/terraform-modules/dns/cloudflare/versions.tf
index 34893cc..2312a24 100644
--- a/terraform-modules/dns/cloudflare/versions.tf
+++ b/terraform-modules/dns/cloudflare/versions.tf
@@ -3,7 +3,8 @@ terraform {
   required_version = ">= 0.13"
   required_providers {
     cloudflare = {
-      source = "terraform-providers/cloudflare"
+      source = " cloudflare/cloudflare"
+      version = "<= 2.10.1"
     }
   }
 }
diff --git a/terraform-modules/providers/openstack/versions.tf b/terraform-modules/providers/openstack/versions.tf
index 2bd38ae..5672b37 100644
--- a/terraform-modules/providers/openstack/versions.tf
+++ b/terraform-modules/providers/openstack/versions.tf
@@ -3,7 +3,7 @@ terraform {
   required_version = ">= 0.13"
   required_providers {
     openstack = {
-      source = "terraform-providers/openstack"
+      source = "terraform-provider-openstack/openstack"
       version = "<= 1.24.0"
     }
     template = {

From ad422b3a1144da29cc2486f0720b697c97d6d75e Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sat, 18 May 2024 22:50:18 +0100
Subject: [PATCH 066/109] EVEN TAYLOR SWIFT IS OK WITH BLANK SPACES

---
 terraform-modules/dns/cloudflare/versions.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-modules/dns/cloudflare/versions.tf b/terraform-modules/dns/cloudflare/versions.tf
index 2312a24..8f19d90 100644
--- a/terraform-modules/dns/cloudflare/versions.tf
+++ b/terraform-modules/dns/cloudflare/versions.tf
@@ -3,7 +3,7 @@ terraform {
   required_version = ">= 0.13"
   required_providers {
     cloudflare = {
-      source = " cloudflare/cloudflare"
+      source = "cloudflare/cloudflare"
       version = "<= 2.10.1"
     }
   }

From 95929b9c482b4a94700734f03fb40fbca59d80cd Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sat, 18 May 2024 22:51:58 +0100
Subject: [PATCH 067/109] increase provider

---
 terraform-modules/providers/openstack/versions.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-modules/providers/openstack/versions.tf b/terraform-modules/providers/openstack/versions.tf
index 5672b37..70f9291 100644
--- a/terraform-modules/providers/openstack/versions.tf
+++ b/terraform-modules/providers/openstack/versions.tf
@@ -4,7 +4,7 @@ terraform {
   required_providers {
     openstack = {
       source = "terraform-provider-openstack/openstack"
-      version = "<= 1.24.0"
+      version = "~> 1.53.0"
     }
     template = {
       source = "hashicorp/template"

From c2cf04ee54cd6657d0c108e6b365d866b9547103 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 11:58:37 +0100
Subject: [PATCH 068/109] convention

---
 terraform-modules/providers/openstack/main.tf    | 2 +-
 terraform-modules/providers/openstack/outputs.tf | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index c7f4fbd..acb01cc 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -241,7 +241,7 @@ resource "openstack_compute_instance_v2" "node" {
 }
 
 resource "openstack_networking_floatingip_v2" "fip_1" {
-  pool       = data.openstack_networking_network_v2.ext_network.name
+  pool = data.openstack_networking_network_v2.ext_network.name
 }
 
 resource "openstack_compute_floatingip_associate_v2" "fip_1" {
diff --git a/terraform-modules/providers/openstack/outputs.tf b/terraform-modules/providers/openstack/outputs.tf
index c01d24e..5cd4634 100644
--- a/terraform-modules/providers/openstack/outputs.tf
+++ b/terraform-modules/providers/openstack/outputs.tf
@@ -1,5 +1,5 @@
 output "public_ip" {
-  value = openstack_networking_floatingip_v2.fip_1.address
+  value = module.openstack.openstack_networking_floatingip_v2.fip_1.address
 }
 
 output "admin_user" {

From a27a705f04b46a488462a474db869aab8b3488e3 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 12:27:41 +0100
Subject: [PATCH 069/109] Change structure

---
 terraform-modules/providers/openstack/main.tf      | 4 ----
 terraform-modules/providers/openstack/outputs.tf   | 2 +-
 terraform-modules/providers/openstack/providers.tf | 1 +
 terraform-modules/providers/openstack/versions.tf  | 3 ---
 4 files changed, 2 insertions(+), 8 deletions(-)
 create mode 100644 terraform-modules/providers/openstack/providers.tf

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index acb01cc..3a2c21b 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -1,5 +1,3 @@
-provider "openstack" {
-
 data "openstack_images_image_v2" "ubuntu" {
   name        = var.image_name
   most_recent = true
@@ -247,6 +245,4 @@ resource "openstack_networking_floatingip_v2" "fip_1" {
 resource "openstack_compute_floatingip_associate_v2" "fip_1" {
   floating_ip = openstack_networking_floatingip_v2.fip_1.address
   instance_id = openstack_compute_instance_v2.master.id
-}
-
 }
\ No newline at end of file
diff --git a/terraform-modules/providers/openstack/outputs.tf b/terraform-modules/providers/openstack/outputs.tf
index 5cd4634..ac36b87 100644
--- a/terraform-modules/providers/openstack/outputs.tf
+++ b/terraform-modules/providers/openstack/outputs.tf
@@ -1,5 +1,5 @@
 output "public_ip" {
-  value = module.openstack.openstack_networking_floatingip_v2.fip_1.address
+  value = openstack.resource.openstack_networking_floatingip_v2.fip_1.address
 }
 
 output "admin_user" {
diff --git a/terraform-modules/providers/openstack/providers.tf b/terraform-modules/providers/openstack/providers.tf
new file mode 100644
index 0000000..5bc5e9e
--- /dev/null
+++ b/terraform-modules/providers/openstack/providers.tf
@@ -0,0 +1 @@
+provider "openstack" {}
\ No newline at end of file
diff --git a/terraform-modules/providers/openstack/versions.tf b/terraform-modules/providers/openstack/versions.tf
index 70f9291..af76733 100644
--- a/terraform-modules/providers/openstack/versions.tf
+++ b/terraform-modules/providers/openstack/versions.tf
@@ -6,8 +6,5 @@ terraform {
       source = "terraform-provider-openstack/openstack"
       version = "~> 1.53.0"
     }
-    template = {
-      source = "hashicorp/template"
-    }
   }
 }

From 9ee327dad59a254f71f65320ac5a6bc8d81be877 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 12:28:56 +0100
Subject: [PATCH 070/109] leftover

---
 terraform-modules/providers/openstack/outputs.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-modules/providers/openstack/outputs.tf b/terraform-modules/providers/openstack/outputs.tf
index ac36b87..c01d24e 100644
--- a/terraform-modules/providers/openstack/outputs.tf
+++ b/terraform-modules/providers/openstack/outputs.tf
@@ -1,5 +1,5 @@
 output "public_ip" {
-  value = openstack.resource.openstack_networking_floatingip_v2.fip_1.address
+  value = openstack_networking_floatingip_v2.fip_1.address
 }
 
 output "admin_user" {

From 0d20483a27f3cbd90591a8b1401a6c8db7062c2a Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 12:35:47 +0100
Subject: [PATCH 071/109] getting there

---
 terraform-modules/dns/cloudflare/main.tf           | 4 +---
 terraform-modules/providers/openstack/providers.tf | 1 -
 terraform-modules/providers/openstack/versions.tf  | 3 +++
 3 files changed, 4 insertions(+), 4 deletions(-)
 delete mode 100644 terraform-modules/providers/openstack/providers.tf

diff --git a/terraform-modules/dns/cloudflare/main.tf b/terraform-modules/dns/cloudflare/main.tf
index 7543439..6d61a7a 100644
--- a/terraform-modules/dns/cloudflare/main.tf
+++ b/terraform-modules/dns/cloudflare/main.tf
@@ -1,10 +1,8 @@
-provider "cloudflare" {
+
 
 resource "cloudflare_record" "domain" {
   zone_id = var.zone_id
   name    = var.domain
   value   = var.public_ip
   type    = "A"
-}
-
 }
\ No newline at end of file
diff --git a/terraform-modules/providers/openstack/providers.tf b/terraform-modules/providers/openstack/providers.tf
deleted file mode 100644
index 5bc5e9e..0000000
--- a/terraform-modules/providers/openstack/providers.tf
+++ /dev/null
@@ -1 +0,0 @@
-provider "openstack" {}
\ No newline at end of file
diff --git a/terraform-modules/providers/openstack/versions.tf b/terraform-modules/providers/openstack/versions.tf
index af76733..70f9291 100644
--- a/terraform-modules/providers/openstack/versions.tf
+++ b/terraform-modules/providers/openstack/versions.tf
@@ -6,5 +6,8 @@ terraform {
       source = "terraform-provider-openstack/openstack"
       version = "~> 1.53.0"
     }
+    template = {
+      source = "hashicorp/template"
+    }
   }
 }

From 21f9afd541bdd91ae560cb84d14ee89a6f805fc9 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 12:41:56 +0100
Subject: [PATCH 072/109] Upgrade opst provider

---
 terraform-modules/providers/openstack/main.tf     | 8 ++++----
 terraform-modules/providers/openstack/versions.tf | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 3a2c21b..25ad82c 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -181,7 +181,7 @@ resource "openstack_compute_keypair_v2" "keypair" {
   public_key = element(var.ssh_authorized_keys, 0)
 }
 
-resource "openstack_blockstorage_volume_v2" "mastervolume" {
+resource "openstack_blockstorage_volume_v3" "mastervolume" {
   name        = "master-volume"
   size        = var.instance_volume_size
   image_id    = data.openstack_images_image_v2.ubuntu.id
@@ -195,7 +195,7 @@ resource "openstack_compute_instance_v2" "master" {
   user_data       = data.template_cloudinit_config.master_config.rendered
 
   block_device {
-    uuid                  = openstack_blockstorage_volume_v2.mastervolume.id
+    uuid                  = openstack_blockstorage_volume_v3.mastervolume.id
     source_type           = "volume"
     destination_type      = "volume"
     boot_index            = 0
@@ -207,7 +207,7 @@ resource "openstack_compute_instance_v2" "master" {
   }
 }
 
-resource "openstack_blockstorage_volume_v2" "nodevolume" {
+resource "openstack_blockstorage_volume_v3" "nodevolume" {
   name        = "node-volume"
   size        = var.instance_volume_size
   image_id    = data.openstack_images_image_v2.ubuntu.id
@@ -226,7 +226,7 @@ resource "openstack_compute_instance_v2" "node" {
   )
 
   block_device {
-    uuid                  = openstack_blockstorage_volume_v2.nodevolume.id
+    uuid                  = openstack_blockstorage_volume_v3.nodevolume.id
     source_type           = "volume"
     destination_type      = "volume"
     boot_index            = 0
diff --git a/terraform-modules/providers/openstack/versions.tf b/terraform-modules/providers/openstack/versions.tf
index 70f9291..3b19d7a 100644
--- a/terraform-modules/providers/openstack/versions.tf
+++ b/terraform-modules/providers/openstack/versions.tf
@@ -4,7 +4,7 @@ terraform {
   required_providers {
     openstack = {
       source = "terraform-provider-openstack/openstack"
-      version = "~> 1.53.0"
+      version = "~> 2.0.0"
     }
     template = {
       source = "hashicorp/template"

From a6ecd4969db44e53f801d8e2840cff5ddf6eb473 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 15:56:56 +0100
Subject: [PATCH 073/109] Use new secgroup

---
 terraform-modules/providers/openstack/main.tf | 74 +----------------
 .../providers/openstack/secgroup-common.tf    | 82 +++++++++++++++++++
 2 files changed, 84 insertions(+), 72 deletions(-)
 create mode 100644 terraform-modules/providers/openstack/secgroup-common.tf

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 25ad82c..ae4c1ca 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -3,75 +3,6 @@ data "openstack_images_image_v2" "ubuntu" {
   most_recent = true
 }
 
-resource "openstack_compute_secgroup_v2" "secgroup_1" {
-  name        = "${var.project_name}-secgroup-new"
-  description = "BinderHub security group - Agah"
-
-  rule {
-    from_port   = -1
-    to_port     = -1
-    ip_protocol = "icmp"
-    self        = true
-  }
-
-  rule {
-    from_port   = 1
-    to_port     = 65535
-    ip_protocol = "tcp"
-    self        = true
-  }
-
-  rule {
-    from_port   = 1
-    to_port     = 65535
-    ip_protocol = "udp"
-    self        = true
-  }
-
-  rule {
-    from_port   = -1
-    to_port     = -1
-    ip_protocol = "icmp"
-    cidr        = "192.168.73.30/32"
-  }
-
-  rule {
-    from_port   = 1
-    to_port     = 65535
-    ip_protocol = "tcp"
-    cidr        = "192.168.73.30/32"
-  }
-
-  rule {
-    from_port   = 1
-    to_port     = 65535
-    ip_protocol = "udp"
-    cidr        = "192.168.73.30/32"
-  }
-
-  rule {
-    from_port   = 22
-    to_port     = 22
-    ip_protocol = "tcp"
-    cidr        = "0.0.0.0/0"
-  }
-
-  rule {
-    from_port   = 443
-    to_port     = 443
-    ip_protocol = "tcp"
-    cidr        = "0.0.0.0/0"
-  }
-
-  rule {
-    from_port   = 80
-    to_port     = 80
-    ip_protocol = "tcp"
-    cidr        = "0.0.0.0/0"
-  }
-}
-
-
 resource "openstack_networking_subnet_v2" "subnet" {
   count = var.is_computecanada ? 0 : 1
 
@@ -84,7 +15,6 @@ resource "openstack_networking_subnet_v2" "subnet" {
 
 resource "openstack_networking_network_v2" "network_1" {
   count = var.is_computecanada ? 0 : 1
-
   name = "${var.project_name}-network"
 }
 
@@ -191,7 +121,7 @@ resource "openstack_compute_instance_v2" "master" {
   name            = "${var.project_name}-master"
   flavor_name     = var.os_flavor_master
   key_pair        = openstack_compute_keypair_v2.keypair.name
-  security_groups = [openstack_compute_secgroup_v2.secgroup_1.name,"neurolibre-secgroup"]
+  security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-secgroup"]
   user_data       = data.template_cloudinit_config.master_config.rendered
 
   block_device {
@@ -219,7 +149,7 @@ resource "openstack_compute_instance_v2" "node" {
 
   flavor_name     = var.os_flavor_node
   key_pair        = openstack_compute_keypair_v2.keypair.name
-  security_groups = [openstack_compute_secgroup_v2.secgroup_1.name,"neurolibre-secgroup"]
+  security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-secgroup"]
   user_data = element(
     data.template_cloudinit_config.node_config.*.rendered,
     count.index,
diff --git a/terraform-modules/providers/openstack/secgroup-common.tf b/terraform-modules/providers/openstack/secgroup-common.tf
new file mode 100644
index 0000000..90d2c60
--- /dev/null
+++ b/terraform-modules/providers/openstack/secgroup-common.tf
@@ -0,0 +1,82 @@
+resource "openstack_networking_secgroup_v2" "common" {
+  name        = "${var.project_name}-secgroup-common"
+  description = "Common security group for BinderHub nodes"
+}
+
+resource "openstack_networking_secgroup_rule_v2" "allow_icmp" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  security_group_id = openstack_networking_secgroup_v2.common.id
+  protocol          = "icmp"
+  remote_ip_prefix  = "192.168.73.0/24"
+}
+
+resource "openstack_networking_secgroup_rule_v2" "allow_ssh" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  security_group_id = openstack_networking_secgroup_v2.common.id
+  protocol          = "tcp"
+  port_range_min    = 22
+  port_range_max    = 22
+  remote_ip_prefix  = "0.0.0.0/0"
+}
+
+resource "openstack_networking_secgroup_rule_v2" "allow_https" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  security_group_id = openstack_networking_secgroup_v2.common.id
+  protocol          = "tcp"
+  port_range_min    = 443
+  port_range_max    = 443
+  remote_ip_prefix  = "0.0.0.0/0"
+}
+
+resource "openstack_networking_secgroup_rule_v2" "allow_http" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  security_group_id = openstack_networking_secgroup_v2.common.id
+  protocol          = "tcp"
+  port_range_min    = 80
+  port_range_max    = 80
+  remote_ip_prefix  = "0.0.0.0/0"
+}
+
+resource "openstack_networking_secgroup_rule_v2" "allow_nodeport_tcp" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  security_group_id = openstack_networking_secgroup_v2.common.id
+  protocol          = "tcp"
+  port_range_min    = 30000
+  port_range_max    = 32767
+  remote_ip_prefix  = "0.0.0.0/0"
+}
+
+resource "openstack_networking_secgroup_rule_v2" "allow_nodeport_udp" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  security_group_id = openstack_networking_secgroup_v2.common.id
+  protocol          = "udp"
+  port_range_min    = 30000
+  port_range_max    = 32767
+  remote_ip_prefix  = "0.0.0.0/0"
+}
+
+resource "openstack_networking_secgroup_rule_v2" "allow_internal_tcp" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  security_group_id = openstack_networking_secgroup_v2.common.id
+  protocol          = "tcp"
+  port_range_min    = 1
+  port_range_max    = 65535
+  remote_ip_prefix  = "192.168.73.0/24"
+}
+
+resource "openstack_networking_secgroup_rule_v2" "allow_internal_udp" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  security_group_id = openstack_networking_secgroup_v2.common.id
+  protocol          = "udp"
+  port_range_min    = 1
+  port_range_max    = 65535
+  remote_ip_prefix  = "192.168.73.0/24"
+}
\ No newline at end of file

From 1916d9c0d0d38ed74f06e99a9a9e63dc7aabfcbd Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 16:00:31 +0100
Subject: [PATCH 074/109] Change api

---
 terraform-modules/providers/openstack/main.tf | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index ae4c1ca..2ff3e0d 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -172,7 +172,8 @@ resource "openstack_networking_floatingip_v2" "fip_1" {
   pool = data.openstack_networking_network_v2.ext_network.name
 }
 
-resource "openstack_compute_floatingip_associate_v2" "fip_1" {
+
+resource "openstack_networking_floatingip_associate_v2" "fip_1" {
   floating_ip = openstack_networking_floatingip_v2.fip_1.address
   instance_id = openstack_compute_instance_v2.master.id
 }
\ No newline at end of file

From 5f3cf62b321b2c58e8c9f78f8e7fdd3949a5d5a0 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 16:15:46 +0100
Subject: [PATCH 075/109] Associate fip to port

---
 terraform-modules/providers/openstack/main.tf | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 2ff3e0d..b02a918 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -168,12 +168,15 @@ resource "openstack_compute_instance_v2" "node" {
   }
 }
 
+resource "openstack_networking_port_v2" "ext_port" {
+  network_id = data.openstack_networking_network_v2.ext_network.id
+}
+
 resource "openstack_networking_floatingip_v2" "fip_1" {
   pool = data.openstack_networking_network_v2.ext_network.name
 }
 
-
-resource "openstack_networking_floatingip_associate_v2" "fip_1" {
+resource "openstack_compute_floatingip_associate_v2" "fip_1" {
   floating_ip = openstack_networking_floatingip_v2.fip_1.address
-  instance_id = openstack_compute_instance_v2.master.id
+  port_id     = openstack_networking_port_v2.ext_port.id
 }
\ No newline at end of file

From 7ba7144b9c9771af91c37d4a8b6b79d9066c507b Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 16:17:08 +0100
Subject: [PATCH 076/109] api..

---
 terraform-modules/providers/openstack/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index b02a918..942f91a 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -176,7 +176,7 @@ resource "openstack_networking_floatingip_v2" "fip_1" {
   pool = data.openstack_networking_network_v2.ext_network.name
 }
 
-resource "openstack_compute_floatingip_associate_v2" "fip_1" {
+resource "openstack_networking_floatingip_associate_v2" "fip_1" {
   floating_ip = openstack_networking_floatingip_v2.fip_1.address
   port_id     = openstack_networking_port_v2.ext_port.id
 }
\ No newline at end of file

From a6490b5ef04f4b1c0f543082108d6938fc06e852 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 16:40:53 +0100
Subject: [PATCH 077/109] replace null_resource

---
 terraform-modules/binderhub/main.tf | 100 ++++++++++++++--------------
 1 file changed, 49 insertions(+), 51 deletions(-)

diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index b9704bb..e227ad0 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -62,66 +62,64 @@ data "template_file" "install-binderhub" {
   }
 }
 
-resource "null_resource" "remote_install" {
-  connection {
-    user = var.admin_user
-    host = var.ip
-    agent= true
-  }
+resource "terraform_data" "binderhub" {
 
-  provisioner "file" {
-    source     = "${path.module}/assets/fill_submission_metadata.bash"
-    destination = "/home/${var.admin_user}/fill_submission_metadata.bash"
-  }
+connection {
+  host = var.ip
+}
 
-  provisioner "file" {
-    source     = "${path.module}/assets/repo2data.bash"
-    destination = "/home/${var.admin_user}/repo2data.bash"
-  }
+provisioner "file" {
+  content     = data.template_file.config.rendered
+  destination = "/home/${var.admin_user}/config.yaml"
+}
 
-  provisioner "file" {
-    source     = "${path.module}/assets/jb_build.bash"
-    destination = "/home/${var.admin_user}/jb_build.bash"
-  }
+provisioner "file" {
+  content     = data.template_file.secrets.rendered
+  destination = "/home/${var.admin_user}/secrets.yaml"
+}
 
-  provisioner "file" {
-    content     = data.template_file.config.rendered
-    destination = "/home/${var.admin_user}/config.yaml"
-  }
+provisioner "file" {
+  content     = data.template_file.pv.rendered
+  destination = "/home/${var.admin_user}/pv.yaml"
+}
 
-  provisioner "file" {
-    content     = data.template_file.secrets.rendered
-    destination = "/home/${var.admin_user}/secrets.yaml"
-  }
+provisioner "file" {
+  content     = data.template_file.nginx-ingress.rendered
+  destination = "/home/${var.admin_user}/nginx-ingress.yaml"
+}
 
-  provisioner "file" {
-    content     = data.template_file.pv.rendered
-    destination = "/home/${var.admin_user}/pv.yaml"
-  }
+provisioner "file" {
+  content     = data.template_file.production-binderhub-issuer.rendered
+  destination = "/home/${var.admin_user}/production-binderhub-issuer.yaml"
+}
 
-  provisioner "file" {
-    content     = data.template_file.nginx-ingress.rendered
-    destination = "/home/${var.admin_user}/nginx-ingress.yaml"
-  }
+provisioner "file" {
+  content     = data.template_file.staging-binderhub-issuer.rendered
+  destination = "/home/${var.admin_user}/staging-binderhub-issuer.yaml"
+}
 
-  provisioner "file" {
-    content     = data.template_file.production-binderhub-issuer.rendered
-    destination = "/home/${var.admin_user}/production-binderhub-issuer.yaml"
-  }
+provisioner "file" {
+  content     = data.template_file.install-binderhub.rendered
+  destination = "/home/${var.admin_user}/install-binderhub.sh"
+}
 
-  provisioner "file" {
-    content     = data.template_file.staging-binderhub-issuer.rendered
-    destination = "/home/${var.admin_user}/staging-binderhub-issuer.yaml"
-  }
 
-  provisioner "file" {
-    content     = data.template_file.install-binderhub.rendered
-    destination = "/home/${var.admin_user}/install-binderhub.sh"
-  }
+provisioner "file" {
+  source     = "${path.module}/assets/fill_submission_metadata.bash"
+  destination = "/home/${var.admin_user}/fill_submission_metadata.bash"
+}
 
-  provisioner "remote-exec" {
-    inline = [
-      "bash /home/${var.admin_user}/install-binderhub.sh",
-    ]
-  }
+provisioner "file" {
+  source     = "${path.module}/assets/repo2data.bash"
+  destination = "/home/${var.admin_user}/repo2data.bash"
+}
+
+provisioner "file" {
+  source     = "${path.module}/assets/jb_build.bash"
+  destination = "/home/${var.admin_user}/jb_build.bash"
+}
+
+provisioner "remote-exec" {
+  inline = ["bash /home/${var.admin_user}/install-binderhub.sh",]
+}
 }
\ No newline at end of file

From 575183498fc24e89fb36b299105a7a28729afc85 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 16:56:11 +0100
Subject: [PATCH 078/109] Open port int

---
 terraform-modules/providers/openstack/main.tf | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 942f91a..acad517 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -168,8 +168,8 @@ resource "openstack_compute_instance_v2" "node" {
   }
 }
 
-resource "openstack_networking_port_v2" "ext_port" {
-  network_id = data.openstack_networking_network_v2.ext_network.id
+resource "openstack_networking_port_v2" "fip_port" {
+  network_id = data.openstack_networking_network_v2.int_network.id
 }
 
 resource "openstack_networking_floatingip_v2" "fip_1" {
@@ -178,5 +178,5 @@ resource "openstack_networking_floatingip_v2" "fip_1" {
 
 resource "openstack_networking_floatingip_associate_v2" "fip_1" {
   floating_ip = openstack_networking_floatingip_v2.fip_1.address
-  port_id     = openstack_networking_port_v2.ext_port.id
+  port_id     = openstack_networking_port_v2.fip_port.id
 }
\ No newline at end of file

From 203207a926d725e45e463742039ca4c05d3911b6 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 17:03:45 +0100
Subject: [PATCH 079/109] To device

---
 terraform-modules/providers/openstack/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index acad517..3ff7228 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -169,7 +169,7 @@ resource "openstack_compute_instance_v2" "node" {
 }
 
 resource "openstack_networking_port_v2" "fip_port" {
-  network_id = data.openstack_networking_network_v2.int_network.id
+  device_id = openstack_compute_instance_v2.master.id
 }
 
 resource "openstack_networking_floatingip_v2" "fip_1" {

From 2864beaf279243a38c902823ac52b4303892719d Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 17:19:50 +0100
Subject: [PATCH 080/109] port issue

---
 terraform-modules/providers/openstack/main.tf | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 3ff7228..6a7131b 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -117,6 +117,13 @@ resource "openstack_blockstorage_volume_v3" "mastervolume" {
   image_id    = data.openstack_images_image_v2.ubuntu.id
 }
 
+
+resource "openstack_networking_port_v2" "master" {
+  name               = "${var.project_name}-master"
+  admin_state_up     = "true"
+  network_id         = openstack_networking_network_v2.int_network.id
+}
+
 resource "openstack_compute_instance_v2" "master" {
   name            = "${var.project_name}-master"
   flavor_name     = var.os_flavor_master
@@ -134,6 +141,7 @@ resource "openstack_compute_instance_v2" "master" {
 
   network {
     name = var.is_computecanada ? data.openstack_networking_network_v2.int_network.name : "${var.project_name}-network"
+    port = openstack_networking_port_v2.master.id
   }
 }
 
@@ -168,15 +176,11 @@ resource "openstack_compute_instance_v2" "node" {
   }
 }
 
-resource "openstack_networking_port_v2" "fip_port" {
-  device_id = openstack_compute_instance_v2.master.id
-}
-
 resource "openstack_networking_floatingip_v2" "fip_1" {
   pool = data.openstack_networking_network_v2.ext_network.name
 }
 
 resource "openstack_networking_floatingip_associate_v2" "fip_1" {
   floating_ip = openstack_networking_floatingip_v2.fip_1.address
-  port_id     = openstack_networking_port_v2.fip_port.id
+  port_id     = openstack_networking_port_v2.master.id
 }
\ No newline at end of file

From fa637a89528e06dfafd2ed8699307656ad975c3f Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 17:22:40 +0100
Subject: [PATCH 081/109] existing network

---
 terraform-modules/providers/openstack/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 6a7131b..818d37b 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -121,7 +121,7 @@ resource "openstack_blockstorage_volume_v3" "mastervolume" {
 resource "openstack_networking_port_v2" "master" {
   name               = "${var.project_name}-master"
   admin_state_up     = "true"
-  network_id         = openstack_networking_network_v2.int_network.id
+  network_id         = data.openstack_networking_network_v2.int_network.id
 }
 
 resource "openstack_compute_instance_v2" "master" {

From 0565a8fe4c8acfaabd30a577d3a14a3f9ad05122 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 17:30:17 +0100
Subject: [PATCH 082/109] add user

---
 terraform-modules/binderhub/main.tf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index e227ad0..ebedaa9 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -66,6 +66,7 @@ resource "terraform_data" "binderhub" {
 
 connection {
   host = var.ip
+  user = var.admin_user
 }
 
 provisioner "file" {

From b76be69d7105813868de2ce4576da2bdb79fa7a7 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 18:24:24 +0100
Subject: [PATCH 083/109] Minor rev

---
 terraform-modules/providers/openstack/main.tf            | 3 ++-
 terraform-modules/providers/openstack/secgroup-common.tf | 8 ++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 818d37b..a62eec3 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -146,7 +146,8 @@ resource "openstack_compute_instance_v2" "master" {
 }
 
 resource "openstack_blockstorage_volume_v3" "nodevolume" {
-  name        = "node-volume"
+  count = var.nb_nodes
+  name        = format("node-%02d-volume", count.index + 1)
   size        = var.instance_volume_size
   image_id    = data.openstack_images_image_v2.ubuntu.id
 }
diff --git a/terraform-modules/providers/openstack/secgroup-common.tf b/terraform-modules/providers/openstack/secgroup-common.tf
index 90d2c60..5a7fdd0 100644
--- a/terraform-modules/providers/openstack/secgroup-common.tf
+++ b/terraform-modules/providers/openstack/secgroup-common.tf
@@ -66,8 +66,8 @@ resource "openstack_networking_secgroup_rule_v2" "allow_internal_tcp" {
   ethertype         = "IPv4"
   security_group_id = openstack_networking_secgroup_v2.common.id
   protocol          = "tcp"
-  port_range_min    = 1
-  port_range_max    = 65535
+  port_range_min    = 0
+  port_range_max    = 0
   remote_ip_prefix  = "192.168.73.0/24"
 }
 
@@ -76,7 +76,7 @@ resource "openstack_networking_secgroup_rule_v2" "allow_internal_udp" {
   ethertype         = "IPv4"
   security_group_id = openstack_networking_secgroup_v2.common.id
   protocol          = "udp"
-  port_range_min    = 1
-  port_range_max    = 65535
+  port_range_min    = 0
+  port_range_max    = 0
   remote_ip_prefix  = "192.168.73.0/24"
 }
\ No newline at end of file

From 796d71b573f13437a8448e4f788aa6895d0e31dd Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 18:31:41 +0100
Subject: [PATCH 084/109] Give idx id

---
 terraform-modules/providers/openstack/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index a62eec3..da2cc2e 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -165,7 +165,7 @@ resource "openstack_compute_instance_v2" "node" {
   )
 
   block_device {
-    uuid                  = openstack_blockstorage_volume_v3.nodevolume.id
+    uuid                  = openstack_blockstorage_volume_v3.nodevolume[count.index].id
     source_type           = "volume"
     destination_type      = "volume"
     boot_index            = 0

From 573a42cb0eb0879f6412f89a828bd195ad0b837b Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 18:47:23 +0100
Subject: [PATCH 085/109] RM count from node data & increase ssh timeout

---
 terraform-modules/binderhub/main.tf           | 1 +
 terraform-modules/providers/openstack/main.tf | 5 +----
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index ebedaa9..814b934 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -67,6 +67,7 @@ resource "terraform_data" "binderhub" {
 connection {
   host = var.ip
   user = var.admin_user
+  timeout = "10m"
 }
 
 provisioner "file" {
diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index da2cc2e..61cfa5d 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -159,10 +159,7 @@ resource "openstack_compute_instance_v2" "node" {
   flavor_name     = var.os_flavor_node
   key_pair        = openstack_compute_keypair_v2.keypair.name
   security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-secgroup"]
-  user_data = element(
-    data.template_cloudinit_config.node_config.*.rendered,
-    count.index,
-  )
+  user_data = data.template_cloudinit_config.node_config.*.rendered
 
   block_device {
     uuid                  = openstack_blockstorage_volume_v3.nodevolume[count.index].id

From 2bdf3cfe66c8715bc805c0e7447a55756eb11e2e Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 18:51:59 +0100
Subject: [PATCH 086/109] Fix iter render

---
 terraform-modules/providers/openstack/main.tf | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 61cfa5d..5ca6e12 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -75,6 +75,8 @@ data "template_file" "kubeadm_common" {
 }
 
 data "template_cloudinit_config" "node_config" {
+  count = var.nb_nodes
+
   part {
     filename     = "common.yaml"
     merge_type   = "list(append)+dict(recurse_array)+str()"
@@ -159,7 +161,7 @@ resource "openstack_compute_instance_v2" "node" {
   flavor_name     = var.os_flavor_node
   key_pair        = openstack_compute_keypair_v2.keypair.name
   security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-secgroup"]
-  user_data = data.template_cloudinit_config.node_config.*.rendered
+  user_data = data.template_cloudinit_config.node_config[count.index].rendered
 
   block_device {
     uuid                  = openstack_blockstorage_volume_v3.nodevolume[count.index].id

From 73e80a2aa944e0a1f7ad76285b1965aab0a85a60 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 19:19:40 +0100
Subject: [PATCH 087/109] issue with node

---
 terraform-modules/providers/openstack/main.tf | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 5ca6e12..2657682 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -161,7 +161,11 @@ resource "openstack_compute_instance_v2" "node" {
   flavor_name     = var.os_flavor_node
   key_pair        = openstack_compute_keypair_v2.keypair.name
   security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-secgroup"]
-  user_data = data.template_cloudinit_config.node_config[count.index].rendered
+  user_data =element(
+              data.template_cloudinit_config.node_config[count.index].rendered,
+              count.index,)
+
+
 
   block_device {
     uuid                  = openstack_blockstorage_volume_v3.nodevolume[count.index].id

From b19bc1cbeedef820cc06fe48cd7e937e800a9205 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 19:22:27 +0100
Subject: [PATCH 088/109] Details

---
 terraform-modules/binderhub/assets/install-binderhub.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/terraform-modules/binderhub/assets/install-binderhub.sh b/terraform-modules/binderhub/assets/install-binderhub.sh
index 61025d5..53c32cf 100755
--- a/terraform-modules/binderhub/assets/install-binderhub.sh
+++ b/terraform-modules/binderhub/assets/install-binderhub.sh
@@ -1,8 +1,12 @@
 #!/bin/bash
 #waiting for k8s initialization
+echo "[Binderhub pre-install] Waiting for the instance boot to finalize"
 while [ ! -f /var/lib/cloud/instance/boot-finished ]; do sleep 10; done
+echo "[Binderhub pre-install] Waiting for k8s init to finalize"
 while [ ! -f /shared/k8s-initialized ]; do sleep 1; done
 
+echo "[Binderhub install] Started"
+
 cd /home/${admin_user}
 
 # node helath monitoring

From b73fa7012b44774f744cbf39cec8ebed103cd3ad Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 19:25:50 +0100
Subject: [PATCH 089/109] Add more print statements

---
 cloud-init/kubeadm/master.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cloud-init/kubeadm/master.yaml b/cloud-init/kubeadm/master.yaml
index c772790..be6cf1b 100644
--- a/cloud-init/kubeadm/master.yaml
+++ b/cloud-init/kubeadm/master.yaml
@@ -4,6 +4,7 @@ packages:
 
 runcmd:
   # NFS data
+  - echo "Mounting data"
   - mkdir -p /DATA
   - echo "192.168.73.179:/DATA      /DATA      nfs rw,noatime,nolock,hard,tcp 0 0" >> /etc/fstab
   - mount -a
@@ -22,6 +23,7 @@ runcmd:
   - cat /tmp/kubeadm-bootstrap/.bashrc >> /home/${admin_user}/.bashrc
   # ssh config for nodes
   # make sure that k8s node is ready, and enable easy ssh
+  - echo "Adding nodes"
   - su ${admin_user} -c "touch /home/${admin_user}/.ssh/config"
   - mv /tmp/kubeadm-bootstrap/add_nodes.bash /home/${admin_user}/
   - chmod u+x /home/${admin_user}/add_nodes.bash

From b3dd1e0f449cbe7bcf682933ca3be9bf765fee1d Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 19:30:24 +0100
Subject: [PATCH 090/109] issue with the node...

---
 terraform-modules/providers/openstack/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 2657682..d82c762 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -162,7 +162,7 @@ resource "openstack_compute_instance_v2" "node" {
   key_pair        = openstack_compute_keypair_v2.keypair.name
   security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-secgroup"]
   user_data =element(
-              data.template_cloudinit_config.node_config[count.index].rendered,
+              data.template_cloudinit_config.node_config.*.rendered,
               count.index,)
 
 

From db05f34f3e49331a509892e5d7540a17caedbbf7 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 21:13:21 +0100
Subject: [PATCH 091/109] secgroups

---
 terraform-modules/providers/openstack/main.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index d82c762..c29151a 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -130,7 +130,7 @@ resource "openstack_compute_instance_v2" "master" {
   name            = "${var.project_name}-master"
   flavor_name     = var.os_flavor_master
   key_pair        = openstack_compute_keypair_v2.keypair.name
-  security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-secgroup"]
+  security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-secgroup","neurolibre-test-secgroup"]
   user_data       = data.template_cloudinit_config.master_config.rendered
 
   block_device {
@@ -160,7 +160,7 @@ resource "openstack_compute_instance_v2" "node" {
 
   flavor_name     = var.os_flavor_node
   key_pair        = openstack_compute_keypair_v2.keypair.name
-  security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-secgroup"]
+  security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-secgroup","neurolibre-test-secgroup"]
   user_data =element(
               data.template_cloudinit_config.node_config.*.rendered,
               count.index,)

From 013db36b16d3da97d510f55506c55284033c6e4c Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 21:46:53 +0100
Subject: [PATCH 092/109] lanet olasi secgrouplar

---
 terraform-modules/providers/openstack/main.tf |  4 +-
 .../providers/openstack/secgroup-common.tf    | 86 +++++++++++--------
 2 files changed, 51 insertions(+), 39 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index c29151a..d82c762 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -130,7 +130,7 @@ resource "openstack_compute_instance_v2" "master" {
   name            = "${var.project_name}-master"
   flavor_name     = var.os_flavor_master
   key_pair        = openstack_compute_keypair_v2.keypair.name
-  security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-secgroup","neurolibre-test-secgroup"]
+  security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-secgroup"]
   user_data       = data.template_cloudinit_config.master_config.rendered
 
   block_device {
@@ -160,7 +160,7 @@ resource "openstack_compute_instance_v2" "node" {
 
   flavor_name     = var.os_flavor_node
   key_pair        = openstack_compute_keypair_v2.keypair.name
-  security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-secgroup","neurolibre-test-secgroup"]
+  security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-secgroup"]
   user_data =element(
               data.template_cloudinit_config.node_config.*.rendered,
               count.index,)
diff --git a/terraform-modules/providers/openstack/secgroup-common.tf b/terraform-modules/providers/openstack/secgroup-common.tf
index 5a7fdd0..f689e45 100644
--- a/terraform-modules/providers/openstack/secgroup-common.tf
+++ b/terraform-modules/providers/openstack/secgroup-common.tf
@@ -1,82 +1,94 @@
 resource "openstack_networking_secgroup_v2" "common" {
-  name        = "${var.project_name}-secgroup-common"
+  name        = "${var.project_name}-secgroup-new"
   description = "Common security group for BinderHub nodes"
 }
 
-resource "openstack_networking_secgroup_rule_v2" "allow_icmp" {
+resource "openstack_networking_secgroup_rule_v2" "icmp_self" {
   direction         = "ingress"
   ethertype         = "IPv4"
-  security_group_id = openstack_networking_secgroup_v2.common.id
   protocol          = "icmp"
-  remote_ip_prefix  = "192.168.73.0/24"
+  port_range_min    = -1
+  port_range_max    = -1
+  security_group_id = openstack_networking_secgroup_v2.common.id
+  remote_group_id   = openstack_networking_secgroup_v2.common.id
 }
 
-resource "openstack_networking_secgroup_rule_v2" "allow_ssh" {
+resource "openstack_networking_secgroup_rule_v2" "tcp_self" {
   direction         = "ingress"
   ethertype         = "IPv4"
-  security_group_id = openstack_networking_secgroup_v2.common.id
   protocol          = "tcp"
-  port_range_min    = 22
-  port_range_max    = 22
-  remote_ip_prefix  = "0.0.0.0/0"
+  port_range_min    = 0
+  port_range_max    = 0
+  security_group_id = openstack_networking_secgroup_v2.common.id
+  remote_group_id   = openstack_networking_secgroup_v2.common.id
 }
 
-resource "openstack_networking_secgroup_rule_v2" "allow_https" {
+resource "openstack_networking_secgroup_rule_v2" "udp_self" {
   direction         = "ingress"
   ethertype         = "IPv4"
+  protocol          = "udp"
+  port_range_min    = 0
+  port_range_max    = 0
   security_group_id = openstack_networking_secgroup_v2.common.id
-  protocol          = "tcp"
-  port_range_min    = 443
-  port_range_max    = 443
-  remote_ip_prefix  = "0.0.0.0/0"
+  remote_group_id   = openstack_networking_secgroup_v2.common.id
 }
 
-resource "openstack_networking_secgroup_rule_v2" "allow_http" {
+resource "openstack_networking_secgroup_rule_v2" "icmp_specific" {
   direction         = "ingress"
   ethertype         = "IPv4"
+  protocol          = "icmp"
+  port_range_min    = -1
+  port_range_max    = -1
   security_group_id = openstack_networking_secgroup_v2.common.id
+  remote_ip_prefix  = "192.168.73.30/32"
+}
+
+resource "openstack_networking_secgroup_rule_v2" "tcp_specific" {
+  direction         = "ingress"
+  ethertype         = "IPv4"
   protocol          = "tcp"
-  port_range_min    = 80
-  port_range_max    = 80
-  remote_ip_prefix  = "0.0.0.0/0"
+  port_range_min    = 0
+  port_range_max    = 0
+  security_group_id = openstack_networking_secgroup_v2.common.id
+  remote_ip_prefix  = "192.168.73.30/32"
 }
 
-resource "openstack_networking_secgroup_rule_v2" "allow_nodeport_tcp" {
+resource "openstack_networking_secgroup_rule_v2" "udp_specific" {
   direction         = "ingress"
   ethertype         = "IPv4"
+  protocol          = "udp"
+  port_range_min    = 0
+  port_range_max    = 0
   security_group_id = openstack_networking_secgroup_v2.common.id
-  protocol          = "tcp"
-  port_range_min    = 30000
-  port_range_max    = 32767
-  remote_ip_prefix  = "0.0.0.0/0"
+  remote_ip_prefix  = "192.168.73.30/32"
 }
 
-resource "openstack_networking_secgroup_rule_v2" "allow_nodeport_udp" {
+resource "openstack_networking_secgroup_rule_v2" "tcp_22" {
   direction         = "ingress"
   ethertype         = "IPv4"
+  protocol          = "tcp"
+  port_range_min    = 22
+  port_range_max    = 22
   security_group_id = openstack_networking_secgroup_v2.common.id
-  protocol          = "udp"
-  port_range_min    = 30000
-  port_range_max    = 32767
   remote_ip_prefix  = "0.0.0.0/0"
 }
 
-resource "openstack_networking_secgroup_rule_v2" "allow_internal_tcp" {
+resource "openstack_networking_secgroup_rule_v2" "tcp_443" {
   direction         = "ingress"
   ethertype         = "IPv4"
-  security_group_id = openstack_networking_secgroup_v2.common.id
   protocol          = "tcp"
-  port_range_min    = 0
-  port_range_max    = 0
-  remote_ip_prefix  = "192.168.73.0/24"
+  port_range_min    = 443
+  port_range_max    = 443
+  security_group_id = openstack_networking_secgroup_v2.common.id
+  remote_ip_prefix  = "0.0.0.0/0"
 }
 
-resource "openstack_networking_secgroup_rule_v2" "allow_internal_udp" {
+resource "openstack_networking_secgroup_rule_v2" "tcp_80" {
   direction         = "ingress"
   ethertype         = "IPv4"
+  protocol          = "tcp"
+  port_range_min    = 80
+  port_range_max    = 80
   security_group_id = openstack_networking_secgroup_v2.common.id
-  protocol          = "udp"
-  port_range_min    = 0
-  port_range_max    = 0
-  remote_ip_prefix  = "192.168.73.0/24"
+  remote_ip_prefix  = "0.0.0.0/0"
 }
\ No newline at end of file

From e19af60f2f6b046d584d962c22a12876150d4392 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 22:01:10 +0100
Subject: [PATCH 093/109] why..

---
 terraform-modules/providers/openstack/main.tf | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index d82c762..660c8f0 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -130,7 +130,7 @@ resource "openstack_compute_instance_v2" "master" {
   name            = "${var.project_name}-master"
   flavor_name     = var.os_flavor_master
   key_pair        = openstack_compute_keypair_v2.keypair.name
-  security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-secgroup"]
+  security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-test-secgroup"]
   user_data       = data.template_cloudinit_config.master_config.rendered
 
   block_device {
@@ -143,7 +143,6 @@ resource "openstack_compute_instance_v2" "master" {
 
   network {
     name = var.is_computecanada ? data.openstack_networking_network_v2.int_network.name : "${var.project_name}-network"
-    port = openstack_networking_port_v2.master.id
   }
 }
 
@@ -160,7 +159,7 @@ resource "openstack_compute_instance_v2" "node" {
 
   flavor_name     = var.os_flavor_node
   key_pair        = openstack_compute_keypair_v2.keypair.name
-  security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-secgroup"]
+  security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-test-secgroup"]
   user_data =element(
               data.template_cloudinit_config.node_config.*.rendered,
               count.index,)

From cded5c3beca385fd984507f133d152250c4f9ae6 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Sun, 19 May 2024 22:03:48 +0100
Subject: [PATCH 094/109] zort

---
 terraform-modules/providers/openstack/secgroup-common.tf | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/terraform-modules/providers/openstack/secgroup-common.tf b/terraform-modules/providers/openstack/secgroup-common.tf
index f689e45..fb2a956 100644
--- a/terraform-modules/providers/openstack/secgroup-common.tf
+++ b/terraform-modules/providers/openstack/secgroup-common.tf
@@ -7,8 +7,8 @@ resource "openstack_networking_secgroup_rule_v2" "icmp_self" {
   direction         = "ingress"
   ethertype         = "IPv4"
   protocol          = "icmp"
-  port_range_min    = -1
-  port_range_max    = -1
+  port_range_min    = 0
+  port_range_max    = 0
   security_group_id = openstack_networking_secgroup_v2.common.id
   remote_group_id   = openstack_networking_secgroup_v2.common.id
 }
@@ -37,8 +37,8 @@ resource "openstack_networking_secgroup_rule_v2" "icmp_specific" {
   direction         = "ingress"
   ethertype         = "IPv4"
   protocol          = "icmp"
-  port_range_min    = -1
-  port_range_max    = -1
+  port_range_min    = 0
+  port_range_max    = 0
   security_group_id = openstack_networking_secgroup_v2.common.id
   remote_ip_prefix  = "192.168.73.30/32"
 }

From ef57dc3e1000401c67d74ef1d168fa6930bdc12d Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Mon, 20 May 2024 07:36:35 +0100
Subject: [PATCH 095/109] Floating IP vs secgroup mutex

---
 terraform-modules/providers/openstack/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 660c8f0..7eaa427 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -142,7 +142,7 @@ resource "openstack_compute_instance_v2" "master" {
   }
 
   network {
-    name = var.is_computecanada ? data.openstack_networking_network_v2.int_network.name : "${var.project_name}-network"
+    port = openstack_networking_port_v2.master.id
   }
 }
 

From 65142bc5d95e86cc264c33dcf94839aa00b683dc Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Mon, 20 May 2024 07:52:44 +0100
Subject: [PATCH 096/109] separate block

---
 terraform-modules/providers/openstack/main.tf | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index 7eaa427..a03f0cd 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -3,6 +3,7 @@ data "openstack_images_image_v2" "ubuntu" {
   most_recent = true
 }
 
+# Define a network, only if not using Compute Canada
 resource "openstack_networking_subnet_v2" "subnet" {
   count = var.is_computecanada ? 0 : 1
 
@@ -13,11 +14,13 @@ resource "openstack_networking_subnet_v2" "subnet" {
   enable_dhcp = true
 }
 
+# Define a subnet within the network, only if not using Compute Canada
 resource "openstack_networking_network_v2" "network_1" {
   count = var.is_computecanada ? 0 : 1
   name = "${var.project_name}-network"
 }
 
+
 data "template_file" "kubeadm_master" {
   template = file("${path.module}/../../../cloud-init/kubeadm/master.yaml")
 
@@ -40,6 +43,7 @@ data "openstack_networking_network_v2" "int_network" {
   name = var.cc_private_network
 }
 
+# Define a router, only if not using Compute Canada
 resource "openstack_networking_router_v2" "router_1" {
   count = var.is_computecanada ? 0 : 1
 
@@ -47,16 +51,15 @@ resource "openstack_networking_router_v2" "router_1" {
   external_network_id = data.openstack_networking_network_v2.ext_network.id
 }
 
+# Attach the subnet to the router, only if not using Compute Canada
 resource "openstack_networking_router_interface_v2" "router_interface_1" {
   count = var.is_computecanada ? 0 : 1
-
   router_id = openstack_networking_router_v2.router_1[0].id
   subnet_id = openstack_networking_subnet_v2.subnet[0].id
 }
 
 data "template_file" "kubeadm_node" {
   template = file("${path.module}/../../../cloud-init/kubeadm/node.yaml")
-
   vars = {
     master_ip       = openstack_compute_instance_v2.master.network[0].fixed_ip_v4
     admin_user      = var.admin_user
@@ -144,6 +147,9 @@ resource "openstack_compute_instance_v2" "master" {
   network {
     port = openstack_networking_port_v2.master.id
   }
+  network {
+    name = var.is_computecanada ? data.openstack_networking_network_v2.int_network.name : "${var.project_name}-network"
+  }
 }
 
 resource "openstack_blockstorage_volume_v3" "nodevolume" {

From 7c42f73ef0792784412f8cd83050cb87854d3df2 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Mon, 20 May 2024 08:07:29 +0100
Subject: [PATCH 097/109] Attach secgroup to the port

---
 terraform-modules/providers/openstack/main.tf | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index a03f0cd..b7f734f 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -127,6 +127,10 @@ resource "openstack_networking_port_v2" "master" {
   name               = "${var.project_name}-master"
   admin_state_up     = "true"
   network_id         = data.openstack_networking_network_v2.int_network.id
+  security_group_ids = [
+    openstack_networking_secgroup_v2.common.id,
+    "neurolibre-test-secgroup"
+  ]
 }
 
 resource "openstack_compute_instance_v2" "master" {
@@ -147,9 +151,6 @@ resource "openstack_compute_instance_v2" "master" {
   network {
     port = openstack_networking_port_v2.master.id
   }
-  network {
-    name = var.is_computecanada ? data.openstack_networking_network_v2.int_network.name : "${var.project_name}-network"
-  }
 }
 
 resource "openstack_blockstorage_volume_v3" "nodevolume" {

From 5eb415d1de0d0b278b6b94608709c41c9a2a71f3 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Mon, 20 May 2024 09:03:47 +0100
Subject: [PATCH 098/109] Fix bugs, improve comments

---
 terraform-modules/providers/openstack/main.tf | 157 ++++++++++++------
 .../providers/openstack/variables.tf          |   4 +
 2 files changed, 109 insertions(+), 52 deletions(-)

diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index b7f734f..c5c4373 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -1,26 +1,13 @@
+# Grab information about the image name 
+# provided in the local (not version ctrld) main.tf
 data "openstack_images_image_v2" "ubuntu" {
   name        = var.image_name
   most_recent = true
 }
 
-# Define a network, only if not using Compute Canada
-resource "openstack_networking_subnet_v2" "subnet" {
-  count = var.is_computecanada ? 0 : 1
-
-  name        = "subnet"
-  network_id  = openstack_networking_network_v2.network_1[0].id
-  ip_version  = 4
-  cidr        = "10.0.1.0/24"
-  enable_dhcp = true
-}
-
-# Define a subnet within the network, only if not using Compute Canada
-resource "openstack_networking_network_v2" "network_1" {
-  count = var.is_computecanada ? 0 : 1
-  name = "${var.project_name}-network"
-}
-
-
+# Pass the keys provided in the local main.tf
+# to create a master.yaml template config on the 
+# instantiated VM.
 data "template_file" "kubeadm_master" {
   template = file("${path.module}/../../../cloud-init/kubeadm/master.yaml")
 
@@ -34,30 +21,7 @@ data "template_file" "kubeadm_master" {
   }
 }
 
-data "openstack_networking_network_v2" "ext_network" {
-  name = var.public_network
-  external = true
-}
-
-data "openstack_networking_network_v2" "int_network" {
-  name = var.cc_private_network
-}
-
-# Define a router, only if not using Compute Canada
-resource "openstack_networking_router_v2" "router_1" {
-  count = var.is_computecanada ? 0 : 1
-
-  name                = "${var.project_name}-router"
-  external_network_id = data.openstack_networking_network_v2.ext_network.id
-}
-
-# Attach the subnet to the router, only if not using Compute Canada
-resource "openstack_networking_router_interface_v2" "router_interface_1" {
-  count = var.is_computecanada ? 0 : 1
-  router_id = openstack_networking_router_v2.router_1[0].id
-  subnet_id = openstack_networking_subnet_v2.subnet[0].id
-}
-
+# Do the same for node.yaml (worker node exclusive)
 data "template_file" "kubeadm_node" {
   template = file("${path.module}/../../../cloud-init/kubeadm/node.yaml")
   vars = {
@@ -69,14 +33,35 @@ data "template_file" "kubeadm_node" {
   }
 }
 
+# Do the same for common.yaml
 data "template_file" "kubeadm_common" {
   template = file("${path.module}/../../../cloud-init/kubeadm/common.yaml")
-
   vars = {
     ssh_authorized_keys = indent(2, join("\n", formatlist("- %s", var.ssh_authorized_keys)))
   }
 }
 
+# =====================================================
+# NOTE: We don't create/destroy these network resources
+# they pre-exist. Hence, we just grab details to proceed.
+
+# Get details of an existing PUBLIC network
+data "openstack_networking_network_v2" "ext_network" {
+  name = var.public_network
+  external = true
+}
+
+# Get details of an existing INTERNAL network
+# (that has its subnet) which is connected to the 
+# Public-Network via a router.
+data "openstack_networking_network_v2" "int_network" {
+  name = var.cc_private_network
+}
+# =====================================================
+
+# Cloud-init configs for the worker node(s)
+# See cloud-init/kubeadm directory
+# Common.yaml + node.yaml
 data "template_cloudinit_config" "node_config" {
   count = var.nb_nodes
 
@@ -95,6 +80,9 @@ data "template_cloudinit_config" "node_config" {
   }
 }
 
+# Cloud-init configs for the master node
+# See cloud-init/kubeadm directory
+# Common.yaml + master.yaml
 data "template_cloudinit_config" "master_config" {
   part {
     filename     = "common.yaml"
@@ -111,33 +99,47 @@ data "template_cloudinit_config" "master_config" {
   }
 }
 
+# Create a keypair to be shared across nodes 
+# This is the first entry of the ssh_authorized_keys
+# This is passed from main.tf LOCALLY (not version controlled)
 resource "openstack_compute_keypair_v2" "keypair" {
   name       = "${var.project_name}-keypair"
   public_key = element(var.ssh_authorized_keys, 0)
 }
 
-resource "openstack_blockstorage_volume_v3" "mastervolume" {
-  name        = "master-volume"
-  size        = var.instance_volume_size
-  image_id    = data.openstack_images_image_v2.ubuntu.id
+# Grab details of a security group that HAS ALREADY BEEN
+# attached to the sftp instance (/DATA)
+data "openstack_networking_secgroup_v2" "neurolibre_sftp_secgroup" {
+  name = var.sftp_secgroup_name
 }
 
-
+# Create a PORT under the internal network which will be attached to 
+# the master node with the security groups defined here.
 resource "openstack_networking_port_v2" "master" {
   name               = "${var.project_name}-master"
   admin_state_up     = "true"
   network_id         = data.openstack_networking_network_v2.int_network.id
   security_group_ids = [
     openstack_networking_secgroup_v2.common.id,
-    "neurolibre-test-secgroup"
+    data.openstack_networking_secgroup_v2.neurolibre_sftp_secgroup.id
   ]
 }
 
+# =====================================================  MASTER NODE START
+# Create a volume for the master node
+resource "openstack_blockstorage_volume_v3" "mastervolume" {
+  name        = "master-volume"
+  size        = var.instance_volume_size
+  image_id    = data.openstack_images_image_v2.ubuntu.id
+}
+
+# Create the master node
 resource "openstack_compute_instance_v2" "master" {
   name            = "${var.project_name}-master"
   flavor_name     = var.os_flavor_master
   key_pair        = openstack_compute_keypair_v2.keypair.name
-  security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-test-secgroup"]
+  security_groups = [openstack_networking_secgroup_v2.common.id,
+                    data.openstack_networking_secgroup_v2.neurolibre_sftp_secgroup.id]
   user_data       = data.template_cloudinit_config.master_config.rendered
 
   block_device {
@@ -152,7 +154,13 @@ resource "openstack_compute_instance_v2" "master" {
     port = openstack_networking_port_v2.master.id
   }
 }
+# =====================================================  MASTER NODE ENDD
+
 
+
+# ===================================================== WORKER NODE(S) START
+# Create worker node storage volume(s). The number of worker volumes is 
+# defined by the count
 resource "openstack_blockstorage_volume_v3" "nodevolume" {
   count = var.nb_nodes
   name        = format("node-%02d-volume", count.index + 1)
@@ -160,13 +168,16 @@ resource "openstack_blockstorage_volume_v3" "nodevolume" {
   image_id    = data.openstack_images_image_v2.ubuntu.id
 }
 
+# Create workers node(s). The number of worker nodes is 
+# defined by the count
 resource "openstack_compute_instance_v2" "node" {
   count = var.nb_nodes
   name  = "${var.project_name}-node${count.index + 1}"
 
   flavor_name     = var.os_flavor_node
   key_pair        = openstack_compute_keypair_v2.keypair.name
-  security_groups = [openstack_networking_secgroup_v2.common.id,"neurolibre-test-secgroup"]
+  security_groups = [openstack_networking_secgroup_v2.common.id,
+                    data.openstack_networking_secgroup_v2.neurolibre_sftp_secgroup.id]
   user_data =element(
               data.template_cloudinit_config.node_config.*.rendered,
               count.index,)
@@ -185,12 +196,54 @@ resource "openstack_compute_instance_v2" "node" {
     name = var.is_computecanada ? data.openstack_networking_network_v2.int_network.name : "${var.project_name}-network"
   }
 }
+# ===================================================== WORKER NODE(S) END
 
+# Create a floating IP in the external network pool
 resource "openstack_networking_floatingip_v2" "fip_1" {
   pool = data.openstack_networking_network_v2.ext_network.name
 }
 
+# Attached the created floating IP to the PORT with security group
+# linked to the master node. So that the floating IP is associated with
+# the master node, as well as that secutiry group.
 resource "openstack_networking_floatingip_associate_v2" "fip_1" {
   floating_ip = openstack_networking_floatingip_v2.fip_1.address
   port_id     = openstack_networking_port_v2.master.id
-}
\ No newline at end of file
+}
+
+
+
+
+# # ================================================== OpenNebula attemps
+# # Define a network, only if not using Compute Canada
+# resource "openstack_networking_subnet_v2" "subnet" {
+#   count = var.is_computecanada ? 0 : 1
+
+#   name        = "subnet"
+#   network_id  = openstack_networking_network_v2.network_1[0].id
+#   ip_version  = 4
+#   cidr        = "10.0.1.0/24"
+#   enable_dhcp = true
+# }
+
+# # Define a subnet within the network, only if not using Compute Canada
+# resource "openstack_networking_network_v2" "network_1" {
+#   count = var.is_computecanada ? 0 : 1
+#   name = "${var.project_name}-network"
+# }
+
+# # Define a router, only if not using Compute Canada
+# resource "openstack_networking_router_v2" "router_1" {
+#   count = var.is_computecanada ? 0 : 1
+
+#   name                = "${var.project_name}-router"
+#   external_network_id = data.openstack_networking_network_v2.ext_network.id
+# }
+
+# # Attach the subnet to the router, only if not using Compute Canada
+# resource "openstack_networking_router_interface_v2" "router_interface_1" {
+#   count = var.is_computecanada ? 0 : 1
+#   router_id = openstack_networking_router_v2.router_1[0].id
+#   subnet_id = openstack_networking_subnet_v2.subnet[0].id
+# }
+# # ==================================================
\ No newline at end of file
diff --git a/terraform-modules/providers/openstack/variables.tf b/terraform-modules/providers/openstack/variables.tf
index 5a21fee..5c0c168 100644
--- a/terraform-modules/providers/openstack/variables.tf
+++ b/terraform-modules/providers/openstack/variables.tf
@@ -47,6 +47,10 @@ variable "docker_registry" {
   default     = "docker.io"
 }
 
+variable "sftp_secgroup_name" {
+  description = "A security group name that already exists on the sftp server."
+}
+
 variable "docker_id" {
   description = "Docker hub username"
 }

From 8b84933cc03793a3f0ce0f921b72954c473f6d69 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Mon, 20 May 2024 12:02:19 +0100
Subject: [PATCH 099/109] Parametrize sftp, modify ip

---
 cloud-init/kubeadm/master.yaml                | 11 +--
 cloud-init/kubeadm/node.yaml                  |  4 +-
 .../binderhub/assets/install-binderhub.sh     | 10 +--
 terraform-modules/providers/openstack/main.tf | 78 ++++++++++---------
 .../providers/openstack/variables.tf          |  8 ++
 5 files changed, 62 insertions(+), 49 deletions(-)

diff --git a/cloud-init/kubeadm/master.yaml b/cloud-init/kubeadm/master.yaml
index be6cf1b..670a957 100644
--- a/cloud-init/kubeadm/master.yaml
+++ b/cloud-init/kubeadm/master.yaml
@@ -3,12 +3,11 @@ packages:
   - nfs-kernel-server
 
 runcmd:
-  # NFS data
-  - echo "Mounting data"
-  - mkdir -p /DATA
-  - echo "192.168.73.179:/DATA      /DATA      nfs rw,noatime,nolock,hard,tcp 0 0" >> /etc/fstab
+  # Mount SFTP to /DATA directory
+  - mkdir -p ${sftp_dir}
+  - echo "${sftp_ip}:${sftp_dir}      ${sftp_dir}      nfs rw,noatime,nolock,hard,tcp 0 0" >> /etc/fstab
   - mount -a
-  # NFS
+  # NFS export /shared directory and make it accessible to all subnet
   - echo "/shared `hostname -I | awk '{print $1}' | grep -Po '(\d+\.){3}'`0/24(rw,sync,no_root_squash,no_all_squash)" > /etc/exports
   - mkdir -p /shared
   - chown ${admin_user} /shared
@@ -16,7 +15,9 @@ runcmd:
   - systemctl restart nfs-kernel-server
   # Kubeadm  
   - echo "SUDO_UID=${admin_user} sudo -E /tmp/kubeadm-bootstrap/init-master.bash" >> /tmp/install.sh
+  # Initialize kubeadm 
   - su ${admin_user} -c "/tmp/install.sh"
+  # Share the join command with all the nodes 
   - kubeadm token create --print-join-command > /shared/kubeadm-join.bash
   - kubectl label nodes `hostname` type=production  
   # k8s commands
diff --git a/cloud-init/kubeadm/node.yaml b/cloud-init/kubeadm/node.yaml
index b061904..df24881 100644
--- a/cloud-init/kubeadm/node.yaml
+++ b/cloud-init/kubeadm/node.yaml
@@ -5,8 +5,8 @@ runcmd:
   # NFS
   - mkdir -p /shared
   - echo "${master_ip}:/shared      /shared      nfs rw,noatime,nolock,hard,tcp 0 0" >> /etc/fstab
-  - mkdir -p /DATA
-  - echo "192.168.73.179:/DATA      /DATA      nfs rw,noatime,nolock,hard,tcp 0 0" >> /etc/fstab
+  - mkdir -p ${sftp_dir}
+  - echo "${sftp_ip}:${sftp_dir}      ${sftp_dir}      nfs rw,noatime,nolock,hard,tcp 0 0" >> /etc/fstab
   - mount -a
   # Kubeadm  
   - echo "while [ ! -f /shared/kubeadm-join.bash ]; do sleep 1; done; sudo bash /shared/kubeadm-join.bash" >> /tmp/install.sh
diff --git a/terraform-modules/binderhub/assets/install-binderhub.sh b/terraform-modules/binderhub/assets/install-binderhub.sh
index 53c32cf..7b02fdd 100755
--- a/terraform-modules/binderhub/assets/install-binderhub.sh
+++ b/terraform-modules/binderhub/assets/install-binderhub.sh
@@ -1,9 +1,9 @@
 #!/bin/bash
-#waiting for k8s initialization
-echo "[Binderhub pre-install] Waiting for the instance boot to finalize"
-while [ ! -f /var/lib/cloud/instance/boot-finished ]; do sleep 10; done
-echo "[Binderhub pre-install] Waiting for k8s init to finalize"
-while [ ! -f /shared/k8s-initialized ]; do sleep 1; done
+
+echo "[Binderhub pre-install] BOOT?"
+while [ ! -f /var/lib/cloud/instance/boot-finished ]; do sleep 10; echo "Waiting for cloud-init on master to finalize"; done
+echo "[Binderhub pre-install] K8S READY?"
+while [ ! -f /shared/k8s-initialized ]; do sleep 5; echo "Waiting for K8S on master to be ready"; done
 
 echo "[Binderhub install] Started"
 
diff --git a/terraform-modules/providers/openstack/main.tf b/terraform-modules/providers/openstack/main.tf
index c5c4373..8ce2f83 100644
--- a/terraform-modules/providers/openstack/main.tf
+++ b/terraform-modules/providers/openstack/main.tf
@@ -5,42 +5,6 @@ data "openstack_images_image_v2" "ubuntu" {
   most_recent = true
 }
 
-# Pass the keys provided in the local main.tf
-# to create a master.yaml template config on the 
-# instantiated VM.
-data "template_file" "kubeadm_master" {
-  template = file("${path.module}/../../../cloud-init/kubeadm/master.yaml")
-
-  vars = {
-    admin_user      = var.admin_user
-    project_name    = var.project_name
-    nb_nodes        = var.nb_nodes
-    docker_registry = var.docker_registry
-    docker_id       = var.docker_id
-    docker_password = var.docker_password
-  }
-}
-
-# Do the same for node.yaml (worker node exclusive)
-data "template_file" "kubeadm_node" {
-  template = file("${path.module}/../../../cloud-init/kubeadm/node.yaml")
-  vars = {
-    master_ip       = openstack_compute_instance_v2.master.network[0].fixed_ip_v4
-    admin_user      = var.admin_user
-    docker_registry = var.docker_registry
-    docker_id       = var.docker_id
-    docker_password = var.docker_password
-  }
-}
-
-# Do the same for common.yaml
-data "template_file" "kubeadm_common" {
-  template = file("${path.module}/../../../cloud-init/kubeadm/common.yaml")
-  vars = {
-    ssh_authorized_keys = indent(2, join("\n", formatlist("- %s", var.ssh_authorized_keys)))
-  }
-}
-
 # =====================================================
 # NOTE: We don't create/destroy these network resources
 # they pre-exist. Hence, we just grab details to proceed.
@@ -103,7 +67,7 @@ data "template_cloudinit_config" "master_config" {
 # This is the first entry of the ssh_authorized_keys
 # This is passed from main.tf LOCALLY (not version controlled)
 resource "openstack_compute_keypair_v2" "keypair" {
-  name       = "${var.project_name}-keypair"
+  name       = "${var.project_name}-keypair-ed25519"
   public_key = element(var.ssh_authorized_keys, 0)
 }
 
@@ -113,6 +77,7 @@ data "openstack_networking_secgroup_v2" "neurolibre_sftp_secgroup" {
   name = var.sftp_secgroup_name
 }
 
+
 # Create a PORT under the internal network which will be attached to 
 # the master node with the security groups defined here.
 resource "openstack_networking_port_v2" "master" {
@@ -212,6 +177,45 @@ resource "openstack_networking_floatingip_associate_v2" "fip_1" {
 }
 
 
+# Pass the keys provided in the local main.tf
+# to create a master.yaml template config on the 
+# instantiated VM.
+data "template_file" "kubeadm_master" {
+  template = file("${path.module}/../../../cloud-init/kubeadm/master.yaml")
+
+  vars = {
+    sftp_ip         = var.sftp_ip_address
+    sftp_dir        = var.sftp_mnt_dir
+    admin_user      = var.admin_user
+    project_name    = var.project_name
+    nb_nodes        = var.nb_nodes
+    docker_registry = var.docker_registry
+    docker_id       = var.docker_id
+    docker_password = var.docker_password
+  }
+}
+
+# Do the same for node.yaml (worker node exclusive)
+data "template_file" "kubeadm_node" {
+  template = file("${path.module}/../../../cloud-init/kubeadm/node.yaml")
+  vars = {
+    master_ip       = openstack_compute_instance_v2.master.access_ip_v4
+    sftp_ip         = var.sftp_ip_address
+    sftp_dir         = var.sftp_mnt_dir
+    admin_user      = var.admin_user
+    docker_registry = var.docker_registry
+    docker_id       = var.docker_id
+    docker_password = var.docker_password
+  }
+}
+
+# Do the same for common.yaml
+data "template_file" "kubeadm_common" {
+  template = file("${path.module}/../../../cloud-init/kubeadm/common.yaml")
+  vars = {
+    ssh_authorized_keys = indent(2, join("\n", formatlist("- %s", var.ssh_authorized_keys)))
+  }
+}
 
 
 # # ================================================== OpenNebula attemps
diff --git a/terraform-modules/providers/openstack/variables.tf b/terraform-modules/providers/openstack/variables.tf
index 5c0c168..91af314 100644
--- a/terraform-modules/providers/openstack/variables.tf
+++ b/terraform-modules/providers/openstack/variables.tf
@@ -51,6 +51,14 @@ variable "sftp_secgroup_name" {
   description = "A security group name that already exists on the sftp server."
 }
 
+variable "sftp_ip_address" {
+  description = "Internal IP address of the SFTP instance on openstack."
+}
+
+variable "sftp_mnt_dir" {
+  description = "Directory on the node where SFTP will be mounted."
+}
+
 variable "docker_id" {
   description = "Docker hub username"
 }

From 3eeb54186ea497a0a8d72752474fa0970f4d5aa6 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Mon, 20 May 2024 15:16:35 +0100
Subject: [PATCH 100/109] nfs issue

---
 cloud-init/kubeadm/master.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cloud-init/kubeadm/master.yaml b/cloud-init/kubeadm/master.yaml
index 670a957..aac6879 100644
--- a/cloud-init/kubeadm/master.yaml
+++ b/cloud-init/kubeadm/master.yaml
@@ -1,6 +1,7 @@
 #cloud-config
 packages:
   - nfs-kernel-server
+  - nfs-common
 
 runcmd:
   # Mount SFTP to /DATA directory

From 9442a420b29f1db36ed5151a8515e5a629a35c87 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Mon, 20 May 2024 16:27:09 +0100
Subject: [PATCH 101/109] Located the problem

---
 cloud-init/kubeadm/master.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cloud-init/kubeadm/master.yaml b/cloud-init/kubeadm/master.yaml
index aac6879..670a957 100644
--- a/cloud-init/kubeadm/master.yaml
+++ b/cloud-init/kubeadm/master.yaml
@@ -1,7 +1,6 @@
 #cloud-config
 packages:
   - nfs-kernel-server
-  - nfs-common
 
 runcmd:
   # Mount SFTP to /DATA directory

From 1aca3d0e5591a3c1d1b46928fff44944ab0f2f30 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Mon, 20 May 2024 21:30:52 +0100
Subject: [PATCH 102/109] Change tls

---
 .../binderhub/assets/cloudflare-secret.yaml         |  7 +++++++
 .../binderhub/assets/install-binderhub.sh           | 11 ++++++-----
 .../assets/production-binderhub-issuer.yaml         | 13 +++++++------
 terraform-modules/binderhub/main.tf                 |  7 +++++++
 terraform-modules/binderhub/variables.tf            |  7 +++++++
 5 files changed, 34 insertions(+), 11 deletions(-)
 create mode 100644 terraform-modules/binderhub/assets/cloudflare-secret.yaml

diff --git a/terraform-modules/binderhub/assets/cloudflare-secret.yaml b/terraform-modules/binderhub/assets/cloudflare-secret.yaml
new file mode 100644
index 0000000..5313f21
--- /dev/null
+++ b/terraform-modules/binderhub/assets/cloudflare-secret.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cloudflare-api-token-secret
+type: Opaque
+stringData:
+  api-token: "${cloudflare_token}"
\ No newline at end of file
diff --git a/terraform-modules/binderhub/assets/install-binderhub.sh b/terraform-modules/binderhub/assets/install-binderhub.sh
index 7b02fdd..6aa53c8 100755
--- a/terraform-modules/binderhub/assets/install-binderhub.sh
+++ b/terraform-modules/binderhub/assets/install-binderhub.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 echo "[Binderhub pre-install] BOOT?"
-while [ ! -f /var/lib/cloud/instance/boot-finished ]; do sleep 10; echo "Waiting for cloud-init on master to finalize"; done
+while [ ! -f /var/lib/cloud/instance/boot-finished ]; do sleep 30; echo "Waiting for cloud-init on master to finalize (could take ~10min)"; done
 echo "[Binderhub pre-install] K8S READY?"
 while [ ! -f /shared/k8s-initialized ]; do sleep 5; echo "Waiting for K8S on master to be ready"; done
 
@@ -22,10 +22,10 @@ kubectl create namespace cert-manager
 sudo helm repo add jetstack https://charts.jetstack.io
 sudo helm repo update
 # running on master node to avoid issues with webhook not in the k8s network
-sudo helm install cert-manager --namespace cert-manager --version v1.9.0 jetstack/cert-manager --set installCRDs=true \
-  --set nodeSelector."node-role\.kubernetes\.io/master=" \
-  --set cainjector.nodeSelector."node-role\.kubernetes\.io/master=" \
-  --set webhook.nodeSelector."node-role\.kubernetes\.io/master=" \
+sudo helm install cert-manager --namespace cert-manager --version v1.12.0 jetstack/cert-manager --set installCRDs=true \
+  --set nodeSelector."node-role\.kubernetes\.io/control-plane=" \
+  --set cainjector.nodeSelector."node-role\.kubernetes\.io/control-plane=" \
+  --set webhook.nodeSelector."node-role\.kubernetes\.io/control-plane=" \
   --kubeconfig ~/.kube/config
 #wait until cert-manager is ready
 kubectl wait --namespace cert-manager \
@@ -35,6 +35,7 @@ kubectl wait --namespace cert-manager \
 # apply the issuer(s)
 kubectl create namespace binderhub
 # kubectl apply -f staging-binderhub-issuer.yaml
+kubectl apply -f cloudflare-secret.yaml -n binderhub
 kubectl apply -f production-binderhub-issuer.yaml
 
 # Binderhub proxy
diff --git a/terraform-modules/binderhub/assets/production-binderhub-issuer.yaml b/terraform-modules/binderhub/assets/production-binderhub-issuer.yaml
index e4a84fc..020e629 100644
--- a/terraform-modules/binderhub/assets/production-binderhub-issuer.yaml
+++ b/terraform-modules/binderhub/assets/production-binderhub-issuer.yaml
@@ -8,14 +8,15 @@ spec:
     # You must replace this email address with your own.
     # Let's Encrypt will use this to contact you about expiring
     # certificates, and issues related to your account.
-    email: ${TLS_email}
+    email: conp.dev@gmail.com
     server: https://acme-v02.api.letsencrypt.org/directory
     privateKeySecretRef:
       # Secret resource used to store the account's private key.
       name: letsencrypt-production
     solvers:
-    # An empty 'selector' means that this solver matches all domains
-    - selector: {}
-      http01:
-        ingress:
-          class: nginx
+    - dns01:
+        cloudflare:
+          email: conp.dev@gmail.com
+          apiTokenSecretRef:
+            name: cloudflare-api-token-secret
+            key: api-token
\ No newline at end of file
diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index 814b934..81f99e6 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -62,6 +62,13 @@ data "template_file" "install-binderhub" {
   }
 }
 
+data "template_file" "cloudflare-secret" {
+  template = file("${path.module}/assets/cloudflare-secret.yaml")
+  vars = {
+    cloudflare_token  = var.cloudflare_token
+  }
+}
+
 resource "terraform_data" "binderhub" {
 
 connection {
diff --git a/terraform-modules/binderhub/variables.tf b/terraform-modules/binderhub/variables.tf
index 880bbf9..b0edd9f 100644
--- a/terraform-modules/binderhub/variables.tf
+++ b/terraform-modules/binderhub/variables.tf
@@ -37,9 +37,16 @@ variable "docker_registry" {
 
 variable "docker_id" {
   description = "Docker hub username"
+  sensitive = true
 }
 
 variable "docker_password" {
   description = "Docker hub password"
+  sensitive = true
+}
+
+variable "cloudflare_token" {
+  description = "Cloudflare token."
+  sensitive = true
 }
 

From 6ef21eac2b2b7700dca1b44ba288ea74f8ccecdd Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Mon, 20 May 2024 22:42:12 +0100
Subject: [PATCH 103/109] umrumda degil

---
 terraform-modules/binderhub/assets/install-binderhub.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/terraform-modules/binderhub/assets/install-binderhub.sh b/terraform-modules/binderhub/assets/install-binderhub.sh
index 6aa53c8..b927745 100755
--- a/terraform-modules/binderhub/assets/install-binderhub.sh
+++ b/terraform-modules/binderhub/assets/install-binderhub.sh
@@ -23,9 +23,9 @@ sudo helm repo add jetstack https://charts.jetstack.io
 sudo helm repo update
 # running on master node to avoid issues with webhook not in the k8s network
 sudo helm install cert-manager --namespace cert-manager --version v1.12.0 jetstack/cert-manager --set installCRDs=true \
-  --set nodeSelector."node-role\.kubernetes\.io/control-plane=" \
-  --set cainjector.nodeSelector."node-role\.kubernetes\.io/control-plane=" \
-  --set webhook.nodeSelector."node-role\.kubernetes\.io/control-plane=" \
+  --set nodeSelector."node-role\.kubernetes\.io/master=" \
+  --set cainjector.nodeSelector."node-role\.kubernetes\.io/master=" \
+  --set webhook.nodeSelector."node-role\.kubernetes\.io/master=" \
   --kubeconfig ~/.kube/config
 #wait until cert-manager is ready
 kubectl wait --namespace cert-manager \

From c8653e2af4466bb0000b5eaf7c402298067ec7ab Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Tue, 21 May 2024 12:58:59 +0100
Subject: [PATCH 104/109] update node affinity

---
 terraform-modules/binderhub/assets/install-binderhub.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/terraform-modules/binderhub/assets/install-binderhub.sh b/terraform-modules/binderhub/assets/install-binderhub.sh
index b927745..6aa53c8 100755
--- a/terraform-modules/binderhub/assets/install-binderhub.sh
+++ b/terraform-modules/binderhub/assets/install-binderhub.sh
@@ -23,9 +23,9 @@ sudo helm repo add jetstack https://charts.jetstack.io
 sudo helm repo update
 # running on master node to avoid issues with webhook not in the k8s network
 sudo helm install cert-manager --namespace cert-manager --version v1.12.0 jetstack/cert-manager --set installCRDs=true \
-  --set nodeSelector."node-role\.kubernetes\.io/master=" \
-  --set cainjector.nodeSelector."node-role\.kubernetes\.io/master=" \
-  --set webhook.nodeSelector."node-role\.kubernetes\.io/master=" \
+  --set nodeSelector."node-role\.kubernetes\.io/control-plane=" \
+  --set cainjector.nodeSelector."node-role\.kubernetes\.io/control-plane=" \
+  --set webhook.nodeSelector."node-role\.kubernetes\.io/control-plane=" \
   --kubeconfig ~/.kube/config
 #wait until cert-manager is ready
 kubectl wait --namespace cert-manager \

From 774f49cd0104cb894c02bfda984dd477ea735661 Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Tue, 21 May 2024 13:02:22 +0100
Subject: [PATCH 105/109] deal with that later

---
 terraform-modules/binderhub/variables.tf | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/terraform-modules/binderhub/variables.tf b/terraform-modules/binderhub/variables.tf
index b0edd9f..ffc76d9 100644
--- a/terraform-modules/binderhub/variables.tf
+++ b/terraform-modules/binderhub/variables.tf
@@ -37,16 +37,13 @@ variable "docker_registry" {
 
 variable "docker_id" {
   description = "Docker hub username"
-  sensitive = true
 }
 
 variable "docker_password" {
   description = "Docker hub password"
-  sensitive = true
 }
 
 variable "cloudflare_token" {
   description = "Cloudflare token."
-  sensitive = true
 }
 

From 06924b9b19ce1c5b3bae44aef05b671ffe960dca Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Tue, 21 May 2024 15:00:02 +0100
Subject: [PATCH 106/109] Add missing file

---
 terraform-modules/binderhub/assets/config.yaml | 4 ++--
 terraform-modules/binderhub/main.tf            | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/terraform-modules/binderhub/assets/config.yaml b/terraform-modules/binderhub/assets/config.yaml
index d8bdb80..c64ced2 100644
--- a/terraform-modules/binderhub/assets/config.yaml
+++ b/terraform-modules/binderhub/assets/config.yaml
@@ -24,8 +24,8 @@ jupyterhub:
   hub:
     baseUrl: /jupyter/
     image:
-      name: jupyterhub/k8s-hub
-      tag: 1.1.3-n141.h28efde1b
+      name: quay.io/jupyterhub/k8s-hub
+      tag: "3.3.5"
   proxy:
     service:
       type: NodePort
diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index 81f99e6..e911cad 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -112,6 +112,10 @@ provisioner "file" {
   destination = "/home/${var.admin_user}/install-binderhub.sh"
 }
 
+provisioner "file" {
+  content     = data.template_file.cloudflare-secret.rendered
+  destination = "/home/${var.admin_user}/cloudflare-secret.yaml"
+}
 
 provisioner "file" {
   source     = "${path.module}/assets/fill_submission_metadata.bash"

From d6f786eeb3b09ae1474565ad6c68564ccf7cc6cc Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Tue, 21 May 2024 18:09:56 +0100
Subject: [PATCH 107/109] Enable Cross-Origin Resource Sharing (CORS)

---
 terraform-modules/binderhub/assets/config.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/terraform-modules/binderhub/assets/config.yaml b/terraform-modules/binderhub/assets/config.yaml
index c64ced2..f3850ae 100644
--- a/terraform-modules/binderhub/assets/config.yaml
+++ b/terraform-modules/binderhub/assets/config.yaml
@@ -26,6 +26,9 @@ jupyterhub:
     image:
       name: quay.io/jupyterhub/k8s-hub
       tag: "3.3.5"
+    config: 
+      BinderSpawner:
+        cors_allow_origin: '*'
   proxy:
     service:
       type: NodePort
@@ -93,6 +96,7 @@ config:
       - ^shishirchoudharygic/mltraining.*
       - ^hmharshit/mltraining.*
   BinderHub:
+    cors_allow_origin: '*'
     hub_url: https://test.conp.cloud/jupyter
     use_registry: true
     image_prefix: binder-registry.conp.cloud/binder-registry.conp.cloud/binder-

From 6313e6e4e0d0703d7f9a9a804d35606432b4977e Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Mon, 27 May 2024 08:59:33 +0100
Subject: [PATCH 108/109] Add condition to skip JBuild on myst

---
 terraform-modules/binderhub/assets/jb_build.bash | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/terraform-modules/binderhub/assets/jb_build.bash b/terraform-modules/binderhub/assets/jb_build.bash
index e0c1c19..cab30a8 100755
--- a/terraform-modules/binderhub/assets/jb_build.bash
+++ b/terraform-modules/binderhub/assets/jb_build.bash
@@ -77,10 +77,15 @@ fi
 if [ -f "$NEUROLIBRE_CUSTOM" ]; then
     BOOK_LAYOUT=$(extract_yaml_field "$NEUROLIBRE_CUSTOM" "book_layout")
     SINGLE_PAGE=$(extract_yaml_field "$NEUROLIBRE_CUSTOM" "single_page")
+    MYST=$(extract_yaml_field "$NEUROLIBRE_CUSTOM" "build_myst")
   else
     echo "YAML file not found: $yaml_file"
 fi
 
+if [ "$MYST" == "true" ]; then
+  echo "Myst build requested, skipping jupyter book build"
+  exit 0
+fi
 
 # building jupyter book
 echo "" 2>&1 | tee -a ${BOOK_BUILD_LOG}

From 4aa15962ca4b9d59ce8c23fe0b305f507cc2d79a Mon Sep 17 00:00:00 2001
From: Agah <agahkarakuzu@gmail.com>
Date: Fri, 13 Sep 2024 23:12:26 -0400
Subject: [PATCH 109/109] Deprecate JB build in user pods

---
 .../binderhub/assets/config.yaml              | 26 +---------------
 .../binderhub/assets/install-binderhub.sh     |  9 ++++--
 .../binderhub/assets/jb_build.bash            |  5 ++++
 terraform-modules/binderhub/main.tf           | 30 ++++++++++---------
 4 files changed, 28 insertions(+), 42 deletions(-)

diff --git a/terraform-modules/binderhub/assets/config.yaml b/terraform-modules/binderhub/assets/config.yaml
index f3850ae..8850156 100644
--- a/terraform-modules/binderhub/assets/config.yaml
+++ b/terraform-modules/binderhub/assets/config.yaml
@@ -37,6 +37,7 @@ jupyterhub:
     every: 30
     concurrency: 5 #to avoid Hub slow down, 5 concurrent processes
     maxAge: 7200 #2h
+  # https://z2jh.jupyter.org/en/latest/resources/reference.html#singleuser  
   singleuser:
     storage:
       type: none
@@ -44,42 +45,17 @@ jupyterhub:
       - name: shared-data
         hostPath:
           path: /DATA
-      - name: book-data
-        hostPath:
-          path: /DATA/book-artifacts
-      - name: repo2data
-        hostPath:
-          path: /DATA
       extraVolumeMounts:
       - name: shared-data
         mountPath: /home/jovyan/data  # where each user can reach shared data
         readOnly : true
-      - name: book-data
-        mountPath: /mnt/books
-      - name: repo2data
-        mountPath: /mnt/data
     memory:
        guarantee: 4G
        limit: 8G
     cpu:
        guarantee: 1
     startTimeout: 3600 #1h
-    #for repo2data https://zero-to-jupyterhub.readthedocs.io/en/latest/resources/reference.html#singleuser-initcontainers
     # https://zero-to-jupyterhub.readthedocs.io/en/latest/jupyterhub/customizing/user-environment.html#about-user-storage-and-adding-files-to-it
-    extraFiles:
-      jb_build:
-        mountPath: /usr/local/share/jb_build.bash
-        mode: 0755
-      repo2data:
-        mountPath: /usr/local/share/repo2data.bash
-        mode: 0755
-      fill_submission_metadata:
-        mountPath: /usr/local/share/fill_submission_metadata.bash
-        mode: 0755
-    lifecycleHooks:
-      postStart:
-        exec:
-          command: ["/bin/sh", "-c", "/usr/local/share/repo2data.bash; /usr/local/share/jb_build.bash"]
 
 # BinderHub config
 config:
diff --git a/terraform-modules/binderhub/assets/install-binderhub.sh b/terraform-modules/binderhub/assets/install-binderhub.sh
index 6aa53c8..144e06f 100755
--- a/terraform-modules/binderhub/assets/install-binderhub.sh
+++ b/terraform-modules/binderhub/assets/install-binderhub.sh
@@ -56,10 +56,13 @@ sudo helm repo add jupyterhub https://jupyterhub.github.io/helm-chart
 sudo helm repo update
 sudo helm install binderhub jupyterhub/binderhub --version=${binder_version} \
   --namespace=binderhub -f config.yaml -f secrets.yaml \
-  --set-file jupyterhub.singleuser.extraFiles.repo2data.stringData=./repo2data.bash \
-  --set-file jupyterhub.singleuser.extraFiles.fill_submission_metadata.stringData=./fill_submission_metadata.bash \
-  --set-file jupyterhub.singleuser.extraFiles.jb_build.stringData=./jb_build.bash \
   --kubeconfig ~/.kube/config
+
+  # DROPPING JB BUILD INSIDE POD SUPPORT
+  # --set-file jupyterhub.singleuser.extraFiles.repo2data.stringData=./repo2data.bash \
+  # --set-file jupyterhub.singleuser.extraFiles.fill_submission_metadata.stringData=./fill_submission_metadata.bash \
+  # --set-file jupyterhub.singleuser.extraFiles.jb_build.stringData=./jb_build.bash \
+  
 # sudo helm upgrade binderhub jupyterhub/binderhub -n binderhub --version=${binder_version} \
 #   -f confgi.yaml -f secrets.yaml \
 #   --set-file jupyterhub.singleuser.extraFiles.repo2data.stringData=./repo2data.bash \
diff --git a/terraform-modules/binderhub/assets/jb_build.bash b/terraform-modules/binderhub/assets/jb_build.bash
index cab30a8..a82351e 100755
--- a/terraform-modules/binderhub/assets/jb_build.bash
+++ b/terraform-modules/binderhub/assets/jb_build.bash
@@ -1,5 +1,10 @@
 #!/bin/bash
 
+
+# ---------------------------------------------------------------
+# DEPRECATED
+# ---------------------------------------------------------------
+
 # repo parameters
 IFS='/'; BINDER_PARAMS=(${BINDER_REF_URL}); unset IFS;
 PROVIDER_NAME=${BINDER_PARAMS[-5]}
diff --git a/terraform-modules/binderhub/main.tf b/terraform-modules/binderhub/main.tf
index e911cad..2ba1869 100644
--- a/terraform-modules/binderhub/main.tf
+++ b/terraform-modules/binderhub/main.tf
@@ -117,20 +117,22 @@ provisioner "file" {
   destination = "/home/${var.admin_user}/cloudflare-secret.yaml"
 }
 
-provisioner "file" {
-  source     = "${path.module}/assets/fill_submission_metadata.bash"
-  destination = "/home/${var.admin_user}/fill_submission_metadata.bash"
-}
-
-provisioner "file" {
-  source     = "${path.module}/assets/repo2data.bash"
-  destination = "/home/${var.admin_user}/repo2data.bash"
-}
-
-provisioner "file" {
-  source     = "${path.module}/assets/jb_build.bash"
-  destination = "/home/${var.admin_user}/jb_build.bash"
-}
+## DEPRECATED
+
+# provisioner "file" {
+#   source     = "${path.module}/assets/fill_submission_metadata.bash"
+#   destination = "/home/${var.admin_user}/fill_submission_metadata.bash"
+# }
+
+# provisioner "file" {
+#   source     = "${path.module}/assets/repo2data.bash"
+#   destination = "/home/${var.admin_user}/repo2data.bash"
+# }
+
+# provisioner "file" {
+#   source     = "${path.module}/assets/jb_build.bash"
+#   destination = "/home/${var.admin_user}/jb_build.bash"
+# }
 
 provisioner "remote-exec" {
   inline = ["bash /home/${var.admin_user}/install-binderhub.sh",]