From 83ace19836f2558640d5f887456e0a55d6647d07 Mon Sep 17 00:00:00 2001 From: remoteeng00 Date: Fri, 8 Jul 2022 00:35:41 +0700 Subject: [PATCH 1/2] Update ES config with heap size 1G, Update biothing studio build command --- Makefile | 1 + ansible_playbook/configure.yml | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/Makefile b/Makefile index 0024dc7..a3a2ced 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ PYTHON_VERSION ?= biothings-studio: docker build $(DOCKER_BUILD_EXTRA_OPTS) \ --build-arg STUDIO_VERSION=$(STUDIO_VERSION) \ + --build-arg BIOTHINGS_REPOSITORY=$(BIOTHINGS_REPOSITORY) \ --build-arg BIOTHINGS_VERSION=$(BIOTHINGS_VERSION) \ --build-arg PROD=1 \ -t biothings-studio:$$(git branch | grep ^\* | sed "s#\* ##") . diff --git a/ansible_playbook/configure.yml b/ansible_playbook/configure.yml index 543a4d3..960868b 100644 --- a/ansible_playbook/configure.yml +++ b/ansible_playbook/configure.yml @@ -177,3 +177,9 @@ - {"line": "", "re": "^cluster.initial_master_nodes.*$"} - {"line": "xpack.security.enabled: false", "re": "^xpack.security.enabled.*$"} +- name: update /etc/elasticsearch/jvm.options again replacing min/max heap size + replace: dest=/etc/elasticsearch/jvm.options replace="{{ item.line }}" regexp="{{ item.re }}" + become: yes + with_items: + - {"line": "-Xms1g", "re": "^## -Xms4g$"} + - {"line": "-Xmx1g", "re": "^## -Xmx4g$"} From 600a9b3dfc880e04825fbbd79a280b210f302934 Mon Sep 17 00:00:00 2001 From: remoteeng00 Date: Wed, 20 Jul 2022 01:55:58 +0700 Subject: [PATCH 2/2] Add customizable & optional parameter: ES_HEAP_SIZE --- .env.example | 1 + Dockerfile | 3 +++ Makefile | 4 +++- ansible_playbook/configure.yml | 21 ++++++++++++++------- ansible_playbook/vars/biothings_studio.yml | 4 ++-- 5 files changed, 23 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index 62a6513..ba937dd 100644 --- a/.env.example +++ b/.env.example @@ -3,3 +3,4 @@ export AWS_SECRET_KEY= export BIOTHINGS_REPOSITORY=https://github.com/biothings/biothings.api.git export BIOTHINGS_VERSION=master export PYTHON_VERSION= +export ES_HEAP_SIZE= diff --git a/Dockerfile b/Dockerfile index 49fc9b5..7e41dc3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -231,6 +231,7 @@ ADD ansible_playbook /tmp/ansible_playbook ADD inventory /etc/ansible/hosts COPY --from=build-webapp --chown=root:www-data /build/src/github.com/biothings/biothings_studio/webapp/dist /srv/www/webapp +ARG ES_HEAP_SIZE WORKDIR /tmp/ansible_playbook RUN if [ -n "$API_NAME" ]; \ @@ -240,12 +241,14 @@ RUN if [ -n "$API_NAME" ]; \ -e "studio_version=$STUDIO_VERSION" \ -e "api_name=$API_NAME" \ -e "api_version=$API_VERSION" \ + -e "es_heap_size=$ES_HEAP_SIZE" \ -c local; \ else \ ansible-playbook biothings_studio.yml \ -e "biothings_version=$BIOTHINGS_VERSION" \ -e "biothings_repository=$BIOTHINGS_REPOSITORY" \ -e "studio_version=$STUDIO_VERSION" \ + -e "es_heap_size=$ES_HEAP_SIZE" \ -c local; \ fi diff --git a/Makefile b/Makefile index a3a2ced..8f08fcb 100644 --- a/Makefile +++ b/Makefile @@ -9,11 +9,11 @@ BIOTHINGS_VERSION ?= master BIOTHINGS_REPOSITORY ?= https://github.com/biothings/biothings.api.git DOCKER_BUILD_EXTRA_OPTS ?= --force-rm PYTHON_VERSION ?= +ES_HEAP_SIZE ?= biothings-studio: docker build $(DOCKER_BUILD_EXTRA_OPTS) \ --build-arg STUDIO_VERSION=$(STUDIO_VERSION) \ - --build-arg BIOTHINGS_REPOSITORY=$(BIOTHINGS_REPOSITORY) \ --build-arg BIOTHINGS_VERSION=$(BIOTHINGS_VERSION) \ --build-arg PROD=1 \ -t biothings-studio:$$(git branch | grep ^\* | sed "s#\* ##") . @@ -76,6 +76,7 @@ demohub: --build-arg STUDIO_VERSION=$(STUDIO_VERSION) \ --build-arg BIOTHINGS_VERSION=$(BIOTHINGS_VERSION) \ --build-arg BIOTHINGS_REPOSITORY=$(BIOTHINGS_REPOSITORY) \ + --build-arg ES_HEAP_SIZE=512m \ --build-arg API_VERSION=master \ --build-arg TEST=1 \ --build-arg AWS_ACCESS_KEY=$(AWS_ACCESS_KEY) \ @@ -87,6 +88,7 @@ demohub-test: --build-arg STUDIO_VERSION=$(STUDIO_VERSION) \ --build-arg BIOTHINGS_VERSION=$(BIOTHINGS_VERSION) \ --build-arg BIOTHINGS_REPOSITORY=$(BIOTHINGS_REPOSITORY) \ + --build-arg ES_HEAP_SIZE=512m \ --build-arg API_VERSION=master \ --build-arg TEST=1 \ --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \ diff --git a/ansible_playbook/configure.yml b/ansible_playbook/configure.yml index 960868b..e1e831c 100644 --- a/ansible_playbook/configure.yml +++ b/ansible_playbook/configure.yml @@ -110,8 +110,10 @@ - name: mongodb data directory file: path="{{ software.common_configurations.mongodb.dbpath }}" state=directory owner=mongodb group=mongodb +#- debug: var=software.web_node + - name: biothings (override requirement_web.txt version) - git: repo={{ biothings_repository }} dest=/home/biothings/biothings.api version={{ software.web_node.biothings_git_version }} + git: repo={{ software.web_node.biothings_repository }} dest=/home/biothings/biothings.api version={{ software.web_node.biothings_git_version }} become_user: "biothings" become: true @@ -177,9 +179,14 @@ - {"line": "", "re": "^cluster.initial_master_nodes.*$"} - {"line": "xpack.security.enabled: false", "re": "^xpack.security.enabled.*$"} -- name: update /etc/elasticsearch/jvm.options again replacing min/max heap size - replace: dest=/etc/elasticsearch/jvm.options replace="{{ item.line }}" regexp="{{ item.re }}" - become: yes - with_items: - - {"line": "-Xms1g", "re": "^## -Xms4g$"} - - {"line": "-Xmx1g", "re": "^## -Xmx4g$"} + +- name: Update /etc/elasticsearch/jvm.options.d/heap.options again replacing min/max heap size + when: software.web_node.elasticsearch.heap_size != '' + shell: + cmd: | + cat > /etc/elasticsearch/jvm.options.d/heap.options << EOF + -Xms{{ software.web_node.elasticsearch.heap_size }} + -Xmx{{ software.web_node.elasticsearch.heap_size }} + EOF + args: + executable: /bin/sh diff --git a/ansible_playbook/vars/biothings_studio.yml b/ansible_playbook/vars/biothings_studio.yml index adc80a8..72eecf5 100644 --- a/ansible_playbook/vars/biothings_studio.yml +++ b/ansible_playbook/vars/biothings_studio.yml @@ -13,9 +13,9 @@ software: plugin: head: "mobz/elasticsearch-head" bigdesk: "lukas-vlcek/bigdesk" - heap_size: "2g" + heap_size: "{{ es_heap_size | default('')}}" node_roles: ['data', 'master'] - biothings_repository: "{{ biothings_repository | default('https://github.com/biothings/biothings.api.git') }}" + biothings_repository: "{{ biothings_repository | default('https://github.com/biothings/biothings.api.git', true) }}" studio_git_version: "{{ studio_version | default('master')}}" biothings_git_version: "{{ biothings_version | default('master')}}" common_configurations: