From e31b6bc81dc5a34ee98f6839ec74ecc160968a33 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Thu, 14 Nov 2024 16:51:09 +0300 Subject: [PATCH 1/2] PMM-7 Remove yml components, update docs --- .gitignore | 2 +- .gitmodules | 3 +-- Makefile | 14 +++++++------- README.md | 34 ++++++++++++++++++---------------- ci-default.yml | 17 +---------------- ci.py | 16 ++++------------ requirements.txt | 2 +- 7 files changed, 33 insertions(+), 55 deletions(-) diff --git a/.gitignore b/.gitignore index 65175e055dea..e8be2c5382b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -build/pmm-server-docker/RPMS/ +build/ *.swp *~ tmp/ diff --git a/.gitmodules b/.gitmodules index 18e1e222d806..67c412fb08c8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,10 +1,10 @@ +# PMM Client and Server Components [submodule "pmm"] path = sources/pmm/src/github.com/percona/pmm url = https://github.com/percona/pmm branch = v3 # PMM Client - [submodule "node_exporter"] path = sources/node_exporter/src/github.com/prometheus/node_exporter url = https://github.com/percona/node_exporter.git @@ -39,7 +39,6 @@ branch = release-v3.5.2 # PMM Server - [submodule "grafana-dashboards"] path = sources/grafana-dashboards url = https://github.com/percona/grafana-dashboards diff --git a/Makefile b/Makefile index 9e3dae692622..e5b6723c4e70 100644 --- a/Makefile +++ b/Makefile @@ -14,18 +14,18 @@ help: ## Display this help message. @grep '^[a-zA-Z]' $(MAKEFILE_LIST) | \ awk -F ':.*?## ' 'NF==2 {printf " %-26s%s\n", $$1, $$2}' -submodules: ## Update all sumodules . +submodules: ## Update all sumodules. git submodule update --init --remote --jobs 10 git submodule status -deps: ## Get deps from repos +deps: ## Get deps from repos. python3 ci.py -trigger: - git commit -m 'Trigger FB' --allow-empty +trigger: ## Make an empty commit to trigger the build. + git commit -m 'chore: trigger FB' --allow-empty git push -prepare: ## Create new FB (new style) +prepare: ## Create a new FB (new style). python3 ci.py -g --prepare $(RUN_ARGS) clean: ## Clean build results. @@ -36,8 +36,8 @@ purge: ## Clean cache and leftovers. Please run this when s git submodule update git submodule foreach 'git reset --hard && git clean -xdff' -fb: ## Creates feature build branch. - # Usage: make fb mainBranch=v3 featureBranch=PMM-XXXX-name submodules="pmm pmm-managed" +fb: ## Create a feature build branch. + # Usage: make fb mainBranch=v3 featureBranch=PMM-XXXX-branch-name submodules="pmm pmm-managed" $(eval MAIN_BRANCH = $(or $(mainBranch),v3)) git checkout $(MAIN_BRANCH) make purge diff --git a/README.md b/README.md index 5dc10f5269ae..64511dda512c 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,9 @@ to our build system managed by Jenkins as it helps pull the right branches from # Installation of dependencies -If you build with Python's script then you need to install the dependencies: +If you intend to build PMM with 'ci.py' Python script, then you need to install the dependencies: -``` +```shell pip install -r requirements.txt ``` @@ -18,29 +18,29 @@ To create a feature build (FB) you have to edit `ci.yml` and specify the branche ```yaml deps: - name: pmm - branch: PMM-0000-fix-everything + branch: PMM-0000-feature-branch - name: pmm-qa - branch: PMM-0000-fix-everything-and-even-more + branch: PMM-0000-bugfix-branch ``` -To build from a fork, you need to specify `url` for the dependency, for example: +To build from a fork, you need to additionally specify `url` for the dependency, for example: ```yaml deps: - name: pmm-server url: https://github.com//pmm-server - branch: PMM-0000-fix-everything + branch: PMM-0000-feature-branch ``` -Next, you will commit changes to git and push them to the repo: +Next, commit your changes to git and push them to the repo: -``` +```shell git add ci.yml git commit -m 'use custom branches' git push ``` -Whenever you commit and push to a feature branch, a Jenkins job will be triggered and it will start building your feature. You can follow its progress right from the PR's actions (at the bottom of each PR). +Whenever you commit and push to a feature branch, a Jenkins job will be triggered, which will start building your feature. You can follow its progress by following a github workflow link displayed at the bottom of each PR. ## Using a Personal Access Token (PAT) @@ -54,21 +54,23 @@ The token requires the following permissions: It is recommended to set an expiration date for your token. -if you use zsh: +To make the Github token available to `ci.py`, add the following lines to your shell script: + +if you use `zsh` shell: ```console echo 'export GITHUB_API_TOKEN=********' >> ~/.zshrc source ~/.zshrc ``` -if you use bash: +if you use `bash` shell: ```console echo 'export GITHUB_API_TOKEN=********' >> ~/.bash_profile source ~/.bash_profile ``` -NOTE: Please make sure you don't commit your PAT to github. Should the PAT accidentally leak out, please revoke it asap and re-create it. +NOTE: Please make sure you DO NOT commit your PAT to github. Should the PAT accidentally leak out, please revoke it asap and re-create it. ## FAQ @@ -77,15 +79,15 @@ NOTE: Please make sure you don't commit your PAT to github. Should the PAT accid If you use the same branch name in all repos then you can run: ```console -make prepare +make prepare ``` -Branches with "you branch name" will be used for all repos or the default branch (usually called `main`) if the branch with this name isn't found in the repo. +Branch "your-branch-name" will be used for every submodule repo. If not found, the repo's default branch (usually `main`) will be used. -If you want to create a FB from a fork, you can pass an environment variable "FORK_OWNER" which should be equal to your username in github and run: +If you want to create a FB from a fork, you can pass an environment variable "FORK_OWNER", which should be set to your github username: ```console -FORK_OWNER= make prepare +FORK_OWNER= make prepare ``` ### I got an error "...branch has no upstream branch" diff --git a/ci-default.yml b/ci-default.yml index 5c44227ae44c..fa632e64fc03 100644 --- a/ci-default.yml +++ b/ci-default.yml @@ -1,89 +1,74 @@ deps: - # COMMON + # CLIENT AND SERVER - name: pmm branch: v3 path: sources/pmm/src/github.com/percona/pmm url: https://github.com/percona/pmm - component: common # CLIENT - name: node_exporter branch: v3 path: sources/node_exporter/src/github.com/prometheus/node_exporter url: https://github.com/percona/node_exporter - component: client - name: mysqld_exporter branch: main path: sources/mysqld_exporter/src/github.com/percona/mysqld_exporter url: https://github.com/percona/mysqld_exporter - component: client - name: mongodb_exporter branch: main path: sources/mongodb_exporter/src/github.com/percona/mongodb_exporter url: https://github.com/percona/mongodb_exporter - component: client - name: postgres_exporter branch: main path: sources/postgres_exporter/src/github.com/percona/postgres_exporter url: https://github.com/percona/postgres_exporter - component: client - name: proxysql_exporter branch: main path: sources/proxysql_exporter/src/github.com/percona/proxysql_exporter url: https://github.com/percona/proxysql_exporter - component: client - name: rds_exporter branch: main path: sources/rds_exporter/src/github.com/percona/rds_exporter url: https://github.com/percona/rds_exporter - component: client - name: azure_metrics_exporter branch: main path: sources/azure_metrics_exporter/src/github.com/percona/azure_metrics_exporter url: https://github.com/percona/azure_metrics_exporter - component: client - name: percona-toolkit branch: release-v3.5.2 path: sources/percona-toolkit/src/github.com/percona/percona-toolkit url: https://github.com/percona/percona-toolkit - component: client # SERVER - - name: grafana-dashboards branch: v3 path: sources/grafana-dashboards url: https://github.com/percona/grafana-dashboards - component: server - name: grafana branch: v3 path: sources/grafana/src/github.com/grafana/grafana url: https://github.com/percona/grafana - component: server - name: pmm-dump branch: main path: sources/pmm-dump url: https://github.com/percona/pmm-dump - component: server # QA - name: pmm-qa branch: v3 path: sources/pmm-qa/src/github.com/percona/pmm-qa url: https://github.com/percona/pmm-qa - component: qa - name: pmm-ui-tests branch: v3 path: sources/pmm-ui-tests/src/github.com/percona/pmm-ui-tests url: https://github.com/percona/pmm-ui-tests - component: qa diff --git a/ci.py b/ci.py index 36b4aa0540f8..682ac9f4265b 100644 --- a/ci.py +++ b/ci.py @@ -232,12 +232,6 @@ def check_deps(self): pull.create_issue_comment(outdated_branches_message) sys.exit(1) - def create_release(self): - pass - - def create_tags(self): - pass - def validate_config(self): for dep in self.config['deps']: if not os.path.abspath(dep['path']).startswith(os.getcwd()): @@ -269,7 +263,7 @@ def get_list_of_submodules(self): def convert_gitmodules_to_yaml(self): yaml_config = Path(self.target) if yaml_config.is_file(): - logging.warning('File {} already exist!'.format(self.target)) + logging.warning('File {} already exists!'.format(self.target)) sys.exit(1) with open(self.target, 'w') as f: yaml.dump(self.submodules, f, sort_keys=False) @@ -277,15 +271,15 @@ def convert_gitmodules_to_yaml(self): def switch_branch(path, branch): - # symbolic-ref works only if we on branch. If we use commit we use rev-parse instead + # 'symbolic-ref' works only if we are on a branch. If we use a commit, we run 'rev-parse' instead. try: cur_branch = check_output('git symbolic-ref --short HEAD'.split(), cwd=path).decode().strip() except CalledProcessError: cur_branch = check_output('git rev-parse HEAD'.split(), cwd=path).decode().strip() if cur_branch != branch: branches = check_output('git ls-remote --heads origin'.split(), cwd=path) - branches = [line.split("/")[-1] - for line in branches.decode().strip().split("\n")] + branches = [line.split("/")[-1] for line in branches.decode().strip().split("\n")] + if branch in branches: print(f'Switch to branch: {branch} (from {cur_branch})') check_call(f'git remote set-branches origin {branch}'.split(), cwd=path) @@ -304,8 +298,6 @@ def main(): parser.add_argument('--global', '-g', dest='global_repo', help='find and use all branches with this name', action='store_true') parser.add_argument('--convert', help='convert .gitmodules to .git-deps.yml', action='store_true') - parser.add_argument('--release', help='create release candidate') - parser.add_argument('--tags', help='create tag') parser.add_argument('--get_branch', help='get branch name for repo') args = parser.parse_args() diff --git a/requirements.txt b/requirements.txt index 1c2b51c053c8..b5b4afc332aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -pyaml==23.12.0 +pyaml==24.9.0 GitPython==3.1.41 PyGithub==2.1.1 From bb9b6cf1a1ce88f1bc13a07ed039e453d8f10a5a Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Thu, 14 Nov 2024 21:04:11 +0300 Subject: [PATCH 2/2] PMM-7 Bump up dependencies --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index b5b4afc332aa..92697e5ab31c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ pyaml==24.9.0 -GitPython==3.1.41 -PyGithub==2.1.1 +GitPython==3.1.43 +PyGithub==2.5.0