Skip to content

Commit

Permalink
Merge pull request #152 from paulzierep/deploy-results-to-different-b…
Browse files Browse the repository at this point in the history 
…ranch

Upload results to results branch
  • Loading branch information
@bebatut
bebatut authored Jul 29, 2024
2 parents 2c5cde7 + e45e087 commit 671ce33
Show file tree
Hide file tree
Showing 57 changed files with 210 additions and 424,315 deletions.
109 changes: 52 additions & 57 deletions .github/workflows/fetch_all_tools.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,42 +16,30 @@ permissions:
contents: write

jobs:
fetch-available-servers:
runs-on: ubuntu-20.04
name: Fetch list of all available servers
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
- name: Install requirement
run: python -m pip install -r requirements.txt
- name: Run script
run: |
python bin/get_public_galaxy_servers.py -o data/available_public_servers.csv
- name: Commit servers
# add or commit any changes in results if there was a change, merge with main, and push as bot
run: |
git config user.name github-actions
git config user.email [email protected]
git pull --no-rebase -s recursive -X ours
git add data
git status
git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot - step get available servers")
git push

fetch-all-tools-stepwise:
runs-on: ubuntu-20.04
environment: fetch-tools
name: Fetch all tool stepwise
strategy:
max-parallel: 1 #need to run one after another, since otherwise there is a chance, that mulitple jobs want to push to the results branch at the same time (which fails due to merge)
matrix:
python-version: [3.8]
python-version: [3.11]
subset:
- repositories01.list
- repositories02.list
- repositories03.list
- repositories04.list
steps:
- uses: actions/checkout@v4
- name: Checkout main
uses: actions/checkout@v4
with:
ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
- name: Checkout results
uses: actions/checkout@v4
with:
ref: results
path: results
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
Expand All @@ -63,43 +51,50 @@ jobs:
env:
GITHUB_API_KEY: ${{ secrets.GH_API_TOKEN }}
- name: Commit all tools
# add or commit any changes in results if there was a change, merge with main and push as bot
run: |
git config user.name github-actions
git config user.email [email protected]
git pull --no-rebase -s recursive -X ours
git add results
git status
git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot - step fetch")
git push
uses: s0/git-publish-subdir-action@develop
env:
BRANCH: results
FOLDER: results
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
REPO: self
SKIP_EMPTY_COMMITS: true
MESSAGE: "Update results step: fetch all tool stepwise - commmit {sha}"

fetch-all-tools-merge:
runs-on: ubuntu-20.04
needs: fetch-all-tools-stepwise
name: Fetch all tools merge
steps:
- uses: actions/checkout@v4
with:
ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
- uses: actions/setup-python@v5
- name: Install requirement
run: |
python -m pip install -r requirements.txt
sudo apt-get install jq
- name: Merge all tools
run: | #merge files with only one header -> https://stackoverflow.com/questions/16890582/unixmerge-multiple-csv-files-with-same-header-by-keeping-the-header-of-the-firs; map(.[]) -> https://stackoverflow.com/questions/42011086/merge-arrays-of-json (get flat array, one tool per entry)
awk 'FNR==1 && NR!=1{next;}{print}' results/repositories*.list_tools.tsv > results/all_tools.tsv
jq -s 'map(.[])' results/repositories*.list_tools.json > results/all_tools.json
- name: Wordcloud and interactive table
run: |
bash bin/format_tools.sh
- name: Commit all tools
# add or commit any changes in results if there was a change, merge with main and push as bot
run: |
git config user.name github-actions
git config user.email [email protected]
git pull --no-rebase -s recursive -X ours
git add results
git status
git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot - step merge")
git push
- name: Checkout main
uses: actions/checkout@v4
with:
ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
- name: Checkout results
uses: actions/checkout@v4
with:
ref: results
path: results
- uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Install requirement
run: |
python -m pip install -r requirements.txt
sudo apt-get install jq
- name: Merge all tools
run: | #merge files with only one header -> https://stackoverflow.com/questions/16890582/unixmerge-multiple-csv-files-with-same-header-by-keeping-the-header-of-the-firs; map(.[]) -> https://stackoverflow.com/questions/42011086/merge-arrays-of-json (get flat array, one tool per entry)
awk 'FNR==1 && NR!=1{next;}{print}' results/repositories*.list_tools.tsv > results/all_tools.tsv
jq -s 'map(.[])' results/repositories*.list_tools.json > results/all_tools.json
- name: Wordcloud and interactive table
run: |
bash bin/format_tools.sh
- name: Commit all tools
# add or commit any changes in results if there was a change, merge with main and push as bot
uses: s0/git-publish-subdir-action@develop
env:
BRANCH: results
FOLDER: results
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
REPO: self
SKIP_EMPTY_COMMITS: true
MESSAGE: "Update results step: fetch all tools merge - commmit {sha}"
27 changes: 17 additions & 10 deletions .github/workflows/fetch_all_tutorials.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,15 @@ jobs:
environment: fetch-tutorials
name: Fetch all tutorials
steps:
- uses: actions/checkout@v4
- name: Checkout main
uses: actions/checkout@v4
with:
ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
- name: Checkout results
uses: actions/checkout@v4
with:
ref: results
path: results
- uses: actions/setup-python@v5
with:
python-version: '3.11'
Expand All @@ -33,12 +41,11 @@ jobs:
env:
PLAUSIBLE_API_KEY: ${{ secrets.PLAUSIBLE_API_TOKEN }}
- name: Commit all tools
# add or commit any changes in results if there was a change, merge with main and push as bot
run: |
git config user.name github-actions
git config user.email [email protected]
git pull --no-rebase -s recursive -X ours
git add results
git status
git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tutorials bot - step fetch")
git push
uses: s0/git-publish-subdir-action@develop
env:
BRANCH: results
FOLDER: results
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
REPO: self
SKIP_EMPTY_COMMITS: true
MESSAGE: "Update results step: fetch all tutorials - commmit {sha}"
88 changes: 57 additions & 31 deletions .github/workflows/filter_communities.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,73 +26,99 @@ permissions:

jobs:
filter-all-tutorials:
name: Filter tutorials
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- name: Checkout main
uses: actions/checkout@v4
with:
ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
- name: Checkout results
uses: actions/checkout@v4
with:
ref: results
path: results
- uses: actions/setup-python@v5
with:
python-version: '3.8'
python-version: '3.11'
- name: Install requirement
run: python -m pip install -r requirements.txt
- name: Run script
run: |
bash bin/get_community_tutorials.sh
- name: Commit results
# commit the new filtered data, only if stuff was changed
run: |
git config user.name github-actions
git config user.email [email protected]
git pull --no-rebase -s recursive -X ours
git add results
git status
git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tutorials / tools bot - step tutorial filter")
git push
uses: s0/git-publish-subdir-action@develop
env:
BRANCH: results
FOLDER: results
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
REPO: self
SKIP_EMPTY_COMMITS: true
MESSAGE: "Update results step: filter tutorials - commmit {sha}"

update-tools-to-keep-exclude:
name: Update tool to keep and exclude
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- name: Checkout main
uses: actions/checkout@v4
with:
ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
- name: Checkout results
uses: actions/checkout@v4
with:
ref: results
path: results
- uses: actions/setup-python@v5
with:
python-version: '3.8'
python-version: '3.11'
- name: Install requirement
run: python -m pip install -r requirements.txt
- name: Run script
run: |
bash bin/update_tools_to_keep_exclude.sh
- name: Commit results
# commit the new filtered data, only if stuff was changed
run: |
git config user.name github-actions
git config user.email [email protected]
git pull --no-rebase -s recursive -X ours
git add results
git status
git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tutorials / tools bot - step exluded/kept tool list update")
git push
uses: s0/git-publish-subdir-action@develop
env:
BRANCH: results
FOLDER: results
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
REPO: self
SKIP_EMPTY_COMMITS: true
MESSAGE: "Update results step: update tool to keep and exclude - commmit {sha}"

filter-all-tools:
name: Filter all tools
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- name: Checkout main
uses: actions/checkout@v4
with:
ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
- name: Checkout results
uses: actions/checkout@v4
with:
ref: results
path: results
- uses: actions/setup-python@v5
with:
python-version: '3.8'
python-version: '3.11'
- name: Install requirement
run: python -m pip install -r requirements.txt
- name: Run script
run: |
bash bin/get_community_tools.sh
- name: Commit results
# commit the new filtered data, only if stuff was changed
run: |
git config user.name github-actions
git config user.email [email protected]
git pull --no-rebase -s recursive -X ours
git add results
git status
git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot - step tool filter")
git push
uses: s0/git-publish-subdir-action@develop
env:
BRANCH: results
FOLDER: results
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
REPO: self
SKIP_EMPTY_COMMITS: true
MESSAGE: "Update results step: filter all tools - commmit {sha}"


22 changes: 19 additions & 3 deletions .github/workflows/run_tests.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,20 @@
name: Run tests
on: [push, pull_request]
on: [push, pull_request_target]

jobs:
test-tools:
runs-on: ubuntu-20.04
environment: fetch-tools
steps:
- uses: actions/checkout@v4
- name: Checkout main
uses: actions/checkout@v4
with:
ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
- name: Checkout results
uses: actions/checkout@v4
with:
ref: results
path: results
- uses: actions/setup-python@v5
with:
python-version: '3.11'
Expand All @@ -26,7 +34,15 @@ jobs:
test-tutorials:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v4
- name: Checkout main
uses: actions/checkout@v4
with:
ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
- name: Checkout results
uses: actions/checkout@v4
with:
ref: results
path: results
- uses: actions/setup-python@v5
with:
python-version: '3.11'
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/static.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v4
with:
ref: results
- name: Setup Pages
uses: actions/configure-pages@v4
- name: Upload artifact
Expand Down
Loading

0 comments on commit 671ce33

Please sign in to comment.