Restructuration for community content (#169)

* Restructure the repository content:

- Move sources (code) to a `source` folder
- Move metadata, resources, and subdomains for communities into communities folder where each community has its own subfolder
- Move results back to the main branch, in the resources folder

* Reorganize GitHub action workflows

- Merge weekly workflows into 1 workflow that wil open a PR with the changes
- Update other workflows to the new structure

* Adapt CRON workflow to limit the subset jobs to a job and use artifacts to share btw jobs

* Update .github/workflows/fetch_filter_resources.yaml

Co-authored-by: paulzierep <[email protected]>

---------

Co-authored-by: paulzierep <[email protected]>

Thank you @bebatut - lets see it in production !

.github/workflows/fetch_filter_resources.yaml

@@ -0,0 +1,121 @@
+name: Weekly resource fetching and community filtering
+
+on:
+  workflow_dispatch:
+  schedule:
+    #Every Sunday at 8:00 am
+    - cron: "0 8 * * 0" 
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "tools"
+  cancel-in-progress: false
+
+jobs:
+  fetch-servers:
+    runs-on: ubuntu-20.04
+    name: Merge tools, fetch tutorials and filter the resources for communities
+      steps:
+      - name: Checkout main
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install requirement
+        run: |
+          python -m pip install -r requirements.txt
+          sudo apt-get install jq
+      - name: Fetch list of all available servers
+          run: | 
+            python sources/bin/get_public_galaxy_servers.py -o sources/data/available_public_servers.csv
+      - name: Archive available servers
+        uses: actions/upload-artifact@v4
+        with:
+          name: available-servers
+          path: sources/data/available_public_servers.csv
+  fetch-tools-stepwise:
+    runs-on: ubuntu-20.04
+    name: Fetch tool stepwise
+    strategy:
+      #max-parallel: 1 #need to run one after another, since otherwise there is a chance, that mulitple jobs want to push to the results branch at the same time (which fails due to merge)
+      matrix:
+        python-version: [3.11]
+        subset:
+         - repositories01.list
+         - repositories02.list
+         - repositories03.list
+         - repositories04.list
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install requirement
+        run: python -m pip install -r requirements.txt
+      - name: Download a single artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: available-servers
+          path: sources/data/
+      - name: Fetch all tool stepwise
+        run: | 
+          bash sources/bin/extract_all_tools.sh "${{ matrix.subset }}"
+        env: 
+          GITHUB_API_KEY: ${{ secrets.GH_API_TOKEN }}
+      - name: Archive tool sublists production artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: tools-${{ matrix.subset }}
+          path: communities/all/resources/repositories${{ matrix.subset }}.list_tools.tsv
+  merge-fetch-filter:
+    runs-on: ubuntu-20.04
+    name: Merge tools, fetch tutorials and filter the resources for communities
+      steps:
+      - name: Checkout main
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install requirement
+        run: |
+          python -m pip install -r requirements.txt
+          sudo apt-get install jq
+      - name: Download All Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: tools
+          pattern: tools-*
+          merge-multiple: true
+          path: communities/all/resources/
+      - name: Merge all tools
+        run: | #merge files with only one header -> https://stackoverflow.com/questions/16890582/unixmerge-multiple-csv-files-with-same-header-by-keeping-the-header-of-the-firs; map(.[]) -> https://stackoverflow.com/questions/42011086/merge-arrays-of-json (get flat array, one tool per entry)
+          awk 'FNR==1 && NR!=1{next;}{print}' communities/all/resources/repositories*.list_tools.tsv > communities/all/resources/tools.tsv
+          jq -s 'map(.[])' communities/all/resources/repositories*.list_tools.json > communities/all/resources/all_tools.json 
+      - name: Generate wordcloud and interactive table
+        run: |
+          bash sources/bin/format_tools.sh
+      - name: Fetch all tutorials
+        run: | 
+          bash bin/extract_all_tutorials.sh
+        env: 
+          PLAUSIBLE_API_KEY: ${{ secrets.PLAUSIBLE_API_TOKEN }}
+      - name: Filter tutorials for communities
+        run: | 
+          bash bin/get_community_tutorials.sh
+      - name: Update tool to keep and exclude for communities
+        run: | 
+          bash bin/update_tools_to_keep_exclude.sh
+      - name: Filter tools for communities
+        run: | 
+          bash bin/get_community_tools.sh
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v4
+        with:
+          commit-message: Update resources
+          title: Automatic resources update
+          body: Automatic resource update done via GitHub Action once a week
+          base: main
+          branch: resource-update
+          delete-branch: true