init

init

init

datapipeline/.github/workflows/cd-pipeline.yaml → .github/workflows/cd-pipeline.yaml

@@ -1,58 +1,58 @@
-name: Deploy
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  # push_to_registry:
-  #   env:
-  #     - NCP_ACCESS_KEY: EFDDD78569302196AECF
-  #     - NCP_SECRET_KEY: 81BF2828B2206EE1EA19875BEA3247460CAA3E2C
-  #     - NCP_CONTAINER_REGISTRY: datapipeline-petfins.ncr.fin-ntruss.com
-
-
-  #   name: Push to ncp container registry
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - name: Checkout
-  #       uses: actions/checkout@v3
-  #     - name: Set up Docker Buildx
-  #       uses: docker/setup-buildx-action@v2
-  #     - name: Login to NCP Container Registry
-  #       uses: docker/login-action@v2
-  #       with:
-  #         registry: datapipeline-petfins.ncr.fin-ntruss.com
-  #         username: EFDDD78569302196AECF
-  #         password: 81BF2828B2206EE1EA19875BEA3247460CAA3E2C
-  #     - name: build and push
-  #       uses: docker/build-push-action@v3
-  #       with:
-  #         context: .
-  #         file: ./Dockerfile
-  #         push: true
-  #         tags:  datapipeline-petfins.ncr.fin-ntruss.com/petfins-datapipeline:latest
-          # cache-from: type=registry,ref=${{ NCP_CONTAINER_REGISTRY }}/petfins-datapipeline:latest
-          # cache-to: type=inline
-
-
-  pull_from_registry:
-    name: Connect server ssh and pull from container registry
-    # needs: push_to_registry
-    runs-on: ubuntu-latest
-    steps:
-    # docker login
-      - name: connect ssh
-        uses: appleboy/ssh-action@master
-        with:
-          host: ${{ secrets.DEV_HOST }}
-          username: ${{ secrets.DEV_USERNAME }}
-          password: ${{ secrets.DEV_PASSWORD }}
-          port: ${{ secrets.DEV_PORT }}
-          script: |
-            docker pull ${{ secrets.NCP_CONTAINER_REGISTRY }}/petfins-datapipeline:latest
-            docker stop $(docker ps -a -q)
-            docker rm $(docker ps -a -q)
-            docker run --rm -it  ${{ secrets.NCP_CONTAINER_REGISTRY }}/petfins-datapipeline:latest
+name: Deploy
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  # push_to_registry:
+  #   env:
+  #     - NCP_ACCESS_KEY: EFDDD78569302196AECF
+  #     - NCP_SECRET_KEY: 81BF2828B2206EE1EA19875BEA3247460CAA3E2C
+  #     - NCP_CONTAINER_REGISTRY: datapipeline-petfins.ncr.fin-ntruss.com
+
+
+  #   name: Push to ncp container registry
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v3
+  #     - name: Set up Docker Buildx
+  #       uses: docker/setup-buildx-action@v2
+  #     - name: Login to NCP Container Registry
+  #       uses: docker/login-action@v2
+  #       with:
+  #         registry: datapipeline-petfins.ncr.fin-ntruss.com
+  #         username: EFDDD78569302196AECF
+  #         password: 81BF2828B2206EE1EA19875BEA3247460CAA3E2C
+  #     - name: build and push
+  #       uses: docker/build-push-action@v3
+  #       with:
+  #         context: .
+  #         file: ./Dockerfile
+  #         push: true
+  #         tags:  datapipeline-petfins.ncr.fin-ntruss.com/petfins-datapipeline:latest
+          # cache-from: type=registry,ref=${{ NCP_CONTAINER_REGISTRY }}/petfins-datapipeline:latest
+          # cache-to: type=inline
+
+
+  pull_from_registry:
+    name: Connect server ssh and pull from container registry
+    # needs: push_to_registry
+    runs-on: ubuntu-latest
+    steps:
+    # docker login
+      - name: connect ssh
+        uses: appleboy/ssh-action@master
+        with:
+          host: ${{ secrets.DEV_HOST }}
+          username: ${{ secrets.DEV_USERNAME }}
+          password: ${{ secrets.DEV_PASSWORD }}
+          port: ${{ secrets.DEV_PORT }}
+          script: |
+            docker pull ${{ secrets.NCP_CONTAINER_REGISTRY }}/petfins-datapipeline:latest
+            docker stop $(docker ps -a -q)
+            docker rm $(docker ps -a -q)
+            docker run --rm -it  ${{ secrets.NCP_CONTAINER_REGISTRY }}/petfins-datapipeline:latest
             docker image prune -f

datapipeline/.github/workflows/ci-pipeline.yam → .github/workflows/ci-pipeline.yam

@@ -1,93 +1,93 @@
-# name: CI Pipeline
-
-# on: 
-#   push:
-#     branches:
-#       - main
-#   pull_request:
-#     branches:
-#       - main
-
-# jobs:
-#   lint:
-#     runs-on: ubuntu-latest
-#     steps:
-#     - uses: actions/checkout@v4
-#     - name: Set up Python
-#       uses: actions/setup-python@v2
-#       with:
-#         python-version: 3.10
-#     #----------------------------------------------
-#     #  -----  install & configure poetry  -----
-#     #----------------------------------------------
-#     - name: Install Poetry
-#       uses: snok/install-poetry@v1
-#       with:
-#         virtualenvs-create: true
-#         virtualenvs-in-project: true
-#         installer-parallel: true
-
-#     #----------------------------------------------
-#     #       load cached venv if cache exists
-#     #----------------------------------------------
-#     # - name: Load cached venv
-#     #   id: cached-poetry-dependencies
-#     #   uses: actions/cache@v3
-#     #   with:
-#     #     path: .venv
-#     #     key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
-#     #----------------------------------------------
-#     # install dependencies if cache does not exist
-#     #----------------------------------------------
-#     - name: Install dependencies
-#       # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-#       run: poetry install --no-interaction --no-root
-#     - name: Lint with pylint
-#       run: |
-#         source .venv/bin/activate
-#         poetry run pylint --disable=R,C **/*.py
-#     - name: Format with black
-#       run: |
-#         poetry run black .
-#     - name: Cache dependencies
-#       uses: actions/cache@v2
-#       with:
-#         path: .venv
-#         key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
-
-#   test:
-#     runs-on: ubuntu-latest
-#     needs: lint
-#     steps:
-#     - uses: actions/checkout@v4
-#     - name: Load cached venv
-#       id: cached-poetry-dependencies
-#       uses: actions/cache@v3
-#       with:
-#         path: .venv
-#         key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
-#     - name: Install Poetry
-#       uses: snok/install-poetry@v1
-#       with:
-#         virtualenvs-create: true
-#         virtualenvs-in-project: true
-#         installer-parallel: true
-
-#     - name: Use Cache from Lint Job
-#       run: |
-#         source .venv/bin/activate
-#         poetry install
-
-#     # - uses: actions/checkout@v2
-#     # - name: Set up Python
-#     #   uses: actions/setup-python@v2
-#     #   with:
-#     #     python-version: 3.10
-#     # - name: Install dependencies
-#     #   run: |
-#     #     python -m pip install --upgrade pip
-#     #     pip install pytest
-#     - name: Test with pytest
-#       run: |
-#         poetry run pytest .
+# name: CI Pipeline
+
+# on: 
+#   push:
+#     branches:
+#       - main
+#   pull_request:
+#     branches:
+#       - main
+
+# jobs:
+#   lint:
+#     runs-on: ubuntu-latest
+#     steps:
+#     - uses: actions/checkout@v4
+#     - name: Set up Python
+#       uses: actions/setup-python@v2
+#       with:
+#         python-version: 3.10
+#     #----------------------------------------------
+#     #  -----  install & configure poetry  -----
+#     #----------------------------------------------
+#     - name: Install Poetry
+#       uses: snok/install-poetry@v1
+#       with:
+#         virtualenvs-create: true
+#         virtualenvs-in-project: true
+#         installer-parallel: true
+
+#     #----------------------------------------------
+#     #       load cached venv if cache exists
+#     #----------------------------------------------
+#     # - name: Load cached venv
+#     #   id: cached-poetry-dependencies
+#     #   uses: actions/cache@v3
+#     #   with:
+#     #     path: .venv
+#     #     key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+#     #----------------------------------------------
+#     # install dependencies if cache does not exist
+#     #----------------------------------------------
+#     - name: Install dependencies
+#       # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+#       run: poetry install --no-interaction --no-root
+#     - name: Lint with pylint
+#       run: |
+#         source .venv/bin/activate
+#         poetry run pylint --disable=R,C **/*.py
+#     - name: Format with black
+#       run: |
+#         poetry run black .
+#     - name: Cache dependencies
+#       uses: actions/cache@v2
+#       with:
+#         path: .venv
+#         key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+#   test:
+#     runs-on: ubuntu-latest
+#     needs: lint
+#     steps:
+#     - uses: actions/checkout@v4
+#     - name: Load cached venv
+#       id: cached-poetry-dependencies
+#       uses: actions/cache@v3
+#       with:
+#         path: .venv
+#         key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+#     - name: Install Poetry
+#       uses: snok/install-poetry@v1
+#       with:
+#         virtualenvs-create: true
+#         virtualenvs-in-project: true
+#         installer-parallel: true
+
+#     - name: Use Cache from Lint Job
+#       run: |
+#         source .venv/bin/activate
+#         poetry install
+
+#     # - uses: actions/checkout@v2
+#     # - name: Set up Python
+#     #   uses: actions/setup-python@v2
+#     #   with:
+#     #     python-version: 3.10
+#     # - name: Install dependencies
+#     #   run: |
+#     #     python -m pip install --upgrade pip
+#     #     pip install pytest
+#     - name: Test with pytest
+#       run: |
+#         poetry run pytest .
 

.gitignore

@@ -6,4 +6,5 @@
 **/__pycache__
 outputs/
 mlruns/
+mlartifacts/
 resources/