restore test setup and add more metadata fields to post (#5)

* restore test setup and add more metadata fields to post
Signed-off-by: vsoch <[email protected]>

.github/workflows/post-jobs-slack.yaml

@@ -1,9 +1,5 @@
 on:
-  push:
-    paths:
-      - 'example/jobs.yaml'
-    branches:
-      - main
+  pull_request: []
 
 jobs:
   slack-poster:
@@ -21,8 +17,32 @@ jobs:
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         with:
           filename: "example/jobs.yaml"
-          key: "url"
+          previous_filename: "example/jobs-previous.yaml"
+          keys: "url"
+          unique: "url"
+
+          deploy: false
+          test: true
+
+
+      - id: multifield_updater
+        name: Job Updater
+        uses: ./
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+        with:
+          filename: "example/jobs.yaml"
+          previous_filename: "example/jobs-previous.yaml"
+          keys: "name,location,url"
+          unique: "url"
+
+          deploy: false
+          test: true
 
       - run: echo ${{ steps.updater.outputs.fields }}
-        name: Show New Jobs
+        name: Show Keys Used
+        shell: bash
+
+      - run: echo ${{ steps.updater.outputs.matrix }}
+        name: Show Matrix
         shell: bash

README.md

@@ -44,7 +44,7 @@ and put it in a safe place. We will want to keep this URL as a secret in our eve
 
 ## 2. Usage
 
-Add an GitHub workflow file in `.github/workflows` to specify the following. Note that
+Add a GitHub workflow file in `.github/workflows` to specify the following. Note that
 the workflow below will do the check and update on any push to main (e.g., a merged pull request).
 
 ### Deploy to Slack
@@ -73,13 +73,15 @@ jobs:
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         with:
           filename: "_data/jobs.yml"
-          key: "url"
+          keys: "url,name"
+          unique: "url"
 
       - run: echo ${{ steps.updater.outputs.fields }}
         name: Show New Jobs
         shell: bash
 ```
 
+In the above, we will include the url and name fields, and use the url field to determine uniqueness (default).
 By default, given that you have the slack webhook in the environment, deployment will
 happen because deploy is true. If you just want to test, then do:
 
@@ -92,7 +94,7 @@ happen because deploy is true. If you just want to test, then do:
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         with:
           filename: "_data/jobs.yml"
-          key: "url"
+          keys: "url"
           deploy: false
 ```
 
@@ -108,7 +110,7 @@ aren't necessarily new) then add test:
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         with:
           filename: "_data/jobs.yml"
-          key: "url"
+          keys: "url"
           test: true
 ```
 
@@ -128,7 +130,7 @@ secrets.
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         with:
           filename: "_data/jobs.yml"
-          key: "url"
+          keys: "url,name"
 
           deploy: true
           test: false

action.yml

@@ -4,9 +4,16 @@ inputs:
   filename:
     description: the filename for the jobs
     required: true    
-  key:
-    description: The key of the list to post (defaults to url)
+  previous_filename:
+    description: the previous filename (usually only for manual tesing)
     required: false
+  keys:
+    description: Comma separated list of keys to post (defaults to url)
+    required: false
+    default: url
+  unique:
+    description: Field to use to determine uniqueness
+    required: true
     default: url
   test:
     description: Test the updater (ensure there are jobs)
@@ -39,7 +46,7 @@ inputs:
 
 outputs:
   fields:
-    description: New fields parsed
+    description: Fields (keys) parsed
     value: ${{ steps.jobs-updater.outputs.fields }}    
   matrix:
     description: Matrix (list of lists) with value (index 1), icon (index 2) and full message (index 3)
@@ -60,8 +67,10 @@ runs:
     - name: Run action entrypoint
       id: jobs-updater
       env:
+        INPUT_PREVIOUS_FILENAME: ${{ inputs.previous_filename }}
         INPUT_FILENAME: ${{ inputs.filename }}
-        INPUT_KEY: ${{ inputs.key }}
+        INPUT_KEYS: ${{ inputs.keys }}
+        INPUT_UNIQUE: ${{ inputs.unique }}
         CURRENT_SHA: ${{ github.sha }}
         ACTION_DIR: ${{ github.action_path }}
         INPUT_REPO: ${{ github.repository }}

entrypoint.sh

@@ -29,6 +29,11 @@ if [[ ! -f "${JOBFILE}" ]]; then
     exit 1
 fi
 
+# If we are verbatim given a previous filename, use it
+if [ ! -z ${INPUT_PREVIOUS_FILENAME} ]; then 
+    JOBFILE="${INPUT_PREVIOUS_FILENAME}"
+fi
+
 # Required to have slack webhook in environment
 if [ -z ${SLACK_WEBHOOK+x} ]; then 
     printf "Warning, SLACK_WEBHOOK not found, will not deploy to slack.\n"
@@ -53,9 +58,9 @@ if [ ! -z ${TWITTER_API_KEY+x} ] && [ ! -z ${TWITTER_API_SECRET+x} ] && [ ! -z $
 fi
 
 if [[ "${DEPLOY}" == "true" ]]; then
-  COMMAND="python ${ACTION_DIR}/find-updates.py update --key ${INPUT_KEY} --original ${JOBFILE} --updated ${INPUT_FILENAME} --deploy"
+  COMMAND="python ${ACTION_DIR}/find-updates.py update --keys ${INPUT_KEYS} --unique ${INPUT_UNIQUE} --original ${JOBFILE} --updated ${INPUT_FILENAME} --deploy"
 else
-  COMMAND="python ${ACTION_DIR}/find-updates.py update --key ${INPUT_KEY} --original ${JOBFILE} --updated ${INPUT_FILENAME}"
+  COMMAND="python ${ACTION_DIR}/find-updates.py update --keys ${INPUT_KEYS} --unique ${INPUT_UNIQUE} --original ${JOBFILE} --updated ${INPUT_FILENAME}"
 fi
 
 if [[ "${INPUT_TEST}" == "true" ]]; then

example/jobs-previous.yaml

@@ -0,0 +1,17 @@
+- expires: 2021-09-30
+  location: Scoot Science - remote in the US or Canada
+  name: Data Engineer & Full Stack Software Engineer
+  url: https://www.scootscience.com/careers/
+- expires: 2020-12-31
+  location: National Center for Supercomputing Applications / University of Illinois,
+    Urbana, IL
+  name: Assistant Research Programmer/Research Programmer/Senior Research Programmer
+  url: https://jobs.illinois.edu/academic-job-board/job-details?jobID=130370&job=research-programmer-national-center-for-supercomputing-applications-130370
+- expires: 2020-12-31
+  location: National Center for Supercomputing Applications / University of Illinois, Urbana, IL
+  name: Assistant Research Programmer/Research Programmer/Senior Research Programmer
+  url: https://jobs.hr.wisc.edu/en-us/job/510571/researcher
+- expires: 2020-12-31
+  location: National Center for Supercomputing Applications / University of Illinois, Urbana, IL
+  name: Assistant Research Programmer/Research Programmer/Senior Research Programmer
+  url: https://jobs.ornl.gov/job/Oak-Ridge-Full-Stack-Software-Engineer-TN-37830/793411000/

find-updates.py

@@ -3,10 +3,9 @@
 # This script does the following.
 # 1. Reads in a current and changed yaml file
 # 2. Finds changes between the two
-# 3. Post them to slack
+# 3. Post them to slack and/or Twitter
 
 import argparse
-from datetime import datetime
 import requests
 import random
 import json
@@ -80,9 +79,16 @@ def get_parser():
     )
 
     update.add_argument(
-        "--key",
-        dest="key",
-        help="The key to post to slack",
+        "--keys",
+        dest="keys",
+        help="The keys (comma separated list) to post to slack or Twitter",
+    )
+
+    update.add_argument(
+        "--unique",
+        dest="unique",
+        help="The key to use to determine uniqueness (defaults to url)",
+        default="url",
     )
 
     return parser
@@ -109,6 +115,20 @@ def get_twitter_client():
     )
 
 
+def prepare_post(entry, keys):
+    """Prepare the slack or tweet. There should be a descriptor for
+    all fields except for url.
+    """
+    post = ""
+    for key in keys:
+        if key in entry:
+            if key == "url":
+                post = post + entry[key] + "\n"
+            else:
+                post = post + key.capitalize() + ": " + entry[key] + "\n"
+    return post
+
+
 def main():
     parser = get_parser()
 
@@ -133,21 +153,25 @@ def help(return_code=0):
     original = read_yaml(args.original)
     updated = read_yaml(args.updated)
 
+    # Parse keys into list
+    keys = [x for x in args.keys.split(",") if x]
+
     # Find new posts in updated
     previous = set()
-    new = set()
-    entries = set()
     for item in original:
-        if args.key in item:
-            previous.add(item[args.key])
+        if args.unique in item:
+            previous.add(item[args.unique])
 
+    # Create a lookup by the unique id
+    new = []
+    entries = []
     for item in updated:
-        if args.key in item and item[args.key] not in previous:
-            new.add(item[args.key])
+        if args.unique in item and item[args.unique] not in previous:
+            new.append(item)
 
         # Also keep list of all for test
-        elif args.key in item and item[args.key]:
-            entries.add(item[args.key])
+        elif args.unique in item and item[args.unique]:
+            entries.append(item)
 
     # Test uses all entries
     if args.test:
@@ -184,20 +208,37 @@ def help(return_code=0):
         "🔥️",
         "💻️",
     ]
-    for name in new:
+
+    for entry in new:
+
+        # Prepare the post
+        post = prepare_post(entry, keys)
+
         choice = random.choice(icons)
-        message = "New Job! %s: %s" % (choice, name)
+        message = "New Job! %s\n%s" % (choice, post)
+        print(message)
+
+        # Convert dates, etc. back to string
+        filtered = {}
+        for k, v in entry.items():
+            try:
+                filtered[k] = json.dumps(v)
+            except:
+                continue
 
         # Add the job name to the matrix
         # IMPORTANT: emojis in output mess up the action
-        matrix.append([name])
+        matrix.append(filtered)
         data = {"text": message, "unfurl_links": True}
-        print(data)
 
         # If we are instructed to deploy to twitter and have a client
         if args.deploy_twitter and client:
-            message = "New #RSEng Job! %s: %s" % (choice, name)
-            client.create_tweet(text=message)
+            message = "New #RSEng Job! %s\n%s" % (choice, post)
+            print(message)
+            try:
+                client.create_tweet(text=message)
+            except Exception as e:
+                print("Issue posting tweet: %s, and length is %s" % (e, len(message)))
 
         # Don't continue if testing
         if not args.deploy or args.test:
@@ -211,11 +252,11 @@ def help(return_code=0):
                 % (response.reason, response.status_code)
             )
 
-    print("::set-output name=fields::%s" % list(new))
+    print("::set-output name=fields::%s" % json.dumps(keys))
     print("::set-output name=matrix::%s" % json.dumps(matrix))
     print("::set-output name=empty_matrix::false")
     print("matrix: %s" % json.dumps(matrix))
-    print("group: %s" % list(new))
+    print("group: %s" % new)
 
 
 if __name__ == "__main__":