diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml new file mode 100644 index 0000000..0e64922 --- /dev/null +++ b/.github/workflows/preview.yml @@ -0,0 +1,49 @@ +name: Create Preview + +on: + pull_request_target: + branches: [main] + types: [opened, synchronize, reopened] + +jobs: + preview: + name: Preview + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Get changed files in posts folder + id: get_changed_files + uses: tj-actions/changed-files@v44 + with: + files: posts/** + json: "true" + + - name: get published files cache + if: steps.get_changed_files.outputs.any_changed == 'true' + run: | + git fetch origin processed_files:processed_files + git checkout processed_files -- processed_files.json + + - name: Set up Python + if: steps.get_changed_files.outputs.any_changed == 'true' + uses: actions/setup-python@v3 + with: + python-version: 3.9 + + - name: Install dependencies + if: steps.get_changed_files.outputs.any_changed == 'true' + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + + - name: Run script to create preview + if: steps.get_changed_files.outputs.any_changed == 'true' + env: + CHANGED_FILES: ${{ steps.get_changed_files.outputs.all_changed_files }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.number }} + run: python -u github_run.py --preview diff --git a/.github/workflows/publish_content.yml b/.github/workflows/publish_content.yml new file mode 100644 index 0000000..f5e8eee --- /dev/null +++ b/.github/workflows/publish_content.yml @@ -0,0 +1,65 @@ +name: Publish Content + +on: + pull_request_target: + branches: [main] + types: [closed] + +jobs: + publish: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Get changed files in posts folder + id: get_changed_files + uses: tj-actions/changed-files@v44 + with: + files: posts/** + json: "true" + + - name: get published files cache + if: steps.get_changed_files.outputs.any_changed == 'true' + run: | + git fetch origin processed_files:processed_files + git checkout processed_files -- processed_files.json + + - name: Set up Python + if: steps.get_changed_files.outputs.any_changed == 'true' + uses: actions/setup-python@v3 + with: + python-version: 3.9 + + - name: Install dependencies + if: steps.get_changed_files.outputs.any_changed == 'true' + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + + - name: Run script to publish contents + if: steps.get_changed_files.outputs.any_changed == 'true' + env: + MASTODON_ACCESS_TOKEN: ${{ secrets.MASTODON_ACCESS_TOKEN }} + BLUESKY_USERNAME: ${{ secrets.BLUESKY_USERNAME }} + BLUESKY_PASSWORD: ${{ secrets.BLUESKY_PASSWORD }} + MATRIX_ACCESS_TOKEN: ${{ secrets.MATRIX_ACCESS_TOKEN }} + MATRIX_ROOM_ID: ${{ secrets.MATRIX_ROOM_ID }} + MATRIX_USER_ID: ${{ secrets.MATRIX_USER_ID }} + SLACK_ACCESS_TOKEN: ${{ secrets.SLACK_ACCESS_TOKEN }} + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + CHANGED_FILES: ${{ steps.get_changed_files.outputs.all_changed_files }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.number }} + run: python -u github_run.py --json-out processed_files.json + + - name: Commit changes + if: steps.get_changed_files.outputs.any_changed == 'true' + uses: stefanzweifel/git-auto-commit-action@v5 + with: + file_pattern: "processed_files.json" + branch: "processed_files" diff --git a/.github/workflows/toot-together.yml b/.github/workflows/toot-together.yml deleted file mode 100644 index cfc265b..0000000 --- a/.github/workflows/toot-together.yml +++ /dev/null @@ -1,25 +0,0 @@ -on: [push, pull_request] -name: Toot, together! -jobs: - preview: - name: Preview - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - steps: - - uses: joschi/toot-together@v1.x - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - toot: - name: Toot - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - steps: - - name: checkout master - uses: actions/checkout@v4 - - name: Toot - uses: joschi/toot-together@v1.x - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # URL to the instance hosting your Mastodon account - MASTODON_URL: "https://bawü.social" - MASTODON_ACCESS_TOKEN: ${{ secrets.MASTODON_ACCESS_TOKEN }} diff --git a/.gitignore b/.gitignore index b6e4761..6c9cc34 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ dist/ downloads/ eggs/ .eggs/ -lib/ +# lib/ lib64/ parts/ sdist/ diff --git a/.schema.yaml b/.schema.yaml new file mode 100644 index 0000000..3e4ae98 --- /dev/null +++ b/.schema.yaml @@ -0,0 +1,32 @@ +type: object +properties: + media: + type: array + items: + type: string + uniqueItems: true + images: + type: array + items: + type: object + properties: + url: + type: string + format: uri + alt_text: + type: string + required: [] + mentions: + type: object + additionalProperties: + type: array + items: + type: string + required: [] + hashtags: + type: object + additionalProperties: + type: array + items: + type: string + required: [] diff --git a/README.md b/README.md index 1f114f0..6bc150e 100644 Binary files a/README.md and b/README.md differ diff --git a/github_run.py b/github_run.py new file mode 100644 index 0000000..d58a628 --- /dev/null +++ b/github_run.py @@ -0,0 +1,97 @@ +import argparse +import fnmatch +import os +import sys + +import requests + +from lib.galaxy_social import galaxy_social + + +class github_run: + def __init__(self): + self.github_token = os.getenv("GITHUB_TOKEN") + self.repo = os.getenv("GITHUB_REPOSITORY") + self.pr_number = os.getenv("PR_NUMBER") + + def comment(self, comment_text): + print(comment_text) + if ( + not comment_text + or not self.github_token + or not self.repo + or not self.pr_number + ): + return + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self.github_token}", + "X-GitHub-Api-Version": "2022-11-28", + } + url = ( + f"https://api.github.com/repos/{self.repo}/issues/{self.pr_number}/comments" + ) + data = {"body": str(comment_text)} + response = requests.post(url, headers=headers, json=data) + if response.status_code == 201: + return True + else: + raise Exception( + f"Failed to create github comment!, {response.json().get('message')}" + ) + + def get_files(self): + url = f"https://api.github.com/repos/{self.repo}/pulls/{self.pr_number}/files" + response = requests.get(url) + if response.status_code == 200: + changed_files = response.json() + for file in changed_files: + raw_url = file["raw_url"] + if raw_url.endswith(".md"): + response = requests.get(raw_url) + if response.status_code == 200: + changed_file_path = file["filename"] + os.makedirs( + os.path.dirname(changed_file_path), + exist_ok=True, + ) + with open(changed_file_path, "w") as f: + f.write(response.text) + + changed_files = os.environ.get("CHANGED_FILES") + files_to_process = [] + if changed_files: + for file_path in eval(changed_files.replace("\\", "")): + if file_path.endswith(".md"): + files_to_process.append(file_path) + else: + for root, _, files in os.walk("posts"): + for filename in fnmatch.filter(files, "*.md"): + file_path = os.path.join(root, filename) + files_to_process.append(file_path) + + return files_to_process + + +if __name__ == "__main__": + github = github_run() + files_to_process = github.get_files() + if not files_to_process: + github.comment("No files to process.") + sys.exit() + + parser = argparse.ArgumentParser(description="Galaxy Social.") + parser.add_argument("--preview", action="store_true", help="Preview the post") + parser.add_argument( + "--json-out", + help="Output json file for processed files", + default="processed_files.json", + ) + args = parser.parse_args() + + gs = galaxy_social(args.preview, args.json_out) + try: + message = gs.process_files(files_to_process) + except Exception as e: + message = e + github.comment(message) diff --git a/lib/galaxy_social.py b/lib/galaxy_social.py new file mode 100644 index 0000000..9108442 --- /dev/null +++ b/lib/galaxy_social.py @@ -0,0 +1,191 @@ +import json +import os +import sys +from argparse import ArgumentParser +from fnmatch import filter +from importlib import import_module + +from bs4 import BeautifulSoup +from jsonschema import validate +from markdown import markdown +from yaml import safe_load as yaml + + +class galaxy_social: + def __init__(self, preview: bool, json_out: str): + self.preview = preview + self.json_out = json_out + plugins_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "plugins.yml" + ) + with open(plugins_path, "r") as file: + self.plugins_config = yaml(file) + + self.plugins = {} + for plugin in self.plugins_config["plugins"]: + if preview and plugin["name"].lower() != "markdown": + continue + + if plugin["enabled"]: + module_name, class_name = plugin["class"].rsplit(".", 1) + try: + module_path = f"{'lib.' if not os.path.dirname(os.path.abspath(sys.argv[0])).endswith('lib') else ''}plugins.{module_name}" + module = import_module(module_path) + plugin_class = getattr(module, class_name) + except Exception as e: + raise Exception( + f"Error with plugin {module_name}.{class_name}.\n{e}" + ) + + config = {} + if plugin.get("config"): + for key, value in plugin["config"].items(): + if isinstance(value, str) and value.startswith("$"): + try: + config[key] = os.environ[value[1:]] + except KeyError: + raise Exception( + f"Missing environment variable {value[1:]}." + ) + else: + config[key] = value + else: + raise Exception(f"Missing config for {module_name}.{class_name}.") + + try: + self.plugins[plugin["name"].lower()] = plugin_class(**config) + except Exception as e: + raise Exception( + f"Invalid config for {module_name}.{class_name}.\nChange configs in plugins.yml.\n{e}" + ) + + def parse_markdown_file(self, file_path): + with open(file_path, "r") as file: + content = file.read() + try: + _, metadata, text = content.split("---\n", 2) + metadata = yaml(metadata) + schema_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", ".schema.yaml" + ) + with open(schema_path, "r") as f: + schema = yaml(f) + validate(instance=metadata, schema=schema) + except Exception as e: + raise Exception(f"Invalid metadata in {file_path}.\n{e}") + + metadata["media"] = [media.lower() for media in metadata["media"]] + + for media in metadata["media"]: + if not any( + item["name"].lower() == media for item in self.plugins_config["plugins"] + ): + raise Exception(f"Invalid media {media}.") + + metadata["mentions"] = ( + {key.lower(): value for key, value in metadata["mentions"].items()} + if metadata.get("mentions") + else {} + ) + metadata["hashtags"] = ( + {key.lower(): value for key, value in metadata["hashtags"].items()} + if metadata.get("hashtags") + else {} + ) + markdown_content = markdown(text.strip()) + plain_content = BeautifulSoup(markdown_content, "html.parser").get_text( + separator="\n" + ) + return plain_content, metadata + + def process_markdown_file(self, file_path, processed_files): + content, metadata = self.parse_markdown_file(file_path) + if self.preview: + try: + _, _, message = self.plugins["markdown"].create_post( + content=content, + mentions=[], + hashtags=[], + images=metadata.get("images", []), + media=metadata["media"], + preview=True, + file_path=file_path, + ) + return processed_files, message + except Exception as e: + raise Exception(f"Failed to create preview for {file_path}.\n{e}") + stats = {} + url = {} + if file_path in processed_files: + stats = processed_files[file_path] + for media in metadata["media"]: + if file_path in processed_files and media in processed_files[file_path]: + continue + mentions = metadata.get("mentions", {}).get(media, []) + hashtags = metadata.get("hashtags", {}).get(media, []) + images = metadata.get("images", []) + stats[media], url[media] = self.plugins[media].create_post( + content, mentions, hashtags, images, file_path=file_path + ) + url_text = "\n".join( + [f"[{media}]({link})" for media, link in url.items() if link] + ) + message = f"Posted to:\n\n{url_text}" if url_text else "No posts created." + + processed_files[file_path] = stats + print(f"Processed {file_path}: {stats}") + return processed_files, message + + def process_files(self, files_to_process): + processed_files = {} + messages = "---\n" + processed_files_path = self.json_out + if os.path.exists(processed_files_path): + with open(processed_files_path, "r") as file: + processed_files = json.load(file) + for file_path in files_to_process: + processed_files, message = self.process_markdown_file( + file_path, processed_files + ) + messages += f"{message}\n\n---\n" + if not self.preview: + with open(processed_files_path, "w") as file: + json.dump(processed_files, file) + return messages + + +if __name__ == "__main__": + parser = ArgumentParser(description="Galaxy Social.") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--files", nargs="+", help="List of files to process") + group.add_argument("--folder", help="Folder containing files to process") + parser.add_argument("--preview", action="store_true", help="Preview the post") + parser.add_argument( + "--json-out", + help="Output json file for processed files", + default="processed_files.json", + ) + args = parser.parse_args() + + if args.files: + files_to_process = args.files + files_not_exist = [ + file for file in files_to_process if not os.path.isfile(file) + ] + if files_not_exist: + raise Exception(f"{', '.join(files_not_exist)} -> not exist.") + elif args.folder: + if not os.path.isdir(args.folder): + raise Exception(f"{args.folder} -> not exist.") + files_to_process = [ + os.path.join(root, filename) + for root, _, files in os.walk(args.folder) + for filename in filter(files, "*.md") + ] + if not files_to_process: + print("No files to process.") + sys.exit() + print(f"Processing {len(files_to_process)} file(s): {files_to_process}\n") + gs = galaxy_social(args.preview, args.json_out) + message = gs.process_files(files_to_process) + print(message) diff --git a/lib/plugins/bluesky.py b/lib/plugins/bluesky.py new file mode 100644 index 0000000..13680ea --- /dev/null +++ b/lib/plugins/bluesky.py @@ -0,0 +1,204 @@ +import re +import textwrap +from typing import Dict, List, Optional, Tuple, cast + +import atproto +import requests +from bs4 import BeautifulSoup + + +class bluesky_client: + def __init__(self, **kwargs): + self.base_url = kwargs.get("base_url", "https://bsky.social") + self.blueskysocial = atproto.Client(self.base_url) + self.blueskysocial.login( + login=kwargs.get("username"), password=kwargs.get("password") + ) + self.max_content_length = kwargs.get("max_content_length", 300) + + def parse_mentions(self, text: str) -> List[Dict]: + spans = [] + mention_regex = rb"[$|\W](@([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)" + text_bytes = text.encode("UTF-8") + for m in re.finditer(mention_regex, text_bytes): + spans.append( + { + "start": m.start(1), + "end": m.end(1), + "handle": m.group(1)[1:].decode("UTF-8"), + } + ) + return spans + + def parse_urls(self, text: str) -> List[Dict]: + spans = [] + url_regex = rb"[$|\W](https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*[-a-zA-Z0-9@%_\+~#//=])?)" + text_bytes = text.encode("UTF-8") + for m in re.finditer(url_regex, text_bytes): + spans.append( + { + "start": m.start(1), + "end": m.end(1), + "url": m.group(1).decode("UTF-8"), + } + ) + return spans + + def parse_hashtags(self, text: str) -> List[Dict]: + spans = [] + hashtag_regex = rb"[$|\W]#(\w+)" + text_bytes = text.encode("UTF-8") + for m in re.finditer(hashtag_regex, text_bytes): + spans.append( + { + "start": m.start(1), + "end": m.end(1), + "tag": m.group(1).decode("UTF-8"), + } + ) + return spans + + def parse_facets(self, text: str) -> Tuple[List[Dict], Optional[str]]: + facets = [] + for h in self.parse_hashtags(text): + facets.append( + { + "index": { + "byteStart": h["start"], + "byteEnd": h["end"], + }, + "features": [ + {"$type": "app.bsky.richtext.facet#tag", "tag": h["tag"]} + ], + } + ) + for m in self.parse_mentions(text): + resp = requests.get( + "https://bsky.social/xrpc/com.atproto.identity.resolveHandle", + params={"handle": m["handle"]}, + ) + if resp.status_code == 400: + continue + did = resp.json()["did"] + facets.append( + { + "index": { + "byteStart": m["start"], + "byteEnd": m["end"], + }, + "features": [ + {"$type": "app.bsky.richtext.facet#mention", "did": did} + ], + } + ) + last_url = None + for u in self.parse_urls(text): + facets.append( + { + "index": { + "byteStart": u["start"], + "byteEnd": u["end"], + }, + "features": [ + { + "$type": "app.bsky.richtext.facet#link", + "uri": u["url"], + } + ], + } + ) + last_url = u["url"] + return facets, last_url + + def handle_url_card( + self, url: str + ) -> Optional[atproto.models.AppBskyEmbedExternal.Main]: + try: + response = requests.get(url) + except: + return None + embed_external = None + if response.status_code == 200: + soup = BeautifulSoup(response.text, "html.parser") + title_tag = soup.find("meta", attrs={"property": "og:title"}) + title_tag_alt = soup.title.string if soup.title else None + description_tag = soup.find("meta", attrs={"property": "og:description"}) + description_tag_alt = soup.find("meta", attrs={"name": "description"}) + image_tag = soup.find("meta", attrs={"property": "og:image"}) + title = title_tag["content"] if title_tag else title_tag_alt + description = ( + description_tag["content"] + if description_tag + else description_tag_alt["content"] if description_tag_alt else "" + ) + uri = url + thumb = ( + self.blueskysocial.upload_blob( + requests.get(image_tag["content"]).content + ).blob + if image_tag + else None + ) + embed_external = atproto.models.AppBskyEmbedExternal.Main( + external=atproto.models.AppBskyEmbedExternal.External( + title=title, + description=description, + uri=uri, + thumb=thumb, + ) + ) + return embed_external + + def create_post( + self, content, mentions, hashtags, images, **kwargs + ) -> Tuple[bool, Optional[str]]: + embed_images = [] + for image in images[:4]: + response = requests.get(image["url"]) + if response.status_code == 200 and response.headers.get( + "Content-Type", "" + ).startswith("image/"): + img_data = response.content + upload = self.blueskysocial.com.atproto.repo.upload_blob(img_data) + embed_images.append( + atproto.models.AppBskyEmbedImages.Image( + alt=image["alt_text"] if "alt_text" in image else "", + image=upload.blob, + ) + ) + embed = ( + atproto.models.AppBskyEmbedImages.Main(images=embed_images) + if embed_images + else None + ) + + status = [] + reply_to = None + mentions = " ".join([f"@{v}" for v in mentions]) + hashtags = " ".join([f"#{v}" for v in hashtags]) + for text in textwrap.wrap( + content + "\n" + mentions + "\n" + hashtags, + self.max_content_length, + replace_whitespace=False, + ): + facets, last_url = self.parse_facets(text) + if not images or reply_to: + embed = self.handle_url_card(cast(str, last_url)) + + post = self.blueskysocial.send_post( + text, facets=facets, embed=embed, reply_to=reply_to + ) + + for _ in range(5): + data = self.blueskysocial.get_posts([post.uri]).posts + if data: + status.append(data[0].record.text == text) + break + + if reply_to is None: + link = f"https://bsky.app/profile/{self.blueskysocial.me.handle}/post/{post.uri.split('/')[-1]}" + root = atproto.models.create_strong_ref(post) + parent = atproto.models.create_strong_ref(post) + reply_to = atproto.models.AppBskyFeedPost.ReplyRef(parent=parent, root=root) + + return all(status), link diff --git a/lib/plugins/linkedin.py b/lib/plugins/linkedin.py new file mode 100644 index 0000000..0c60ac4 --- /dev/null +++ b/lib/plugins/linkedin.py @@ -0,0 +1,47 @@ +# this file is not working! It is just a template for the final implementation... +import requests + + +class linkedin_client: + def __init__( + self, access_token=None, api_base_url="https://api.linkedin.com/rest/" + ): + self.api_base_url = api_base_url + self.headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json", + "X-Restli-Protocol-Version": "2.0.0", + "LinkedIn-Version": "202403", + } + + def linkedin_post(self, content): + url = self.api_base_url + "posts" + data = { + "author": "urn:li:organization:5515715", + "commentary": content, + "visibility": "PUBLIC", + "distribution": { + "feedDistribution": "MAIN_FEED", + "targetEntities": [], + "thirdPartyDistributionChannels": [], + }, + "lifecycleState": "PUBLISHED", + "isReshareDisabledByAuthor": False, + } + response = requests.post(url, headers=self.headers, json=data) + return response.json() + + def get_profile(self): + url = self.api_base_url + "people/~" + response = requests.get(url, headers=self.headers) + return response.json() + + # This method is not implemented + def create_post(self, content): + self.linkedin_post(content) + return True + linkedin_posts = self.get_profile() + for post in linkedin_posts["posts"]["values"]: + if content in post["commentary"]: + return True + return False diff --git a/lib/plugins/markdown.py b/lib/plugins/markdown.py new file mode 100644 index 0000000..49e3567 --- /dev/null +++ b/lib/plugins/markdown.py @@ -0,0 +1,45 @@ +import os +import time + + +class markdown_client: + def __init__(self, **kwargs): + self.save_path = kwargs.get("save_path") and ( + kwargs["save_path"] + if os.path.isabs(kwargs["save_path"]) + else os.path.join(os.getcwd(), kwargs["save_path"]) + ) + + def create_post(self, content, mentions, hashtags, images, **kwargs): + try: + _images = "\n".join( + [f'![{image.get("alt_text", "")}]({image["url"]})' for image in images] + ) + mentions = " ".join([f"@{v}" for v in mentions]) + hashtags = " ".join([f"#{v}" for v in hashtags]) + text = f"{content}\n{mentions}\n{hashtags}\n{_images}" + if self.save_path: + os.makedirs(self.save_path, exist_ok=True) + prefix = ( + kwargs.get("file_path", "").replace("/", "-").replace(".md", "") + ) + file_name = ( + f"{self.save_path}/{prefix}_{time.strftime('%Y%m%d-%H%M%S')}.md" + ) + with open(file_name, "w") as f: + f.write(text) + if kwargs.get("preview"): + social_media = ", ".join(kwargs.get("media", [])) + pre_comment_text = "" + if len(images) > 4 and ( + "mastodon" in social_media or "bluesky" in social_media + ): + pre_comment_text = f"Please note that Mastodon and Bluesky only support up to 4 images in a single post. The first 4 images will be included in the post, and the rest will be ignored.\n" + comment_text = f"File: {kwargs.get('file_path')}\n{pre_comment_text}This is a preview that will be posted to {social_media}:\n\n{text}" + return True, None, comment_text + return True, None + except Exception as e: + if kwargs.get("preview", False): + print(e) + return False, None, e + return False, None diff --git a/lib/plugins/mastodon.py b/lib/plugins/mastodon.py new file mode 100644 index 0000000..6ffb816 --- /dev/null +++ b/lib/plugins/mastodon.py @@ -0,0 +1,63 @@ +import tempfile +import textwrap + +import requests +from bs4 import BeautifulSoup +from mastodon import Mastodon + + +class mastodon_client: + def __init__(self, **kwargs): + self.base_url = kwargs.get("base_url", "https://mstdn.science") + self.mastodon_handle = Mastodon( + access_token=kwargs.get("access_token"), api_base_url=self.base_url + ) + self.max_content_length = kwargs.get("max_content_length", 500) + + def create_post(self, content, mentions, hashtags, images, **kwargs): + media_ids = [] + for image in images[:4]: + response = requests.get(image["url"]) + if response.status_code == 200 and response.headers.get( + "Content-Type", "" + ).startswith("image/"): + with tempfile.NamedTemporaryFile() as temp: + temp.write(response.content) + temp.flush() + media_uploaded = self.mastodon_handle.media_post( + media_file=temp.name, + description=image["alt_text"] if "alt_text" in image else None, + ) + media_ids.append(media_uploaded["id"]) + + toot_id = None + status = [] + mentions = " ".join([f"@{v}" for v in mentions]) + hashtags = " ".join([f"#{v}" for v in hashtags]) + for text in textwrap.wrap( + content + "\n" + mentions + "\n" + hashtags, + self.max_content_length, + replace_whitespace=False, + ): + toot = self.mastodon_handle.status_post( + status=text, + in_reply_to_id=toot_id, + media_ids=media_ids if (media_ids != [] and toot_id == None) else None, + ) + + if not toot_id: + link = f"{self.base_url}/@{toot['account']['acct']}/{toot['id']}" + toot_id = toot["id"] + + for _ in range(3): + post = self.mastodon_handle.status(toot_id) + if post.content: + post_content = BeautifulSoup(post.content, "html.parser").get_text( + separator=" " + ) + status.append( + "".join(post_content.split()) == "".join(text.split()) + ) + break + + return all(status), link diff --git a/lib/plugins/matrix.py b/lib/plugins/matrix.py new file mode 100644 index 0000000..e3e438d --- /dev/null +++ b/lib/plugins/matrix.py @@ -0,0 +1,94 @@ +import asyncio +import tempfile + +import aiofiles.os +import magic +import requests +from nio import AsyncClient, UploadResponse +from PIL import Image + + +class matrix_client: + + def __init__(self, **kwargs): + self.base_url = kwargs.get("base_url", "https://matrix.org") + self.client = AsyncClient(self.base_url) + self.client.access_token = kwargs.get("access_token") + self.client.user_id = kwargs.get("user_id") + self.client.device_id = kwargs.get("device_id") + self.room_id = kwargs.get("room_id") + + async def async_create_post(self, text, mentions, images): + for image in images: + response = requests.get(image["url"]) + if response.status_code != 200: + continue + image_name = image["url"].split("/")[-1] + temp = tempfile.NamedTemporaryFile() + temp.write(response.content) + temp.flush() + mime_type = magic.from_file(temp.name, mime=True) + if not mime_type.startswith("image/"): + continue + + width, height = Image.open(temp.name).size + file_stat = await aiofiles.os.stat(temp.name) + async with aiofiles.open(temp.name, "r+b") as f: + resp, _ = await self.client.upload( + f, + content_type=mime_type, + filename=image_name, + filesize=file_stat.st_size, + ) + + if not isinstance(resp, UploadResponse): + continue + + content = { + "body": image_name, + "info": { + "size": file_stat.st_size, + "mimetype": mime_type, + "thumbnail_info": None, + "w": width, + "h": height, + "thumbnail_url": None, + }, + "msgtype": "m.image", + "url": resp.content_uri, + } + + try: + await self.client.room_send( + self.room_id, message_type="m.room.message", content=content + ) + except: + return False, None + + if mentions: + text = ( + text + + "\n\n" + + " ".join([f"https://matrix.to/#/@{mention}" for mention in mentions]) + ) + content = { + "msgtype": "m.text", + "format": "org.matrix.custom.html", + "body": text, + } + try: + response = await self.client.room_send( + self.room_id, message_type="m.room.message", content=content + ) + await self.client.close() + message_id = response.event_id + link = f"https://matrix.to/#/{self.room_id}/{message_id}" + except: + return False, None + + return True, link + + def create_post(self, content, mentions, hashtags, images, **kwargs): + # hashtags and alt_texts are not used in this function + result, link = asyncio.run(self.async_create_post(content, mentions, images)) + return result, link diff --git a/lib/plugins/slack.py b/lib/plugins/slack.py new file mode 100644 index 0000000..7619523 --- /dev/null +++ b/lib/plugins/slack.py @@ -0,0 +1,66 @@ +import textwrap + +import requests +from slack_sdk import WebClient + + +class slack_client: + def __init__(self, **kwargs): + self.client = WebClient(token=kwargs.get("access_token")) + self.channel_id = kwargs.get("channel_id") + self.max_content_length = kwargs.get("max_content_length", 40000) + + def upload_images(self, images): + uploaded_files = [] + for image in images: + filename = image["url"].split("/")[-1] + + with requests.get(image["url"]) as response: + if response.status_code != 200 or not response.headers.get( + "Content-Type", "" + ).startswith("image/"): + continue + image_content = response.content + + response = self.client.files_getUploadURLExternal( + filename=filename, + length=len(image_content), + alt_txt=image["alt_text"] if "alt_text" in image else None, + ) + + with requests.post( + response["upload_url"], files={"file": image_content} + ) as upload_response: + if upload_response.status_code != 200: + continue + uploaded_files.append({"id": response["file_id"]}) + + response = self.client.files_completeUploadExternal( + files=uploaded_files, channel_id=self.channel_id + ) + return response + + def create_post(self, text, mentions, hashtags, images, **kwargs): + status = [] + link = None + parent_ts = None + for text in textwrap.wrap( + text, + self.max_content_length, + replace_whitespace=False, + ): + response = self.client.chat_postMessage( + channel=self.channel_id, + text=text, + thread_ts=parent_ts if parent_ts else None, + ) + if not parent_ts: + parent_ts = response["ts"] + link = self.client.chat_getPermalink( + channel=self.channel_id, message_ts=parent_ts + )["permalink"] + status.append(response["ok"]) + if images: + response = self.upload_images(images) + status.append(response["ok"]) + return all(status), link diff --git a/plugins.yml b/plugins.yml new file mode 100644 index 0000000..7e43191 --- /dev/null +++ b/plugins.yml @@ -0,0 +1,40 @@ +plugins: + - name: mastodon + class: mastodon.mastodon_client + enabled: true + config: + base_url: "https://mstdn.science" + access_token: $MASTODON_ACCESS_TOKEN + max_content_length: 500 + + - name: bluesky + class: bluesky.bluesky_client + enabled: true + config: + base_url: "https://bsky.social" + username: $BLUESKY_USERNAME + password: $BLUESKY_PASSWORD + max_content_length: 300 + + - name: matrix + class: matrix.matrix_client + enabled: true + config: + base_url: "https://matrix.org" + access_token: $MATRIX_ACCESS_TOKEN + room_id: $MATRIX_ROOM_ID + user_id: $MATRIX_USER_ID + + - name: slack + class: slack.slack_client + enabled: true + config: + access_token: $SLACK_ACCESS_TOKEN + channel_id: $SLACK_CHANNEL_ID + max_content_length: 40000 + + - name: markdown + class: markdown.markdown_client + enabled: true + config: + save_path: "markdown_cache" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9cc0bbd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +aiofiles==23.2.1 +atproto==0.0.46 +beautifulsoup4==4.12.3 +Markdown==3.6 +Mastodon.py==1.8.1 +matrix-nio==0.24.0 +Pillow==10.3.0 +PyYAML==6.0.1 +slack_sdk==3.27.1 +jsonschema==4.21.1 \ No newline at end of file diff --git a/toots/2023/2023-11-11-esg-psa.toot b/toots/2023/2023-11-11-esg-psa.toot deleted file mode 100644 index 8250ed6..0000000 --- a/toots/2023/2023-11-11-esg-psa.toot +++ /dev/null @@ -1,4 +0,0 @@ -News from the #EOSC EuroScienceGateway project: New EGI Check-in and WLCG IAM backends available in python-social-auth. -#usegalaxy integration included :) - -https://galaxyproject.org/news/2023-11-08-esg-psa/ diff --git a/toots/2023/2023-11-11-gga-updates.toot b/toots/2023/2023-11-11-gga-updates.toot deleted file mode 100644 index 1c6b9b2..0000000 --- a/toots/2023/2023-11-11-gga-updates.toot +++ /dev/null @@ -1,5 +0,0 @@ -As presented at #GCC2023 and #EGD2023, a lot of new exciting developments have been made in #usegalaxy for the annotation of genomes! - -We have written down a small update blog post: https://galaxyproject.org/news/2023-10-30-gga-update - -#SciWorkflows included! diff --git a/toots/2023/2023-11-11-pulsar-im.toot b/toots/2023/2023-11-11-pulsar-im.toot deleted file mode 100644 index f23d21b..0000000 --- a/toots/2023/2023-11-11-pulsar-im.toot +++ /dev/null @@ -1,4 +0,0 @@ -The #EOSC EuroScienceGateway presents an easy deployment of #usegalaxy Pulsar endpoints with the #EGI Infrastructure Manager - -Check out our demo from Sebastian LunaValero demo 👉 https://www.youtube.com/watch?v=5EMXzD_JDjw -And read more about it 👉 https://galaxyproject.org/news/2023-10-31-esg-byoc-im diff --git a/toots/README.md b/toots/README.md deleted file mode 100644 index b2c6f32..0000000 --- a/toots/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# The toots/ folder - -To create a new toot create a new `*.toot` file in this `toots/` folder. - -[Create new toot](../../../new/master/?filename=toots/.toot) - -## Example - -Create a new file `toots/hello-world.toot` with the content - -``` -Hello, world! -``` - -You can use subfolders, e.g. `toots/2020-11/hello-world.toot`, as long as the file is in the `toots/` folder and has the `.toot` file extension - -## Create a toot with a poll - -A toot including a poll must end with 2-4 options in the following format - -``` -Here is some text - -[ ] option A -[ ] option B -[ ] option C -[ ] option D -``` - -## Notes - -- Only newly created files are handled, deletions, updates or renames are ignored. -- Toots posted from other Mastodon clients will not be imported as `*.toot` files. -- If you need to rename an existing toot file, please do so locally using [`git mv old_filename new_filename`](https://help.github.com/en/articles/renaming-a-file-using-the-command-line), otherwise it may occur as deleted and added which would trigger a new toot. -- Your message must fit into a single toot (typically 500 characters). - -## Questions? - -If you have any further questions or suggestions, please create an issue at https://github.com/joschi/toot-together/issues/new