From 89b2e03fa15ab5267f48b0cb3abfeaa8b4d8d164 Mon Sep 17 00:00:00 2001 From: Arash Date: Mon, 22 Apr 2024 13:54:19 +0200 Subject: [PATCH 01/13] :tada: --- .github/workflows/preview.yml | 49 +++++++++++++++++++ .github/workflows/publish_content.yml | 65 +++++++++++++++++++++++++ .github/workflows/toot-together.yml | 25 ---------- .schema.yaml | 32 ++++++++++++ README.md | Bin 749 -> 4108 bytes plugins.yml | 40 +++++++++++++++ requirements.txt | 10 ++++ toots/2023/2023-11-11-esg-psa.toot | 4 -- toots/2023/2023-11-11-gga-updates.toot | 5 -- toots/2023/2023-11-11-pulsar-im.toot | 4 -- toots/README.md | 39 --------------- 11 files changed, 196 insertions(+), 77 deletions(-) create mode 100644 .github/workflows/preview.yml create mode 100644 .github/workflows/publish_content.yml delete mode 100644 .github/workflows/toot-together.yml create mode 100644 .schema.yaml create mode 100644 plugins.yml create mode 100644 requirements.txt delete mode 100644 toots/2023/2023-11-11-esg-psa.toot delete mode 100644 toots/2023/2023-11-11-gga-updates.toot delete mode 100644 toots/2023/2023-11-11-pulsar-im.toot delete mode 100644 toots/README.md diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml new file mode 100644 index 0000000..e53688e --- /dev/null +++ b/.github/workflows/preview.yml @@ -0,0 +1,49 @@ +name: Create Preview + +on: + pull_request_target: + branches: [main] + types: [opened, synchronize, reopened] + +jobs: + preview: + name: Preview + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Get changed files in posts folder + id: get_changed_files + uses: tj-actions/changed-files@v44 + with: + files: posts/** + json: "true" + + - name: get published files cache + if: steps.get_changed_files.outputs.any_changed == 'true' + run: | + git fetch origin processed_files:processed_files + git checkout processed_files -- processed_files.json + + - name: Set up Python + if: steps.get_changed_files.outputs.any_changed == 'true' + uses: actions/setup-python@v3 + with: + python-version: 3.9 + + - name: Install dependencies + if: steps.get_changed_files.outputs.any_changed == 'true' + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + + - name: Run script to create preview + if: steps.get_changed_files.outputs.any_changed == 'true' + env: + CHANGED_FILES: ${{ steps.get_changed_files.outputs.all_changed_files }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.number }} + run: python -u lib/galaxy-social.py preview diff --git a/.github/workflows/publish_content.yml b/.github/workflows/publish_content.yml new file mode 100644 index 0000000..228c8ee --- /dev/null +++ b/.github/workflows/publish_content.yml @@ -0,0 +1,65 @@ +name: Publish Content + +on: + pull_request_target: + branches: [main] + types: [closed] + +jobs: + publish: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Get changed files in posts folder + id: get_changed_files + uses: tj-actions/changed-files@v44 + with: + files: posts/** + json: "true" + + - name: get published files cache + if: steps.get_changed_files.outputs.any_changed == 'true' + run: | + git fetch origin processed_files:processed_files + git checkout processed_files -- processed_files.json + + - name: Set up Python + if: steps.get_changed_files.outputs.any_changed == 'true' + uses: actions/setup-python@v3 + with: + python-version: 3.9 + + - name: Install dependencies + if: steps.get_changed_files.outputs.any_changed == 'true' + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + + - name: Run script to publish contents + if: steps.get_changed_files.outputs.any_changed == 'true' + env: + MASTODON_ACCESS_TOKEN: ${{ secrets.MASTODON_ACCESS_TOKEN }} + BLUESKY_USERNAME: ${{ secrets.BLUESKY_USERNAME }} + BLUESKY_PASSWORD: ${{ secrets.BLUESKY_PASSWORD }} + MATRIX_ACCESS_TOKEN: ${{ secrets.MATRIX_ACCESS_TOKEN }} + MATRIX_ROOM_ID: ${{ secrets.MATRIX_ROOM_ID }} + MATRIX_USER_ID: ${{ secrets.MATRIX_USER_ID }} + SLACK_ACCESS_TOKEN: ${{ secrets.SLACK_ACCESS_TOKEN }} + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + CHANGED_FILES: ${{ steps.get_changed_files.outputs.all_changed_files }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.number }} + run: python -u lib/galaxy-social.py + + - name: Commit changes + if: steps.get_changed_files.outputs.any_changed == 'true' + uses: stefanzweifel/git-auto-commit-action@v5 + with: + file_pattern: "processed_files.json" + branch: "processed_files" diff --git a/.github/workflows/toot-together.yml b/.github/workflows/toot-together.yml deleted file mode 100644 index cfc265b..0000000 --- a/.github/workflows/toot-together.yml +++ /dev/null @@ -1,25 +0,0 @@ -on: [push, pull_request] -name: Toot, together! -jobs: - preview: - name: Preview - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - steps: - - uses: joschi/toot-together@v1.x - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - toot: - name: Toot - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - steps: - - name: checkout master - uses: actions/checkout@v4 - - name: Toot - uses: joschi/toot-together@v1.x - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # URL to the instance hosting your Mastodon account - MASTODON_URL: "https://bawü.social" - MASTODON_ACCESS_TOKEN: ${{ secrets.MASTODON_ACCESS_TOKEN }} diff --git a/.schema.yaml b/.schema.yaml new file mode 100644 index 0000000..3e4ae98 --- /dev/null +++ b/.schema.yaml @@ -0,0 +1,32 @@ +type: object +properties: + media: + type: array + items: + type: string + uniqueItems: true + images: + type: array + items: + type: object + properties: + url: + type: string + format: uri + alt_text: + type: string + required: [] + mentions: + type: object + additionalProperties: + type: array + items: + type: string + required: [] + hashtags: + type: object + additionalProperties: + type: array + items: + type: string + required: [] diff --git a/README.md b/README.md index 1f114f03a55ebbf08fddb3c025d7e541b7073dec..f233fec526a4f7882bc84dc992480e849953dbe6 100644 GIT binary patch literal 4108 zcmbuCTW=dh6vyWoiSMu^JhTYAqP+DjDODRTrD|LxgplucTqkje?X(8*)q(%-oE*>W z)~-QlW$*6HxnKTs_V2%rQw}6s-;mvF&6DIJuzuW{3^9Y^>l~CwMnUL=eM*rDJzqR z4KBAzM_b)XJ%{#@sBa}14UbL3S-O_qSz;%1-$*XDHnrf#hhAQb7oR4Qb7^wk=?P1+ z#Id4>m#%H0J2tk{l{9n5qOZ-OH}_EqPs+H>#5XmmV{zU{TSObG+d^MN9(&Q|x>rSn zdx;CZExUNSl7%D7t!JW=#Fl;oY4S+VRRd7Q&Xp`6e+%)Ur@6P1RWtcst}o5*VVc`2 zJ2PqS%QGsXu6Zr~ORZIk=1P}W$)og0wC}|eQD~c=S6c5Yif=_X-Ri_vDtMK;`gtah zc(SFy$A713Ea^mgDoq1D-&jV%yM3BYx1JJVvN5x0mEP84oTCtt#;<|mLk8Nq*Sip^ zW-fNg2uq=j-0Y%zV$y;le1@Cfg}9fN5wiB*ya)@zufaA8hwUG3X@h%YE4hL@_xVLj zu5jK?)SqOZE&-*6)}J5i^u97h`wo1na4$ZY{e=0U<}K+rXBka{pGV^jc5UAObl1F%J6o_TFPh8sc#* zz7x}q6_w;Tij|c`O$8#r!*3tf2S2E^R#jjb(b>q>weHl+ouq}I?z!uJB`TulxEQ_$ zWyH0VOk5oq9n(FsQ06>Lr1!d(y`J9k8P!$X{Cs+#^%5nqyVA z+-54DG(>!;NYO|5;<-mYwdAJ=x#ulVpS^2~CSGv@IF@Aoh}6Cux_xD}dWBPiQK7jC za~`R>jr3ibC%aFi3)$(G)XyeI7b++BV|4Ge{KX5)Y8zjUxrTc6osM3}`>r&*bw`c0Bjd?Y_3k8U zph(2Cr?QvD+IqFqDpQi z`N#J*GiU9S;`xQMB`f#_;ygkoGv>znC!&lDpAPVis4;`&#d)fLn%V#x m7RnH)MRm+YLv4O$zM}VBR#M}0JIi7^h%b$-b(cMdHT(mup`T^| literal 749 zcmbVK&5qMB5WeRrM%@D~>c(ifK^lZ$VL2e7vbUd4hdK9)WS% zUJxguNU=xxX1<@Pv^fA;)A`k`C7&qF>H?O2;+`UxQiN|m|VCw5KmuL06K$r*w2DwlvtSgoK7L^hvby3{F zN3{6k1Yfx|Xp7>pHxU9298KbIf|!CiP!v39a@+dbSmMo zCqUysnhBD8E7Y(fXxUn9ICo*c1VJU^yCn=n;Wp602vy!sY$Kr8;KsHG$AQY?lLyop zkda3MXSzP+5;Wq$^miPGfkVnOaZK#aWdF%AkHI5FcA1rPqZi_G@*=ny^AHi2hhu~< z>$tqSXywQO=pgY$U=K;8ZZ0l9jLml5r!>S@N_CU7L*Q@JrjkRa7Nt0T0}q_@m_!!E zZkOz99d?Uv(JwAB7ng20v41|T1?1fUlZi_@n^K)uXgfcD7qDlmDU|R* nWM=0j_=Ltr7%ZV=FCN9wipdjxO{g7R(d+xS??2y{PK(q(`6mJ- diff --git a/plugins.yml b/plugins.yml new file mode 100644 index 0000000..596b253 --- /dev/null +++ b/plugins.yml @@ -0,0 +1,40 @@ +plugins: + - name: mastodon + class: mastodon.mastodon_client + enabled: true + config: + base_url: "https://mstdn.science" + access_token: MASTODON_ACCESS_TOKEN + max_content_length: 500 + + - name: bluesky + class: bluesky.bluesky_client + enabled: true + config: + base_url: "https://bsky.social" + username: BLUESKY_USERNAME + password: BLUESKY_PASSWORD + max_content_length: 300 + + - name: matrix + class: matrix.matrix_client + enabled: true + config: + base_url: "https://matrix.org" + access_token: MATRIX_ACCESS_TOKEN + room_id: MATRIX_ROOM_ID + user_id: MATRIX_USER_ID + + - name: slack + class: slack.slack_client + enabled: true + config: + access_token: SLACK_ACCESS_TOKEN + channel_id: SLACK_CHANNEL_ID + max_content_length: 40000 + + - name: markdown + class: markdown.markdown_client + enabled: true + config: + save_path: "markdown_cache" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9cc0bbd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +aiofiles==23.2.1 +atproto==0.0.46 +beautifulsoup4==4.12.3 +Markdown==3.6 +Mastodon.py==1.8.1 +matrix-nio==0.24.0 +Pillow==10.3.0 +PyYAML==6.0.1 +slack_sdk==3.27.1 +jsonschema==4.21.1 \ No newline at end of file diff --git a/toots/2023/2023-11-11-esg-psa.toot b/toots/2023/2023-11-11-esg-psa.toot deleted file mode 100644 index 8250ed6..0000000 --- a/toots/2023/2023-11-11-esg-psa.toot +++ /dev/null @@ -1,4 +0,0 @@ -News from the #EOSC EuroScienceGateway project: New EGI Check-in and WLCG IAM backends available in python-social-auth. -#usegalaxy integration included :) - -https://galaxyproject.org/news/2023-11-08-esg-psa/ diff --git a/toots/2023/2023-11-11-gga-updates.toot b/toots/2023/2023-11-11-gga-updates.toot deleted file mode 100644 index 1c6b9b2..0000000 --- a/toots/2023/2023-11-11-gga-updates.toot +++ /dev/null @@ -1,5 +0,0 @@ -As presented at #GCC2023 and #EGD2023, a lot of new exciting developments have been made in #usegalaxy for the annotation of genomes! - -We have written down a small update blog post: https://galaxyproject.org/news/2023-10-30-gga-update - -#SciWorkflows included! diff --git a/toots/2023/2023-11-11-pulsar-im.toot b/toots/2023/2023-11-11-pulsar-im.toot deleted file mode 100644 index f23d21b..0000000 --- a/toots/2023/2023-11-11-pulsar-im.toot +++ /dev/null @@ -1,4 +0,0 @@ -The #EOSC EuroScienceGateway presents an easy deployment of #usegalaxy Pulsar endpoints with the #EGI Infrastructure Manager - -Check out our demo from Sebastian LunaValero demo 👉 https://www.youtube.com/watch?v=5EMXzD_JDjw -And read more about it 👉 https://galaxyproject.org/news/2023-10-31-esg-byoc-im diff --git a/toots/README.md b/toots/README.md deleted file mode 100644 index b2c6f32..0000000 --- a/toots/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# The toots/ folder - -To create a new toot create a new `*.toot` file in this `toots/` folder. - -[Create new toot](../../../new/master/?filename=toots/.toot) - -## Example - -Create a new file `toots/hello-world.toot` with the content - -``` -Hello, world! -``` - -You can use subfolders, e.g. `toots/2020-11/hello-world.toot`, as long as the file is in the `toots/` folder and has the `.toot` file extension - -## Create a toot with a poll - -A toot including a poll must end with 2-4 options in the following format - -``` -Here is some text - -[ ] option A -[ ] option B -[ ] option C -[ ] option D -``` - -## Notes - -- Only newly created files are handled, deletions, updates or renames are ignored. -- Toots posted from other Mastodon clients will not be imported as `*.toot` files. -- If you need to rename an existing toot file, please do so locally using [`git mv old_filename new_filename`](https://help.github.com/en/articles/renaming-a-file-using-the-command-line), otherwise it may occur as deleted and added which would trigger a new toot. -- Your message must fit into a single toot (typically 500 characters). - -## Questions? - -If you have any further questions or suggestions, please create an issue at https://github.com/joschi/toot-together/issues/new From 934a75375b262fd5f7414b5650fbd54a33541f22 Mon Sep 17 00:00:00 2001 From: Arash Date: Mon, 29 Apr 2024 16:13:29 +0200 Subject: [PATCH 02/13] Ignore lib/ directory in .gitignore to add the lib folder to PR & add more documentation. --- .gitignore | 2 +- README.md | Bin 4108 -> 11376 bytes lib/galaxy-social.py | 152 ++++++++++++++++++++++++++++++ lib/github_comment.py | 25 +++++ lib/plugins/bluesky.py | 202 ++++++++++++++++++++++++++++++++++++++++ lib/plugins/linkedin.py | 47 ++++++++++ lib/plugins/markdown.py | 33 +++++++ lib/plugins/mastodon.py | 60 ++++++++++++ lib/plugins/matrix.py | 92 ++++++++++++++++++ lib/plugins/slack.py | 69 ++++++++++++++ 10 files changed, 681 insertions(+), 1 deletion(-) create mode 100644 lib/galaxy-social.py create mode 100644 lib/github_comment.py create mode 100644 lib/plugins/bluesky.py create mode 100644 lib/plugins/linkedin.py create mode 100644 lib/plugins/markdown.py create mode 100644 lib/plugins/mastodon.py create mode 100644 lib/plugins/matrix.py create mode 100644 lib/plugins/slack.py diff --git a/.gitignore b/.gitignore index b6e4761..6c9cc34 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ dist/ downloads/ eggs/ .eggs/ -lib/ +# lib/ lib64/ parts/ sdist/ diff --git a/README.md b/README.md index f233fec526a4f7882bc84dc992480e849953dbe6..4f9a0840997c6bd25dabd09f91e472e42096127d 100644 GIT binary patch literal 11376 zcmb`N-A`P}5ykIwr2G#zI1k=c7%SN%5Bp|paIAQ31mmn$QN-|VY`}y8+hmo0J<0j? znVRZ*XD$Y^8UZtXySu(lovLo;zyEpEw9SiVw>j0%m*z&FwwsU5esfaBe%<`4;pdTF zZSxmB`K-TJ`r9_on#<-;v##{J)!(++)c91NW7W9K+h3S>F`L)w@440u%m1C89%u`5=i018YWeJN_`Y|D}K~xYl9uUBPJ{IjUzJE`|#v}o}t&{qD?FPuVvlp%+G#0ZN6#n zeflC|d_L3QXr@6ceJnfW9tufB=3IN9ND4Zw$euIVfgZ#Fi}v;GOfS~sQ+^v;>9gAJ z>7DQ9-gzq-=oU5y2HR$=@iTb^>a9NEy_4o&os~7tBJ!-O_plG~UdRg%+nVzt ziQkFHNyF<#TAWELxv^SaygSqW2hwDBHow)*tNPi~%xgWH3N1HfO*}c%TyXzU>+meQ z)X%OI+Y|Y}9-kH;>}xMZ`8ko~U11vu)-?72sizuig`q3GoLx_vCz|`E*nme5<$p)- ziN;3Fd(AtZ%VaMY<*M1ykI}%6PvfI zUIWzW%Ir-yTy!=q+xT!>z!6lOGJp;{j3NL!mhn} z6uz$i{(Nq%J}4uJ;Gd+a*)1a7&xBSQe^eCUNqC8>k(uo5Ku7rawPvypbIlu(XJd`l zndoe09avg#K9!wsv>Hw}Z|`aBO8j)FSIvuzQNbp&Oq}TbZqbXFvW8gM)5%u&A6rig zcd>WWfuDrzADUPCY=(EN(?&F0DLXsY;lRhumUj53_B$z(55&0S$htIzk?~TpO!7pl z=P-$-<0o?2IT=`YeZdRxFN~f$P0HMB#l+7?;um;nQpLpj)*NAt-)L=Q68UY7UrNrt zK3&fw!K~-v8gw(8qXC*=6TDyVJksZJ!NnqHFw;or%7q`VwYTeTR1so~q=N-Z!Y{!A zSQ9m9MY6lOb(^glLO{P4p@LpWEl&Q7{BvT;`) z#V=G_FqM4*z649mfKkAd*@7NvNCWyA_ZW06wH;{qNk2x9V-waK&X@(fr^RUCm?9C+ zS)Uqve=PPb??g9+Uu&=>{b2zzgs#C|=^I60Ut@Lm7~YPo2#GfXo!Z4*1~XGeWM#w~ zwjyJ@J#HfKPjfpqiR(hR#S1O2pAq}o{ z8li6`4&WsE%#7`&o}U-pU+Q^{-iw|cA-RwIjBw2{*`>z1A*+&Q$h2dj1RV31J@6jq zEf9Vw{9YE0U{7ZoopzmB&N&$7%WRwxXA?+)<>(oR<+1FJ-r>(x{O}k$z(j46tn#dr z^AW?hCBs2RckhhVkmcQYZk9-;+^)2TM070uu^ALmE$KDM$&9|ewX2+ttae6EWE0*d zyQX^K;hnm`cj5;8_}stj1iD>(=v#tT-4A}`6Sn|eo(~+Z z3%3`=9%i~j3(sOF&-U~LoM0ukm?@30_s?$=7681Uq6_ZxUVI=j3hprX)O zt`&Jm9`@O$d`V1xkB*#Ar_^GV~0I(~>d9IXQP(cxSBL++XBw-A+1KH>wbR zpr3?&Bd3vU4;@J!wJ{k#`eC9n(GxfU20c_>2l5|l9Ge*BH#yet3K{G7f{fMODxdo& zTkcBep54cWt$i(%nUF!A!Z(4FKAr;ia07D_*XR1oDa8Fj*npq9F(LwW&wckSNCwC| zC@R6h%&X3ujC!0P0%zP&asPv_t~=`zKV96!4lr(_Br~NxUBn*u&+g?p8oh6~|IU2# z`Lw>hiIZ^Pj@*j(JBrOwx%Hqv?G%g!FOt2i<(2qmb=bT!-)B_K$kBQSZ~HTxPe_2L0o3~<+SNgQA z&TB_K+8d3J%lz@e&Tu1ldpnYB?N+2+J+@NT6U$=ZV&l zYi5i@OPrZqPjcLytfgEj67*qb1-sGRPVO9^<+7=4t`_{JMw;&<>M)AycA_E?6?Y0X zQzYj)SI6QlxFr39CV_08!THznulq^YfbhYjLMiKf^Whp*OZ821qLn+0;Mb{Y!MEZ- zZg{^bdk{Z32wU+eGHbpY;-h4ryyJDhXcTiZf(PC22j{#j7N$it?y(8Wvu40d^_kuM zgS+uKm*j25C&_c%oNaKeabh32TcIj&qZ~{fy3|Zp=Y6+ca0WvQR@ZE|tEhP=s@92~ zrR{b8P77iq!&+tOb6JR;;F;gvh4+1|uXcQa4CeR*PJM9%>5SnIe)447J&A1>FXKVF z@rX@!kM?k~j{7@wTIS=$NQBpy`_}cMtC8a#8g+e6Rs`dq33Gj8hNtPV9gpCI+VjpW z@9w)g_v5~qM;d3rDJ5BgnTeahU?ZgqzpJze;7qaX*CBvWqh4ezoq-p&f@+*)K$ti*P1 zr(nnPxi1zu$Nh%0l{MVwc2B8DalZs2b_++~QRek;rTSQG)ju(ocmUsvHv)I>Lr2PU zN68jufA{}%a`dCWZGP6vsL$LVsoT4)LwupCGgfa%pGXI}t{!nqzsGvtHviU(+L2mk z@5lI-eB;U#z23v6fsMQOfOVwE38UDj(c2IZ{1nybA^9B@^_RFK!VdqB**9CRNLWDT z58X5j^YUKU|VsEZ>`*ZA5~-Hx%TnR8hYU=PVtfD5uKNslXr5&*-UC*xUbHvn-P9B?f2{b>o?KztHE{*?i7fm$7~h-pAwjw>A+Ax@{d<0cyB!?tVwfy*DRs zB!TH#_eA#4Ez9dUj^ccn$jt8gnW{O@`7THL2zka}_q@A^1nO6f0b`nMIFiObHP!q2 zylj?Sb|NGHLtl2OIX#etjHsi_csl;Obhooldh$LeMouN~OQ%cp&=@?`=cHj9m!FT= z1%?ZLFh}$e`n3q8HF?U4&LI(D>~2b5P#C$gC4Uq3vw}vSIig1l-zPGHCHg1fhvdas z<;l~+d(MdZ)*&L@-#>ZL9W>v&xV=k-@1s+Nfnbd|$wdX{1mg^)lQVzNn^QUXc0yu0 zcB(lzyafjM?_A`;-T5Ws_)FCGq5uCZq6ri}wDPcWS@o&&HSk=c#&-r$2f3qw|H%d4 zJ`874eXHR5-mT+_4S06h_gtvad|zcvk?nqCj?o{AF6f`%mW%QDt`!|GS;BuM7Qgia zZ`a>r%bJt_7r^IWgxrTgKoXXJC|Emgep5PMx}Oi_#ue-6IpTXM<{=n)nLc-u z@M?x{_Zq(04Xu`YkWBTrX#(}|I{%Hvk^{ZlRW0DXYYEkunLdT*8O*2DZg#`|bqDc3 D=W~(^ delta 852 zcmah{L2DCX5S`t6N!o4crfIUoCf^chLP?@DC>HS$g@T@2u^yyLOwH0|BiW?YT>Jxy zUFIzK13Z~upa&0n(Q6Ol9}s%?pl`lynj91f`|W;{d2i;unX?Bk)k!seq~f}>hIPvh_LUYcekUuQAl^QDG8|(_XU0@XTaYi(x0Femy9v$${w8K9Z zx4E&jUTz~VDK3m$&);@t)0NEaO- zA_O4flwIfRD~pz*14IMFI@ITundSGk&0p;i{t>2$EwPm_%CS!Ea(&G;YTV10`SV(Jcj|-8 zY3gAcjfZI8hv@$b*6jpn`W(-$ljEIA@kJXk5A|gVGlFqT4D91;)>NO{Q%TlFCMPC&Se?YpWRawOU!w=?7b^U& za5g_#Hpt=E#VZ*-@q`R#j_|- List[Dict]: + spans = [] + mention_regex = rb"[$|\W](@([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)" + text_bytes = text.encode("UTF-8") + for m in re.finditer(mention_regex, text_bytes): + spans.append( + { + "start": m.start(1), + "end": m.end(1), + "handle": m.group(1)[1:].decode("UTF-8"), + } + ) + return spans + + def parse_urls(self, text: str) -> List[Dict]: + spans = [] + url_regex = rb"[$|\W](https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*[-a-zA-Z0-9@%_\+~#//=])?)" + text_bytes = text.encode("UTF-8") + for m in re.finditer(url_regex, text_bytes): + spans.append( + { + "start": m.start(1), + "end": m.end(1), + "url": m.group(1).decode("UTF-8"), + } + ) + return spans + + def parse_hashtags(self, text: str) -> List[Dict]: + spans = [] + hashtag_regex = rb"[$|\W]#(\w+)" + text_bytes = text.encode("UTF-8") + for m in re.finditer(hashtag_regex, text_bytes): + spans.append( + { + "start": m.start(1), + "end": m.end(1), + "tag": m.group(1).decode("UTF-8"), + } + ) + return spans + + def parse_facets(self, text: str) -> Tuple[List[Dict], Optional[str]]: + facets = [] + for h in self.parse_hashtags(text): + facets.append( + { + "index": { + "byteStart": h["start"], + "byteEnd": h["end"], + }, + "features": [ + {"$type": "app.bsky.richtext.facet#tag", "tag": h["tag"]} + ], + } + ) + for m in self.parse_mentions(text): + resp = requests.get( + "https://bsky.social/xrpc/com.atproto.identity.resolveHandle", + params={"handle": m["handle"]}, + ) + if resp.status_code == 400: + continue + did = resp.json()["did"] + facets.append( + { + "index": { + "byteStart": m["start"], + "byteEnd": m["end"], + }, + "features": [ + {"$type": "app.bsky.richtext.facet#mention", "did": did} + ], + } + ) + last_url = None + for u in self.parse_urls(text): + facets.append( + { + "index": { + "byteStart": u["start"], + "byteEnd": u["end"], + }, + "features": [ + { + "$type": "app.bsky.richtext.facet#link", + "uri": u["url"], + } + ], + } + ) + last_url = u["url"] + return facets, last_url + + def handle_url_card( + self, url: str + ) -> Optional[atproto.models.AppBskyEmbedExternal.Main]: + try: + response = requests.get(url) + except: + return None + embed_external = None + if response.status_code == 200: + soup = BeautifulSoup(response.text, "html.parser") + title_tag = soup.find("meta", attrs={"property": "og:title"}) + title_tag_alt = soup.title.string + description_tag = soup.find("meta", attrs={"property": "og:description"}) + description_tag_alt = soup.find("meta", attrs={"name": "description"}) + image_tag = soup.find("meta", attrs={"property": "og:image"}) + title = title_tag["content"] if title_tag else title_tag_alt + description = ( + description_tag["content"] + if description_tag + else description_tag_alt["content"] if description_tag_alt else None + ) + uri = url + thumb = ( + self.blueskysocial.upload_blob( + requests.get(image_tag["content"]).content + ).blob + if image_tag + else None + ) + embed_external = atproto.models.AppBskyEmbedExternal.Main( + external=atproto.models.AppBskyEmbedExternal.External( + title=title, + description=description, + uri=uri, + thumb=thumb, + ) + ) + return embed_external + + def create_post( + self, content, mentions, hashtags, images + ) -> Tuple[bool, Optional[str]]: + embed_images = [] + for image in images[:4]: + response = requests.get(image["url"]) + if response.status_code == 200: + img_data = response.content + upload = self.blueskysocial.com.atproto.repo.upload_blob(img_data) + embed_images.append( + atproto.models.AppBskyEmbedImages.Image( + alt=image["alt_text"] if "alt_text" in image else "", + image=upload.blob, + ) + ) + embed = ( + atproto.models.AppBskyEmbedImages.Main(images=embed_images) + if embed_images + else None + ) + + status = [] + reply_to = None + mentions = " ".join([f"@{v}" for v in mentions]) + hashtags = " ".join([f"#{v}" for v in hashtags]) + for text in textwrap.wrap( + content + "\n" + mentions + "\n" + hashtags, + self.max_content_length, + replace_whitespace=False, + ): + facets, last_url = self.parse_facets(text) + if not images or reply_to: + embed = self.handle_url_card(cast(str, last_url)) + + post = self.blueskysocial.send_post( + text, facets=facets, embed=embed, reply_to=reply_to + ) + + for _ in range(5): + data = self.blueskysocial.get_posts([post.uri]).posts + if data: + status.append(data[0].record.text == text) + break + + if reply_to is None: + link = f"https://bsky.app/profile/{self.blueskysocial.me.handle}/post/{post.uri.split('/')[-1]}" + root = atproto.models.create_strong_ref(post) + parent = atproto.models.create_strong_ref(post) + reply_to = atproto.models.AppBskyFeedPost.ReplyRef(parent=parent, root=root) + + return all(status), link diff --git a/lib/plugins/linkedin.py b/lib/plugins/linkedin.py new file mode 100644 index 0000000..0c60ac4 --- /dev/null +++ b/lib/plugins/linkedin.py @@ -0,0 +1,47 @@ +# this file is not working! It is just a template for the final implementation... +import requests + + +class linkedin_client: + def __init__( + self, access_token=None, api_base_url="https://api.linkedin.com/rest/" + ): + self.api_base_url = api_base_url + self.headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json", + "X-Restli-Protocol-Version": "2.0.0", + "LinkedIn-Version": "202403", + } + + def linkedin_post(self, content): + url = self.api_base_url + "posts" + data = { + "author": "urn:li:organization:5515715", + "commentary": content, + "visibility": "PUBLIC", + "distribution": { + "feedDistribution": "MAIN_FEED", + "targetEntities": [], + "thirdPartyDistributionChannels": [], + }, + "lifecycleState": "PUBLISHED", + "isReshareDisabledByAuthor": False, + } + response = requests.post(url, headers=self.headers, json=data) + return response.json() + + def get_profile(self): + url = self.api_base_url + "people/~" + response = requests.get(url, headers=self.headers) + return response.json() + + # This method is not implemented + def create_post(self, content): + self.linkedin_post(content) + return True + linkedin_posts = self.get_profile() + for post in linkedin_posts["posts"]["values"]: + if content in post["commentary"]: + return True + return False diff --git a/lib/plugins/markdown.py b/lib/plugins/markdown.py new file mode 100644 index 0000000..0e74b50 --- /dev/null +++ b/lib/plugins/markdown.py @@ -0,0 +1,33 @@ +import os +import sys +import time + +from github_comment import comment_to_github + + +class markdown_client: + def __init__(self, **kwargs): + self.save_path = kwargs.get("save_path") if "save_path" in kwargs else None + + def create_post(self, content, mentions, hashtags, images, **kwargs): + try: + medias = "\n".join( + [f'![{image.get("alt_text", "")}]({image["url"]})' for image in images] + ) + mentions = " ".join([f"@{v}" for v in mentions]) + hashtags = " ".join([f"#{v}" for v in hashtags]) + social_media = ", ".join(kwargs.get("media")) + text = f"{content}\n{mentions}\n{hashtags}\n{medias}" + if self.save_path: + os.makedirs(self.save_path, exist_ok=True) + with open( + f"{self.save_path}/{time.strftime('%Y%m%d-%H%M%S')}.md", "w" + ) as f: + f.write(text) + if "preview" in sys.argv: + comment_text = f"This is a preview that will be posted to {social_media}:\n\n{text}" + comment_to_github(comment_text) + return True, None + except Exception as e: + print(e) + return False, None diff --git a/lib/plugins/mastodon.py b/lib/plugins/mastodon.py new file mode 100644 index 0000000..bdb31ff --- /dev/null +++ b/lib/plugins/mastodon.py @@ -0,0 +1,60 @@ +import textwrap + +import requests +from bs4 import BeautifulSoup +from mastodon import Mastodon + + +class mastodon_client: + def __init__(self, **kwargs): + self.base_url = kwargs.get("base_url", "https://mstdn.science") + self.mastodon_handle = Mastodon( + access_token=kwargs.get("access_token"), api_base_url=self.base_url + ) + self.max_content_length = kwargs.get("max_content_length", 500) + + def create_post(self, content, mentions, hashtags, images): + media_ids = [] + for image in images[:4]: + response = requests.get(image["url"]) + filename = image["url"].split("/")[-1] + if response.status_code == 200: + with open(filename, "wb") as f: + f.write(response.content) + media_uploaded = self.mastodon_handle.media_post( + media_file=filename, + description=image["alt_text"] if "alt_text" in image else None, + ) + media_ids.append(media_uploaded["id"]) + + toot_id = None + status = [] + mentions = " ".join([f"@{v}" for v in mentions]) + hashtags = " ".join([f"#{v}" for v in hashtags]) + for text in textwrap.wrap( + content + "\n" + mentions + "\n" + hashtags, + self.max_content_length, + replace_whitespace=False, + ): + toot = self.mastodon_handle.status_post( + status=text, + in_reply_to_id=toot_id, + media_ids=media_ids if (media_ids != [] and toot_id == None) else None, + ) + + if not toot_id: + link = f"{self.base_url}/@{toot['account']['acct']}/{toot['id']}" + toot_id = toot["id"] + + for _ in range(3): + post = self.mastodon_handle.status(toot_id) + if post.content: + post_content = BeautifulSoup(post.content, "html.parser").get_text( + separator=" " + ) + status.append( + "".join(post_content.split()) == "".join(text.split()) + ) + break + + return all(status), link diff --git a/lib/plugins/matrix.py b/lib/plugins/matrix.py new file mode 100644 index 0000000..ec8d353 --- /dev/null +++ b/lib/plugins/matrix.py @@ -0,0 +1,92 @@ +import asyncio +import os + +import aiofiles.os +import magic +import requests +from nio import AsyncClient, UploadResponse +from PIL import Image + + +class matrix_client: + + def __init__(self, **kwargs): + self.base_url = kwargs.get("base_url", "https://matrix.org") + self.client = AsyncClient(self.base_url) + self.client.access_token = kwargs.get("access_token") + self.client.user_id = kwargs.get("user_id") + self.client.device_id = kwargs.get("device_id") + self.room_id = kwargs.get("room_id") + + async def async_create_post(self, text, mentions, images): + for image in images: + response = requests.get(image["url"]) + filename = image["url"].split("/")[-1] + if response.status_code == 200: + with open(filename, "wb") as f: + f.write(response.content) + mime_type = magic.from_file(filename, mime=True) + if not mime_type.startswith("image/"): + continue + + (width, height) = Image.open(filename).size + file_stat = await aiofiles.os.stat(filename) + async with aiofiles.open(filename, "r+b") as f: + resp, _ = await self.client.upload( + f, + content_type=mime_type, + filename=os.path.basename(filename), + filesize=file_stat.st_size, + ) + + if not isinstance(resp, UploadResponse): + continue + + content = { + "body": os.path.basename(filename), + "info": { + "size": file_stat.st_size, + "mimetype": mime_type, + "thumbnail_info": None, + "w": width, + "h": height, + "thumbnail_url": None, + }, + "msgtype": "m.image", + "url": resp.content_uri, + } + + try: + await self.client.room_send( + self.room_id, message_type="m.room.message", content=content + ) + except: + return False, None + + if mentions: + text = ( + text + + "\n\n" + + " ".join([f"https://matrix.to/#/@{mention}" for mention in mentions]) + ) + content = { + "msgtype": "m.text", + "format": "org.matrix.custom.html", + "body": text, + } + try: + response = await self.client.room_send( + self.room_id, message_type="m.room.message", content=content + ) + await self.client.close() + message_id = response.event_id + link = f"https://matrix.to/#/{self.room_id}/{message_id}" + except: + return False, None + + return True, link + + def create_post(self, content, mentions, hashtags, images): + # hashtags and alt_texts are not used in this function + result, link = asyncio.run(self.async_create_post(content, mentions, images)) + return result, link diff --git a/lib/plugins/slack.py b/lib/plugins/slack.py new file mode 100644 index 0000000..0fda5f3 --- /dev/null +++ b/lib/plugins/slack.py @@ -0,0 +1,69 @@ +import os +import textwrap + +import requests +from slack_sdk import WebClient + + +class slack_client: + def __init__(self, **kwargs): + self.client = WebClient(token=kwargs.get("access_token")) + self.channel_id = kwargs.get("channel_id") + self.max_content_length = kwargs.get("max_content_length", 40000) + + def upload_images(self, images): + uploaded_files = [] + for image in images: + filename = image["url"].split("/")[-1] + + with requests.get(image["url"]) as response: + if response.status_code != 200: + continue + content_length = len(response.content) + with open(filename, "wb") as temp_file: + temp_file.write(response.content) + + response = self.client.files_getUploadURLExternal( + filename=filename, + length=content_length, + alt_txt=image["alt_text"] if "alt_text" in image else None, + ) + upload_url = response.data["upload_url"] + with open(filename, "rb") as temp_file: + with requests.post( + upload_url, files={"file": temp_file} + ) as upload_response: + if upload_response.status_code != 200: + continue + uploaded_files.append({"id": response.data["file_id"]}) + os.remove(filename) + + response = self.client.files_completeUploadExternal( + files=uploaded_files, channel_id=self.channel_id + ) + return response + + def create_post(self, text, mentions, hashtags, images): + status = [] + link = None + parent_ts = None + for text in textwrap.wrap( + text, + self.max_content_length, + replace_whitespace=False, + ): + response = self.client.chat_postMessage( + channel=self.channel_id, + text=text, + thread_ts=parent_ts if parent_ts else None, + ) + if not parent_ts: + parent_ts = response["ts"] + link = self.client.chat_getPermalink( + channel=self.channel_id, message_ts=parent_ts + )["permalink"] + status.append(response["ok"]) + if images: + response = self.upload_images(images) + status.append(response["ok"]) + return all(status), link From 1206509c87056baa471431f767be97e2875218eb Mon Sep 17 00:00:00 2001 From: Arash Date: Fri, 3 May 2024 14:53:28 +0200 Subject: [PATCH 03/13] Refactoring to usable without github and fixing get variable for plugins. --- .github/workflows/preview.yml | 2 +- .github/workflows/publish_content.yml | 2 +- README.md | Bin 11376 -> 12396 bytes github_run.py | 76 +++++++++++ lib/galaxy-social.py | 152 --------------------- lib/galaxy_social.py | 181 ++++++++++++++++++++++++++ lib/github_comment.py | 25 ---- lib/plugins/bluesky.py | 2 +- lib/plugins/markdown.py | 20 ++- lib/plugins/mastodon.py | 2 + lib/plugins/matrix.py | 1 + plugins.yml | 16 +-- 12 files changed, 284 insertions(+), 195 deletions(-) create mode 100644 github_run.py delete mode 100644 lib/galaxy-social.py create mode 100644 lib/galaxy_social.py delete mode 100644 lib/github_comment.py diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index e53688e..0e64922 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -46,4 +46,4 @@ jobs: CHANGED_FILES: ${{ steps.get_changed_files.outputs.all_changed_files }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ github.event.number }} - run: python -u lib/galaxy-social.py preview + run: python -u github_run.py --preview diff --git a/.github/workflows/publish_content.yml b/.github/workflows/publish_content.yml index 228c8ee..492541c 100644 --- a/.github/workflows/publish_content.yml +++ b/.github/workflows/publish_content.yml @@ -55,7 +55,7 @@ jobs: CHANGED_FILES: ${{ steps.get_changed_files.outputs.all_changed_files }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ github.event.number }} - run: python -u lib/galaxy-social.py + run: python -u github_run.py - name: Commit changes if: steps.get_changed_files.outputs.any_changed == 'true' diff --git a/README.md b/README.md index 4f9a0840997c6bd25dabd09f91e472e42096127d..87f74ee9a6aa7c62721a9b3cb0f921ef27e07450 100644 GIT binary patch delta 860 zcmZWn+e*Vg6g&k*P%B<56(lPtRk8Xg=!=31f+F@wQL$+oYcNeqdNWVOKM?l|`~#sM zB7TDJg8B!}Y#J{}$YyhP&YUxI_T%I&|1_LR%~xI$JNbv$ILjkuW}9ok|TQ4-$(>|xhNK3?nf7zKNC*&4b^?*_xD1 zH5;s-g^M&pg?6vUC9 zMseIFFt;IV)!0qTbKs-Yin*DA%*NL&E*=k=ZnX4VdP3hPC-r$|WPn!??(5gl*)d-I Q8r}f02Gob$yXE)97Y|XULjV8( delta 39 vcmaEp@F8LYkJw}*CArNyVk|6^L%4-D_edwPPL>jt+8m^KjcaqC-aSSD6NL?4 diff --git a/github_run.py b/github_run.py new file mode 100644 index 0000000..008db21 --- /dev/null +++ b/github_run.py @@ -0,0 +1,76 @@ +import argparse +import fnmatch +import os + +import requests +from lib.galaxy_social import galaxy_social + + +class github_run: + def __init__(self): + self.github_token = os.getenv("GITHUB_TOKEN") + self.repo = os.getenv("GITHUB_REPOSITORY") + self.pr_number = os.getenv("PR_NUMBER") + + def comment(self, comment_text): + if not comment_text: + return + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self.github_token}", + "X-GitHub-Api-Version": "2022-11-28", + } + url = ( + f"https://api.github.com/repos/{self.repo}/issues/{self.pr_number}/comments" + ) + data = {"body": comment_text} + response = requests.post(url, headers=headers, json=data) + if response.status_code == 201: + return True + else: + raise Exception( + f"Failed to create github comment!, {response.json().get('message')}" + ) + + def get_files(self): + url = f"https://api.github.com/repos/{self.repo}/pulls/{self.pr_number}/files" + response = requests.get(url) + if response.status_code == 200: + changed_files = response.json() + for file in changed_files: + raw_url = file["raw_url"] + if raw_url.endswith(".md"): + response = requests.get(raw_url) + if response.status_code == 200: + changed_file_path = file["filename"] + with open(changed_file_path, "w") as f: + f.write(response.text) + + changed_files = os.environ.get("CHANGED_FILES") + files_to_process = [] + if changed_files: + for file_path in eval(changed_files.replace("\\", "")): + if file_path.endswith(".md"): + files_to_process.append(file_path) + else: + for root, _, files in os.walk("posts"): + for filename in fnmatch.filter(files, "*.md"): + file_path = os.path.join(root, filename) + files_to_process.append(file_path) + + return files_to_process + +if __name__ == "__main__": + github = github_run() + files_to_process = github.get_files() + + parser = argparse.ArgumentParser(description="Process Markdown files.") + parser.add_argument("--preview", action="store_true", help="Preview the post") + args = parser.parse_args() + + gs = galaxy_social(args.preview) + try: + message = gs.process_files(files_to_process) + except Exception as e: + message = e + github.comment(message) \ No newline at end of file diff --git a/lib/galaxy-social.py b/lib/galaxy-social.py deleted file mode 100644 index 178d69f..0000000 --- a/lib/galaxy-social.py +++ /dev/null @@ -1,152 +0,0 @@ -import fnmatch -import importlib -import json -import os -import sys - -import jsonschema -import markdown -import requests -import yaml -from bs4 import BeautifulSoup -from github_comment import comment_to_github - -with open("plugins.yml", "r") as file: - plugins_config = yaml.safe_load(file) - -plugins = {} -for plugin in plugins_config["plugins"]: - if "preview" in sys.argv and plugin["name"].lower() != "markdown": - continue - - if plugin["enabled"]: - module_name, class_name = plugin["class"].rsplit(".", 1) - - try: - module = importlib.import_module(f"plugins.{module_name}") - plugin_class = getattr(module, class_name) - except: - comment_to_github( - f"Error with plugin {module_name}.{class_name}.", error=True - ) - - try: - config = { - key: os.environ.get(value) - for key, value in plugin["config"].items() - if (not isinstance(value, int) and os.environ.get(value) is not None) - } - except: - comment_to_github( - f"Missing config for {module_name}.{class_name}.", error=True - ) - - try: - plugins[plugin["name"].lower()] = plugin_class(**config) - except: - comment_to_github( - f"Invalid config for {module_name}.{class_name}.", error=True - ) - - -def parse_markdown_file(file_path): - with open(file_path, "r") as file: - content = file.read() - _, metadata, text = content.split("---\n", 2) - try: - metadata = yaml.safe_load(metadata) - with open(".schema.yaml", "r") as f: - schema = yaml.safe_load(f) - jsonschema.validate(instance=metadata, schema=schema) - except: - comment_to_github(f"Invalid metadata in {file_path}.", error=True) - - metadata["media"] = [media.lower() for media in metadata["media"]] - - for media in metadata["media"]: - if not any(item["name"].lower() == media for item in plugins_config["plugins"]): - comment_to_github(f"Invalid media {media}.", error=True) - - metadata["mentions"] = ( - {key.lower(): value for key, value in metadata["mentions"].items()} - if metadata.get("mentions") - else {} - ) - metadata["hashtags"] = ( - {key.lower(): value for key, value in metadata["hashtags"].items()} - if metadata.get("hashtags") - else {} - ) - markdown_content = markdown.markdown(text.strip()) - plain_content = BeautifulSoup(markdown_content, "html.parser").get_text( - separator="\n" - ) - return plain_content, metadata - - -def process_markdown_file(file_path, processed_files): - content, metadata = parse_markdown_file(file_path) - if "preview" in sys.argv: - try: - plugins["markdown"].create_post( - content, [], [], metadata.get("images", []), media=metadata["media"] - ) - return processed_files - except: - comment_to_github(f"Failed to create preview for {file_path}.", error=True) - stats = {} - url = {} - for media in metadata["media"]: - if file_path in processed_files and media in processed_files[file_path]: - stats[media] = processed_files[file_path][media] - continue - mentions = metadata.get("mentions", {}).get(media, []) - hashtags = metadata.get("hashtags", {}).get(media, []) - images = metadata.get("images", []) - stats[media], url[media] = plugins[media].create_post( - content, mentions, hashtags, images - ) - url_text = "\n".join([f"[{media}]({link})" for media, link in url.items() if link]) - comment_to_github(f"Posted to:\n\n{url_text}") - - processed_files[file_path] = stats - print(f"Processed {file_path}: {stats}") - return processed_files - - -def main(): - repo = os.getenv("GITHUB_REPOSITORY") - pr_number = os.getenv("PR_NUMBER") - url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}/files" - response = requests.get(url) - print(response.json(), url) - if response.status_code == 200: - changed_files = response.json() - for file in changed_files: - raw_url = file["raw_url"] - if raw_url.endswith(".md"): - response = requests.get(raw_url) - if response.status_code == 200: - with open(file["filename"], "w") as f: - f.write(response.text) - - processed_files = {} - if os.path.exists("processed_files.json"): - with open("processed_files.json", "r") as file: - processed_files = json.load(file) - changed_files = os.environ.get("CHANGED_FILES") - if changed_files: - for file_path in eval(changed_files.replace("\\", "")): - if file_path.endswith(".md"): - processed_files = process_markdown_file(file_path, processed_files) - else: - for root, _, files in os.walk("posts"): - for filename in fnmatch.filter(files, "*.md"): - file_path = os.path.join(root, filename) - processed_files = process_markdown_file(file_path, processed_files) - with open("processed_files.json", "w") as file: - json.dump(processed_files, file) - - -if __name__ == "__main__": - main() diff --git a/lib/galaxy_social.py b/lib/galaxy_social.py new file mode 100644 index 0000000..d353473 --- /dev/null +++ b/lib/galaxy_social.py @@ -0,0 +1,181 @@ +from argparse import ArgumentParser +from fnmatch import filter +from importlib import import_module +import json +import os +import sys + +from jsonschema import validate +from markdown import markdown +from yaml import safe_load as yaml +from bs4 import BeautifulSoup + + +class galaxy_social: + def __init__(self, preview: bool = False): + self.preview = preview + plugins_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "plugins.yml" + ) + with open(plugins_path, "r") as file: + self.plugins_config = yaml(file) + + self.plugins = {} + for plugin in self.plugins_config["plugins"]: + if preview and plugin["name"].lower() != "markdown": + continue + + if plugin["enabled"]: + module_name, class_name = plugin["class"].rsplit(".", 1) + print(os.path.dirname(os.path.abspath(__file__))) + try: + module_path = f"{'lib.' if not os.path.dirname(os.path.abspath(sys.argv[0])).endswith('lib') else ''}plugins.{module_name}" + module = import_module(module_path) + plugin_class = getattr(module, class_name) + except Exception as e: + raise Exception( + f"Error with plugin {module_name}.{class_name}.\n{e}" + ) + + try: + config = {} + for key, value in plugin["config"].items(): + if isinstance(value, str) and value.startswith("$"): + if os.environ.get(value[1:]): + config[key] = os.environ.get(value[1:]) + else: + raise Exception( + f"Missing environment variable {value[1:]}." + ) + else: + config[key] = value + except Exception as e: + raise Exception( + f"Missing config for {module_name}.{class_name}.\n{e}" + ) + + try: + self.plugins[plugin["name"].lower()] = plugin_class(**config) + except Exception as e: + raise Exception( + f"Invalid config for {module_name}.{class_name}.\nChange configs in plugins.yml.\n{e}" + ) + + def parse_markdown_file(self, file_path): + with open(file_path, "r") as file: + content = file.read() + _, metadata, text = content.split("---\n", 2) + try: + metadata = yaml(metadata) + schema_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", ".schema.yaml" + ) + with open(schema_path, "r") as f: + schema = yaml(f) + validate(instance=metadata, schema=schema) + except Exception as e: + raise Exception(f"Invalid metadata in {file_path}.\n{e}") + + metadata["media"] = [media.lower() for media in metadata["media"]] + + for media in metadata["media"]: + if not any( + item["name"].lower() == media for item in self.plugins_config["plugins"] + ): + raise Exception(f"Invalid media {media}.") + + metadata["mentions"] = ( + {key.lower(): value for key, value in metadata["mentions"].items()} + if metadata.get("mentions") + else {} + ) + metadata["hashtags"] = ( + {key.lower(): value for key, value in metadata["hashtags"].items()} + if metadata.get("hashtags") + else {} + ) + markdown_content = markdown(text.strip()) + plain_content = BeautifulSoup(markdown_content, "html.parser").get_text( + separator="\n" + ) + return plain_content, metadata + + def process_markdown_file(self, file_path, processed_files): + content, metadata = self.parse_markdown_file(file_path) + if self.preview: + try: + _, _, message = self.plugins["markdown"].create_post( + content, + [], + [], + metadata.get("images", []), + media=metadata["media"], + preview=True, + ) + return processed_files, message + except Exception as e: + raise Exception(f"Failed to create preview for {file_path}.\n{e}") + stats = {} + url = {} + for media in metadata["media"]: + if file_path in processed_files and media in processed_files[file_path]: + stats[media] = processed_files[file_path][media] + continue + mentions = metadata.get("mentions", {}).get(media, []) + hashtags = metadata.get("hashtags", {}).get(media, []) + images = metadata.get("images", []) + stats[media], url[media] = self.plugins[media].create_post( + content, mentions, hashtags, images + ) + url_text = "\n".join( + [f"[{media}]({link})" for media, link in url.items() if link] + ) + message = f"Posted to:\n\n{url_text}" + + processed_files[file_path] = stats + print(f"Processed {file_path}: {stats}") + return processed_files, message + + def process_files(self, files_to_process): + processed_files = {} + message = "" + processed_files_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "processed_files.json" + ) + if os.path.exists(processed_files_path): + with open(processed_files_path, "r") as file: + processed_files = json.load(file) + for file_path in files_to_process: + processed_files, message = self.process_markdown_file( + file_path, processed_files + ) + with open(processed_files_path, "w") as file: + json.dump(processed_files, file) + return message + + +if __name__ == "__main__": + parser = ArgumentParser(description="Process Markdown files.") + parser.add_argument("--files", nargs="+", help="List of files to process") + parser.add_argument("--folder", help="Folder containing files to process") + parser.add_argument("--preview", action="store_true", help="Preview the post") + args = parser.parse_args() + + if args.files: + files_to_process = args.files + elif args.folder: + files_to_process = [ + os.path.join(root, filename) + for root, _, files in os.walk(args.folder) + for filename in filter(files, "*.md") + ] + else: + parser.print_help() + exit(1) + if not files_to_process: + print("No files to process.") + exit(0) + print(f"Processing {len(files_to_process)} file(s): {files_to_process}\n") + gs = galaxy_social(args.preview) + message = gs.process_files(files_to_process) + print(message) diff --git a/lib/github_comment.py b/lib/github_comment.py deleted file mode 100644 index 2ab1388..0000000 --- a/lib/github_comment.py +++ /dev/null @@ -1,25 +0,0 @@ -import os - -import requests - - -def comment_to_github(comment_text, error=False): - github_token = os.getenv("GITHUB_TOKEN") - repo_owner, repo_name = os.getenv("GITHUB_REPOSITORY").split("/") - pr_number = os.getenv("PR_NUMBER") - headers = { - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {github_token}", - "X-GitHub-Api-Version": "2022-11-28", - } - url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/issues/{pr_number}/comments" - data = {"body": comment_text} - response = requests.post(url, headers=headers, json=data) - if response.status_code != 201: - raise Exception( - f"Failed to create github comment!, {response.json().get('message')}" - ) - else: - if error: - raise Exception(comment_text) - return True diff --git a/lib/plugins/bluesky.py b/lib/plugins/bluesky.py index 2724230..4289a2b 100644 --- a/lib/plugins/bluesky.py +++ b/lib/plugins/bluesky.py @@ -121,7 +121,7 @@ def handle_url_card( if response.status_code == 200: soup = BeautifulSoup(response.text, "html.parser") title_tag = soup.find("meta", attrs={"property": "og:title"}) - title_tag_alt = soup.title.string + title_tag_alt = soup.title.string if soup.title else None description_tag = soup.find("meta", attrs={"property": "og:description"}) description_tag_alt = soup.find("meta", attrs={"name": "description"}) image_tag = soup.find("meta", attrs={"property": "og:image"}) diff --git a/lib/plugins/markdown.py b/lib/plugins/markdown.py index 0e74b50..4c30d89 100644 --- a/lib/plugins/markdown.py +++ b/lib/plugins/markdown.py @@ -1,13 +1,19 @@ import os -import sys import time -from github_comment import comment_to_github - class markdown_client: def __init__(self, **kwargs): - self.save_path = kwargs.get("save_path") if "save_path" in kwargs else None + self.save_path = ( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "..", + kwargs.get("save_path"), + ) + if "save_path" in kwargs + else None + ) def create_post(self, content, mentions, hashtags, images, **kwargs): try: @@ -16,7 +22,6 @@ def create_post(self, content, mentions, hashtags, images, **kwargs): ) mentions = " ".join([f"@{v}" for v in mentions]) hashtags = " ".join([f"#{v}" for v in hashtags]) - social_media = ", ".join(kwargs.get("media")) text = f"{content}\n{mentions}\n{hashtags}\n{medias}" if self.save_path: os.makedirs(self.save_path, exist_ok=True) @@ -24,9 +29,10 @@ def create_post(self, content, mentions, hashtags, images, **kwargs): f"{self.save_path}/{time.strftime('%Y%m%d-%H%M%S')}.md", "w" ) as f: f.write(text) - if "preview" in sys.argv: + if kwargs.get("preview"): + social_media = ", ".join(kwargs.get("media")) comment_text = f"This is a preview that will be posted to {social_media}:\n\n{text}" - comment_to_github(comment_text) + return True, None, comment_text return True, None except Exception as e: print(e) diff --git a/lib/plugins/mastodon.py b/lib/plugins/mastodon.py index bdb31ff..1d9dde1 100644 --- a/lib/plugins/mastodon.py +++ b/lib/plugins/mastodon.py @@ -1,3 +1,4 @@ +import os import textwrap import requests @@ -26,6 +27,7 @@ def create_post(self, content, mentions, hashtags, images): description=image["alt_text"] if "alt_text" in image else None, ) media_ids.append(media_uploaded["id"]) + os.remove(filename) toot_id = None status = [] diff --git a/lib/plugins/matrix.py b/lib/plugins/matrix.py index ec8d353..c0ecf20 100644 --- a/lib/plugins/matrix.py +++ b/lib/plugins/matrix.py @@ -38,6 +38,7 @@ async def async_create_post(self, text, mentions, images): filename=os.path.basename(filename), filesize=file_stat.st_size, ) + os.remove(filename) if not isinstance(resp, UploadResponse): continue diff --git a/plugins.yml b/plugins.yml index 596b253..7e43191 100644 --- a/plugins.yml +++ b/plugins.yml @@ -4,7 +4,7 @@ plugins: enabled: true config: base_url: "https://mstdn.science" - access_token: MASTODON_ACCESS_TOKEN + access_token: $MASTODON_ACCESS_TOKEN max_content_length: 500 - name: bluesky @@ -12,8 +12,8 @@ plugins: enabled: true config: base_url: "https://bsky.social" - username: BLUESKY_USERNAME - password: BLUESKY_PASSWORD + username: $BLUESKY_USERNAME + password: $BLUESKY_PASSWORD max_content_length: 300 - name: matrix @@ -21,16 +21,16 @@ plugins: enabled: true config: base_url: "https://matrix.org" - access_token: MATRIX_ACCESS_TOKEN - room_id: MATRIX_ROOM_ID - user_id: MATRIX_USER_ID + access_token: $MATRIX_ACCESS_TOKEN + room_id: $MATRIX_ROOM_ID + user_id: $MATRIX_USER_ID - name: slack class: slack.slack_client enabled: true config: - access_token: SLACK_ACCESS_TOKEN - channel_id: SLACK_CHANNEL_ID + access_token: $SLACK_ACCESS_TOKEN + channel_id: $SLACK_CHANNEL_ID max_content_length: 40000 - name: markdown From 0da0003c153e6b89e7f25c383e19d2baea1008f8 Mon Sep 17 00:00:00 2001 From: Arash Date: Fri, 3 May 2024 14:59:48 +0200 Subject: [PATCH 04/13] sort imports and format --- github_run.py | 6 ++++-- lib/galaxy_social.py | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/github_run.py b/github_run.py index 008db21..d55dadc 100644 --- a/github_run.py +++ b/github_run.py @@ -3,6 +3,7 @@ import os import requests + from lib.galaxy_social import galaxy_social @@ -60,10 +61,11 @@ def get_files(self): return files_to_process + if __name__ == "__main__": github = github_run() files_to_process = github.get_files() - + parser = argparse.ArgumentParser(description="Process Markdown files.") parser.add_argument("--preview", action="store_true", help="Preview the post") args = parser.parse_args() @@ -73,4 +75,4 @@ def get_files(self): message = gs.process_files(files_to_process) except Exception as e: message = e - github.comment(message) \ No newline at end of file + github.comment(message) diff --git a/lib/galaxy_social.py b/lib/galaxy_social.py index d353473..177654b 100644 --- a/lib/galaxy_social.py +++ b/lib/galaxy_social.py @@ -1,14 +1,14 @@ -from argparse import ArgumentParser -from fnmatch import filter -from importlib import import_module import json import os import sys +from argparse import ArgumentParser +from fnmatch import filter +from importlib import import_module +from bs4 import BeautifulSoup from jsonschema import validate from markdown import markdown from yaml import safe_load as yaml -from bs4 import BeautifulSoup class galaxy_social: From 9aafd3d5bab12fcd2851fd0d38e6dd1436cd7382 Mon Sep 17 00:00:00 2001 From: Arash Date: Mon, 6 May 2024 17:48:50 +0200 Subject: [PATCH 05/13] add support for --json-out, save_path could be rel or abs, file will be saved under tmp, warning about ignored images, refactoring --- .github/workflows/publish_content.yml | 2 +- README.md | Bin 12396 -> 12620 bytes github_run.py | 15 ++++- lib/galaxy_social.py | 79 +++++++++++++++----------- lib/plugins/bluesky.py | 6 +- lib/plugins/markdown.py | 40 +++++++------ lib/plugins/mastodon.py | 25 ++++---- lib/plugins/matrix.py | 27 ++++----- lib/plugins/slack.py | 29 +++++----- 9 files changed, 127 insertions(+), 96 deletions(-) diff --git a/.github/workflows/publish_content.yml b/.github/workflows/publish_content.yml index 492541c..f5e8eee 100644 --- a/.github/workflows/publish_content.yml +++ b/.github/workflows/publish_content.yml @@ -55,7 +55,7 @@ jobs: CHANGED_FILES: ${{ steps.get_changed_files.outputs.all_changed_files }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ github.event.number }} - run: python -u github_run.py + run: python -u github_run.py --json-out processed_files.json - name: Commit changes if: steps.get_changed_files.outputs.any_changed == 'true' diff --git a/README.md b/README.md index 87f74ee9a6aa7c62721a9b3cb0f921ef27e07450..6c6c10d583b814c59d03ed74c6c11533f7682b76 100644 GIT binary patch delta 247 zcmaEpa3*PkjIN*?LncEGLn=ctg93vd2u{}2O%zRF&;`N*h9aO$8Bj$kLpejjiQfkQxP`8juMPeOX}nJfI2r45bVulePI3>-895q6tu)i9r1d z49Q?MISeU4b`sd6WT2^u40#OcU|u`ZxGA{rC delta 52 zcmX?;^d@10j4p39gBwF8Lk>eKLoq|_WL@1vL0tx2h608nphy`|CY7Ol@?u@*&HMEF Fm;i~44!i&W diff --git a/github_run.py b/github_run.py index d55dadc..27e7f9a 100644 --- a/github_run.py +++ b/github_run.py @@ -14,7 +14,8 @@ def __init__(self): self.pr_number = os.getenv("PR_NUMBER") def comment(self, comment_text): - if not comment_text: + print(comment_text) + if not comment_text or not self.github_token or not self.repo or not self.pr_number: return headers = { "Accept": "application/vnd.github+json", @@ -65,12 +66,20 @@ def get_files(self): if __name__ == "__main__": github = github_run() files_to_process = github.get_files() + if not files_to_process: + github.comment("No files to process.") + exit() - parser = argparse.ArgumentParser(description="Process Markdown files.") + parser = argparse.ArgumentParser(description="Galaxy Social.") parser.add_argument("--preview", action="store_true", help="Preview the post") + parser.add_argument( + "--json-out", + help="Output json file for processed files", + default="processed_files.json", + ) args = parser.parse_args() - gs = galaxy_social(args.preview) + gs = galaxy_social(args.preview, args.json_out) try: message = gs.process_files(files_to_process) except Exception as e: diff --git a/lib/galaxy_social.py b/lib/galaxy_social.py index 177654b..2d92a47 100644 --- a/lib/galaxy_social.py +++ b/lib/galaxy_social.py @@ -12,8 +12,9 @@ class galaxy_social: - def __init__(self, preview: bool = False): + def __init__(self, preview: bool, json_out: str): self.preview = preview + self.json_out = json_out plugins_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "plugins.yml" ) @@ -27,7 +28,6 @@ def __init__(self, preview: bool = False): if plugin["enabled"]: module_name, class_name = plugin["class"].rsplit(".", 1) - print(os.path.dirname(os.path.abspath(__file__))) try: module_path = f"{'lib.' if not os.path.dirname(os.path.abspath(sys.argv[0])).endswith('lib') else ''}plugins.{module_name}" module = import_module(module_path) @@ -39,16 +39,17 @@ def __init__(self, preview: bool = False): try: config = {} - for key, value in plugin["config"].items(): - if isinstance(value, str) and value.startswith("$"): - if os.environ.get(value[1:]): - config[key] = os.environ.get(value[1:]) + if plugin.get("config"): + for key, value in plugin["config"].items(): + if isinstance(value, str) and value.startswith("$"): + if os.environ.get(value[1:]): + config[key] = os.environ.get(value[1:]) + else: + raise Exception( + f"Missing environment variable {value[1:]}." + ) else: - raise Exception( - f"Missing environment variable {value[1:]}." - ) - else: - config[key] = value + config[key] = value except Exception as e: raise Exception( f"Missing config for {module_name}.{class_name}.\n{e}" @@ -64,8 +65,8 @@ def __init__(self, preview: bool = False): def parse_markdown_file(self, file_path): with open(file_path, "r") as file: content = file.read() - _, metadata, text = content.split("---\n", 2) try: + _, metadata, text = content.split("---\n", 2) metadata = yaml(metadata) schema_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", ".schema.yaml" @@ -105,32 +106,34 @@ def process_markdown_file(self, file_path, processed_files): if self.preview: try: _, _, message = self.plugins["markdown"].create_post( - content, - [], - [], - metadata.get("images", []), + content=content, + mentions=[], + hashtags=[], + images=metadata.get("images", []), media=metadata["media"], preview=True, + file_path=file_path, ) return processed_files, message except Exception as e: raise Exception(f"Failed to create preview for {file_path}.\n{e}") stats = {} url = {} + if file_path in processed_files: + stats = processed_files[file_path] for media in metadata["media"]: if file_path in processed_files and media in processed_files[file_path]: - stats[media] = processed_files[file_path][media] continue mentions = metadata.get("mentions", {}).get(media, []) hashtags = metadata.get("hashtags", {}).get(media, []) images = metadata.get("images", []) stats[media], url[media] = self.plugins[media].create_post( - content, mentions, hashtags, images + content, mentions, hashtags, images, file_path=file_path ) url_text = "\n".join( [f"[{media}]({link})" for media, link in url.items() if link] ) - message = f"Posted to:\n\n{url_text}" + message = f"Posted to:\n\n{url_text}" if url_text else "No posts created." processed_files[file_path] = stats print(f"Processed {file_path}: {stats}") @@ -138,9 +141,9 @@ def process_markdown_file(self, file_path, processed_files): def process_files(self, files_to_process): processed_files = {} - message = "" + messages = "---\n" processed_files_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "..", "processed_files.json" + os.path.dirname(os.path.abspath(__file__)), "..", self.json_out ) if os.path.exists(processed_files_path): with open(processed_files_path, "r") as file: @@ -149,33 +152,45 @@ def process_files(self, files_to_process): processed_files, message = self.process_markdown_file( file_path, processed_files ) - with open(processed_files_path, "w") as file: - json.dump(processed_files, file) - return message + messages += f"{message}\n\n---\n" + if not self.preview: + with open(processed_files_path, "w") as file: + json.dump(processed_files, file) + return messages if __name__ == "__main__": - parser = ArgumentParser(description="Process Markdown files.") - parser.add_argument("--files", nargs="+", help="List of files to process") - parser.add_argument("--folder", help="Folder containing files to process") + parser = ArgumentParser(description="Galaxy Social.") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--files", nargs="+", help="List of files to process") + group.add_argument("--folder", help="Folder containing files to process") parser.add_argument("--preview", action="store_true", help="Preview the post") + parser.add_argument( + "--json-out", + help="Output json file for processed files", + default="processed_files.json", + ) args = parser.parse_args() if args.files: files_to_process = args.files + files_not_exist = [ + file for file in files_to_process if not os.path.exists(file) + ] + if files_not_exist: + raise Exception(f"{', '.join(files_not_exist)} -> not exist.") elif args.folder: + if not os.path.exists(args.folder): + raise Exception(f"{args.folder} -> not exist.") files_to_process = [ os.path.join(root, filename) for root, _, files in os.walk(args.folder) for filename in filter(files, "*.md") ] - else: - parser.print_help() - exit(1) if not files_to_process: print("No files to process.") - exit(0) + exit() print(f"Processing {len(files_to_process)} file(s): {files_to_process}\n") - gs = galaxy_social(args.preview) + gs = galaxy_social(args.preview, args.json_out) message = gs.process_files(files_to_process) print(message) diff --git a/lib/plugins/bluesky.py b/lib/plugins/bluesky.py index 4289a2b..ed37392 100644 --- a/lib/plugins/bluesky.py +++ b/lib/plugins/bluesky.py @@ -150,12 +150,14 @@ def handle_url_card( return embed_external def create_post( - self, content, mentions, hashtags, images + self, content, mentions, hashtags, images, **kwargs ) -> Tuple[bool, Optional[str]]: embed_images = [] for image in images[:4]: response = requests.get(image["url"]) - if response.status_code == 200: + if response.status_code == 200 and response.headers.get( + "Content-Type", "" + ).startswith("image/"): img_data = response.content upload = self.blueskysocial.com.atproto.repo.upload_blob(img_data) embed_images.append( diff --git a/lib/plugins/markdown.py b/lib/plugins/markdown.py index 4c30d89..49e3567 100644 --- a/lib/plugins/markdown.py +++ b/lib/plugins/markdown.py @@ -4,36 +4,42 @@ class markdown_client: def __init__(self, **kwargs): - self.save_path = ( - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "..", - "..", - kwargs.get("save_path"), - ) - if "save_path" in kwargs - else None + self.save_path = kwargs.get("save_path") and ( + kwargs["save_path"] + if os.path.isabs(kwargs["save_path"]) + else os.path.join(os.getcwd(), kwargs["save_path"]) ) def create_post(self, content, mentions, hashtags, images, **kwargs): try: - medias = "\n".join( + _images = "\n".join( [f'![{image.get("alt_text", "")}]({image["url"]})' for image in images] ) mentions = " ".join([f"@{v}" for v in mentions]) hashtags = " ".join([f"#{v}" for v in hashtags]) - text = f"{content}\n{mentions}\n{hashtags}\n{medias}" + text = f"{content}\n{mentions}\n{hashtags}\n{_images}" if self.save_path: os.makedirs(self.save_path, exist_ok=True) - with open( - f"{self.save_path}/{time.strftime('%Y%m%d-%H%M%S')}.md", "w" - ) as f: + prefix = ( + kwargs.get("file_path", "").replace("/", "-").replace(".md", "") + ) + file_name = ( + f"{self.save_path}/{prefix}_{time.strftime('%Y%m%d-%H%M%S')}.md" + ) + with open(file_name, "w") as f: f.write(text) if kwargs.get("preview"): - social_media = ", ".join(kwargs.get("media")) - comment_text = f"This is a preview that will be posted to {social_media}:\n\n{text}" + social_media = ", ".join(kwargs.get("media", [])) + pre_comment_text = "" + if len(images) > 4 and ( + "mastodon" in social_media or "bluesky" in social_media + ): + pre_comment_text = f"Please note that Mastodon and Bluesky only support up to 4 images in a single post. The first 4 images will be included in the post, and the rest will be ignored.\n" + comment_text = f"File: {kwargs.get('file_path')}\n{pre_comment_text}This is a preview that will be posted to {social_media}:\n\n{text}" return True, None, comment_text return True, None except Exception as e: - print(e) + if kwargs.get("preview", False): + print(e) + return False, None, e return False, None diff --git a/lib/plugins/mastodon.py b/lib/plugins/mastodon.py index 1d9dde1..6ffb816 100644 --- a/lib/plugins/mastodon.py +++ b/lib/plugins/mastodon.py @@ -1,4 +1,4 @@ -import os +import tempfile import textwrap import requests @@ -14,20 +14,21 @@ def __init__(self, **kwargs): ) self.max_content_length = kwargs.get("max_content_length", 500) - def create_post(self, content, mentions, hashtags, images): + def create_post(self, content, mentions, hashtags, images, **kwargs): media_ids = [] for image in images[:4]: response = requests.get(image["url"]) - filename = image["url"].split("/")[-1] - if response.status_code == 200: - with open(filename, "wb") as f: - f.write(response.content) - media_uploaded = self.mastodon_handle.media_post( - media_file=filename, - description=image["alt_text"] if "alt_text" in image else None, - ) - media_ids.append(media_uploaded["id"]) - os.remove(filename) + if response.status_code == 200 and response.headers.get( + "Content-Type", "" + ).startswith("image/"): + with tempfile.NamedTemporaryFile() as temp: + temp.write(response.content) + temp.flush() + media_uploaded = self.mastodon_handle.media_post( + media_file=temp.name, + description=image["alt_text"] if "alt_text" in image else None, + ) + media_ids.append(media_uploaded["id"]) toot_id = None status = [] diff --git a/lib/plugins/matrix.py b/lib/plugins/matrix.py index c0ecf20..e3e438d 100644 --- a/lib/plugins/matrix.py +++ b/lib/plugins/matrix.py @@ -1,5 +1,5 @@ import asyncio -import os +import tempfile import aiofiles.os import magic @@ -21,30 +21,31 @@ def __init__(self, **kwargs): async def async_create_post(self, text, mentions, images): for image in images: response = requests.get(image["url"]) - filename = image["url"].split("/")[-1] - if response.status_code == 200: - with open(filename, "wb") as f: - f.write(response.content) - mime_type = magic.from_file(filename, mime=True) + if response.status_code != 200: + continue + image_name = image["url"].split("/")[-1] + temp = tempfile.NamedTemporaryFile() + temp.write(response.content) + temp.flush() + mime_type = magic.from_file(temp.name, mime=True) if not mime_type.startswith("image/"): continue - (width, height) = Image.open(filename).size - file_stat = await aiofiles.os.stat(filename) - async with aiofiles.open(filename, "r+b") as f: + width, height = Image.open(temp.name).size + file_stat = await aiofiles.os.stat(temp.name) + async with aiofiles.open(temp.name, "r+b") as f: resp, _ = await self.client.upload( f, content_type=mime_type, - filename=os.path.basename(filename), + filename=image_name, filesize=file_stat.st_size, ) - os.remove(filename) if not isinstance(resp, UploadResponse): continue content = { - "body": os.path.basename(filename), + "body": image_name, "info": { "size": file_stat.st_size, "mimetype": mime_type, @@ -87,7 +88,7 @@ async def async_create_post(self, text, mentions, images): return True, link - def create_post(self, content, mentions, hashtags, images): + def create_post(self, content, mentions, hashtags, images, **kwargs): # hashtags and alt_texts are not used in this function result, link = asyncio.run(self.async_create_post(content, mentions, images)) return result, link diff --git a/lib/plugins/slack.py b/lib/plugins/slack.py index 0fda5f3..7619523 100644 --- a/lib/plugins/slack.py +++ b/lib/plugins/slack.py @@ -1,4 +1,3 @@ -import os import textwrap import requests @@ -17,33 +16,31 @@ def upload_images(self, images): filename = image["url"].split("/")[-1] with requests.get(image["url"]) as response: - if response.status_code != 200: + if response.status_code != 200 or not response.headers.get( + "Content-Type", "" + ).startswith("image/"): continue - content_length = len(response.content) - with open(filename, "wb") as temp_file: - temp_file.write(response.content) + image_content = response.content response = self.client.files_getUploadURLExternal( filename=filename, - length=content_length, + length=len(image_content), alt_txt=image["alt_text"] if "alt_text" in image else None, ) - upload_url = response.data["upload_url"] - with open(filename, "rb") as temp_file: - with requests.post( - upload_url, files={"file": temp_file} - ) as upload_response: - if upload_response.status_code != 200: - continue - uploaded_files.append({"id": response.data["file_id"]}) - os.remove(filename) + + with requests.post( + response["upload_url"], files={"file": image_content} + ) as upload_response: + if upload_response.status_code != 200: + continue + uploaded_files.append({"id": response["file_id"]}) response = self.client.files_completeUploadExternal( files=uploaded_files, channel_id=self.channel_id ) return response - def create_post(self, text, mentions, hashtags, images): + def create_post(self, text, mentions, hashtags, images, **kwargs): status = [] link = None parent_ts = None From c110cbb943d9f64362c05c3b19390d00b12b303c Mon Sep 17 00:00:00 2001 From: Arash Date: Mon, 6 May 2024 18:15:45 +0200 Subject: [PATCH 06/13] Add some links about github pr to readme file --- README.md | Bin 12620 -> 14244 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/README.md b/README.md index 6c6c10d583b814c59d03ed74c6c11533f7682b76..6bc150e6159594549c8dd36713078f3dcda21b1a 100644 GIT binary patch delta 1269 zcmds$!AiqG5Qb-q7pb?j^q>ooQY309J$O(M(bp)VNt)VV+tip+JczF#>;qK6t043# z3i>Kuym<7Vjob8~#ZwPb(#-D8%>Mai=GA|SUrItWl_jd`bK~G{Ktt-{=Tbr;U11f{ zHKlm!6i}?sJn~^BGD6BcF*35asDCxakB5QXIQ?gp5bhGt843nl4!oW5+154sUO+Sp=NdBYDw%y9uBK9|d+`qOw}l3aKQ*3Q z7cM}8H&?zs!~fI8k%r4&JzYZ&yuoyi~xg=H}_6VBteP{AqkgE_eE$%FV;If*p z!Q_9Nf+J0r=jaeLLbTS9A~lxfO`e``L8PNTid*|O`0)(&(1zUPG4gR-g7tix+Pa1| eHI2XWMEM^<-dhBjn=FHrje%)?XR^N5sC@xkObkH) delta 60 zcmV-C0K@;JZ_Hw_6akZxFcy<60SuGG0xpyE0xXjt1n!d%1u&DcE)=s81_%j}WE7Jy S4;GWW5GIp=BowoT5*{UkdK0Mt From 56d64ca25450918f4d880f2d1c7c32213ecdc532 Mon Sep 17 00:00:00 2001 From: Arash Kadkhodaei Date: Tue, 7 May 2024 13:21:45 +0200 Subject: [PATCH 07/13] change path.exists to path.isfile for checking the arg files. Co-authored-by: Wolfgang Maier --- lib/galaxy_social.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy_social.py b/lib/galaxy_social.py index 2d92a47..7713a28 100644 --- a/lib/galaxy_social.py +++ b/lib/galaxy_social.py @@ -175,7 +175,7 @@ def process_files(self, files_to_process): if args.files: files_to_process = args.files files_not_exist = [ - file for file in files_to_process if not os.path.exists(file) + file for file in files_to_process if not os.path.isfile(file) ] if files_not_exist: raise Exception(f"{', '.join(files_not_exist)} -> not exist.") From 27ea4bcf74829f247b1aad942c9b39c5d384e429 Mon Sep 17 00:00:00 2001 From: Arash Kadkhodaei Date: Tue, 7 May 2024 13:23:35 +0200 Subject: [PATCH 08/13] change path.exists to path.isdir for the args folder. Co-authored-by: Wolfgang Maier --- lib/galaxy_social.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy_social.py b/lib/galaxy_social.py index 7713a28..fbc99ff 100644 --- a/lib/galaxy_social.py +++ b/lib/galaxy_social.py @@ -180,7 +180,7 @@ def process_files(self, files_to_process): if files_not_exist: raise Exception(f"{', '.join(files_not_exist)} -> not exist.") elif args.folder: - if not os.path.exists(args.folder): + if not os.path.isdir(args.folder): raise Exception(f"{args.folder} -> not exist.") files_to_process = [ os.path.join(root, filename) From e0d0d755f1d9741f52546226c9d49809f4a14dee Mon Sep 17 00:00:00 2001 From: Arash Kadkhodaei Date: Tue, 7 May 2024 13:26:42 +0200 Subject: [PATCH 09/13] change config check to use try except instead of if condition. Co-authored-by: Wolfgang Maier --- lib/galaxy_social.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy_social.py b/lib/galaxy_social.py index fbc99ff..1a350b6 100644 --- a/lib/galaxy_social.py +++ b/lib/galaxy_social.py @@ -42,9 +42,9 @@ def __init__(self, preview: bool, json_out: str): if plugin.get("config"): for key, value in plugin["config"].items(): if isinstance(value, str) and value.startswith("$"): - if os.environ.get(value[1:]): - config[key] = os.environ.get(value[1:]) - else: + try: + config[key] = os.environ[value[1:]] + except KeyError: raise Exception( f"Missing environment variable {value[1:]}." ) From baade6795b23a3b5e7198fcfe486861bebba8d30 Mon Sep 17 00:00:00 2001 From: Arash Date: Tue, 7 May 2024 13:28:03 +0200 Subject: [PATCH 10/13] use sys.exit instead of exit. --- github_run.py | 3 ++- lib/galaxy_social.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/github_run.py b/github_run.py index 27e7f9a..6ec635a 100644 --- a/github_run.py +++ b/github_run.py @@ -1,6 +1,7 @@ import argparse import fnmatch import os +import sys import requests @@ -68,7 +69,7 @@ def get_files(self): files_to_process = github.get_files() if not files_to_process: github.comment("No files to process.") - exit() + sys.exit() parser = argparse.ArgumentParser(description="Galaxy Social.") parser.add_argument("--preview", action="store_true", help="Preview the post") diff --git a/lib/galaxy_social.py b/lib/galaxy_social.py index 1a350b6..184964d 100644 --- a/lib/galaxy_social.py +++ b/lib/galaxy_social.py @@ -189,7 +189,7 @@ def process_files(self, files_to_process): ] if not files_to_process: print("No files to process.") - exit() + sys.exit() print(f"Processing {len(files_to_process)} file(s): {files_to_process}\n") gs = galaxy_social(args.preview, args.json_out) message = gs.process_files(files_to_process) From 2bc84092d7f6b49ea630147e986fca4e0130fff6 Mon Sep 17 00:00:00 2001 From: Arash Date: Tue, 7 May 2024 13:32:28 +0200 Subject: [PATCH 11/13] improve plugin configuration handling --- lib/galaxy_social.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/lib/galaxy_social.py b/lib/galaxy_social.py index 184964d..03296bb 100644 --- a/lib/galaxy_social.py +++ b/lib/galaxy_social.py @@ -37,23 +37,20 @@ def __init__(self, preview: bool, json_out: str): f"Error with plugin {module_name}.{class_name}.\n{e}" ) - try: - config = {} - if plugin.get("config"): - for key, value in plugin["config"].items(): - if isinstance(value, str) and value.startswith("$"): - try: - config[key] = os.environ[value[1:]] - except KeyError: - raise Exception( - f"Missing environment variable {value[1:]}." - ) - else: - config[key] = value - except Exception as e: - raise Exception( - f"Missing config for {module_name}.{class_name}.\n{e}" - ) + config = {} + if plugin.get("config"): + for key, value in plugin["config"].items(): + if isinstance(value, str) and value.startswith("$"): + try: + config[key] = os.environ[value[1:]] + except KeyError: + raise Exception( + f"Missing environment variable {value[1:]}." + ) + else: + config[key] = value + else: + raise Exception(f"Missing config for {module_name}.{class_name}.") try: self.plugins[plugin["name"].lower()] = plugin_class(**config) From 7b37bc3d6e17af1fbc793313dca74d1a1c5f7dd5 Mon Sep 17 00:00:00 2001 From: Arash Date: Tue, 7 May 2024 13:39:16 +0200 Subject: [PATCH 12/13] Use json_out as supplied. --- lib/galaxy_social.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/galaxy_social.py b/lib/galaxy_social.py index 03296bb..9108442 100644 --- a/lib/galaxy_social.py +++ b/lib/galaxy_social.py @@ -139,9 +139,7 @@ def process_markdown_file(self, file_path, processed_files): def process_files(self, files_to_process): processed_files = {} messages = "---\n" - processed_files_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "..", self.json_out - ) + processed_files_path = self.json_out if os.path.exists(processed_files_path): with open(processed_files_path, "r") as file: processed_files = json.load(file) From b6b76b785575ae24e9df4d6556195974710defd5 Mon Sep 17 00:00:00 2001 From: Arash Date: Fri, 10 May 2024 15:25:26 +0200 Subject: [PATCH 13/13] fix some bugs --- github_run.py | 13 +++++++++++-- lib/plugins/bluesky.py | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/github_run.py b/github_run.py index 6ec635a..d58a628 100644 --- a/github_run.py +++ b/github_run.py @@ -16,7 +16,12 @@ def __init__(self): def comment(self, comment_text): print(comment_text) - if not comment_text or not self.github_token or not self.repo or not self.pr_number: + if ( + not comment_text + or not self.github_token + or not self.repo + or not self.pr_number + ): return headers = { "Accept": "application/vnd.github+json", @@ -26,7 +31,7 @@ def comment(self, comment_text): url = ( f"https://api.github.com/repos/{self.repo}/issues/{self.pr_number}/comments" ) - data = {"body": comment_text} + data = {"body": str(comment_text)} response = requests.post(url, headers=headers, json=data) if response.status_code == 201: return True @@ -46,6 +51,10 @@ def get_files(self): response = requests.get(raw_url) if response.status_code == 200: changed_file_path = file["filename"] + os.makedirs( + os.path.dirname(changed_file_path), + exist_ok=True, + ) with open(changed_file_path, "w") as f: f.write(response.text) diff --git a/lib/plugins/bluesky.py b/lib/plugins/bluesky.py index ed37392..13680ea 100644 --- a/lib/plugins/bluesky.py +++ b/lib/plugins/bluesky.py @@ -129,7 +129,7 @@ def handle_url_card( description = ( description_tag["content"] if description_tag - else description_tag_alt["content"] if description_tag_alt else None + else description_tag_alt["content"] if description_tag_alt else "" ) uri = url thumb = (