Replace dependency on noembed with python-oembed and list of providers

CHANGES.rst

@@ -4,6 +4,12 @@ Changelog
 2.3.3 (unreleased)
 ------------------
 
+- Embed: Replace use of noembed.com (or custom noembed instance) with python-oembed and embedded provider configuration generated (with included script) from https://github.com/iamcal/oembed/.
+  [datakurre]
+
+- Embed: Add Maximum width and Maximum height configuration parameters, which are supported by many oEmbed providers.
+  [datakure]
+
 - Content listing: Allow to use collection query parameters from context.
   Content listing: Include query parameters from request.
   Content listing: Add "tile_class".

Makefile

@@ -0,0 +1,5 @@
+all: plone/app/standardtiles/embed_providers.py
+
+plone/app/standardtiles/embed_providers.py: generate_embed_providers.py
+	./generate_embed_providers.py > plone/app/standardtiles/embed_providers.py
+

generate_embed_providers.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -p "python3.withPackages(ps: with ps; [ requests pyyaml ])" -i python3
+
+import io
+import json
+import requests
+import tarfile
+import yaml
+import datetime
+
+URL = "https://github.com/iamcal/oembed/archive/master.tar.gz"
+
+r = requests.get(URL, allow_redirects=True)
+fp = io.BytesIO(r.content)
+fp.seek(0)
+
+endpoints = []
+
+tar = tarfile.open(fileobj=fp, mode="r:gz")
+for info in tar:
+    if all([
+        info.isfile(),
+        "providers" in info.path,
+        info.path.endswith(".yml")
+    ]):
+        fb_ = tar.extractfile(info)
+        data = yaml.safe_load(fb_)
+        for provider in data:
+            for e in provider['endpoints']:
+                endpoints.append([
+                    e['url'],
+                    e.get('schemes', [provider['provider_url'] + '*'])
+                ])
+
+print(f"""\
+\"\"\"oEmbed providers from https://github.com/iamcal/oembed\"\"\"
+# {datetime.datetime.now().isoformat()}
+
+PROVIDERS = {json.dumps(sorted(endpoints), indent=4)}
+""")

plone/app/standardtiles/embed.py

@@ -1,14 +1,32 @@
 # -*- coding: utf-8 -*-
 from plone.app.standardtiles import _PMF as _
+from plone.app.standardtiles import embed_providers
 from plone.memoize import ram
 from plone.supermodel.model import Schema
 from plone.tiles import Tile
 from zope import schema
 
-import requests
+import oembed
 
+try:
+    from urllib.error import HTTPError
+except ImportError:  # py2
+    from urllib2 import HTTPError
 
-NOEMBED_ENDPOINT = 'https://noembed.com/embed?url='
+
+CONSUMER = oembed.OEmbedConsumer()
+ENABLED_PROVIDERS = []
+DISABLED_PROVIDERS = []
+
+for endpoint, scheme in embed_providers.PROVIDERS:
+    try:
+        oe = oembed.OEmbedEndpoint(endpoint, scheme)
+        ENABLED_PROVIDERS.append(endpoint)
+    except:  # noqa
+        # Some endpoints fail to register
+        DISABLED_PROVIDERS.append(endpoint)
+        continue
+    CONSUMER.addEndpoint(oe)
 
 
 class IEmbedTile(Schema):
@@ -20,16 +38,39 @@ class IEmbedTile(Schema):
         required=True
     )
 
+    maxwidth = schema.Int(
+        title=_(u"Maximum width"),
+        required=False
+    )
+
+    maxheight = schema.Int(
+        title=_(u"Maximum height"),
+        required=False
+    )
+
 
 class EmbedTile(Tile):
     """ A tile that embeds media.
     """
 
-    @ram.cache(lambda method, obj: obj.data.get('media_url'))
+    @ram.cache(lambda method, obj: (obj.data.get('media_url'),
+                                    obj.data.get('maxwidth'),
+                                    obj.data.get('maxheight')))
     def __call__(self):
         media_url = self.data.get('media_url')
-        url = NOEMBED_ENDPOINT + media_url
-        rr = requests.get(url)
-        if rr.ok:
-            data = rr.json()
+        maxwidth = self.data.get('maxwidth') or None
+        maxheight = self.data.get('maxheight') or None
+
+        try:
+            response = CONSUMER.embed(
+                media_url,
+                maxwidth=maxwidth,
+                maxheight=maxheight,
+            )
+            data = response.getData()
+        except (oembed.OEmbedError, HTTPError) as e:
+            data = {
+                "html": "<pre>" + str(e) + "</pre>"
+            }
+
         return u"<html><body>%s</body></html>" % data['html']