diff --git a/README.md b/README.md index 5c8f9af..4dd1004 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ + # oembed-parser Extract eEmbed content from given URL. @@ -15,19 +16,28 @@ Extract eEmbed content from given URL. - [Example FaaS](https://us-central1-technews-251304.cloudfunctions.net/oembed-parser?url=https://www.youtube.com/watch?v=8jPQjjsBbIc) -### Installation +## Installation ```bash -npm install oembed-parser +$ npm install oembed-parser + +# or pnpm +$ pnpm install oembed-parser + +# or yarn +$ yarn add oembed-parser ``` -### Usage + +## Usage ```js -import { - extract -} from 'oembed-parser' +const { extract } = require('oembed-parser') +// es6 module syntax +import { extract } from 'oembed-parser' + +// test const url = 'https://www.youtube.com/watch?v=8jPQjjsBbIc' extract(url).then((oembed) => { @@ -37,21 +47,16 @@ extract(url).then((oembed) => { }) ``` -### APIs +## APIs #### .extract(String url [, Object params]) -Extract oEmbed data from specified url. -Return: a Promise - -Optional argument `params` is an object with it we can set `maxwidth` and/or `maxheight` those are used to scale embed size to fit your container size. Please refer [oEmbed/Full Spec/Consumer Request](https://oembed.com/#section2) for more info. +Load and extract oembed data. -Here is how we can use `oembed-parser` in async/await style: +Example: ```js -import { - extract -} from 'oembed-parser' +const { extract } = require('oembed-parser') const getOembed = async (url) => { try { @@ -59,6 +64,7 @@ const getOembed = async (url) => { return oembed } catch (err) { console.trace(err) + return null } } @@ -66,67 +72,114 @@ const data = getOembed('your url') console.log(data) ``` +Optional argument `params` is an object with it we can set `maxwidth` and/or `maxheight` those are used to scale embed size to fit your container size. Please refer [oEmbed/Full Spec/Consumer Request](https://oembed.com/#section2) for more info. #### .hasProvider(String URL) -Return boolean. True if the URL matches with any provider in the list. +Check if a URL matches with any provider in the list. + +Examples: + +```js +const { hasProvider } = require('oembed-parser') + +hasProvider('https://www.youtube.com/watch?v=ciS8aCrX-9s') // return true +hasProvider('https://trello.com/b/BO3bg7yn/notes') // return false +``` #### .findProvider(String URL) -Return provider which is relevant to given URL. +Get the provider which is relevant to given URL. For example: ```js -import { - findProvider -} from 'oembed-parser' +const { findProvider } = require('oembed-parser') findProvider('https://www.facebook.com/video.php?v=999999999') +``` -// get something like below: +Result looks like below: -// { -// fetchEndpoint: 'https://graph.facebook.com/v10.0/oembed_video', -// providerName: 'Facebook', -// providerUrl: 'https://www.facebook.com/' -// } +```json +{ + fetchEndpoint: 'https://graph.facebook.com/v10.0/oembed_video', + providerName: 'Facebook', + providerUrl: 'https://www.facebook.com/' +} ``` +#### .setProviderList(Array providers) -#### .setProviderList(Array of provider definitions) - -Sets the list of providers to use, overriding the defaults. +Apply a list of providers to use, overriding the [default](https://raw.githubusercontent.com/ndaidong/oembed-parser/master/src/utils/providers.json). This can be useful for whitelisting only certain providers, or for adding custom providers. -For the expected format, see the -[default list](https://raw.githubusercontent.com/ndaidong/oembed-parser/master/src/utils/providers.json). +Default list of resource providers is synchronized from [oembed.com](http://oembed.com/providers.json). +For example: + +```js +const { setProviderList } = require('oembed-parser') + +const providers = [ + { + provider_name: 'Alpha', + provider_url: 'https://alpha.com', + endpoints: [ + // endpoint definition here + ] + }, + { + provider_name: 'Beta', + provider_url: 'https://beta.com', + endpoints: [ + // endpoint definition here + ] + } +] -### Provider list +setProviderList(providers) +``` + +#### .setRequestOptions(Object requestOptions) +Set options for request method. -List of resource providers is a clone of [oembed.com](http://oembed.com/providers.json) and available [here](https://raw.githubusercontent.com/ndaidong/oembed-parser/master/src/utils/providers.json). +`oembed-parser` is using [axios]() to send HTTP requests. Please refer [axios' request config](https://axios-http.com/docs/req_config) for more info. + +Default option: + +```js +{ + headers: { + 'user-agent': 'Mozilla/5.0 (X11; Linux i686; rv:94.0) Gecko/20100101 Firefox/94.0', + accept: 'application/json; charset=utf-8' + }, + responseType: 'json', + responseEncoding: 'utf8', + timeout: 6e4, + maxRedirects: 3 +} +``` ## Facebook and Instagram Since October 24 2020, Facebook have deprecated their legacy urls and applied a new Facebook oEmbed endpoints. -Please update your `oembed-parser` version to v1.4.2 or later to be able to extract oembed data from Instagram and Facebook. - -Technically, now we have to use Facebook Graph API, with the access token from a valid and live Facebook app. +Technically, now we have to use Facebook Graph API, with the access token from a valid and live Facebook app. `oembed-parser` will try to get these values from environment variables, so please define them, for example: ```bash export FACEBOOK_APP_ID=your_app_id export FACEBOOK_CLIENT_TOKEN=your_client_token ``` -For more info, please refer: +References: +- [oEmbed Read](https://developers.facebook.com/docs/features-reference/oembed-read) - [Facebook oEmbed](https://developers.facebook.com/docs/plugins/oembed) - +- [Facebook Graph API](https://developers.facebook.com/docs/graph-api/overview) ## Test @@ -140,6 +193,7 @@ npm test npm run eval {URL_TO_PARSE_OEMBED} ``` -# License - +## License The MIT License (MIT) + +--- diff --git a/index.d.ts b/index.d.ts index 7306618..f287c92 100644 --- a/index.d.ts +++ b/index.d.ts @@ -7,10 +7,14 @@ export function extract(url: string, params?: any): Promise; -export function hasProvider(url: string): boolean; +export function hasProvider(url: string): boolean export function setProviderList(providers: Provider[]): void +export function setRequestOptions(options: object): void + +export function getRequestOptions(): object + export interface Endpoint { schemes?: string[]; url: string; diff --git a/package.json b/package.json index b9aa1b3..b966fef 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "version": "2.0.0rc2", + "version": "2.0.0", "name": "oembed-parser", "description": "Get oEmbed data from given URL.", "homepage": "https://www.npmjs.com/package/oembed-parser", @@ -24,10 +24,11 @@ "reset": "node reset" }, "dependencies": { - "got": "^11.8.3" + "axios": "^0.24.0", + "bellajs": "^10.0.2" }, "devDependencies": { - "jest": "^27.4.3", + "jest": "^27.4.5", "nock": "^13.2.1" }, "keywords": [ diff --git a/reset.js b/reset.js index 8add6e2..87a90e6 100644 --- a/reset.js +++ b/reset.js @@ -18,6 +18,7 @@ const dirs = [ const files = [ 'yarn.lock', + 'pnpm-lock.yaml', 'package-lock.json', 'coverage.lcov', 'coverage.lcov' diff --git a/src/config.js b/src/config.js index dd358d0..d05882d 100644 --- a/src/config.js +++ b/src/config.js @@ -1,15 +1,23 @@ // config -const fetchOptions = { +const { clone, copies } = require('bellajs') + +const requestOptions = { headers: { 'user-agent': 'Mozilla/5.0 (X11; Linux i686; rv:94.0) Gecko/20100101 Firefox/94.0', accept: 'application/json; charset=utf-8' }, responseType: 'json', - timeout: 30 * 1e3, - redirect: 'follow' + responseEncoding: 'utf8', + timeout: 6e4, // 1 minute + maxRedirects: 3 } module.exports = { - fetchOptions + getRequestOptions: () => { + return clone(requestOptions) + }, + setRequestOptions: (opts) => { + copies(opts, requestOptions) + } } diff --git a/src/main.js b/src/main.js index 4a871b7..b0a3b59 100644 --- a/src/main.js +++ b/src/main.js @@ -8,6 +8,10 @@ const fetchEmbed = require('./utils/fetchEmbed') const provider = require('./utils/provider') +const { + setRequestOptions +} = require('./config') + const extract = async (url, params = {}) => { if (!isValidURL(url)) { throw new Error('Invalid input URL') @@ -24,5 +28,6 @@ module.exports = { extract, hasProvider: provider.has, findProvider: provider.find, - setProviderList: provider.set + setProviderList: provider.set, + setRequestOptions } diff --git a/src/main.test.js b/src/main.test.js index d0b325f..be5775e 100644 --- a/src/main.test.js +++ b/src/main.test.js @@ -7,9 +7,14 @@ const { extract, hasProvider, findProvider, - setProviderList + setProviderList, + setRequestOptions } = require('./main') +const { + getRequestOptions +} = require('./config') + const required = [ 'type', 'version' @@ -251,3 +256,23 @@ test('test .setProviderList() method', () => { expect(hasProvider('http://www.example.org/media/abcdef')).toBe(true) expect(hasProvider('https://www.youtube.com/watch?v=ciS8aCrX-9s')).toBe(false) }) + +test('Testing setRequestOptions()', () => { + setRequestOptions({ + headers: { + authorization: 'bearer ' + }, + timeout: 20, + somethingElse: 1000 + }) + + const actual = getRequestOptions() + const expectedHeader = { + authorization: 'bearer ', + 'user-agent': 'Mozilla/5.0 (X11; Linux i686; rv:94.0) Gecko/20100101 Firefox/94.0', + accept: 'application/json; charset=utf-8' + } + + expect(actual.headers).toEqual(expectedHeader) + expect(actual.timeout).toEqual(20) +}) diff --git a/src/utils/retrieve.js b/src/utils/retrieve.js index e5f5ed2..cb97953 100644 --- a/src/utils/retrieve.js +++ b/src/utils/retrieve.js @@ -1,19 +1,19 @@ // utils -> retrieve -const got = require('got') +const axios = require('axios') -const { fetchOptions } = require('../config') +const { getRequestOptions } = require('../config') module.exports = async (url) => { try { - const { headers, body } = await got(url, fetchOptions) + const res = await axios.get(url, getRequestOptions()) - const contentType = headers['content-type'] || '' + const contentType = res.headers['content-type'] || '' if (!contentType || !contentType.includes('application/json')) { return null } - return body + return res.data } catch (err) { return null } diff --git a/src/utils/retrieve.test.js b/src/utils/retrieve.test.js index d2fcc56..5cd05a3 100644 --- a/src/utils/retrieve.test.js +++ b/src/utils/retrieve.test.js @@ -28,7 +28,7 @@ test('test retrieve() from bad source', async () => { const url = 'https://some.where/good/page' const { baseUrl, path } = parseUrl(url) const scope = nock(baseUrl) - scope.get(path).reply(500, { data: { name: 'oembed-parser' } }, { + scope.get(path).reply(500, '', { 'Content-Type': 'application/json' }) const result = await retrieve(url)