diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d5333cc..e371364 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -27,6 +27,11 @@ jobs: args: --timeout=3m skip-pkg-cache: true skip-build-cache: true + - name: Setup CloudQuery + if: github.event_name == 'pull_request' + uses: cloudquery/setup-cloudquery@v3 + with: + version: v3.28.0 - name: Get dependencies run: go get -t -d ./... - name: Build @@ -38,4 +43,4 @@ jobs: run: make gen - name: Fail if generation updated files if: github.event_name == 'pull_request' - run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1) + run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1) \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1bcd07d..35f6464 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ cq-source-coinpaprika .cq coinpaprika.yml +cloudquery dist *.log diff --git a/Makefile b/Makefile index 638c40c..3ba5519 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,32 @@ +.PHONY: build +build: + go build + .PHONY: test test: - go test -timeout 3m ./... + go test -race -timeout 3m ./... .PHONY: lint lint: @if test ! -e ./bin/golangci-lint; then \ curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh; \ fi - @./bin/golangci-lint run --timeout 3m + @./bin/golangci-lint run --timeout 3m --verbose .PHONY: gen-docs -gen-docs: - rm -rf ./docs/tables/* - go run main.go doc ./docs/tables +gen-docs: build + @command -v cloudquery >/dev/null 2>&1 || { \ + echo "Error: 'cloudquery' command not found. Please install it before running gen-docs."; \ + echo "You can install it by following the instructions at: https://www.cloudquery.io/docs/quickstart"; \ + exit 1; \ + } + rm -rf docs/tables + cloudquery tables --format markdown --output-dir docs/ test/config.yml + mv -vf docs/coinpaprika docs/tables + +.PHONY: dist +dist: + go run main.go package -m "Release ${VERSION}" ${VERSION} . .PHONY: gen-mocks gen-mocks: diff --git a/README.md b/README.md index e4a90da..a36412a 100644 --- a/README.md +++ b/README.md @@ -19,100 +19,9 @@ It provides a streamlined approach to accessing, querying, and manipulating data - [CloudQuery Quickstart Guide](https://www.cloudquery.io/docs/quickstart) - [Supported Tables](docs/tables/README.md) +## Using the plugin -## Configuration - -The following source configuration file will sync to a PostgreSQL database. See [the CloudQuery Quickstart](https://www.cloudquery.io/docs/quickstart) for more information on how to configure the source and destination. - -1. Without API token, `Free` plan (25 000 calls/month) minimal interval 1h, see [available history range depending on the selected API plan](https://api.coinpaprika.com/#tag/Tickers/operation/getTickersHistoricalById). - - ```yaml - kind: source - spec: - name: "coinpaprika" - path: "coinpaprika/coinpaprika" - version: "v2.0.0" - backend_options: - table_name: "cq_state_coinpaprika" - connection: "@@plugins.sqlite.connection" - tables: - [ "*" ] - destinations: - - "sqlite" - spec: - api_debug: true - start_date: "2023-05-15T08:00:00Z" # for free plan up to 1 year ago - interval: 24h - rate_duration: 30d - rate_number: 25000 - tickers: - ["btc-bitcoin"] - --- - kind: destination - spec: - name: sqlite - path: cloudquery/sqlite - registry: cloudquery - version: "v2.4.15" - spec: - connection_string: ./db.sql - ``` - -2. With API token rate limited for `Bussines` plan (3 000 000 calls/month). API token can be generated at [coinpaprika.com/api](https://coinpaprika.com/api). - - ```yaml - kind: source - spec: - name: "coinpaprika" - path: "coinpaprika/coinpaprika" - version: "v2.0.0" - backend_options: - table_name: "cq_state_coinpaprika" - connection: "@@plugins.sqlite.connection" - tables: - [ "*" ] - destinations: - - "sqlite" - spec: - start_date: "2023-05-15T08:00:00Z" - interval: 5m - access_token: "${COINPAPRIKA_API_TOKEN}" - api_debug: true - rate_duration: 30d - rate_number: 3000000 - tickers: - ["*-bitcoin", "eth-ethereum"] - --- - kind: destination - spec: - name: sqlite - path: cloudquery/sqlite - registry: cloudquery - version: "v2.4.15" - spec: - connection_string: ./db.sql - ``` - -| Spec fields | Description | Default value | Optional | -|---------------|----------------------------------------------------------------------------------------------------------------------------|---------------|----------| -| start_date | Start date for synchronizing data in RFC3339 format. | | NO | -| end_date | End date for synchronizing data in RFC3339 format. | NOW | YES | -| interval | Intervals for historic data [possible values](https://api.coinpaprika.com/#tag/Tickers/operation/getTickersHistoricalById) | | NO | -| access_token | Coinpaprika [API token](https://coinpaprika.com/api). | | YES | -| api_debug | Enable request log. | false | YES | -| rate_duration | Unit of rate in time of request rate, go duration format. | 30 | YES | -| rate_number | Number of request in `rate_duration`. | 30 | YES | -| tickers | list of globe pattern ticker ids to synchronize. | * | YES | - - -The Coinpaprika plugin supports incremental syncing for historical tickers, only new tickers will be fetched. This is done by storing last timestamp of fetched ticker in CloudQuery backend. To enable this, `backend` option must be set in the spec. - -## Running -```bash -# https://www.cloudquery.io/docs -brew install cloudquery/tap/cloudquery -cloudquery sync conf.yml -``` +See [overview.md](docs/overview.md). ## Development @@ -141,3 +50,21 @@ make gen-docs Once the tag is pushed, a new GitHub Actions workflow will be triggered to build the release binaries and create the new release on GitHub. To customize the release notes, see the Go releaser [changelog configuration docs](https://goreleaser.com/customization/changelog/#changelog). + +### Publish a new version to the Cloudquery Hub + +After tagging a release, you can build and publish a new version to the [Cloudquery Hub](https://hub.cloudquery.io/) by running the following commands. +Replace `v1.0.0` with the new version number. + +```bash +# "make dist" uses the README as main documentation and adds a generic release note. Output is created in dist/ +VERSION=v1.0.0 make dist + +# Login to cloudquery hub and publish the new version +cloudquery login +cloudquery plugin publish --finalize +``` + +After publishing the new version, it will show up in the [hub](https://hub.cloudquery.io/). + +For more information please refer to the official [Publishing a Plugin to the Hub](https://www.cloudquery.io/docs/developers/publishing-a-plugin-to-the-hub) guide. \ No newline at end of file diff --git a/client/spec.go b/client/spec.go index a026fbc..895dd27 100644 --- a/client/spec.go +++ b/client/spec.go @@ -9,4 +9,5 @@ type Spec struct { ApiDebug bool `json:"api_debug"` RateNumber int `json:"rate_number"` RateDuration string `json:"rate_duration"` + Concurrency int `json:"concurrency"` } diff --git a/docs/overview.md b/docs/overview.md new file mode 100644 index 0000000..af6a204 --- /dev/null +++ b/docs/overview.md @@ -0,0 +1,99 @@ +# CloudQuery Coinpaprika Source Plugin + +A Coinpaprika source plugin for CloudQuery that loads data from [Coinpaprika API](https://api.coinpaprika.com) to any database, data warehouse or data lake supported by [CloudQuery](https://www.cloudquery.io/), such as PostgreSQL, BigQuery, Athena, and many more. + +## Configuration + +The following source configuration file will sync to a PostgreSQL database. See [the CloudQuery Quickstart](https://www.cloudquery.io/docs/quickstart) for more information on how to configure the source and destination. + +1. Without API token, `Free` plan (25 000 calls/month) minimal interval 1h, see [available history range depending on the selected API plan](https://api.coinpaprika.com/#tag/Tickers/operation/getTickersHistoricalById). + + ```yaml copy + kind: source + spec: + name: "coinpaprika" + path: "coinpaprika/coinpaprika" + registry: "cloudquery" + version: "v2.0.0" + backend_options: + table_name: "cq_state_coinpaprika" + connection: "@@plugins.sqlite.connection" + tables: + [ "*" ] + destinations: + - "sqlite" + spec: + api_debug: true + start_date: "2023-05-15T08:00:00Z" # for free plan up to 1 year ago + interval: 24h + rate_duration: 30d + rate_number: 25000 + tickers: + ["btc-bitcoin"] + --- + kind: destination + spec: + name: sqlite + path: cloudquery/sqlite + registry: cloudquery + version: "v2.4.16" + spec: + connection_string: ./db.sql + ``` + +2. With API token rate limited for `Business` plan (3 000 000 calls/month). API token can be generated at [coinpaprika.com/api](https://coinpaprika.com/api). + + ```yaml copy + kind: source + spec: + name: "coinpaprika" + path: "coinpaprika/coinpaprika" + registry: "cloudquery" + version: "v2.0.0" + backend_options: + table_name: "cq_state_coinpaprika" + connection: "@@plugins.sqlite.connection" + tables: + [ "*" ] + destinations: + - "sqlite" + spec: + start_date: "2023-05-15T08:00:00Z" + interval: 5m + access_token: "${COINPAPRIKA_API_TOKEN}" + api_debug: true + rate_duration: 30d + rate_number: 3000000 + tickers: + ["*-bitcoin", "eth-ethereum"] + --- + kind: destination + spec: + name: sqlite + path: cloudquery/sqlite + registry: cloudquery + version: "v2.4.16" + spec: + connection_string: ./db.sql + ``` + +| Spec fields | Description | Default value | Optional | +|---------------|----------------------------------------------------------------------------------------------------------------------------|---------------|----------| +| start_date | Start date for synchronizing data in RFC3339 format. | | NO | +| end_date | End date for synchronizing data in RFC3339 format. | NOW | YES | +| interval | Intervals for historic data [possible values](https://api.coinpaprika.com/#tag/Tickers/operation/getTickersHistoricalById) | | NO | +| access_token | Coinpaprika [API token](https://coinpaprika.com/api). | | YES | +| api_debug | Enable request log. | false | YES | +| rate_duration | Unit of rate in time of request rate, go duration format. | 30 | YES | +| rate_number | Number of request in `rate_duration`. | 30 | YES | +| tickers | list of globe pattern ticker ids to synchronize. | * | YES | +| concurrency | Best effort maximum number of Go routines to use. Lower this number to reduce memory usage. | 1000 | YES | + +The Coinpaprika plugin supports incremental syncing for historical tickers, only new tickers will be fetched. This is done by storing last timestamp of fetched ticker in CloudQuery backend. To enable this, `backend` option must be set in the spec. + +## Running +```bash +# https://www.cloudquery.io/docs +brew install cloudquery/tap/cloudquery +cloudquery sync conf.yml +``` \ No newline at end of file diff --git a/docs/tables/coinpaprika_coins.md b/docs/tables/coinpaprika_coins.md index 1cbc933..b56f5e0 100644 --- a/docs/tables/coinpaprika_coins.md +++ b/docs/tables/coinpaprika_coins.md @@ -1,7 +1,5 @@ # Table: coinpaprika_coins -This table shows data for Coinpaprika Coins. - https://api.coinpaprika.com/#tag/Coins/paths/~1coins/get The primary key for this table is **id**. diff --git a/docs/tables/coinpaprika_exchanges.md b/docs/tables/coinpaprika_exchanges.md index 8546783..f8caabc 100644 --- a/docs/tables/coinpaprika_exchanges.md +++ b/docs/tables/coinpaprika_exchanges.md @@ -1,7 +1,5 @@ # Table: coinpaprika_exchanges -This table shows data for Coinpaprika Exchanges. - The primary key for this table is **id**. ## Columns diff --git a/docs/tables/coinpaprika_tickers.md b/docs/tables/coinpaprika_tickers.md index 9fae20d..fbcabdb 100644 --- a/docs/tables/coinpaprika_tickers.md +++ b/docs/tables/coinpaprika_tickers.md @@ -1,7 +1,5 @@ # Table: coinpaprika_tickers -This table shows data for Coinpaprika Tickers. - https://api.coinpaprika.com/#tag/Tickers/operation/getTickersHistoricalById The composite primary key for this table is (**coin_id**, **timestamp**). diff --git a/plugin/client.go b/plugin/client.go index 3a04bba..97d968a 100644 --- a/plugin/client.go +++ b/plugin/client.go @@ -108,6 +108,7 @@ func Configure(_ context.Context, logger zerolog.Logger, specBytes []byte, opts tables: tables, scheduler: scheduler.NewScheduler( scheduler.WithLogger(logger), + scheduler.WithConcurrency(config.Concurrency), ), }, nil } diff --git a/test/config.yml b/test/config.yml new file mode 100644 index 0000000..4a3d1b4 --- /dev/null +++ b/test/config.yml @@ -0,0 +1,15 @@ +kind: source +spec: + name: "coinpaprika" + path: "./cq-source-coinpaprika" + registry: "local" + version: "v1.0.1" + tables: ["*"] + destinations: + - "test" +--- +kind: destination +spec: + name: test + path: cloudquery/test + version: "v2.2.3" \ No newline at end of file