feat: Add documentation and plugin publishing commands

.github/workflows/test.yaml

@@ -27,6 +27,11 @@ jobs:
           args: --timeout=3m
           skip-pkg-cache: true
           skip-build-cache: true
+      - name: Setup CloudQuery
+#        if: github.event_name == 'pull_request'
+        uses: cloudquery/setup-cloudquery@v3
+        with:
+          version: v3.28.0
       - name: Get dependencies
         run: go get -t -d ./...
       - name: Build
@@ -38,4 +43,4 @@ jobs:
         run: make gen
       - name: Fail if generation updated files
         if: github.event_name == 'pull_request'
-        run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1)
+        run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1)

Makefile

@@ -1,18 +1,32 @@
+.PHONY: build
+build:
+	go build
+
 .PHONY: test
 test:
-	go test -timeout 3m ./...
+	go test -race -timeout 3m ./...
 
 .PHONY: lint
 lint:
 	@if test ! -e ./bin/golangci-lint; then \
     	curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh; \
     fi
-	@./bin/golangci-lint run --timeout 3m
+	@./bin/golangci-lint run --timeout 3m --verbose
 
 .PHONY: gen-docs
-gen-docs:
-	rm -rf ./docs/tables/*
-	go run main.go doc ./docs/tables
+gen-docs: build
+	@command -v cloudquery >/dev/null 2>&1 || { \
+		echo "Error: 'cloudquery' command not found. Please install it before running gen-docs."; \
+		echo "You can install it by following the instructions at: https://www.cloudquery.io/docs/quickstart"; \
+		exit 1; \
+	}
+	rm -rf docs/tables
+	cloudquery tables --format markdown --output-dir docs/ test/config.yml
+	mv -vf docs/coinpaprika docs/tables
+
+.PHONY: dist
+dist:
+	go run main.go package -m "Release ${VERSION}" ${VERSION} .
 
 .PHONY: gen-mocks
 gen-mocks:

README.md

@@ -19,100 +19,9 @@ It provides a streamlined approach to accessing, querying, and manipulating data
  - [CloudQuery Quickstart Guide](https://www.cloudquery.io/docs/quickstart)
  - [Supported Tables](docs/tables/README.md)
 
+## Using the plugin
 
-## Configuration
-
-The following source configuration file will sync to a PostgreSQL database. See [the CloudQuery Quickstart](https://www.cloudquery.io/docs/quickstart) for more information on how to configure the source and destination.
-
-1. Without API token, `Free` plan (25 000 calls/month) minimal interval 1h, see  [available history range depending on the selected API plan](https://api.coinpaprika.com/#tag/Tickers/operation/getTickersHistoricalById).
-
-     ```yaml
-     kind: source
-     spec:
-       name: "coinpaprika"
-       path: "coinpaprika/coinpaprika"
-       version: "v2.0.0"
-       backend_options:
-         table_name: "cq_state_coinpaprika"
-         connection: "@@plugins.sqlite.connection"
-       tables:
-         [ "*" ]
-       destinations:
-         - "sqlite"
-       spec:
-         api_debug: true
-         start_date: "2023-05-15T08:00:00Z" # for free plan up to 1 year ago
-         interval: 24h
-         rate_duration: 30d
-         rate_number: 25000
-         tickers:
-           ["btc-bitcoin"]
-     ---
-     kind: destination
-     spec:
-       name: sqlite
-       path: cloudquery/sqlite
-       registry: cloudquery
-       version: "v2.4.15"
-       spec:
-         connection_string: ./db.sql    
-     ```
-
-2. With API token rate limited for `Bussines` plan (3 000 000 calls/month). API token can be generated at [coinpaprika.com/api](https://coinpaprika.com/api).
-
-    ```yaml
-    kind: source
-    spec:
-      name: "coinpaprika"
-      path: "coinpaprika/coinpaprika"
-      version: "v2.0.0"
-      backend_options:
-        table_name: "cq_state_coinpaprika"
-        connection: "@@plugins.sqlite.connection"
-      tables:
-        [ "*" ]
-      destinations:
-        - "sqlite"
-      spec: 
-        start_date: "2023-05-15T08:00:00Z"
-        interval: 5m 
-        access_token: "${COINPAPRIKA_API_TOKEN}"
-        api_debug: true
-        rate_duration: 30d
-        rate_number: 3000000
-        tickers: 
-          ["*-bitcoin", "eth-ethereum"]
-    ---
-    kind: destination
-    spec:
-      name: sqlite
-      path: cloudquery/sqlite
-      registry: cloudquery
-      version: "v2.4.15"
-      spec:
-        connection_string: ./db.sql 
-    ```
-
-| Spec fields   | Description                                                                                                                | Default value | Optional |
-|---------------|----------------------------------------------------------------------------------------------------------------------------|---------------|----------|
-| start_date    | Start date for synchronizing data in RFC3339 format.                                                                       |               | NO       |
-| end_date      | End date for synchronizing data in RFC3339 format.                                                                         | NOW           | YES      |
-| interval      | Intervals for historic data [possible values](https://api.coinpaprika.com/#tag/Tickers/operation/getTickersHistoricalById) |               | NO       |
-| access_token  | Coinpaprika [API token](https://coinpaprika.com/api).                                                                      |               | YES      |
-| api_debug     | Enable request log.                                                                                                        | false         | YES      |
-| rate_duration | Unit of rate in time of request rate, go duration format.                                                                  | 30            | YES      |
-| rate_number   | Number of request in `rate_duration`.                                                                                      | 30            | YES      |
-| tickers       | list of globe pattern ticker ids to synchronize.                                                                           | *             | YES      |
-
-
-The Coinpaprika plugin supports incremental syncing for historical tickers, only new tickers will be fetched. This is done by storing last timestamp of fetched ticker in CloudQuery backend. To enable this, `backend` option must be set in the spec. 
-
-## Running
-```bash
-# https://www.cloudquery.io/docs
-brew install cloudquery/tap/cloudquery 
-cloudquery sync conf.yml
-```
+See [overview.md](docs/overview.md).
 
 ## Development
 
@@ -141,3 +50,21 @@ make gen-docs
 
 Once the tag is pushed, a new GitHub Actions workflow will be triggered to build the release binaries and create the new release on GitHub.
 To customize the release notes, see the Go releaser [changelog configuration docs](https://goreleaser.com/customization/changelog/#changelog).
+
+### Publish a new version to the Cloudquery Hub
+
+After tagging a release, you can build and publish a new version to the [Cloudquery Hub](https://hub.cloudquery.io/) by running the following commands.
+Replace `v1.0.0` with the new version number.
+
+```bash
+# "make dist" uses the README as main documentation and adds a generic release note. Output is created in dist/
+VERSION=v1.0.0 make dist
+
+# Login to cloudquery hub and publish the new version
+cloudquery login
+cloudquery plugin publish --finalize
+```
+
+After publishing the new version, it will show up in the [hub](https://hub.cloudquery.io/).
+
+For more information please refer to the official [Publishing a Plugin to the Hub](https://www.cloudquery.io/docs/developers/publishing-a-plugin-to-the-hub) guide.

client/spec.go

@@ -9,4 +9,5 @@ type Spec struct {
 	ApiDebug     bool     `json:"api_debug"`
 	RateNumber   int      `json:"rate_number"`
 	RateDuration string   `json:"rate_duration"`
+	Concurrency  int      `json:"concurrency"`
 }

docs/overview.md

@@ -0,0 +1,97 @@
+# CloudQuery Coinpaprika Source Plugin
+
+A Coinpaprika source plugin for CloudQuery that loads data from [Coinpaprika API](https://api.coinpaprika.com) to any database, data warehouse or data lake supported by [CloudQuery](https://www.cloudquery.io/), such as PostgreSQL, BigQuery, Athena, and many more.
+
+## Configuration
+
+The following source configuration file will sync to a PostgreSQL database. See [the CloudQuery Quickstart](https://www.cloudquery.io/docs/quickstart) for more information on how to configure the source and destination.
+
+1. Without API token, `Free` plan (25 000 calls/month) minimal interval 1h, see  [available history range depending on the selected API plan](https://api.coinpaprika.com/#tag/Tickers/operation/getTickersHistoricalById).
+
+     ```yaml
+     kind: source
+     spec:
+       name: "coinpaprika"
+       path: "coinpaprika/coinpaprika"
+       version: "v2.0.0"
+       backend_options:
+         table_name: "cq_state_coinpaprika"
+         connection: "@@plugins.sqlite.connection"
+       tables:
+         [ "*" ]
+       destinations:
+         - "sqlite"
+       spec:
+         api_debug: true
+         start_date: "2023-05-15T08:00:00Z" # for free plan up to 1 year ago
+         interval: 24h
+         rate_duration: 30d
+         rate_number: 25000
+         tickers:
+           ["btc-bitcoin"]
+     ---
+     kind: destination
+     spec:
+       name: sqlite
+       path: cloudquery/sqlite
+       registry: cloudquery
+       version: "v2.4.15"
+       spec:
+         connection_string: ./db.sql    
+     ```
+
+2. With API token rate limited for `Bussines` plan (3 000 000 calls/month). API token can be generated at [coinpaprika.com/api](https://coinpaprika.com/api).
+
+    ```yaml
+    kind: source
+    spec:
+      name: "coinpaprika"
+      path: "coinpaprika/coinpaprika"
+      version: "v2.0.0"
+      backend_options:
+        table_name: "cq_state_coinpaprika"
+        connection: "@@plugins.sqlite.connection"
+      tables:
+        [ "*" ]
+      destinations:
+        - "sqlite"
+      spec: 
+        start_date: "2023-05-15T08:00:00Z"
+        interval: 5m 
+        access_token: "${COINPAPRIKA_API_TOKEN}"
+        api_debug: true
+        rate_duration: 30d
+        rate_number: 3000000
+        tickers: 
+          ["*-bitcoin", "eth-ethereum"]
+    ---
+    kind: destination
+    spec:
+      name: sqlite
+      path: cloudquery/sqlite
+      registry: cloudquery
+      version: "v2.4.15"
+      spec:
+        connection_string: ./db.sql 
+    ```
+
+| Spec fields   | Description                                                                                                                | Default value | Optional |
+|---------------|----------------------------------------------------------------------------------------------------------------------------|---------------|----------|
+| start_date    | Start date for synchronizing data in RFC3339 format.                                                                       |               | NO       |
+| end_date      | End date for synchronizing data in RFC3339 format.                                                                         | NOW           | YES      |
+| interval      | Intervals for historic data [possible values](https://api.coinpaprika.com/#tag/Tickers/operation/getTickersHistoricalById) |               | NO       |
+| access_token  | Coinpaprika [API token](https://coinpaprika.com/api).                                                                      |               | YES      |
+| api_debug     | Enable request log.                                                                                                        | false         | YES      |
+| rate_duration | Unit of rate in time of request rate, go duration format.                                                                  | 30            | YES      |
+| rate_number   | Number of request in `rate_duration`.                                                                                      | 30            | YES      |
+| tickers       | list of globe pattern ticker ids to synchronize.                                                                           | *             | YES      |
+| concurrency   | Best effort maximum number of Go routines to use. Lower this number to reduce memory usage.                                | 1000          | YES      |
+
+The Coinpaprika plugin supports incremental syncing for historical tickers, only new tickers will be fetched. This is done by storing last timestamp of fetched ticker in CloudQuery backend. To enable this, `backend` option must be set in the spec.
+
+## Running
+```bash
+# https://www.cloudquery.io/docs
+brew install cloudquery/tap/cloudquery 
+cloudquery sync conf.yml
+```

docs/tables/coinpaprika_coins.md

@@ -1,7 +1,5 @@
 # Table: coinpaprika_coins
 
-This table shows data for Coinpaprika Coins.
-
 https://api.coinpaprika.com/#tag/Coins/paths/~1coins/get
 
 The primary key for this table is **id**.

docs/tables/coinpaprika_exchanges.md

@@ -1,7 +1,5 @@
 # Table: coinpaprika_exchanges
 
-This table shows data for Coinpaprika Exchanges.
-
 The primary key for this table is **id**.
 
 ## Columns

docs/tables/coinpaprika_tickers.md

@@ -1,7 +1,5 @@
 # Table: coinpaprika_tickers
 
-This table shows data for Coinpaprika Tickers.
-
 https://api.coinpaprika.com/#tag/Tickers/operation/getTickersHistoricalById
 
 The composite primary key for this table is (**coin_id**, **timestamp**).

plugin/client.go

@@ -108,6 +108,7 @@ func Configure(_ context.Context, logger zerolog.Logger, specBytes []byte, opts
 		tables:  tables,
 		scheduler: scheduler.NewScheduler(
 			scheduler.WithLogger(logger),
+			scheduler.WithConcurrency(config.Concurrency),
 		),
 	}, nil
 }

test/config.yml

@@ -0,0 +1,15 @@
+kind: source
+spec:
+  name: "coinpaprika"
+  path: "./cq-source-coinpaprika"
+  registry: "local"
+  version: "v1.0.1"
+  tables: ["*"]
+  destinations:
+    - "test"
+---
+kind: destination
+spec:
+  name: test
+  path: cloudquery/test
+  version: "v2.2.3"