From 8a10f5b838b4bb63548f0bb403d6f9b1e83d4144 Mon Sep 17 00:00:00 2001 From: Ben Smith Date: Sat, 17 Feb 2024 21:47:45 +0100 Subject: [PATCH] all methods for get result csv with tests --- package.json | 10 ++-- src/api/client.ts | 100 +++++++++++++++++++++++++++------ src/api/execution.ts | 17 +++++- src/api/query.ts | 3 +- src/api/router.ts | 52 ++++++++++------- tests/e2e/client.spec.ts | 12 ++++ tests/e2e/executionAPI.spec.ts | 50 ++++++++++++++++- tests/e2e/util.ts | 1 - yarn.lock | 23 +++----- 9 files changed, 206 insertions(+), 62 deletions(-) diff --git a/package.json b/package.json index a7f3ecd..89ca933 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,11 @@ "fmt": "prettier --write \"./**/*.ts\"", "lint": "eslint" }, + "dependencies": { + "cross-fetch": "^4.0.0", + "deprecated": "^0.0.2", + "loglevel": "^1.8.0" + }, "devDependencies": { "@types/chai": "^4.3.3", "@types/mocha": "^10.0.6", @@ -29,10 +34,5 @@ "ts-mocha": "^10.0.0", "ts-node": "^10.9.1", "typescript": "^5.3.3" - }, - "dependencies": { - "cross-fetch": "^4.0.0", - "deprecated": "^0.0.2", - "loglevel": "^1.8.0" } } diff --git a/src/api/client.ts b/src/api/client.ts index f94b0db..c888f31 100644 --- a/src/api/client.ts +++ b/src/api/client.ts @@ -1,4 +1,10 @@ -import { DuneError, ResultsResponse, ExecutionState, QueryParameter } from "../types"; +import { + DuneError, + ResultsResponse, + ExecutionState, + QueryParameter, + GetStatusResponse, +} from "../types"; import { ageInHours, sleep } from "../utils"; import log from "loglevel"; import { logPrefix } from "../utils"; @@ -27,22 +33,11 @@ export class DuneClient { params?: ExecutionParams, pingFrequency: number = POLL_FREQUENCY_SECONDS, ): Promise { - log.info( - logPrefix, - `refreshing query https://dune.com/queries/${queryID} with parameters ${JSON.stringify( - params, - )}`, + let { state, execution_id: jobID } = await this._runInner( + queryID, + params, + pingFrequency, ); - const { execution_id: jobID } = await this.exec.executeQuery(queryID, params); - let { state } = await this.exec.getExecutionStatus(jobID); - while (!TERMINAL_STATES.includes(state)) { - log.info( - logPrefix, - `waiting for query execution ${jobID} to complete: current state ${state}`, - ); - await sleep(pingFrequency); - state = (await this.exec.getExecutionStatus(jobID)).state; - } if (state === ExecutionState.COMPLETED) { return this.exec.getExecutionResults(jobID); } else { @@ -52,6 +47,27 @@ export class DuneClient { throw new DuneError(message); } } + + async runQueryCSV( + queryID: number, + params?: ExecutionParams, + pingFrequency: number = POLL_FREQUENCY_SECONDS, + ): Promise { + let { state, execution_id: jobID } = await this._runInner( + queryID, + params, + pingFrequency, + ); + if (state === ExecutionState.COMPLETED) { + return this.exec.getResultCSV(jobID); + } else { + const message = `refresh (execution ${jobID}) yields incomplete terminal state ${state}`; + // TODO - log the error in constructor + log.error(logPrefix, message); + throw new DuneError(message); + } + } + /** * Goes a bit beyond the internal call which returns that last execution results. * Here contains additional logic to refresh the results if they are too old. @@ -77,6 +93,58 @@ export class DuneClient { return results; } + /** + * Get the lastest execution results in CSV format. + * @param queryId - query to get results of. + * @param parameters - parameters for which they were called. + * @param maxAgeHours - oldest acceptable results (if expired results are refreshed) + * @returns Latest execution results for the given parameters. + */ + async getLatestResultCSV( + queryId: number, + parameters?: QueryParameter[], + maxAgeHours: number = THREE_MONTHS_IN_HOURS, + ): Promise { + const lastResults = await this.exec.getLastExecutionResults(queryId, parameters); + const lastRun: Date = lastResults.execution_ended_at!; + let results: Promise; + if (lastRun !== undefined && ageInHours(lastRun) > maxAgeHours) { + log.info( + logPrefix, + `results (from ${lastRun}) older than ${maxAgeHours} hours, re-running query.`, + ); + results = this.runQueryCSV(queryId, { query_parameters: parameters }); + } else { + // TODO (user cost savings): transform the lastResults into CSV instead of refetching + results = this.exec.getLastResultCSV(queryId, parameters); + } + return results; + } + + private async _runInner( + queryID: number, + params?: ExecutionParams, + pingFrequency: number = POLL_FREQUENCY_SECONDS, + ): Promise { + log.info( + logPrefix, + `refreshing query https://dune.com/queries/${queryID} with parameters ${JSON.stringify( + params, + )}`, + ); + const { execution_id: jobID } = await this.exec.executeQuery(queryID, params); + let status = await this.exec.getExecutionStatus(jobID); + while (!TERMINAL_STATES.includes(status.state)) { + log.info( + logPrefix, + `waiting for query execution ${jobID} to complete: current state ${status.state}`, + ); + await sleep(pingFrequency); + status = await this.exec.getExecutionStatus(jobID); + } + return status; + } + /** * @deprecated since version 0.0.2 Use runQuery */ diff --git a/src/api/execution.ts b/src/api/execution.ts index 591ba94..adc9f8b 100644 --- a/src/api/execution.ts +++ b/src/api/execution.ts @@ -52,11 +52,26 @@ export class ExecutionAPI extends Router { return response as ResultsResponse; } + async getResultCSV(executionId: string): Promise { + const response = await this._get(`execution/${executionId}/results/csv`); + log.debug(logPrefix, `get_result response ${JSON.stringify(response)}`); + return response; + } + async getLastExecutionResults( queryId: number, parameters?: QueryParameter[], ): Promise { - return await this._get(`query/${queryId}/results`, { + return this._get(`query/${queryId}/results`, { + query_parameters: parameters ? parameters : [], + }); + } + + async getLastResultCSV( + queryId: number, + parameters?: QueryParameter[], + ): Promise { + return this._get(`query/${queryId}/results/csv`, { query_parameters: parameters ? parameters : [], }); } diff --git a/src/api/query.ts b/src/api/query.ts index 32108e3..99b4c7c 100644 --- a/src/api/query.ts +++ b/src/api/query.ts @@ -2,6 +2,7 @@ import { Router } from "./router"; import { DuneQuery, QueryParameter, CreateQueryResponse, DuneError } from "../types"; import { CreateQueryPayload, UpdateQueryPayload } from "../types/requestPayload"; +import log from "loglevel"; interface EditQueryResponse { query_id: number; @@ -56,7 +57,7 @@ export class QueryAPI extends Router { if (params !== undefined) parameters.query_parameters = params; if (Object.keys(parameters).length === 0) { - console.warn("Called updateQuery with no proposed changes."); + log.warn("updateQuery: called with no proposed changes."); return queryId; } diff --git a/src/api/router.ts b/src/api/router.ts index 138911b..799e0b1 100644 --- a/src/api/router.ts +++ b/src/api/router.ts @@ -21,29 +21,39 @@ export class Router { } protected async _handleResponse(responsePromise: Promise): Promise { - const apiResponse = await responsePromise - .then((response) => { - if (!response.ok) { - log.error( - logPrefix, - `response error ${response.status} - ${response.statusText}`, - ); - } - return response.json(); - }) - .catch((error) => { - log.error(logPrefix, `caught unhandled response error ${JSON.stringify(error)}`); - throw error; - }); - if (apiResponse.error) { - log.error(logPrefix, `error contained in response ${JSON.stringify(apiResponse)}`); - if (apiResponse.error instanceof Object) { - throw new DuneError(apiResponse.error.type); - } else { - throw new DuneError(apiResponse.error); + let result; + try { + const response = await responsePromise; + + if (!response.ok) { + log.error( + logPrefix, + `response error ${response.status} - ${response.statusText}`, + ); + } + const clonedResponse = response.clone(); + try { + // Attempt to parse JSON + result = await response.json(); + } catch { + // Fallback to text if JSON parsing fails + // This fallback is used for CSV retrieving methods. + result = await clonedResponse.text(); + } + + // Check for error in result after parsing + if (result.error) { + log.error(logPrefix, `error contained in response ${JSON.stringify(result)}`); + // Assuming DuneError is a custom Error you'd like to throw + throw new DuneError( + result.error instanceof Object ? result.error.type : result.error, + ); } + } catch (error) { + log.error(logPrefix, `caught unhandled response error ${JSON.stringify(error)}`); + throw error; } - return apiResponse; + return result; } protected async _request( diff --git a/tests/e2e/client.spec.ts b/tests/e2e/client.spec.ts index b4cec28..737a6d3 100644 --- a/tests/e2e/client.spec.ts +++ b/tests/e2e/client.spec.ts @@ -34,4 +34,16 @@ describe("DuneClient Extensions", () => { ]); expect(results.result?.rows.length).to.be.greaterThan(0); }); + + it("getsLatestResultsCSV", async () => { + // https://dune.com/queries/1215383 + const resultCSV = await client.getLatestResultCSV(1215383, [ + QueryParameter.text("TextField", "Plain Text"), + ]); + const expectedRows = [ + "text_field,number_field,date_field,list_field\n", + "Plain Text,3.1415926535,2022-05-04 00:00:00.000,Option 1\n", + ]; + expect(resultCSV).to.be.eq(expectedRows.join("")); + }); }); diff --git a/tests/e2e/executionAPI.spec.ts b/tests/e2e/executionAPI.spec.ts index 53cdd10..4a10a18 100644 --- a/tests/e2e/executionAPI.spec.ts +++ b/tests/e2e/executionAPI.spec.ts @@ -3,14 +3,17 @@ import { QueryParameter, ExecutionState, ExecutionAPI } from "../../src/"; import log from "loglevel"; import { ExecutionPerformance } from "../../src/types/requestPayload"; import { BASIC_KEY, expectAsyncThrow } from "./util"; +import { sleep } from "../../src/utils"; log.setLevel("silent", true); describe("ExecutionAPI: native routes", () => { let client: ExecutionAPI; + let testQueryId: number; beforeEach(() => { client = new ExecutionAPI(BASIC_KEY); + testQueryId = 1215383; }); // This doesn't work if run too many times at once: @@ -45,7 +48,6 @@ describe("ExecutionAPI: native routes", () => { }); it("successfully executes with query parameters", async () => { - const queryID = 1215383; const parameters = [ QueryParameter.text("TextField", "Plain Text"), QueryParameter.number("NumberField", 3.1415926535), @@ -53,14 +55,14 @@ describe("ExecutionAPI: native routes", () => { QueryParameter.enum("ListField", "Option 1"), ]; // Execute and check state - const execution = await client.executeQuery(queryID, { + const execution = await client.executeQuery(testQueryId, { query_parameters: parameters, }); expect(execution.execution_id).is.not.null; }); it("execute with Large tier performance", async () => { - const execution = await client.executeQuery(1215383, { + const execution = await client.executeQuery(testQueryId, { performance: ExecutionPerformance.Large, }); expect(execution.execution_id).is.not.null; @@ -83,6 +85,48 @@ describe("ExecutionAPI: native routes", () => { cancelled_at: "2022-10-04T12:08:48.790331Z", }); }); + + it("getResults", async () => { + const execution = await client.executeQuery(testQueryId); + await sleep(1); + // expect basic query has completed after 1s + let status = await client.getExecutionStatus(execution.execution_id); + expect(status.state).to.be.eq(ExecutionState.COMPLETED); + + // let resultJSON = await client.getExecutionResults(execution.execution_id); + await expect(() => client.getExecutionResults(execution.execution_id)).to.not.throw(); + + let resultCSV = await client.getResultCSV(execution.execution_id); + const expectedRows = [ + "text_field,number_field,date_field,list_field\n", + "Plain Text,3.1415926535,2022-05-04 00:00:00.000,Option 1\n", + ]; + expect(resultCSV).to.be.eq(expectedRows.join("")); + }); + + it("getLastResult", async () => { + // https://dune.com/queries/1215383 + const resultCSV = await client.getLastResultCSV(1215383, [ + QueryParameter.text("TextField", "Plain Text"), + ]); + const expectedRows = [ + "text_field,number_field,date_field,list_field\n", + "Plain Text,3.1415926535,2022-05-04 00:00:00.000,Option 1\n", + ]; + expect(resultCSV).to.be.eq(expectedRows.join("")); + }); + + it("getLastResultCSV", async () => { + // https://dune.com/queries/1215383 + const resultCSV = await client.getLastResultCSV(1215383, [ + QueryParameter.text("TextField", "Plain Text"), + ]); + const expectedRows = [ + "text_field,number_field,date_field,list_field\n", + "Plain Text,3.1415926535,2022-05-04 00:00:00.000,Option 1\n", + ]; + expect(resultCSV).to.be.eq(expectedRows.join("")); + }); }); describe("ExecutionAPI: Errors", () => { diff --git a/tests/e2e/util.ts b/tests/e2e/util.ts index 1d4a310..1f558a4 100644 --- a/tests/e2e/util.ts +++ b/tests/e2e/util.ts @@ -11,7 +11,6 @@ if (PLUS_API_KEY === undefined) { export const BASIC_KEY: string = BASIC_API_KEY!; export const PLUS_KEY: string = PLUS_API_KEY!; - export const expectAsyncThrow = async ( promise: Promise, message?: string | object, diff --git a/yarn.lock b/yarn.lock index bd4b085..385f128 100644 --- a/yarn.lock +++ b/yarn.lock @@ -66,9 +66,9 @@ integrity sha512-6EwiSjwWYP7pTckG6I5eyFANjPhmPjUX9JRLUSfNPC7FX7zK9gyZAfUEaECL6ALTpGX5AjnBq3C9XmVWPitNpw== "@jridgewell/resolve-uri@^3.0.3": - version "3.1.1" - resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz#c08679063f279615a3326583ba3a90d1d82cc721" - integrity sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA== + version "3.1.2" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz#7a0ee601f60f99a20c7c7c5ff0c80388c1189bd6" + integrity sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw== "@jridgewell/sourcemap-codec@^1.4.10": version "1.4.15" @@ -140,9 +140,9 @@ integrity sha512-dJvrYWxP/UcXm36Qn36fxhUKu8A/xMRXVT2cliFF1Z7UA9liG5Psj3ezNSZw+5puH2czDXRLcXQxf8JbJt0ejg== "@types/node@^20.11.17": - version "20.11.17" - resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.17.tgz#cdd642d0e62ef3a861f88ddbc2b61e32578a9292" - integrity sha512-QmgQZGWu1Yw9TDyAP9ZzpFJKynYNeOvwMJmaxABfieQoVoiVOS6MN1WSpqpRcbeA5+RW82kraAVxCCJg+780Qw== + version "20.11.19" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.19.tgz#b466de054e9cb5b3831bee38938de64ac7f81195" + integrity sha512-7xMnVEcZFu0DikYjWOlRq7NTPETrm7teqUT2WkQjrTIkEgUyyGdWsj/Zg8bEJt5TNklzbPD1X3fqfsHw3SpapQ== dependencies: undici-types "~5.26.4" @@ -422,9 +422,9 @@ doctrine@^3.0.0: esutils "^2.0.2" dotenv@^16.0.3: - version "16.4.1" - resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.1.tgz#1d9931f1d3e5d2959350d1250efab299561f7f11" - integrity sha512-CjA3y+Dr3FyFDOAMnxZEGtnW9KBR2M0JvvUtXNW+dYJL5ROWxP9DUHCwgFqpMk0OXCc0ljhaNTr2w/kutYIcHQ== + version "16.4.4" + resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.4.tgz#a26e7bb95ebd36272ebb56edb80b826aecf224c1" + integrity sha512-XvPXc8XAQThSjAbY6cQ/9PcBXmFoWuw1sQ3b8HqUCR6ziGXjkTi//kB9SWa2UwqlgdAIuRqAa/9hVljzPehbYg== emoji-regex@^8.0.0: version "8.0.0" @@ -1199,11 +1199,6 @@ tsconfig-paths@^3.5.0: minimist "^1.2.6" strip-bom "^3.0.0" -tslint-config-prettier@^1.18.0: - version "1.18.0" - resolved "https://registry.yarnpkg.com/tslint-config-prettier/-/tslint-config-prettier-1.18.0.tgz#75f140bde947d35d8f0d238e0ebf809d64592c37" - integrity sha512-xPw9PgNPLG3iKRxmK7DWr+Ea/SzrvfHtjFt5LBl61gk2UBG/DB9kCXRjv+xyIU1rUtnayLeMUVJBcMX8Z17nDg== - type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1"