Skip to content

Commit

Permalink
Initial Historical API endpoints
Browse files Browse the repository at this point in the history
  • Loading branch information
neomantra committed May 16, 2024
1 parent 2f91ce6 commit 714c925
Show file tree
Hide file tree
Showing 6 changed files with 602 additions and 6 deletions.
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@

This repository contains Golang bindings to [Databento's](https://databento.com) file format [Databento Binary Encoding (DBN)](https://databento.com/docs/knowledge-base/new-users/dbn-encoding).

Support for the [Databento Historical API](https://databento.com/docs/api-reference-historical) is in-progress. See [`/hist`](./hist) for details.


## Open Collaboration

Expand All @@ -23,7 +25,7 @@ We welcome contributions and feedback. Please adhere to our [Code of Conduct](.

Released under the [Apache License, version 2.0](https://www.apache.org/licenses/LICENSE-2.0), see [LICENSE.txt](./LICENSE.txt).

Portions adapted from [`databento/dbn`](https://github.com/databento/dbn) under the same Apache license.
Portions adapted from [`databento/dbn`](https://github.com/databento/dbn) [`databendo/databento-rs`](https://github.com/databento/databento-rs) under the same Apache license.

Copyright (c) 2024 [Neomantra Corp](https://www.neomantra.com).

Expand Down
136 changes: 131 additions & 5 deletions consts.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@

package dbn

import (
"fmt"
"strings"
)

// Side
type Side uint8

Expand Down Expand Up @@ -117,6 +122,30 @@ const (
SType_Cms SType = 6
)

// Returns the string representation of the SType, or empty string if unknown.
func (s SType) String() string {
switch s {
case SType_InstrumentId:
return "instrument_id"
case SType_RawSymbol:
return "raw_symbol"
case SType_Smart:
return "smart"
case SType_Continuous:
return "continuous"
case SType_Parent:
return "parent"
case SType_Nasdaq:
return "nasdaq"
case SType_Cms:
return "cms"
default:
return ""
}
}

///////////////////////////////////////////////////////////////////////////////

type RType uint8

const (
Expand All @@ -142,6 +171,8 @@ const (
RType_Unknown RType = 0xFF // Golang-only: Unknown or invalid record type
)

///////////////////////////////////////////////////////////////////////////////

type Schema uint16

const (
Expand Down Expand Up @@ -179,26 +210,121 @@ const (
Schema_OhlcvEod Schema = 13
)

// Returns the string representation of the Schema, or empty string if unknown.
func (s Schema) String() string {
switch s {
case Schema_Mixed:
return "mixed"
case Schema_Mbo:
return "mbo"
case Schema_Mbp1:
return "mbp-1"
case Schema_Mbp10:
return "mbp-10"
case Schema_Tbbo:
return "tbbo"
case Schema_Trades:
return "trades"
case Schema_Ohlcv1S:
return "ohlcv-1s"
case Schema_Ohlcv1M:
return "ohlcv-1m"
case Schema_Ohlcv1H:
return "ohlcv-1h"
case Schema_Ohlcv1D:
return "ohlcv-1d"
case Schema_Definition:
return "definition"
case Schema_Statistics:
return "statistics"
case Schema_Status:
return "status"
case Schema_Imbalance:
return "imbalance"
case Schema_OhlcvEod:
return "ohlcv-eod"
default:
return ""
}
}

// SchemaFromString converts a string to a Schema.
func SchemaFromString(str string) (Schema, error) {
str = strings.ToLower(str)
switch str {
case "mixed":
return Schema_Mixed, nil
case "mbo":
return Schema_Mbo, nil
case "mbp-1":
return Schema_Mbp1, nil
case "mbp-10":
return Schema_Mbp10, nil
case "tbbo":
return Schema_Tbbo, nil
case "trades":
return Schema_Trades, nil
case "ohlcv-1s":
return Schema_Ohlcv1S, nil
case "ohlcv-1m":
return Schema_Ohlcv1M, nil
case "ohlcv-1h":
return Schema_Ohlcv1H, nil
case "ohlcv-1d":
return Schema_Ohlcv1D, nil
case "definition":
return Schema_Definition, nil
case "statistics":
return Schema_Statistics, nil
case "status":
return Schema_Status, nil
case "imbalance":
return Schema_Imbalance, nil
case "ohlcv-eod":
return Schema_OhlcvEod, nil
default:
return Schema_Mixed, fmt.Errorf("unknown schema: %s", str)
}
}

///////////////////////////////////////////////////////////////////////////////

// / Encoding A data encoding format.
type Encoding uint8

const (
/// Databento Binary Encoding.
Dbn Encoding = 0
Encoding_Dbn Encoding = 0
/// Comma-separated values.
Csv Encoding = 1
Encoding_Csv Encoding = 1
/// JavaScript object notation.
Json Encoding = 2
Encoding_Json Encoding = 2
)

// Returns the string representation of the Encoding, or empty string if unknown.
func (e Encoding) String() string {
switch e {
case Encoding_Dbn:
return "dbn"
case Encoding_Csv:
return "csv"
case Encoding_Json:
return "json"
default:
return ""
}
}

///////////////////////////////////////////////////////////////////////////////

// / A compression format or none if uncompressed.
type Compression uint8

const (
/// Uncompressed.
None Compression = 0
Compress_None Compression = 0
/// Zstandard compressed.
ZStd Compression = 1
Compress_ZStd Compression = 1
)

// / Constants for the bit flag record fields.
Expand Down
106 changes: 106 additions & 0 deletions hist/hist.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
// Copyright (c) 2024 Neomantra Corp

package dbn_hist

import (
"encoding/base64"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
)

// A **half**-closed date interval with an inclusive start date and an exclusive end date.
type DateRange struct {
// The start date (inclusive).
Start time.Time `json:"start"`
// The end date (exclusive).
End time.Time `json:"end"`
}

type RequestError struct {
Case string `json:"case"`
Message string `json:"message"`
StatusCode int `json:"status_code"`
Docs string `json:"docs,omitempty"`
Payload string `json:"payload,omitempty"`
}

type RequestErrorResp struct {
Detail RequestError `json:"detail"`
}

//////////////////////////////////////////////////////////////////////////////

func databentoGetRequest(urlStr string, apiKey string) ([]byte, error) {
apiUrl, err := url.Parse(urlStr)
if err != nil {
return nil, err
}
req, err := http.NewRequest("GET", apiUrl.String(), nil)
if err != nil {
return nil, err
}

auth := base64.StdEncoding.EncodeToString([]byte(apiKey + ":"))
req.Header.Add("Authorization", "Basic "+auth)

client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()

body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}

return body, nil
}

//////////////////////////////////////////////////////////////////////////////

func databentoPostFormRequest(urlStr string, apiKey string, form url.Values) ([]byte, error) {
apiUrl, err := url.Parse(urlStr)
if err != nil {
return nil, err
}

formBody := strings.NewReader(form.Encode())
req, err := http.NewRequest("POST", apiUrl.String(), formBody)
if err != nil {
return nil, err
}

req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

auth := base64.StdEncoding.EncodeToString([]byte(apiKey + ":"))
req.Header.Add("Authorization", "Basic "+auth)

client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()

badStatusCode := (resp.StatusCode != http.StatusOK)

body, err := io.ReadAll(resp.Body)
if err != nil {
if badStatusCode {
return nil, fmt.Errorf("HTTP %d %s %s %w", resp.StatusCode, resp.Status, string(body), err)
}
return nil, err
}

if badStatusCode {
return nil, fmt.Errorf("HTTP %d %s %s", resp.StatusCode, resp.Status, string(body))
}

return body, nil
}
75 changes: 75 additions & 0 deletions hist/hist_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
// Copyright (c) 2024 Neomantra Corp

package dbn_hist_test

import (
"os"
"testing"
"time"

dbn "github.com/NimbleMarkets/dbn-go"
dbn_hist "github.com/NimbleMarkets/dbn-go/hist"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

// Test Launcher
func TestDbnHist(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "dbn-go hist suite")
}

var databentoApiKey string

var _ = BeforeSuite(func() {
databentoApiKey = os.Getenv("DATABENTO_API_KEY")
if databentoApiKey == "" {
Fail("DATABENTO_API_KEY not set")
}
})

var _ = Describe("DbnHist", func() {
Context("metadata", func() {
It("should ListPublishers", func() {
publishers, err := dbn_hist.ListPublishers(databentoApiKey)
Expect(err).To(BeNil())
Expect(publishers).ToNot(BeEmpty())
})
It("should ListDatasets, ListSchemas, ListFields, ListUnitPrices", func() {
datasets, err := dbn_hist.ListDatasets(databentoApiKey, dbn_hist.DateRange{})
Expect(err).To(BeNil())
Expect(datasets).ToNot(BeEmpty())

schemas, err := dbn_hist.ListSchemas(databentoApiKey, datasets[0])
Expect(err).To(BeNil())
Expect(schemas).ToNot(BeEmpty())

schema, err := dbn.SchemaFromString(schemas[0])
Expect(err).To(BeNil())

fields, err := dbn_hist.ListFields(databentoApiKey, dbn.Encoding_Dbn, schema)
Expect(err).To(BeNil())
Expect(fields).ToNot(BeEmpty())

unitPrices, err := dbn_hist.ListUnitPrices(databentoApiKey, datasets[0])
Expect(err).To(BeNil())
Expect(unitPrices).ToNot(BeEmpty())
})
It("should ResolveSymbology", func() {
resolveParams := dbn_hist.ResolveParams{
Dataset: "XNAS.ITCH",
Symbols: []string{"AAPL"},
StypeIn: dbn.SType_RawSymbol,
StypeOut: dbn.SType_InstrumentId,
DateRange: dbn_hist.DateRange{
Start: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
End: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC),
},
}
resolveResp, err := dbn_hist.SymbologyResolve(databentoApiKey, resolveParams)
Expect(err).To(BeNil())
Expect(resolveResp).ToNot(BeNil())
Expect(resolveResp.Mappings).ToNot(BeEmpty())
})
})
})
Loading

0 comments on commit 714c925

Please sign in to comment.