From 2fc263e73fc4cdaad74536eb45c3abaa8d6f5ac1 Mon Sep 17 00:00:00 2001 From: Onur Cinar Date: Fri, 22 Dec 2023 00:30:41 -0800 Subject: [PATCH] Tiingo added. --- asset/README.md | 185 +++++++++++++++++++++- asset/file_system_repository.go | 16 +- asset/file_system_repository_test.go | 34 ++++- asset/repository.go | 4 + asset/testdata/since.csv | 9 ++ asset/tiingo_repository.go | 221 +++++++++++++++++++++++++++ asset/tiingo_repository_test.go | 138 +++++++++++++++++ helper/README.md | 4 +- helper/filter.go | 2 +- 9 files changed, 603 insertions(+), 10 deletions(-) create mode 100644 asset/testdata/since.csv create mode 100644 asset/tiingo_repository.go create mode 100644 asset/tiingo_repository_test.go diff --git a/asset/README.md b/asset/README.md index 8c76c1f..d3ee0d1 100644 --- a/asset/README.md +++ b/asset/README.md @@ -29,9 +29,20 @@ The information provided on this project is strictly for informational purposes - [func \(r \*FileSystemRepository\) Append\(name string, snapshots \<\-chan \*Snapshot\) error](<#FileSystemRepository.Append>) - [func \(r \*FileSystemRepository\) Assets\(\) \(\[\]string, error\)](<#FileSystemRepository.Assets>) - [func \(r \*FileSystemRepository\) Get\(name string\) \(\<\-chan \*Snapshot, error\)](<#FileSystemRepository.Get>) + - [func \(r \*FileSystemRepository\) GetSince\(name string, date time.Time\) \(\<\-chan \*Snapshot, error\)](<#FileSystemRepository.GetSince>) - [func \(r \*FileSystemRepository\) LastDate\(name string\) \(time.Time, error\)](<#FileSystemRepository.LastDate>) - [type Repository](<#Repository>) - [type Snapshot](<#Snapshot>) +- [type TiingoEndOfDay](<#TiingoEndOfDay>) + - [func \(e \*TiingoEndOfDay\) ToSnapshot\(\) \*Snapshot](<#TiingoEndOfDay.ToSnapshot>) +- [type TiingoMeta](<#TiingoMeta>) +- [type TiingoRepository](<#TiingoRepository>) + - [func NewTiingoRepository\(apiKey string\) \*TiingoRepository](<#NewTiingoRepository>) + - [func \(r \*TiingoRepository\) Append\(\_ string, \_ \<\-chan \*Snapshot\) error](<#TiingoRepository.Append>) + - [func \(r \*TiingoRepository\) Assets\(\) \(\[\]string, error\)](<#TiingoRepository.Assets>) + - [func \(r \*TiingoRepository\) Get\(name string\) \(\<\-chan \*Snapshot, error\)](<#TiingoRepository.Get>) + - [func \(r \*TiingoRepository\) GetSince\(name string, date time.Time\) \(\<\-chan \*Snapshot, error\)](<#TiingoRepository.GetSince>) + - [func \(r \*TiingoRepository\) LastDate\(name string\) \(time.Time, error\)](<#TiingoRepository.LastDate>) @@ -56,7 +67,7 @@ func NewFileSystemRepository(base string) *FileSystemRepository NewFileSystemRepository initializes a file system repository with the given base directory. -### func \(\*FileSystemRepository\) [Append]() +### func \(\*FileSystemRepository\) [Append]() ```go func (r *FileSystemRepository) Append(name string, snapshots <-chan *Snapshot) error @@ -80,10 +91,19 @@ Assets returns the names of all assets in the repository. func (r *FileSystemRepository) Get(name string) (<-chan *Snapshot, error) ``` -Get attempts to return a channel of snapshots fo the asset with the given name. +Get attempts to return a channel of snapshots for the asset with the given name. + + +### func \(\*FileSystemRepository\) [GetSince]() + +```go +func (r *FileSystemRepository) GetSince(name string, date time.Time) (<-chan *Snapshot, error) +``` + +GetSince attempts to return a channel of snapshots for the asset with the given name since the given date. -### func \(\*FileSystemRepository\) [LastDate]() +### func \(\*FileSystemRepository\) [LastDate]() ```go func (r *FileSystemRepository) LastDate(name string) (time.Time, error) @@ -92,7 +112,7 @@ func (r *FileSystemRepository) LastDate(name string) (time.Time, error) LastDate returns the date of the last snapshot for the asset with the given name. -## type [Repository]() +## type [Repository]() Repository serves as a centralized storage and retrieval location for asset snapshots. @@ -105,6 +125,10 @@ type Repository interface { // the asset with the given name. Get(name string) (<-chan *Snapshot, error) + // GetSince attempts to return a channel of snapshots for + // the asset with the given name since the given date. + GetSince(name string, date time.Time) (<-chan *Snapshot, error) + // LastDate returns the date of the last snapshot for // the asset with the given name. LastDate(name string) (time.Time, error) @@ -147,4 +171,157 @@ type Snapshot struct { } ``` + +## type [TiingoEndOfDay]() + +TiingoEndOfDay is the repose from the end\-of\-day endpoint. https://www.tiingo.com/documentation/end-of-day + +```go +type TiingoEndOfDay struct { + // Date is the date this data pertains to. + Date time.Time `json:"date"` + + // Open is the opening price. + Open float64 `json:"open"` + + // High is the highest price. + High float64 `json:"high"` + + // Low is the lowest price. + Low float64 `json:"low"` + + // Close is the closing price. + Close float64 `json:"close"` + + // Volume is the total volume. + Volume int64 `json:"volume"` + + // AdjOpen is the adjusted opening price. + AdjOpen float64 `json:"adjOpen"` + + // AdjHigh is the adjusted highest price. + AdjHigh float64 `json:"adjHigh"` + + // AdjLow is the adjusted lowest price. + AdjLow float64 `json:"adjLow"` + + // AdjClose is the adjusted closing price. + AdjClose float64 `json:"adjClose"` + + // AdjVolume is the adjusted total volume. + AdjVolume int64 `json:"adjVolume"` + + // Dividend is the dividend paid out. + Dividend float64 `json:"divCash"` + + // Split to adjust values after a split. + Split float64 `json:"splitFactor"` +} +``` + + +### func \(\*TiingoEndOfDay\) [ToSnapshot]() + +```go +func (e *TiingoEndOfDay) ToSnapshot() *Snapshot +``` + +ToSnapshot converts the Tiingo end\-of\-day to a snapshot. + + +## type [TiingoMeta]() + +TiingoMeta is the response from the meta endpoint. https://www.tiingo.com/documentation/end-of-day + +```go +type TiingoMeta struct { + // Ticker related to the asset. + Ticker string `json:"ticker"` + + // Name is the full name of the asset. + Name string `json:"name"` + + // ExchangeCode is the exchange where the asset is listed on. + ExchangeCode string `json:"exchangeCode"` + + // Description is the description of the asset. + Description string `json:"description"` + + // StartDate is the earliest date for the asset data. + StartDate time.Time `json:"startDate"` + + // EndDate is the latest date for the asset data. + EndDate time.Time `json:"endDate"` +} +``` + + +## type [TiingoRepository]() + +TiingoRepository provides access to financial market data, retrieving asset snapshots, by interacting with the Tiingo Stock & Financial Markets API. To use this repository, you'll need a valid API key from https://www.tiingo.com. + +```go +type TiingoRepository struct { + Repository + + // BaseURL is the Tiingo API URL. + BaseURL string + // contains filtered or unexported fields +} +``` + + +### func [NewTiingoRepository]() + +```go +func NewTiingoRepository(apiKey string) *TiingoRepository +``` + +NewTiingoRepository initializes a file system repository with the given API key. + + +### func \(\*TiingoRepository\) [Append]() + +```go +func (r *TiingoRepository) Append(_ string, _ <-chan *Snapshot) error +``` + +Append adds the given snapshows to the asset with the given name. + + +### func \(\*TiingoRepository\) [Assets]() + +```go +func (r *TiingoRepository) Assets() ([]string, error) +``` + +Assets returns the names of all assets in the repository. + + +### func \(\*TiingoRepository\) [Get]() + +```go +func (r *TiingoRepository) Get(name string) (<-chan *Snapshot, error) +``` + +Get attempts to return a channel of snapshots for the asset with the given name. + + +### func \(\*TiingoRepository\) [GetSince]() + +```go +func (r *TiingoRepository) GetSince(name string, date time.Time) (<-chan *Snapshot, error) +``` + +GetSince attempts to return a channel of snapshots for the asset with the given name since the given date. + + +### func \(\*TiingoRepository\) [LastDate]() + +```go +func (r *TiingoRepository) LastDate(name string) (time.Time, error) +``` + +LastDate returns the date of the last snapshot for the asset with the given name. + Generated by [gomarkdoc]() diff --git a/asset/file_system_repository.go b/asset/file_system_repository.go index 6135ac7..a7fd650 100644 --- a/asset/file_system_repository.go +++ b/asset/file_system_repository.go @@ -53,11 +53,25 @@ func (r *FileSystemRepository) Assets() ([]string, error) { return assets, nil } -// Get attempts to return a channel of snapshots fo the asset with the given name. +// Get attempts to return a channel of snapshots for the asset with the given name. func (r *FileSystemRepository) Get(name string) (<-chan *Snapshot, error) { return helper.ReadFromCsvFile[Snapshot](r.getCsvFileName(name), true) } +// GetSince attempts to return a channel of snapshots for the asset with the given name since the given date. +func (r *FileSystemRepository) GetSince(name string, date time.Time) (<-chan *Snapshot, error) { + snapshots, err := helper.ReadFromCsvFile[Snapshot](r.getCsvFileName(name), true) + if err != nil { + return nil, err + } + + snapshots = helper.Filter(snapshots, func(s *Snapshot) bool { + return s.Date.Equal(date) || s.Date.After(date) + }) + + return snapshots, nil +} + // LastDate returns the date of the last snapshot for the asset with the given name. func (r *FileSystemRepository) LastDate(name string) (time.Time, error) { var last time.Time diff --git a/asset/file_system_repository_test.go b/asset/file_system_repository_test.go index 12e3411..fef19ab 100644 --- a/asset/file_system_repository_test.go +++ b/asset/file_system_repository_test.go @@ -17,7 +17,7 @@ import ( func TestFileSystemRepositoryAssets(t *testing.T) { repository := asset.NewFileSystemRepository("testdata") - expected := []string{"brk-b", "empty"} + expected := []string{"brk-b", "empty", "since"} actual, err := repository.Assets() if err != nil { @@ -50,9 +50,39 @@ func TestFileSystemRepositoryGet(t *testing.T) { } func TestFileSystemRepositoryGetNonExisting(t *testing.T) { + repository := asset.NewFileSystemRepository("testdata/non_existing") + + _, err := repository.Get("brk-b") + if err == nil { + t.Fatal("expected error") + } +} + +func TestFileSystemRepositoryGetSince(t *testing.T) { + repository := asset.NewFileSystemRepository("testdata") + + date := time.Date(2022, 12, 20, 0, 0, 0, 0, time.UTC) + actual, err := repository.GetSince("brk-b", date) + if err != nil { + t.Fatal(err) + } + + expected, err := helper.ReadFromCsvFile[asset.Snapshot]("testdata/since.csv", true) + if err != nil { + t.Fatal(err) + } + + err = helper.CheckEquals(actual, expected) + if err != nil { + t.Fatal(err) + } +} + +func TestFileSystemRepositoryGetSinceNonExisting(t *testing.T) { repository := asset.NewFileSystemRepository("testdata") - _, err := repository.Get("brk") + date := time.Date(2022, 12, 01, 0, 0, 0, 0, time.UTC) + _, err := repository.GetSince("brk", date) if err == nil { t.Fatal("expected error") } diff --git a/asset/repository.go b/asset/repository.go index 7292123..40f3f03 100644 --- a/asset/repository.go +++ b/asset/repository.go @@ -16,6 +16,10 @@ type Repository interface { // the asset with the given name. Get(name string) (<-chan *Snapshot, error) + // GetSince attempts to return a channel of snapshots for + // the asset with the given name since the given date. + GetSince(name string, date time.Time) (<-chan *Snapshot, error) + // LastDate returns the date of the last snapshot for // the asset with the given name. LastDate(name string) (time.Time, error) diff --git a/asset/testdata/since.csv b/asset/testdata/since.csv new file mode 100644 index 0000000..b6af201 --- /dev/null +++ b/asset/testdata/since.csv @@ -0,0 +1,9 @@ +Date,Open,High,Low,Close,Adj Close,Volume +2022-12-20,300.089996,304.190002,297,302,302,3090700 +2022-12-21,304.380005,308.540009,304.160004,307.820007,307.820007,3264600 +2022-12-22,306.100006,306.5,297.640015,302.690002,302.690002,3560100 +2022-12-23,302.880005,306.570007,300.929993,306.48999,306.48999,2460400 +2022-12-27,306.450012,308.579987,304.649994,305.549988,305.549988,2730900 +2022-12-28,304.769989,307.459991,303.26001,303.429993,303.429993,2628200 +2022-12-29,305.940002,309.380005,305.23999,309.059998,309.059998,2846200 +2022-12-30,306.950012,309.040009,305.619995,308.899994,308.899994,3298300 diff --git a/asset/tiingo_repository.go b/asset/tiingo_repository.go new file mode 100644 index 0000000..aff9120 --- /dev/null +++ b/asset/tiingo_repository.go @@ -0,0 +1,221 @@ +// Copyright (c) 2023 Onur Cinar. All Rights Reserved. +// The source code is provided under MIT License. +// https://github.com/cinar/indicator + +package asset + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "time" +) + +// TiingoMeta is the response from the meta endpoint. +// https://www.tiingo.com/documentation/end-of-day +type TiingoMeta struct { + // Ticker related to the asset. + Ticker string `json:"ticker"` + + // Name is the full name of the asset. + Name string `json:"name"` + + // ExchangeCode is the exchange where the asset is listed on. + ExchangeCode string `json:"exchangeCode"` + + // Description is the description of the asset. + Description string `json:"description"` + + // StartDate is the earliest date for the asset data. + StartDate time.Time `json:"startDate"` + + // EndDate is the latest date for the asset data. + EndDate time.Time `json:"endDate"` +} + +// TiingoEndOfDay is the repose from the end-of-day endpoint. +// https://www.tiingo.com/documentation/end-of-day +type TiingoEndOfDay struct { + // Date is the date this data pertains to. + Date time.Time `json:"date"` + + // Open is the opening price. + Open float64 `json:"open"` + + // High is the highest price. + High float64 `json:"high"` + + // Low is the lowest price. + Low float64 `json:"low"` + + // Close is the closing price. + Close float64 `json:"close"` + + // Volume is the total volume. + Volume int64 `json:"volume"` + + // AdjOpen is the adjusted opening price. + AdjOpen float64 `json:"adjOpen"` + + // AdjHigh is the adjusted highest price. + AdjHigh float64 `json:"adjHigh"` + + // AdjLow is the adjusted lowest price. + AdjLow float64 `json:"adjLow"` + + // AdjClose is the adjusted closing price. + AdjClose float64 `json:"adjClose"` + + // AdjVolume is the adjusted total volume. + AdjVolume int64 `json:"adjVolume"` + + // Dividend is the dividend paid out. + Dividend float64 `json:"divCash"` + + // Split to adjust values after a split. + Split float64 `json:"splitFactor"` +} + +// ToSnapshot converts the Tiingo end-of-day to a snapshot. +func (e *TiingoEndOfDay) ToSnapshot() *Snapshot { + return &Snapshot{ + Date: e.Date, + Open: e.AdjOpen, + High: e.AdjHigh, + Low: e.AdjLow, + Close: e.AdjClose, + Volume: float64(e.AdjVolume), + } +} + +// TiingoRepository provides access to financial market data, retrieving +// asset snapshots, by interacting with the Tiingo Stock & Financial +// Markets API. To use this repository, you'll need a valid API key +// from https://www.tiingo.com. +type TiingoRepository struct { + Repository + + // apiKey is the Tiingo API key. + apiKey string + + // Client is the HTTP client. + client *http.Client + + // BaseURL is the Tiingo API URL. + BaseURL string +} + +// NewTiingoRepository initializes a file system repository with +// the given API key. +func NewTiingoRepository(apiKey string) *TiingoRepository { + return &TiingoRepository{ + apiKey: apiKey, + client: &http.Client{}, + BaseURL: "https://api.tiingo.com", + } +} + +// Assets returns the names of all assets in the repository. +func (r *TiingoRepository) Assets() ([]string, error) { + return nil, errors.ErrUnsupported +} + +// Get attempts to return a channel of snapshots for the asset with the given name. +func (r *TiingoRepository) Get(name string) (<-chan *Snapshot, error) { + return r.GetSince(name, time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)) +} + +// GetSince attempts to return a channel of snapshots for the asset with the given name since the given date. +func (r *TiingoRepository) GetSince(name string, date time.Time) (<-chan *Snapshot, error) { + url := fmt.Sprintf("%s/tiingo/daily/%s/prices?startDate=%s&token=%s", + r.BaseURL, + name, + date.Format("2006-01-02"), + r.apiKey) + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + res, err := r.client.Do(req) + if err != nil { + return nil, err + } + + snapshots := make(chan *Snapshot) + + go func() { + defer res.Body.Close() + defer close(snapshots) + + decoder := json.NewDecoder(res.Body) + + _, err = decoder.Token() + if err != nil { + log.Print(err) + return + } + + for decoder.More() { + var data TiingoEndOfDay + + err = decoder.Decode(&data) + if err != nil { + log.Print(err) + break + } + + snapshots <- data.ToSnapshot() + } + + _, err = decoder.Token() + if err != nil { + log.Printf("GetSince failed with %v", err) + return + } + }() + + return snapshots, nil +} + +// LastDate returns the date of the last snapshot for the asset with the given name. +func (r *TiingoRepository) LastDate(name string) (time.Time, error) { + var lastDate time.Time + + url := fmt.Sprintf("%s/tiingo/daily/%s?token=%s", r.BaseURL, name, r.apiKey) + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return lastDate, err + } + + res, err := r.client.Do(req) + if err != nil { + return lastDate, err + } + + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + return lastDate, err + } + + var meta TiingoMeta + + err = json.Unmarshal(body, &meta) + if err != nil { + return lastDate, err + } + + return meta.EndDate, nil +} + +// Append adds the given snapshows to the asset with the given name. +func (r *TiingoRepository) Append(_ string, _ <-chan *Snapshot) error { + return errors.ErrUnsupported +} diff --git a/asset/tiingo_repository_test.go b/asset/tiingo_repository_test.go new file mode 100644 index 0000000..9dd4369 --- /dev/null +++ b/asset/tiingo_repository_test.go @@ -0,0 +1,138 @@ +// Copyright (c) 2023 Onur Cinar. All Rights Reserved. +// The source code is provided under MIT License. +// https://github.com/cinar/indicator + +package asset_test + +import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "reflect" + "testing" + "time" + + "github.com/cinar/indicator/asset" +) + +func TestTiingoRepositoryAssets(t *testing.T) { + repository := asset.NewTiingoRepository("1234") + + _, err := repository.Assets() + if err != errors.ErrUnsupported { + t.Fatal(err) + } +} + +func TestTiingoRepositoryGet(t *testing.T) { + data := []asset.TiingoEndOfDay{ + { + Date: time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), + AdjOpen: 10, + AdjHigh: 30, + AdjLow: 5, + AdjClose: 20, + AdjVolume: 100, + }, + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := json.Marshal(data) + if err != nil { + t.Fatal(err) + } + + w.Write(body) + })) + + repository := asset.NewTiingoRepository("1234") + repository.BaseURL = server.URL + + snapshots, err := repository.Get("A") + if err != nil { + t.Fatal(err) + } + + expected := data[0].ToSnapshot() + actual := <-snapshots + + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("actual %v expected %v", actual, expected) + } +} + +func TestTiingoRepositoryGetNotReachable(t *testing.T) { + repository := asset.NewTiingoRepository("1234") + repository.BaseURL = "abcd://a.b.c.d" + + _, err := repository.Get("A") + if err == nil { + t.Fatal("expected error") + } +} + +func TestTiingoRepositoryLastDate(t *testing.T) { + meta := asset.TiingoMeta{ + Ticker: "A", + Name: "N", + ExchangeCode: "E", + StartDate: time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), + EndDate: time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC), + Description: "D", + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := json.Marshal(meta) + if err != nil { + t.Fatal(err) + } + + w.Write(body) + })) + + repository := asset.NewTiingoRepository("1234") + repository.BaseURL = server.URL + + lastDate, err := repository.LastDate("A") + if err != nil { + t.Fatal(err) + } + + if lastDate != meta.EndDate { + t.Fatalf("actual %v expected %v", lastDate, meta.EndDate) + } +} + +func TestTiingoRepositoryLastDateInvalid(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + })) + + repository := asset.NewTiingoRepository("1234") + repository.BaseURL = server.URL + + _, err := repository.LastDate("A") + if err == nil { + t.Fatal("expected error") + } +} + +func TestTiingoRepositoryLastDateNotReachable(t *testing.T) { + repository := asset.NewTiingoRepository("1234") + repository.BaseURL = "abcd://a.b.c.d" + + _, err := repository.LastDate("A") + if err == nil { + t.Fatal("expected error") + } +} + +func TestTiingoRepositoryAppend(t *testing.T) { + repository := asset.NewTiingoRepository("1234") + + err := repository.Append("A", nil) + if err != errors.ErrUnsupported { + t.Fatal(err) + } +} diff --git a/helper/README.md b/helper/README.md index dad9d4c..f436c58 100644 --- a/helper/README.md +++ b/helper/README.md @@ -40,7 +40,7 @@ The information provided on this project is strictly for informational purposes - [func Drain\[T any\]\(c \<\-chan T\)](<#Drain>) - [func Duplicate\[T any\]\(input \<\-chan T, count int\) \[\]\<\-chan T](<#Duplicate>) - [func Field\[T, S any\]\(c \<\-chan \*S, name string\) \(\<\-chan T, error\)](<#Field>) -- [func Filter\[T Number\]\(c \<\-chan T, p func\(T\) bool\) \<\-chan T](<#Filter>) +- [func Filter\[T any\]\(c \<\-chan T, p func\(T\) bool\) \<\-chan T](<#Filter>) - [func First\[T any\]\(c \<\-chan T, count int\) \<\-chan T](<#First>) - [func Head\[T Number\]\(c \<\-chan T, count int\) \<\-chan T](<#Head>) - [func IncrementBy\[T Number\]\(c \<\-chan T, i T\) \<\-chan T](<#IncrementBy>) @@ -347,7 +347,7 @@ Field extracts a specific field from a channel of struct pointers and delivers i ## func [Filter]() ```go -func Filter[T Number](c <-chan T, p func(T) bool) <-chan T +func Filter[T any](c <-chan T, p func(T) bool) <-chan T ``` Filter filters the items from the input channel based on the provided predicate function. The predicate function takes a float64 value as input and returns a boolean value indicating whether the value should be included in the output channel. diff --git a/helper/filter.go b/helper/filter.go index 214a9b3..d469d04 100644 --- a/helper/filter.go +++ b/helper/filter.go @@ -14,7 +14,7 @@ package helper // even := helper.Filter(c, func(n int) bool { // return n%2 == 0 // }) -func Filter[T Number](c <-chan T, p func(T) bool) <-chan T { +func Filter[T any](c <-chan T, p func(T) bool) <-chan T { fc := make(chan T) go func() {