-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
3 changed files
with
257 additions
and
74 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,117 @@ | ||
/** | ||
* Copyright (c) 2020-present Snowplow Analytics Ltd. | ||
* All rights reserved. | ||
* | ||
* This software is made available by Snowplow Analytics, Ltd., | ||
* under the terms of the Snowplow Limited Use License Agreement, Version 1.0 | ||
* located at https://docs.snowplow.io/limited-use-license-1.0 | ||
* BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION | ||
* OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. | ||
*/ | ||
|
||
package inmemory | ||
|
||
import ( | ||
"time" | ||
|
||
"github.com/pkg/errors" | ||
log "github.com/sirupsen/logrus" | ||
"github.com/snowplow/snowbridge/config" | ||
"github.com/snowplow/snowbridge/pkg/models" | ||
"github.com/snowplow/snowbridge/pkg/source/sourceiface" | ||
"github.com/twinj/uuid" | ||
) | ||
|
||
// ConfigPair is passed to configuration to determine when to build in memory source. | ||
func ConfigPair(messages chan []string) config.ConfigurationPair { | ||
return config.ConfigurationPair{ | ||
Name: "inMemory", | ||
Handle: adapterGenerator(configfunction(messages)), | ||
} | ||
} | ||
|
||
type configuration struct{} | ||
|
||
type inMemorySource struct { | ||
messages chan []string | ||
log *log.Entry | ||
exitSignal chan struct{} | ||
} | ||
|
||
func configfunction(messages chan []string) func(c *configuration) (sourceiface.Source, error) { | ||
return func(c *configuration) (sourceiface.Source, error) { | ||
return newInMemorySource(messages) | ||
} | ||
} | ||
|
||
type adapter func(i interface{}) (interface{}, error) | ||
|
||
func (f adapter) Create(i interface{}) (interface{}, error) { | ||
return f(i) | ||
} | ||
|
||
func (f adapter) ProvideDefault() (interface{}, error) { | ||
cfg := &configuration{} | ||
|
||
return cfg, nil | ||
} | ||
|
||
func adapterGenerator(f func(c *configuration) (sourceiface.Source, error)) adapter { | ||
return func(i interface{}) (interface{}, error) { | ||
cfg, ok := i.(*configuration) | ||
if !ok { | ||
return nil, errors.New("invalid input") | ||
} | ||
|
||
return f(cfg) | ||
} | ||
} | ||
|
||
func newInMemorySource(messages chan []string) (*inMemorySource, error) { | ||
return &inMemorySource{ | ||
log: log.WithFields(log.Fields{"source": "in_memory"}), | ||
messages: messages, | ||
exitSignal: make(chan struct{}), | ||
}, nil | ||
} | ||
|
||
func (ss *inMemorySource) Read(sf *sourceiface.SourceFunctions) error { | ||
ss.log.Infof("Reading messages from in memory buffer") | ||
|
||
processing: | ||
for { | ||
select { | ||
case <-ss.exitSignal: | ||
break processing | ||
case input := <-ss.messages: | ||
timeNow := time.Now().UTC() | ||
var messages []*models.Message | ||
for _, single := range input { | ||
message := models.Message{ | ||
Data: []byte(single), | ||
PartitionKey: uuid.NewV4().String(), | ||
TimeCreated: timeNow, | ||
TimePulled: timeNow, | ||
} | ||
messages = append(messages, &message) | ||
} | ||
|
||
err := sf.WriteToTarget(messages) | ||
if err != nil { | ||
ss.log.WithFields(log.Fields{"error": err}).Error(err) | ||
} | ||
} | ||
} | ||
|
||
ss.log.Infof("Done with processing") | ||
return nil | ||
} | ||
|
||
func (ss *inMemorySource) Stop() { | ||
ss.log.Warn("Stopping in memory source") | ||
ss.exitSignal <- struct{}{} | ||
} | ||
|
||
func (ss *inMemorySource) GetID() string { | ||
return "inMemory" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
/** | ||
* Copyright (c) 2020-present Snowplow Analytics Ltd. | ||
* All rights reserved. | ||
* | ||
* This software is made available by Snowplow Analytics, Ltd., | ||
* under the terms of the Snowplow Limited Use License Agreement, Version 1.0 | ||
* located at https://docs.snowplow.io/limited-use-license-1.0 | ||
* BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION | ||
* OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. | ||
*/ | ||
|
||
package inmemory | ||
|
||
import ( | ||
"sync" | ||
"testing" | ||
|
||
"github.com/snowplow/snowbridge/pkg/models" | ||
"github.com/snowplow/snowbridge/pkg/source/sourceiface" | ||
"github.com/stretchr/testify/assert" | ||
) | ||
|
||
func TestInMemorySource_ReadSuccess(t *testing.T) { | ||
assert := assert.New(t) | ||
|
||
wg := sync.WaitGroup{} | ||
inputChannel := make(chan []string) | ||
source, err := newInMemorySource(inputChannel) | ||
assert.NotNil(source) | ||
assert.Nil(err) | ||
assert.Equal("inMemory", source.GetID()) | ||
defer source.Stop() | ||
|
||
var out []string | ||
|
||
writeFunc := func(messages []*models.Message) error { | ||
for _, msg := range messages { | ||
out = append(out, string(msg.Data)) | ||
wg.Done() | ||
} | ||
return nil | ||
} | ||
|
||
sf := sourceiface.SourceFunctions{ | ||
WriteToTarget: writeFunc, | ||
} | ||
|
||
go func() { | ||
err1 := source.Read(&sf) | ||
assert.Nil(err1) | ||
}() | ||
|
||
wg.Add(6) | ||
inputChannel <- []string{"m1", "m2"} | ||
inputChannel <- []string{"m3", "m4", "m5"} | ||
inputChannel <- []string{"m6"} | ||
wg.Wait() | ||
|
||
assert.Equal([]string{"m1", "m2", "m3", "m4", "m5", "m6"}, out) | ||
} |