diff --git a/README.md b/README.md
index 3f2dd48..03c5dd0 100644
--- a/README.md
+++ b/README.md
@@ -18,14 +18,14 @@ gratum with a couple of beliefs about data transformations.
For Gradle:
- compile group: 'com.github.chubbard', name: 'gratum', version: '1.1.7'
+ compile group: 'com.github.chubbard', name: 'gratum', version: '1.1.8'
For Maven:
com.github.chubbard
gratum
- 1.1.7
+ 1.1.8
## Oh Shell Yeah!
diff --git a/build.gradle b/build.gradle
index bcc76a3..68da47e 100644
--- a/build.gradle
+++ b/build.gradle
@@ -9,8 +9,8 @@ targetCompatibility = 1.8
sourceCompatibility = 1.8
group = 'com.github.chubbard'
-version = '1.1.7'
-//version = '1.1.6-SNAPSHOT'
+//version = '1.1.7'
+version = '1.1.8'
description = """
A simplified standalone ETL engine for groovy. Gratum is groovy + datum.
"""
@@ -24,7 +24,7 @@ dependencies {
implementation group: 'commons-codec', name: 'commons-codec', version: '1.15'
implementation group: 'commons-cli', name: 'commons-cli', version: '1.4'
- implementation('org.codehaus.groovy:groovy-all:2.5.18') {
+ implementation('org.codehaus.groovy:groovy-all:2.5.23') {
exclude group: 'org.codehaus.groovy', module:'groovy-swing'
exclude group: 'org.codehaus.groovy', module:'groovy-testng'
exclude group: 'org.codehaus.groovy', module:'groovy-console'
diff --git a/src/main/groovy/gratum/concurrency/LocalConcurrentContext.groovy b/src/main/groovy/gratum/concurrency/LocalConcurrentContext.groovy
index b2694a8..9e711a5 100644
--- a/src/main/groovy/gratum/concurrency/LocalConcurrentContext.groovy
+++ b/src/main/groovy/gratum/concurrency/LocalConcurrentContext.groovy
@@ -5,14 +5,15 @@ import gratum.etl.Pipeline
import gratum.etl.Rejection
import gratum.etl.RejectionCategory
import gratum.source.ChainedSource
-import gratum.source.ClosureSource
import groovy.transform.CompileStatic
+import groovy.transform.stc.ClosureParams
+import groovy.transform.stc.FromString
import java.util.concurrent.ArrayBlockingQueue
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
-@CompileStatic
+//@CompileStatic
public class LocalConcurrentContext implements ConcurrentContext {
final int workerSize
@@ -33,13 +34,19 @@ public class LocalConcurrentContext implements ConcurrentContext {
latch = new CountDownLatch(workers+1)
}
- public LocalConcurrentContext spread(@DelegatesTo(LocalConcurrentContext) Closure workerClosure ) {
+ public LocalConcurrentContext spread(
+ @DelegatesTo(LocalConcurrentContext)
+ @ClosureParams(value = FromString, options = ["gratum.etl.Pipeline"])
+ Closure workerClosure ) {
this.workerClosure = workerClosure
this.workerClosure.delegate = this
return this
}
- public LocalConcurrentContext collect(@DelegatesTo(LocalConcurrentContext) Closure resultsClosure ) {
+ public LocalConcurrentContext collect(
+ @DelegatesTo(LocalConcurrentContext)
+ @ClosureParams(value = FromString, options = ["gratum.etl.Pipeline"])
+ Closure resultsClosure ) {
this.resultProcessorClosure = resultsClosure
this.resultProcessorClosure.delegate = this
return this
@@ -49,7 +56,7 @@ public class LocalConcurrentContext implements ConcurrentContext {
return { Pipeline pipeline ->
createWorkers()
createResultProcessor()
- pipeline.addStep("Queue to Workers") { Map row ->
+ pipeline.addStep("Queue to Workers") { row ->
eventQueue.put( row )
return row
}
@@ -85,7 +92,7 @@ public class LocalConcurrentContext implements ConcurrentContext {
for( int i = 0; i < workerSize; i++ ) {
workers << new PipelineWorker("Worker-${i+1}", {
try {
- Pipeline pipeline = ClosureSource.of({ Pipeline pipeline ->
+ Pipeline pipeline = Pipeline.create("Worker") { pipeline ->
boolean done = false
while (!done && !Thread.interrupted()) {
Map row = eventQueue.poll()
@@ -96,21 +103,21 @@ public class LocalConcurrentContext implements ConcurrentContext {
pipeline.process(row)
}
}
- }).name("Worker").into()
- .onRejection { Pipeline rej ->
- rej.addStep { Map row ->
- // so when we play this down the rejections pipeline it'll expect a REJECT_KEY to be there so we recreate it
- // because at this point the REJECTED_KEY property has been removed so we re-add it. Not great.
- Rejection reject = new Rejection(row.rejectionReason as String, row.rejectionCategory as RejectionCategory, row.rejectionStep as String)
- row[Pipeline.REJECTED_KEY] = reject
- resultQueue.put(row)
- return row
- }
- return
- }
+ }
+ pipeline.onRejection { rej ->
+ rej.addStep("replaceRejectKey") { row ->
+ // so when we play this down the rejections pipeline it'll expect a REJECT_KEY to be there so we recreate it
+ // because at this point the REJECTED_KEY property has been removed so we re-add it. Not great.
+ Rejection reject = new Rejection(row["rejectionReason"] as String, row["rejectionCategory"] as RejectionCategory, row["rejectionStep"] as String)
+ row[Pipeline.REJECTED_KEY] = reject
+ resultQueue.put((Map)row)
+ return row
+ }
+ return
+ }
- LoadStatistic stat = ((Pipeline) workerClosure(pipeline))
- .addStep("Queue to Results") { Map row ->
+ LoadStatistic stat = workerClosure.call(pipeline)
+ .addStep("Queue to Results") { row ->
resultQueue.put(row)
return row
}
@@ -127,7 +134,7 @@ public class LocalConcurrentContext implements ConcurrentContext {
void createResultProcessor() {
resultProcessor = new PipelineWorker( "Results Processor", {
try {
- Pipeline pipeline = ClosureSource.of({ Pipeline pipeline ->
+ Pipeline pipeline = Pipeline.create("Result Processor") { pipeline ->
boolean done = false
while (!done && !Thread.interrupted()) {
Map row = resultQueue.poll(10, TimeUnit.SECONDS)
@@ -141,7 +148,7 @@ public class LocalConcurrentContext implements ConcurrentContext {
done = true
}
}
- }).name("Result Processor").into()
+ }
LoadStatistic stats = resultProcessorClosure.call(pipeline)
.go()
return stats
diff --git a/src/main/groovy/gratum/etl/Pipeline.groovy b/src/main/groovy/gratum/etl/Pipeline.groovy
index 4193716..7061b83 100644
--- a/src/main/groovy/gratum/etl/Pipeline.groovy
+++ b/src/main/groovy/gratum/etl/Pipeline.groovy
@@ -89,7 +89,10 @@ public class Pipeline {
* passed to send a row into the pipeline.
* @return The pipeline attached to results of the startClosure.
*/
- public static Pipeline create( String name, @DelegatesTo(Pipeline) Closure startClosure ) {
+ public static Pipeline create( String name,
+ @DelegatesTo(Pipeline)
+ @ClosureParams( value = FromString, options = ["gratum.etl.Pipeline"])
+ Closure startClosure ) {
ClosureSource.of( startClosure ).name( name ).into()
}
@@ -120,7 +123,10 @@ public class Pipeline {
* @param step The code used to process each row on the Pipeline.
* @return this Pipeline.
*/
- public Pipeline prependStep( String name = null, @DelegatesTo(Pipeline) Closure