Skip to content

Commit

Permalink
Merge branch 'feature/unit-tests' into 'main'
Browse files Browse the repository at this point in the history 
Feature/unit tests

Closes #6

See merge request NGWPC/flows2fim!5
  • Loading branch information
Abdul Siddiqui committed Sep 3, 2024
2 parents 343596a + ac184ee commit f468270
Show file tree
Hide file tree
Showing 12 changed files with 334 additions and 140 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,5 +36,8 @@ go.work
# Data
data/

# Unit test data
!testdata/unit_tests/flow_files/*.csv



4 changes: 1 addition & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,7 @@ Dependencies:

### Testing

1. Provide access to S3 fimc-data bucket to GDAL.

2. Run `go test ./...` to run automated tests.
1. Run `go test ./...` to run automated tests.

### Building

Expand Down
10 changes: 8 additions & 2 deletions cmd/controls/controls.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,10 @@ import (

var usage string = `Usage of controls:
Given a flow file and a reach database. Create controls table of reach flows and downstream boundary conditions.
Database file must have a table rating_curves and contain following coloumns
Flow file's first coloumn values must be reach ids, and second coloumn must be discharges in cfs. Invalid lines are skipped.
Database file must have a table 'rating_curves' and contain following coloumns
reach_id INTEGER
us_flow REAL
us_depth REAL
Expand All @@ -27,6 +30,9 @@ Database file must have a table rating_curves and contain following coloumns
ds_wse REAL
boundary_condition TEXT CHECK(boundary_condition IN ('nd','kwse'))
UNIQUE(reach_id, us_flow, ds_wse, boundary_condition)
Database file must have a table 'conflation' and contain following coloumns
reach_id INTEGER
conflation_to_id INTEGER
CLI flag syntax. The following forms are permitted:
-flag
Expand Down Expand Up @@ -72,7 +78,7 @@ func ReadFlows(filePath string) (map[int]float32, error) {
for scanner.Scan() {
line := scanner.Text()
parts := strings.Split(line, ",")
if len(parts) != 2 {
if len(parts) < 2 {
continue // Skip invalid lines
}
reachID, err := strconv.Atoi(parts[0])
Expand Down
77 changes: 68 additions & 9 deletions cmd/controls/controls_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,85 @@
package controls

import (
"reflect"
"testing"

_ "modernc.org/sqlite"
)

func TestRun(t *testing.T) {
func TestReadFlows(t *testing.T) {
tests := []struct {
name string
args []string
wantErr bool
name string
filePath string
wantFlows map[int]float32
wantErr bool
}{
{"positive", []string{"-db=/app/testdata/reach_data.db", "-f", "/app/testdata/flows_100yr.csv", "-c", "/app/testdata/outputs/controls.csv", "-sid", "8489318", "-scs", "0.0"}, false},
{"db file does not exist", []string{"-db=/app/testdata/not_exist.db", "-f", "/app/testdata/flows_100yr.csv", "-c", "/app/testdata/outputs/controls.csv", "-sid", "8489318", "-scs", "0.0"}, true},
{"flow file does not exist", []string{"-db=/app/testdata/reach_data.db", "-f", "/app/testdata/flows_not_exist.csv", "-c", "/app/testdata/outputs/controls.csv", "-sid", "8489318", "-scs", "0.0"}, true},
{
name: "empty flow lines should be skipped",
filePath: "/app/testdata/unit_tests/flow_files/empty_flow_values_no_header.csv",
wantFlows: map[int]float32{
2820118: 29171.14,
2820116: 35.31,
},
wantErr: false,
},
{
name: "file does not exist",
filePath: "/app/testdata/non_existent_file.csv",
wantErr: true,
},
{
name: "reach_id and flow coloumn swapped",
filePath: "/app/testdata/unit_tests/flow_files/coloumns_swapped.csv",
wantFlows: map[int]float32{},
wantErr: false,
},
{
name: "empty file",
filePath: "/app/testdata/unit_tests/flow_files/empty_file.csv",
wantFlows: map[int]float32{},
wantErr: false,
},
// {
// name: "negative flows should be skipped",
// filePath: "testdata/negative_flows.csv",
// wantFlows: map[int]float32{
// 2820002: -100.0,
// 2820006: -200.0,
// },
// wantErr: false,
// },
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := Run(tt.args); (err != nil) != tt.wantErr {
t.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr)
flows, err := ReadFlows(tt.filePath)
if (err != nil) != tt.wantErr {
t.Errorf("ReadFlows() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !tt.wantErr && !reflect.DeepEqual(flows, tt.wantFlows) {
t.Errorf("ReadFlows() = %v, want %v", flows, tt.wantFlows)
}
})
}
}

// func TestRun(t *testing.T) {
// tests := []struct {
// name string
// args []string
// wantErr bool
// }{
// {"positive", []string{"-db=/app/testdata/reach_data.db", "-f", "/app/testdata/flows_100yr.csv", "-c", "/app/testdata/outputs/controls.csv", "-sid", "8489318", "-scs", "0.0"}, false},
// {"db file does not exist", []string{"-db=/app/testdata/not_exist.db", "-f", "/app/testdata/flows_100yr.csv", "-c", "/app/testdata/outputs/controls.csv", "-sid", "8489318", "-scs", "0.0"}, true},
// {"flow file does not exist", []string{"-db=/app/testdata/reach_data.db", "-f", "/app/testdata/flows_not_exist.csv", "-c", "/app/testdata/outputs/controls.csv", "-sid", "8489318", "-scs", "0.0"}, true},
// }
// for _, tt := range tests {
// t.Run(tt.name, func(t *testing.T) {
// if err := Run(tt.args); (err != nil) != tt.wantErr {
// t.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr)
// }
// })
// }
// }
17 changes: 16 additions & 1 deletion cmd/fim/fim.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,22 @@ import (
var usage string = `Usage of fim:
Given a control table and a fim library folder, create a flood inundation VRT or a merged TIFF for the control conditions.
GDAL VSI paths can be used, given GDAL must have access to cloud creds.
Does not support relative cloud paths.
GDAL does not support relative cloud paths.
FIM library folder must have this structure:
.
├── 2821866
│ ├── z_nd
│ │ ├── f_10283.tif
│ │ ├── f_104569.tif
│ │ ├── f_11199.tif
│ │ ├── f_112807.tif
│ ├── z_53_5
│ ├── f_102921.tif
│ ├── f_10485.tif
│ ├── f_111159.tif
│ ├── f_11309.tif
CLI flag syntax. The following forms are permitted:
-flag
--flag // double dashes are also permitted
Expand Down
222 changes: 111 additions & 111 deletions cmd/fim/fim_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,114 +1,114 @@
package fim

import (
"reflect"
"testing"
)
// import (
// "reflect"
// "testing"
// )

func TestRun(t *testing.T) {
tests := []struct {
name string
args []string
wantGdalArgs []string
wantErr bool
}{
{"controls file not exist", []string{"-c", "/app/testdata/outputs/not_exist.csv", "-lib", "/app/testdata/library", "-o", "output.vrt"}, []string{}, true},
{"output in current dir", []string{"-c", "/app/testdata/outputs/controls.csv", "-lib", "/app/testdata/library", "-o", "output.vrt"}, []string{
"/app/cmd/fim/output.vrt",
"../../testdata/library/8489318/z0_0/f_1560.tif",
"../../testdata/library/8490370/z0_0/f_130.tif",
"../../testdata/library/8490230/z0_0/f_373.tif",
"../../testdata/library/8489322/z0_0/f_100.tif",
"../../testdata/library/8489330/z0_0/f_1023.tif",
"../../testdata/library/8489352/z5_0/f_190.tif",
"../../testdata/library/8489296/z0_0/f_1843.tif",
"../../testdata/library/8489316/z0_0/f_1449.tif",
"../../testdata/library/8489306/z0_0/f_1415.tif",
"../../testdata/library/8490350/z0_0/f_1560.tif",
"../../testdata/library/8490228/z5_6/f_435.tif",
"../../testdata/library/8489308/z6_5/f_456.tif",
"../../testdata/library/8489320/z6_4/f_520.tif",
"../../testdata/library/8490352/z5_0/f_250.tif",
}, false},
{"relative", []string{"-c", "/app/testdata/outputs/controls.csv", "-lib", "/app/testdata/library", "-o", "/app/testdata/outputs/relative.vrt"}, []string{
"/app/testdata/outputs/relative.vrt",
"../library/8489318/z0_0/f_1560.tif",
"../library/8490370/z0_0/f_130.tif",
"../library/8490230/z0_0/f_373.tif",
"../library/8489322/z0_0/f_100.tif",
"../library/8489330/z0_0/f_1023.tif",
"../library/8489352/z5_0/f_190.tif",
"../library/8489296/z0_0/f_1843.tif",
"../library/8489316/z0_0/f_1449.tif",
"../library/8489306/z0_0/f_1415.tif",
"../library/8490350/z0_0/f_1560.tif",
"../library/8490228/z5_6/f_435.tif",
"../library/8489308/z6_5/f_456.tif",
"../library/8489320/z6_4/f_520.tif",
"../library/8490352/z5_0/f_250.tif",
}, false},
{"absolute", []string{"-c", "/app/testdata/outputs/controls.csv", "-lib", "/app/testdata/library", "-o", "/app/testdata/outputs/absolute.vrt", "-rel=False"}, []string{
"/app/testdata/outputs/absolute.vrt",
"/app/testdata/library/8489318/z0_0/f_1560.tif",
"/app/testdata/library/8490370/z0_0/f_130.tif",
"/app/testdata/library/8490230/z0_0/f_373.tif",
"/app/testdata/library/8489322/z0_0/f_100.tif",
"/app/testdata/library/8489330/z0_0/f_1023.tif",
"/app/testdata/library/8489352/z5_0/f_190.tif",
"/app/testdata/library/8489296/z0_0/f_1843.tif",
"/app/testdata/library/8489316/z0_0/f_1449.tif",
"/app/testdata/library/8489306/z0_0/f_1415.tif",
"/app/testdata/library/8490350/z0_0/f_1560.tif",
"/app/testdata/library/8490228/z5_6/f_435.tif",
"/app/testdata/library/8489308/z6_5/f_456.tif",
"/app/testdata/library/8489320/z6_4/f_520.tif",
"/app/testdata/library/8490352/z5_0/f_250.tif",
}, false},
{"disk_vrt_s3_lib", []string{"-c", "/app/testdata/outputs/controls.csv", "-lib", "/vsis3/fimc-data/fim2d/prototype/2024_03_13/", "-o", "/app/testdata/outputs/disk_vrt_s3_lib.vrt"}, []string{
"/app/testdata/outputs/disk_vrt_s3_lib.vrt",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489318/z0_0/f_1560.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490370/z0_0/f_130.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490230/z0_0/f_373.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489322/z0_0/f_100.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489330/z0_0/f_1023.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489352/z5_0/f_190.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489296/z0_0/f_1843.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489316/z0_0/f_1449.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489306/z0_0/f_1415.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490350/z0_0/f_1560.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490228/z5_6/f_435.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489308/z6_5/f_456.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489320/z6_4/f_520.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490352/z5_0/f_250.tif",
}, false},
{"s3_vrt_s3_lib", []string{"-c", "/app/testdata/outputs/controls.csv", "-lib", "/vsis3/fimc-data/fim2d/prototype/2024_03_13/", "-o", "/vsis3/fimc-data/flows2fim/testdata/s3_vrt_s3_lib.vrt"}, []string{
"/vsis3/fimc-data/flows2fim/testdata/s3_vrt_s3_lib.vrt",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489318/z0_0/f_1560.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490370/z0_0/f_130.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490230/z0_0/f_373.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489322/z0_0/f_100.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489330/z0_0/f_1023.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489352/z5_0/f_190.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489296/z0_0/f_1843.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489316/z0_0/f_1449.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489306/z0_0/f_1415.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490350/z0_0/f_1560.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490228/z5_6/f_435.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489308/z6_5/f_456.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489320/z6_4/f_520.tif",
"/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490352/z5_0/f_250.tif",
}, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotGdalArgs, err := Run(tt.args)
if (err != nil) != tt.wantErr {
t.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(gotGdalArgs, tt.wantGdalArgs) {
t.Errorf("Run() = %v, want %v", gotGdalArgs, tt.wantGdalArgs)
}
})
}
}
// func TestRun(t *testing.T) {
// tests := []struct {
// name string
// args []string
// wantGdalArgs []string
// wantErr bool
// }{
// {"controls file not exist", []string{"-c", "/app/testdata/outputs/not_exist.csv", "-lib", "/app/testdata/library", "-o", "output.vrt"}, []string{}, true},
// {"output in current dir", []string{"-c", "/app/testdata/outputs/controls.csv", "-lib", "/app/testdata/library", "-o", "output.vrt"}, []string{
// "/app/cmd/fim/output.vrt",
// "../../testdata/library/8489318/z0_0/f_1560.tif",
// "../../testdata/library/8490370/z0_0/f_130.tif",
// "../../testdata/library/8490230/z0_0/f_373.tif",
// "../../testdata/library/8489322/z0_0/f_100.tif",
// "../../testdata/library/8489330/z0_0/f_1023.tif",
// "../../testdata/library/8489352/z5_0/f_190.tif",
// "../../testdata/library/8489296/z0_0/f_1843.tif",
// "../../testdata/library/8489316/z0_0/f_1449.tif",
// "../../testdata/library/8489306/z0_0/f_1415.tif",
// "../../testdata/library/8490350/z0_0/f_1560.tif",
// "../../testdata/library/8490228/z5_6/f_435.tif",
// "../../testdata/library/8489308/z6_5/f_456.tif",
// "../../testdata/library/8489320/z6_4/f_520.tif",
// "../../testdata/library/8490352/z5_0/f_250.tif",
// }, false},
// {"relative", []string{"-c", "/app/testdata/outputs/controls.csv", "-lib", "/app/testdata/library", "-o", "/app/testdata/outputs/relative.vrt"}, []string{
// "/app/testdata/outputs/relative.vrt",
// "../library/8489318/z0_0/f_1560.tif",
// "../library/8490370/z0_0/f_130.tif",
// "../library/8490230/z0_0/f_373.tif",
// "../library/8489322/z0_0/f_100.tif",
// "../library/8489330/z0_0/f_1023.tif",
// "../library/8489352/z5_0/f_190.tif",
// "../library/8489296/z0_0/f_1843.tif",
// "../library/8489316/z0_0/f_1449.tif",
// "../library/8489306/z0_0/f_1415.tif",
// "../library/8490350/z0_0/f_1560.tif",
// "../library/8490228/z5_6/f_435.tif",
// "../library/8489308/z6_5/f_456.tif",
// "../library/8489320/z6_4/f_520.tif",
// "../library/8490352/z5_0/f_250.tif",
// }, false},
// {"absolute", []string{"-c", "/app/testdata/outputs/controls.csv", "-lib", "/app/testdata/library", "-o", "/app/testdata/outputs/absolute.vrt", "-rel=False"}, []string{
// "/app/testdata/outputs/absolute.vrt",
// "/app/testdata/library/8489318/z0_0/f_1560.tif",
// "/app/testdata/library/8490370/z0_0/f_130.tif",
// "/app/testdata/library/8490230/z0_0/f_373.tif",
// "/app/testdata/library/8489322/z0_0/f_100.tif",
// "/app/testdata/library/8489330/z0_0/f_1023.tif",
// "/app/testdata/library/8489352/z5_0/f_190.tif",
// "/app/testdata/library/8489296/z0_0/f_1843.tif",
// "/app/testdata/library/8489316/z0_0/f_1449.tif",
// "/app/testdata/library/8489306/z0_0/f_1415.tif",
// "/app/testdata/library/8490350/z0_0/f_1560.tif",
// "/app/testdata/library/8490228/z5_6/f_435.tif",
// "/app/testdata/library/8489308/z6_5/f_456.tif",
// "/app/testdata/library/8489320/z6_4/f_520.tif",
// "/app/testdata/library/8490352/z5_0/f_250.tif",
// }, false},
// {"disk_vrt_s3_lib", []string{"-c", "/app/testdata/outputs/controls.csv", "-lib", "/vsis3/fimc-data/fim2d/prototype/2024_03_13/", "-o", "/app/testdata/outputs/disk_vrt_s3_lib.vrt"}, []string{
// "/app/testdata/outputs/disk_vrt_s3_lib.vrt",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489318/z0_0/f_1560.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490370/z0_0/f_130.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490230/z0_0/f_373.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489322/z0_0/f_100.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489330/z0_0/f_1023.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489352/z5_0/f_190.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489296/z0_0/f_1843.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489316/z0_0/f_1449.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489306/z0_0/f_1415.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490350/z0_0/f_1560.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490228/z5_6/f_435.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489308/z6_5/f_456.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489320/z6_4/f_520.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490352/z5_0/f_250.tif",
// }, false},
// {"s3_vrt_s3_lib", []string{"-c", "/app/testdata/outputs/controls.csv", "-lib", "/vsis3/fimc-data/fim2d/prototype/2024_03_13/", "-o", "/vsis3/fimc-data/flows2fim/testdata/s3_vrt_s3_lib.vrt"}, []string{
// "/vsis3/fimc-data/flows2fim/testdata/s3_vrt_s3_lib.vrt",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489318/z0_0/f_1560.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490370/z0_0/f_130.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490230/z0_0/f_373.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489322/z0_0/f_100.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489330/z0_0/f_1023.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489352/z5_0/f_190.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489296/z0_0/f_1843.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489316/z0_0/f_1449.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489306/z0_0/f_1415.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490350/z0_0/f_1560.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490228/z5_6/f_435.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489308/z6_5/f_456.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8489320/z6_4/f_520.tif",
// "/vsis3/fimc-data/fim2d/prototype/2024_03_13/8490352/z5_0/f_250.tif",
// }, false},
// }
// for _, tt := range tests {
// t.Run(tt.name, func(t *testing.T) {
// gotGdalArgs, err := Run(tt.args)
// if (err != nil) != tt.wantErr {
// t.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr)
// return
// }
// if !reflect.DeepEqual(gotGdalArgs, tt.wantGdalArgs) {
// t.Errorf("Run() = %v, want %v", gotGdalArgs, tt.wantGdalArgs)
// }
// })
// }
// }
Loading

0 comments on commit f468270

Please sign in to comment.