Allow cancellation while decoding json

README.md

@@ -75,9 +75,9 @@ For more information on how to configure your system for Fyne please see: [Getti
 
 ### What is the largest JSON file that I can load?
 
-The largest JSON file you can load on your computer depends mainly on how much RAM you have. As a very rough rule of thumb you need about 3x the amount of RAM compared to the size of your JSON file. The actual figures can vary depending on operating system and the structure of the JSON file.
+The largest JSON file you can load on your computer depends mainly on how much RAM you have and on the particular JSON file. The main driver for memory consumption is the number of elements in a JSON document.
 
-For comparison we did a load test on one of our developer notebooks. It has 8 GB RAM and runs Ubuntu 22.04 LTS. We were able to load a JSON files successfully with up to 2.5 GB in size and 42 million elements.
+For comparison we did a load test on one of our developer notebooks. It has 8 GB RAM and runs Ubuntu 22.04 LTS. We were able to load a JSON files successfully with up to 45 million elements. The size of our test file was about 2.5 GB.
 
 ## Attributions
 

internal/jsondocument/jsondocument.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"log/slog"
 	"regexp"
 	"slices"
@@ -145,7 +146,7 @@ func (j *JSONDocument) Value(uid widget.TreeNodeID) Node {
 // It reports it's current progress to the caller via updates to progressInfo.
 func (j *JSONDocument) Load(ctx context.Context, reader fyne.URIReadCloser, progressInfo binding.Untyped) error {
 	j.progressInfo = progressInfo
-	data, err := j.load(reader)
+	data, err := j.load(ctx, reader)
 	if err != nil {
 		return err
 	}
@@ -211,17 +212,39 @@ func (j *JSONDocument) Size() int {
 	return int(j.n)
 }
 
-func (j *JSONDocument) load(reader fyne.URIReadCloser) (any, error) {
+// readCloserCtx adds context to a ReadCloser and allows a stream to be canceled.
+type readCloserCtx struct {
+	ctx context.Context
+	r   io.ReadCloser
+}
+
+func (r *readCloserCtx) Read(p []byte) (n int, err error) {
+	if err := r.ctx.Err(); err != nil {
+		return 0, err
+	}
+	return r.r.Read(p)
+}
+
+func (r *readCloserCtx) Close() error {
+	return r.r.Close()
+}
+
+// newReaderContext returns a new readCloser with a context.
+func newReaderContext(ctx context.Context, r io.ReadCloser) io.ReadCloser {
+	return &readCloserCtx{ctx: ctx, r: r}
+}
+
+func (j *JSONDocument) load(ctx context.Context, reader io.ReadCloser) (any, error) {
 	defer reader.Close()
 	if err := j.setProgressInfo(ProgressInfo{CurrentStep: 1}); err != nil {
 		return nil, err
 	}
 	var data any
-	dec := json.NewDecoder(reader)
+	reader2 := newReaderContext(ctx, reader)
+	dec := json.NewDecoder(reader2)
 	if err := dec.Decode(&data); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal data: %s", err)
+		return nil, err
 	}
-	slog.Info("Completed loading file", "uri", reader.URI())
 	return data, nil
 }
 

internal/jsondocument/jsondocument_internal_test.go

@@ -1,47 +1,44 @@
 package jsondocument
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"runtime"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
-// func TestLoadFile(t *testing.T) {
-// 	ctx := context.Background()
-// 	j := New()
-// 	t.Run("should return unmarshaled data from stream", func(t *testing.T) {
-// 		// given
-// 		data := map[string]any{"alpha": "two"}
-// 		dat, err := json.Marshal(data)
-// 		if err != nil {
-// 			t.Fatal(err)
-// 		}
-// 		r := MakeURIReadCloser(bytes.NewReader(dat), "test")
-// 		// when
-// 		byt, err := j.loadFile(r)
-// 		if assert.NoError(t, err) {
-// 			got, err := j.parseFile(ctx, byt)
-// 			// then
-// 			if assert.NoError(t, err) {
-// 				assert.Equal(t, data, got)
-// 			}
-// 		}
-// 	})
-// 	t.Run("should return error when stream can not be unmarshaled", func(t *testing.T) {
-// 		// given
-// 		r := MakeURIReadCloser(strings.NewReader("invalid JSON"), "test")
-// 		// when
-// 		byt, err := j.loadFile(r)
-// 		if assert.NoError(t, err) {
-// 			_, err := j.parseFile(ctx, byt)
-// 			// then
-// 			assert.Error(t, err)
-// 		}
-// 	})
-// }
+func TestLoadFile(t *testing.T) {
+	ctx := context.Background()
+	j := New()
+	t.Run("should return load and unmarshaled data from stream", func(t *testing.T) {
+		// given
+		data := map[string]any{"alpha": "two"}
+		dat, err := json.Marshal(data)
+		if err != nil {
+			t.Fatal(err)
+		}
+		r := MakeURIReadCloser(bytes.NewReader(dat), "test")
+		// when
+		got, err := j.load(ctx, r)
+		// then
+		if assert.NoError(t, err) {
+			assert.Equal(t, data, got)
+		}
+
+	})
+	t.Run("should return error when stream can not be unmarshaled", func(t *testing.T) {
+		// given
+		r := MakeURIReadCloser(strings.NewReader("invalid JSON"), "test")
+		// when
+		_, err := j.load(ctx, r)
+		// then
+		assert.Error(t, err)
+	})
+}
 
 func TestAddNode(t *testing.T) {
 	ctx := context.TODO()

internal/jsondocument/jsondocument_test.go

@@ -156,8 +156,8 @@ func TestJsonDocumentLoad(t *testing.T) {
 			if assert.NoError(t, err) {
 				p := x.(jsondocument.ProgressInfo)
 				assert.Equal(t, 3, p.Size)
-				assert.Equal(t, 4, p.CurrentStep)
-				assert.Equal(t, 4, p.TotalSteps)
+				assert.Equal(t, 3, p.CurrentStep)
+				assert.Equal(t, 3, p.TotalSteps)
 			}
 		}
 	})

internal/ui/ui.go

@@ -268,10 +268,8 @@ func (u *UI) loadDocument(reader fyne.URIReadCloser) {
 		case 1:
 			text = fmt.Sprintf("Loading file from disk: %s", name)
 		case 2:
-			text = fmt.Sprintf("Parsing file: %s", name)
-		case 3:
 			text = fmt.Sprintf("Calculating document size: %s", name)
-		case 4:
+		case 3:
 			if pb2.Hidden {
 				pb1.Stop()
 				pb1.Hide()

tools/multiply_json/main.go

@@ -2,14 +2,17 @@
 package main
 
 import (
-	"encoding/json"
 	"flag"
 	"fmt"
 	"log"
 	"os"
 	"path/filepath"
+
+	jsoniter "github.com/json-iterator/go"
 )
 
+var json = jsoniter.ConfigCompatibleWithStandardLibrary
+
 const (
 	factorDefault = 3
 )
@@ -43,14 +46,17 @@ func main() {
 		k := fmt.Sprintf("clone_%03d", i)
 		target[k] = source
 	}
-	fmt.Println("Marshalling into JSON...")
-	data, err = json.MarshalIndent(target, "", "    ")
+	filename := "out.json"
+	fmt.Printf("Writing file: %s ...", filename)
+
+	f, err := os.Create(filename)
 	if err != nil {
 		log.Fatal(err)
 	}
-	filename := "out.json"
-	fmt.Printf("Writing file: %s ...", filename)
-	if err := os.WriteFile(filename, data, 0644); err != nil {
+	defer f.Close()
+	enc := json.NewEncoder(f)
+	enc.SetIndent("", "    ")
+	if err := enc.Encode(target); err != nil {
 		log.Fatal(err)
 	}
 	fmt.Println("DONE")