Basic Parent-Child entanglement coding

entanglement/encode_test.go

@@ -0,0 +1,37 @@
+package entanglement
+
+import (
+	"context"
+	"testing"
+
+	format "github.com/ipfs/go-ipld-format"
+	"github.com/ipfs/go-merkledag"
+	dstest "github.com/ipfs/go-merkledag/test"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEncode(t *testing.T) {
+	ctx := context.Background()
+	dag := dstest.Mock()
+
+	in := merkledag.NodeWithData([]byte("1234567890"))
+	in2 := merkledag.NodeWithData([]byte("987654321"))
+	in3 := merkledag.NodeWithData([]byte("1khgk234509876"))
+	in.AddNodeLink("link", in2)
+	in.AddNodeLink("link", in3)
+	dag.AddMany(ctx, []format.Node{in, in2, in3})
+
+	nd, err := Encode(ctx, dag, in, 3)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, nd.Recoverability())
+
+	for _, r := range nd.RecoveryLinks() {
+		assert.NotNil(t, r)
+	}
+	r1, _ := nd.RecoveryLinks()[0].GetNode(ctx, dag)
+	r2, _ := nd.RecoveryLinks()[1].GetNode(ctx, dag)
+	rec, _ := XORByteSlice(r1.RawData(), r2.RawData())
+
+	assert.Equal(t, in2.RawData(), r1.RawData()[1:12])
+	assert.Equal(t, in3.RawData(), rec[1:])
+}

entanglement/encoder.go

@@ -0,0 +1,70 @@
+package entanglement
+
+import (
+	"context"
+
+	format "github.com/ipfs/go-ipld-format"
+	"github.com/ipfs/go-merkledag"
+	"github.com/multiformats/go-varint"
+
+	recovery "github.com/Wondertan/go-ipfs-recovery"
+)
+
+// Encode applies Reed-Solomon coding on the given IPLD Node promoting it to a recovery Node.
+// Use `r` to specify needed amount of generated recovery Nodes.
+func Encode(ctx context.Context, dag format.DAGService, nd format.Node, r recovery.Recoverability) (*Node, error) {
+	var err error
+	ls := nd.Links()
+	rd := NewNode(nd.(*merkledag.ProtoNode))
+
+	nds, s := make([]format.Node, len(rd.Links())), 0
+	for i, l := range rd.Links() {
+		nds[i], err = l.GetNode(ctx, dag)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(nds[i].RawData()) > s { // finding the largest child
+			s = len(nds[i].RawData())
+		}
+	}
+
+	// encode size of redundant data
+	s += varint.UvarintSize(uint64(s))
+	ln := len(ls)
+	bs := make([][]byte, ln*2)
+	for i := range bs {
+		bs[i] = make([]byte, s)
+		if i < ln {
+			l := len(nds[i].RawData())
+			n := varint.PutUvarint(bs[i], uint64(l))
+			copy(bs[i][n:], nds[i].RawData())
+		}
+	}
+
+	for i := range bs[ln:] {
+		if i == 0 {
+			bs[ln] = bs[0]
+		} else {
+			bs[ln+i], err = XORByteSlice(bs[ln+i-1], bs[i])
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		rnd := merkledag.NewRawNode(bs[ln+i])
+		err = dag.Add(ctx, rnd)
+		if err != nil {
+			return nil, err
+		}
+
+		rd.AddRedundantNode(rnd)
+	}
+
+	err = dag.Add(ctx, rd)
+	if err != nil {
+		return nil, err
+	}
+
+	return rd, dag.Remove(ctx, nd.Cid())
+}

entanglement/entanglement.go

@@ -0,0 +1,66 @@
+package entanglement
+
+import (
+	"context"
+	"fmt"
+
+	recovery "github.com/Wondertan/go-ipfs-recovery"
+	"github.com/ipfs/go-cid"
+	format "github.com/ipfs/go-ipld-format"
+)
+
+// Custom codec for Entanglement recovery Nodes.
+const Codec = 0x701 // random number // TODO Register in IPFS codec table.
+
+func init() {
+	// register global decoder
+	format.Register(Codec, DecodeNode)
+
+	// register codec
+	cid.Codecs["recovery-entanglement`"] = Codec
+	cid.CodecToStr[Codec] = "recovery-entanglement`"
+
+}
+
+type entangler struct {
+	dag format.DAGService
+}
+
+// NewRestorer creates a new Entanglement Recoverer.
+func NewRecoverer(dag format.DAGService) recovery.Recoverer {
+	return &entangler{dag: dag}
+}
+
+// NewEncoder creates new Entanglement Encoder.
+func NewEncoder(dag format.DAGService) recovery.Encoder {
+	return &entangler{dag: dag}
+}
+
+func (ent *entangler) Recover(ctx context.Context, nd recovery.Node, rids ...cid.Cid) ([]format.Node, error) {
+	pnd, ok := nd.(*Node)
+	if !ok {
+		return nil, fmt.Errorf("Entanglement: wrong Node type")
+	}
+
+	return Recover(ctx, ent.dag, pnd, rids...)
+}
+
+func (ent *entangler) Encode(ctx context.Context, nd format.Node, r recovery.Recoverability) (recovery.Node, error) {
+
+	return nil, nil
+}
+
+// XORByteSlice returns an XOR slice of 2 input slices
+func XORByteSlice(a []byte, b []byte) ([]byte, error) {
+	if len(a) != len(b) {
+		return nil, fmt.Errorf("length of byte slices is not equivalent: %d != %d", len(a), len(b))
+	}
+
+	buf := make([]byte, len(a))
+
+	for i := range a {
+		buf[i] = a[i] ^ b[i]
+	}
+
+	return buf, nil
+}

entanglement/entanglement_test.go

@@ -0,0 +1,82 @@
+package entanglement
+
+import (
+	"context"
+	"testing"
+
+	recovery "github.com/Wondertan/go-ipfs-recovery"
+	"github.com/Wondertan/go-ipfs-recovery/test"
+	format "github.com/ipfs/go-ipld-format"
+	"github.com/ipfs/go-merkledag"
+	dstest "github.com/ipfs/go-merkledag/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEncodeRecover(t *testing.T) {
+	ctx := context.Background()
+	dag := dstest.Mock()
+
+	in := merkledag.NodeWithData([]byte("1234567890"))
+	in2 := merkledag.NodeWithData([]byte("03243423423423"))
+	in3 := merkledag.NodeWithData([]byte("123450"))
+	in4 := merkledag.NodeWithData([]byte("1234509876"))
+	in.AddNodeLink("link", in2)
+	in.AddNodeLink("link", in3)
+	in.AddNodeLink("link", in4)
+	dag.AddMany(ctx, []format.Node{in, in2, in3, in4})
+
+	enc, err := Encode(ctx, dag, in, 1)
+	require.NoError(t, err)
+
+	dag.Remove(ctx, in2.Cid())
+	dag.Remove(ctx, in4.Cid())
+
+	out, err := Recover(ctx, dag, enc, in2.Cid(), in4.Cid())
+	require.NoError(t, err)
+	assert.Equal(t, in2.RawData(), out[0].RawData())
+	assert.Equal(t, in4.RawData(), out[1].RawData())
+
+	out2, err := dag.Get(ctx, in2.Cid())
+	assert.NoError(t, err)
+	assert.Equal(t, in2.RawData(), out2.RawData())
+
+	out4, err := dag.Get(ctx, in4.Cid())
+	assert.NoError(t, err)
+	assert.Equal(t, in4.RawData(), out4.RawData())
+}
+
+func TestUnixFS(t *testing.T) {
+	ctx := context.Background()
+	dag := dstest.Mock()
+	dr := test.NewFSDagger(t, ctx, &merkledag.ComboService{
+		Read:  recovery.NewNodeGetter(dag, NewRecoverer(dag)),
+		Write: dag,
+	})
+	dr.Morpher = func(nd format.Node) (format.Node, error) {
+		if len(nd.Links()) == 0 {
+			return nd, nil
+		}
+
+		return Encode(ctx, dag, nd, 3)
+	}
+
+	root :=
+		dr.NewDir("root",
+			dr.RandNode("file1"),
+			dr.RandNode("file2"),
+			dr.NewDir("dir1",
+				dr.RandNode("file3"),
+				dr.RandNode("file4"),
+			),
+			dr.NewDir("dir2",
+				dr.RandNode("file5"),
+			),
+		)
+
+	dr.Remove("file1")
+	dr.Remove("dir2")
+	dr.Remove("file4")
+
+	root.Validate()
+}