From e01e7abcefc0447175c94f2b9a2cd4b55638c619 Mon Sep 17 00:00:00 2001 From: gov Date: Thu, 30 Jul 2020 04:00:21 -0400 Subject: [PATCH 1/8] added alpha entanglement --- ae/AlphaEntanglement.go | 79 ++++++++++++++++++++++++++++++++++++ ae/AlphaEntanglement_test.go | 34 ++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 ae/AlphaEntanglement.go create mode 100644 ae/AlphaEntanglement_test.go diff --git a/ae/AlphaEntanglement.go b/ae/AlphaEntanglement.go new file mode 100644 index 0000000..59967aa --- /dev/null +++ b/ae/AlphaEntanglement.go @@ -0,0 +1,79 @@ +package ae + +import ( + "context" + "fmt" + + "github.com/Wondertan/go-ipfs-restore/rs" + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" +) + +type AlphaRestorer struct { + dag format.DAGService +} + +const ( + alpha = 1 + s = 1 + p = 1 +) + +func xorByteSlice(b1, b2 []byte) { + +} +func XORByteSlice(a []byte, b []byte) ([]byte, error) { + if len(a) != len(b) { + return nil, fmt.Errorf("length of byte slices is not equivalent: %d != %d", len(a), len(b)) + } + + buf := make([]byte, len(a)) + + for i, _ := range a { + buf[i] = a[i] ^ b[i] + } + + return buf, nil +} +func (ar *AlphaRestorer) Encode(ctx context.Context, nd format.Node) (format.Node, error) { + pnd, err := rs.ValidateNode(nd) + if err != nil { + return nil, err + } + + var nodes []format.Node + var reds [][]byte + + nodes = append(nodes, nd) + ndps := format.GetDAG(ctx, ar.dag, pnd) + for _, ndp := range ndps { + nd, err = ndp.Get(ctx) + if err != nil { + return nil, err + } + + nodes = append(nodes, nd) + } + + lastRedundancy := make([]byte, len(nodes[0].RawData())) + + for _, node := range nodes { + red, err := XORByteSlice(lastRedundancy, node.RawData()) + if err != nil { + return nil, err + } + reds = append(reds, red) + + redNode := merkledag.NodeWithData(red) + nodeProto, err := rs.ValidateNode(node) + if err != nil { + return nil, err + } + + nodeProto.AddNodeLink("redundancy", redNode) + + ar.dag.Add(ctx, redNode) + } + + return pnd, nil +} diff --git a/ae/AlphaEntanglement_test.go b/ae/AlphaEntanglement_test.go new file mode 100644 index 0000000..8a45428 --- /dev/null +++ b/ae/AlphaEntanglement_test.go @@ -0,0 +1,34 @@ +package ae + +import ( + "context" + "testing" + + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" + dstest "github.com/ipfs/go-merkledag/test" +) + +func NewRestorer() *AlphaRestorer { + ds := dstest.Mock() + r := &AlphaRestorer{ + dag: ds, + } + return r +} + +func TestEncode(t *testing.T) { + // Arrange + in := merkledag.NodeWithData([]byte("1234567890")) + in2 := merkledag.NodeWithData([]byte("0987654321")) + in3 := merkledag.NodeWithData([]byte("1234509876")) + r := NewRestorer() + ctx := context.Background() + in.AddNodeLink("link", in2) + in.AddNodeLink("link", in3) + r.dag.AddMany(ctx, []format.Node{in, in2, in3}) + + // Act + + // Assert +} From 36f466b9e456ca3272871448924216d6ec486e44 Mon Sep 17 00:00:00 2001 From: gov Date: Mon, 3 Aug 2020 02:39:57 -0400 Subject: [PATCH 2/8] draft entanglement recovery --- .../alpha_entanglement.go | 31 ++- .../alpha_entanglement_test.go | 30 ++- entanglement/node.go | 247 ++++++++++++++++++ entanglement/node_test.go | 69 +++++ entanglement/recover.go | 96 +++++++ entanglement/recover_test.go | 121 +++++++++ 6 files changed, 575 insertions(+), 19 deletions(-) rename ae/AlphaEntanglement.go => entanglement/alpha_entanglement.go (63%) rename ae/AlphaEntanglement_test.go => entanglement/alpha_entanglement_test.go (50%) create mode 100644 entanglement/node.go create mode 100644 entanglement/node_test.go create mode 100644 entanglement/recover.go create mode 100644 entanglement/recover_test.go diff --git a/ae/AlphaEntanglement.go b/entanglement/alpha_entanglement.go similarity index 63% rename from ae/AlphaEntanglement.go rename to entanglement/alpha_entanglement.go index 59967aa..739ccf5 100644 --- a/ae/AlphaEntanglement.go +++ b/entanglement/alpha_entanglement.go @@ -1,9 +1,10 @@ -package ae +package entanglement import ( "context" "fmt" + restore "github.com/Wondertan/go-ipfs-restore" "github.com/Wondertan/go-ipfs-restore/rs" format "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" @@ -19,10 +20,14 @@ const ( p = 1 ) -func xorByteSlice(b1, b2 []byte) { - +func NewAlphaRestorer(ds format.DAGService) *AlphaRestorer { + r := &AlphaRestorer{ + dag: ds, + } + return r } -func XORByteSlice(a []byte, b []byte) ([]byte, error) { + +func xorByteSlice(a []byte, b []byte) ([]byte, error) { if len(a) != len(b) { return nil, fmt.Errorf("length of byte slices is not equivalent: %d != %d", len(a), len(b)) } @@ -35,7 +40,9 @@ func XORByteSlice(a []byte, b []byte) ([]byte, error) { return buf, nil } + func (ar *AlphaRestorer) Encode(ctx context.Context, nd format.Node) (format.Node, error) { + // get protonode pnd, err := rs.ValidateNode(nd) if err != nil { return nil, err @@ -44,6 +51,7 @@ func (ar *AlphaRestorer) Encode(ctx context.Context, nd format.Node) (format.Nod var nodes []format.Node var reds [][]byte + // get all links of the node nodes = append(nodes, nd) ndps := format.GetDAG(ctx, ar.dag, pnd) for _, ndp := range ndps { @@ -55,22 +63,29 @@ func (ar *AlphaRestorer) Encode(ctx context.Context, nd format.Node) (format.Nod nodes = append(nodes, nd) } + // for all links, create an alpha entanglement lastRedundancy := make([]byte, len(nodes[0].RawData())) - for _, node := range nodes { - red, err := XORByteSlice(lastRedundancy, node.RawData()) + // n.b. how does this work in terms of ordering? + for _, linkedNode := range nodes { + + red, err := xorByteSlice(lastRedundancy, linkedNode.RawData()) + lastRedundancy = red if err != nil { return nil, err } reds = append(reds, red) redNode := merkledag.NodeWithData(red) - nodeProto, err := rs.ValidateNode(node) + + linkedNodeProto, err := rs.ValidateNode(linkedNode) + + rnode := restore.NewNode(linkedNodeProto) if err != nil { return nil, err } - nodeProto.AddNodeLink("redundancy", redNode) + rnode.AddRedundantNode(redNode) ar.dag.Add(ctx, redNode) } diff --git a/ae/AlphaEntanglement_test.go b/entanglement/alpha_entanglement_test.go similarity index 50% rename from ae/AlphaEntanglement_test.go rename to entanglement/alpha_entanglement_test.go index 8a45428..e4f7905 100644 --- a/ae/AlphaEntanglement_test.go +++ b/entanglement/alpha_entanglement_test.go @@ -1,34 +1,42 @@ -package ae +package entanglement import ( "context" + "fmt" "testing" format "github.com/ipfs/go-ipld-format" + ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" dstest "github.com/ipfs/go-merkledag/test" + assert "github.com/stretchr/testify/assert" ) -func NewRestorer() *AlphaRestorer { - ds := dstest.Mock() - r := &AlphaRestorer{ - dag: ds, - } - return r -} - func TestEncode(t *testing.T) { // Arrange in := merkledag.NodeWithData([]byte("1234567890")) in2 := merkledag.NodeWithData([]byte("0987654321")) in3 := merkledag.NodeWithData([]byte("1234509876")) - r := NewRestorer() + ar := NewAlphaRestorer(dstest.Mock()) ctx := context.Background() in.AddNodeLink("link", in2) in.AddNodeLink("link", in3) - r.dag.AddMany(ctx, []format.Node{in, in2, in3}) + ar.dag.AddMany(ctx, []format.Node{in, in2, in3}) + + var ng ipld.NodeGetter = ar.dag + + ng = merkledag.NewSession(ctx, ng) // Act + pnd, err := ar.Encode(ctx, in) // Assert + assert.Nil(t, err) + assert.NotNil(t, pnd) + + for _, link := range pnd.Links() { + nd, err := link.GetNode(ctx, ng) + assert.Nil(t, err) + fmt.Println(nd.RawData()) + } } diff --git a/entanglement/node.go b/entanglement/node.go new file mode 100644 index 0000000..4a381d3 --- /dev/null +++ b/entanglement/node.go @@ -0,0 +1,247 @@ +package entanglement + +import ( + "fmt" + "sort" + "strconv" + + "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" + + "github.com/Wondertan/go-ipfs-recovery" + cpb "github.com/Wondertan/go-ipfs-recovery/reedsolomon/pb" +) + +// TODO Do not save sizes of all links independently as they are always the same. + +// Node is a recovery Node based ob Reed-Solomon coding. +type Node struct { + *merkledag.ProtoNode + Left []*Block + Right []*Block + Position int + Data []byte + Class StrandClass + Identifier string + + // recovery []*format.Link + cache []byte + cid cid.Cid +} + +func NewNode(proto *merkledag.ProtoNode) *Node { + nd := &Node{ProtoNode: proto.Copy().(*merkledag.ProtoNode)} + nd.SetCidBuilder(nd.CidBuilder().WithCodec(Codec)) + return nd +} + +func (n *Node) Recoverability() recovery.Recoverability { + return len(n.recovery) +} + +func (n *Node) RecoveryLinks() []*format.Link { + return n.recovery +} + +func (n *Node) AddRedundantNode(nd format.Node) { + if nd == nil { + return + } + + n.cache = nil + n.recovery = append(n.recovery, &format.Link{ + Name: strconv.Itoa(len(n.recovery)), + Size: uint64(len(nd.RawData())), + Cid: nd.Cid(), + }) +} + +func (n *Node) RemoveRedundantNode(id cid.Cid) { + if !id.Defined() { + return + } + + ref := n.recovery[:0] + for _, v := range n.recovery { + if v.Cid.Equals(id) { + n.cache = nil + } else { + ref = append(ref, v) + } + } + n.recovery = ref +} + +func (n *Node) RawData() []byte { + if n.cache != nil { + return n.cache + } + + var err error + n.cache, err = MarshalNode(n) + if err != nil { + panic(fmt.Sprintf("can't marshal Node: %s", err)) + } + + return n.cache +} + +func (n *Node) Cid() cid.Cid { + if n.cache != nil && n.cid.Defined() { + return n.cid + } + + var err error + n.cid, err = n.CidBuilder().Sum(n.RawData()) + if err != nil { + panic(fmt.Sprintf("can't form CID: %s", err)) + } + + return n.cid +} + +func (n *Node) String() string { + return n.Cid().String() +} + +func (n *Node) Copy() format.Node { + nd := new(Node) + nd.ProtoNode = n.ProtoNode.Copy().(*merkledag.ProtoNode) + l := len(n.recovery) + if l > 0 { + nd.recovery = make([]*format.Link, l) + for i, r := range n.recovery { + nd.recovery[i] = &format.Link{ + Name: r.Name, + Size: r.Size, + Cid: r.Cid, + } + } + } + + return nd +} + +func (n *Node) Stat() (*format.NodeStat, error) { + l := len(n.RawData()) + cumSize, err := n.Size() + if err != nil { + return nil, err + } + + return &format.NodeStat{ + Hash: n.Cid().String(), + NumLinks: len(n.Links()), + BlockSize: l, + DataSize: len(n.Data()), + CumulativeSize: int(cumSize), + }, nil +} + +func (n *Node) Size() (uint64, error) { + s := uint64(len(n.RawData())) + for _, l := range n.Links() { + s += l.Size + } + for _, l := range n.recovery { + s += l.Size + } + return s, nil +} + +func MarshalNode(n *Node) ([]byte, error) { + var err error + pb := &cpb.PBNode{} + pb.Proto, err = n.ProtoNode.Marshal() + if err != nil { + return nil, err + } + + l := len(n.recovery) + if l > 0 { + sort.Stable(merkledag.LinkSlice(n.recovery)) + pb.Recovery = make([]*cpb.PBLink, l) + for i, r := range n.recovery { + pb.Recovery[i] = &cpb.PBLink{ + Name: r.Name, + Size_: r.Size, + Hash: r.Cid.Bytes(), + } + } + } + + return pb.Marshal() +} + +func UnmarshalNode(data []byte) (*Node, error) { + nd := &Node{} + pb := &cpb.PBNode{} + err := pb.Unmarshal(data) + if err != nil { + return nil, err + } + + nd.ProtoNode, err = merkledag.DecodeProtobuf(pb.Proto) + if err != nil { + return nil, err + } + + l := len(pb.Recovery) + if l > 0 { + nd.recovery = make([]*format.Link, l) + for i, r := range pb.Recovery { + nd.recovery[i] = &format.Link{ + Name: r.Name, + Size: r.Size_, + } + + nd.recovery[i].Cid, err = cid.Cast(r.Hash) + if err != nil { + return nil, err + } + } + sort.Stable(merkledag.LinkSlice(nd.recovery)) + } + + return nd, nil +} + +func DecodeNode(b blocks.Block) (format.Node, error) { + id := b.Cid() + if id.Prefix().Codec != Codec { + return nil, fmt.Errorf("can only decode restorable node") + } + + nd, err := UnmarshalNode(b.RawData()) + if err != nil { + return nil, err + } + + nd.cid = b.Cid() + nd.SetCidBuilder(b.Cid().Prefix()) + return nd, nil +} + +// Shadowed methods to reset caching. +// +func (n *Node) AddRawLink(name string, l *format.Link) error { + n.cache = nil + return n.ProtoNode.AddRawLink(name, l) +} + +func (n *Node) AddNodeLink(name string, that format.Node) error { + n.cache = nil + return n.ProtoNode.AddNodeLink(name, that) +} + +func (n *Node) RemoveNodeLink(name string) error { + n.cache = nil + return n.ProtoNode.RemoveNodeLink(name) +} + +func (n *Node) SetData(d []byte) { + n.cache = nil + n.ProtoNode.SetData(d) +} diff --git a/entanglement/node_test.go b/entanglement/node_test.go new file mode 100644 index 0000000..5465965 --- /dev/null +++ b/entanglement/node_test.go @@ -0,0 +1,69 @@ +package entanglement + +import ( + "testing" + + "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNodeRedundant(t *testing.T) { + nd := NewNode(merkledag.NodeWithData([]byte("1234567890"))) + r := merkledag.NewRawNode([]byte("12345")) + nd.AddRedundantNode(r) + assert.Len(t, nd.RecoveryLinks(), 1) + nd.RemoveRedundantNode(r.Cid()) + assert.Len(t, nd.RecoveryLinks(), 0) +} + +func TestNodeMarshalUnmarshal(t *testing.T) { + in := NewNode(merkledag.NodeWithData([]byte("1234567890"))) + red := merkledag.NewRawNode([]byte("12345")) + in.AddRedundantNode(red) + + data, err := MarshalNode(in) + require.NoError(t, err) + + out, err := UnmarshalNode(data) + require.NoError(t, err) + + out.SetCidBuilder(in.CidBuilder()) + assert.True(t, in.Cid().Equals(out.Cid())) + assert.Equal(t, in.RecoveryLinks(), out.RecoveryLinks()) +} + +func TestNodeDecode(t *testing.T) { + in := NewNode(merkledag.NodeWithData([]byte("1234567890"))) + out, err := format.Decode(in) + require.NoError(t, err) + assert.True(t, in.Cid().Equals(out.Cid())) +} + +func TestNodeCache(t *testing.T) { + nd := NewNode(merkledag.NodeWithData([]byte("1234567890"))) + nd.AddRedundantNode(merkledag.NewRawNode([]byte("12345"))) + assert.Zero(t, nd.cid) + assert.Nil(t, nd.cache) + + nd.Cid() + assert.NotZero(t, nd.cid) + assert.NotNil(t, nd.cache) + + nd.AddNodeLink("", merkledag.NewRawNode([]byte("12345"))) + assert.Nil(t, nd.cache) +} + +func TestNode_Copy(t *testing.T) { + nd := NewNode(merkledag.NodeWithData([]byte("1234567890"))) + r := merkledag.NewRawNode([]byte("12345")) + nd.AddRedundantNode(r) + + cp := nd.Copy() + nd.SetData([]byte{}) + nd.RemoveRedundantNode(r.Cid()) + + assert.NotNil(t, cp.(*Node).RecoveryLinks()) + assert.NotNil(t, cp.(*Node).Data()) +} diff --git a/entanglement/recover.go b/entanglement/recover.go new file mode 100644 index 0000000..9a012b4 --- /dev/null +++ b/entanglement/recover.go @@ -0,0 +1,96 @@ +package entanglement + +import ( + "context" + + "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-ipld-format" + "github.com/templexxx/reedsolomon" + + "github.com/Wondertan/go-ipfs-recovery" +) + +// Recover tries to recompute all lost IPLD Nodes using Reed-Solomon coded recovery Node. +// Pass known lost ids explicitly to avoid re-requesting them and to return corresponding Nodes on success. +func Recover(ctx context.Context, dag format.DAGService, pnd *Node, lost ...cid.Cid) ([]format.Node, error) { + // collect ids of all linked nodes. + lpnd := len(pnd.Links()) + lrpnd := len(pnd.RecoveryLinks()) + ids := make([]cid.Cid, lpnd+lrpnd) + +outer: + for i, l := range pnd.Links() { + // exclude known lost ids. + for _, id := range lost { + if l.Cid.Equals(id) { + ids[i] = cid.Undef + break outer + } + } + + ids[i] = l.Cid + } + + for i, l := range pnd.RecoveryLinks() { + ids[i+lpnd] = l.Cid + } + + // track `lost` or `not lost` blocks indexes with actual data. + var lst, nlst []int + bs := make([][]byte, lpnd+lrpnd) + + for i, ndp := range format.GetNodes(ctx, dag, ids) { + nd, err := ndp.Get(ctx) + switch err { + case context.DeadlineExceeded, context.Canceled: + return nil, err + case nil: + bs[i] = nd.RawData() + lst = append(lst, i) + default: + bs[i] = make([]byte, pnd.Links()[0].Size) // the size is always the same, validated by Encode. + nlst = append(nlst, i) + } + } + + if lrpnd < len(nlst) { + return nil, recovery.ErrRecoveryExceeded + } + + rs, err := reedsolomon.New(lpnd, lrpnd) + if err != nil { + return nil, err + } + + err = rs.Reconst(bs, lst, nlst) + if err != nil { + return nil, err + } + + // decode and save recomputed nodes, filter and return known to be lost. + nds := make([]format.Node, 0, len(lost)) + for _, i := range nlst { + id := ids[i] + if !id.Defined() { + id = pnd.Links()[i].Cid + } + + b, _ := blocks.NewBlockWithCid(bs[i], id) + nd, err := format.Decode(b) + if err != nil { + return nil, err + } + + err = dag.Add(ctx, nd) + if err != nil { + return nil, err + } + + if !ids[i].Defined() { + nds = append(nds, nd) + } + } + + return nds, nil +} diff --git a/entanglement/recover_test.go b/entanglement/recover_test.go new file mode 100644 index 0000000..284ca97 --- /dev/null +++ b/entanglement/recover_test.go @@ -0,0 +1,121 @@ +package entanglement + +import ( + "context" + "testing" + + "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" + dstest "github.com/ipfs/go-merkledag/test" + "github.com/stretchr/testify/assert" +) + +func TestRecover(t *testing.T) { + // Arrange + ctx := context.Background() + + in := merkledag.NodeWithData([]byte("1234567890")) + in2 := merkledag.NodeWithData([]byte("0987654321")) + in3 := merkledag.NodeWithData([]byte("1234509876")) + + dag := dstest.Mock() + dag.AddMany(ctx, []format.Node{in, in2, in3}) + + e := NewEncoder(dag) + + in.AddNodeLink("link", in2) + in.AddNodeLink("link", in3) + + rnd, err := e.Encode(ctx, in, 2) + + nd, ok := rnd.(*Node) + + // Act + failures, err := Recover(ctx, dag, nd) + + // Assert + assert.Nil(t, err) + assert.NotNil(t, failures) + assert.True(t, ok) +} + +func TestRecoverOneFailure(t *testing.T) { + // Arrange + ctx := context.Background() + + in := merkledag.NodeWithData([]byte("1234567890")) + in2 := merkledag.NodeWithData([]byte("0987654321")) + in3 := merkledag.NodeWithData([]byte("1234509876")) + + dag := dstest.Mock() + dag.AddMany(ctx, []format.Node{in, in2, in3}) + + e := NewEncoder(dag) + + in.AddNodeLink("link", in2) + in.AddNodeLink("link", in3) + + rnd, err := e.Encode(ctx, in, 2) + + nd, ok := rnd.(*Node) + + // Act + dag.Remove(ctx, in2.Cid()) + failures, err := Recover(ctx, dag, nd) + + // Assert + assert.Nil(t, err) + assert.NotNil(t, failures) + assert.True(t, ok) + assert.Empty(t, failures) + + lnk := nd.Links()[0] + r, err := dag.Get(ctx, lnk.Cid) + + assert.Equal(t, r.RawData(), in2.RawData()) + + assert.Nil(t, err) +} + +func TestRecoverTwoFailures(t *testing.T) { + // Arrange + ctx := context.Background() + + in := merkledag.NodeWithData([]byte("1234567890")) + in2 := merkledag.NodeWithData([]byte("0987654321")) + in3 := merkledag.NodeWithData([]byte("1234509876")) + + dag := dstest.Mock() + dag.AddMany(ctx, []format.Node{in, in2, in3}) + + e := NewEncoder(dag) + + in.AddNodeLink("link", in2) + in.AddNodeLink("link", in3) + + rnd, err := e.Encode(ctx, in, 2) + + nd, ok := rnd.(*Node) + + // Act + dag.Remove(ctx, in2.Cid()) + dag.Remove(ctx, in3.Cid()) + failures, err := Recover(ctx, dag, nd) + + // Assert + assert.Nil(t, err) + assert.NotNil(t, failures) + assert.True(t, ok) + assert.Empty(t, failures) + + lnk := nd.Links()[0] + r, err := dag.Get(ctx, lnk.Cid) + + lnk1 := nd.Links()[1] + r1, err := dag.Get(ctx, lnk1.Cid) + + assert.Equal(t, r.RawData(), in2.RawData()) + assert.Equal(t, r1.RawData(), in3.RawData()) + + assert.Nil(t, err) +} From 573f9ac7f8cf077e5024dd2e8bf5d4d336f57ace Mon Sep 17 00:00:00 2001 From: gov Date: Wed, 5 Aug 2020 02:13:56 -0400 Subject: [PATCH 3/8] WIP --- .../{alpha_entanglement.go => encode.go} | 76 +++++----- entanglement/entanglement.go | 50 ++++++ ...anglement_test.go => entanglement_test.go} | 0 entanglement/node.go | 143 ++++++++++++------ go.sum | 1 + 5 files changed, 187 insertions(+), 83 deletions(-) rename entanglement/{alpha_entanglement.go => encode.go} (64%) create mode 100644 entanglement/entanglement.go rename entanglement/{alpha_entanglement_test.go => entanglement_test.go} (100%) diff --git a/entanglement/alpha_entanglement.go b/entanglement/encode.go similarity index 64% rename from entanglement/alpha_entanglement.go rename to entanglement/encode.go index 739ccf5..1dfa38d 100644 --- a/entanglement/alpha_entanglement.go +++ b/entanglement/encode.go @@ -4,46 +4,14 @@ import ( "context" "fmt" - restore "github.com/Wondertan/go-ipfs-restore" - "github.com/Wondertan/go-ipfs-restore/rs" + recovery "github.com/Wondertan/go-ipfs-recovery" format "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" ) -type AlphaRestorer struct { - dag format.DAGService -} - -const ( - alpha = 1 - s = 1 - p = 1 -) - -func NewAlphaRestorer(ds format.DAGService) *AlphaRestorer { - r := &AlphaRestorer{ - dag: ds, - } - return r -} - -func xorByteSlice(a []byte, b []byte) ([]byte, error) { - if len(a) != len(b) { - return nil, fmt.Errorf("length of byte slices is not equivalent: %d != %d", len(a), len(b)) - } - - buf := make([]byte, len(a)) - - for i, _ := range a { - buf[i] = a[i] ^ b[i] - } - - return buf, nil -} - -func (ar *AlphaRestorer) Encode(ctx context.Context, nd format.Node) (format.Node, error) { +func Encode(ctx context.Context, dag format.DAGService, nd format.Node, r recovery.Recoverability) (*Node, error) { // get protonode - pnd, err := rs.ValidateNode(nd) + err := ValidateNode(nd) if err != nil { return nil, err } @@ -53,7 +21,7 @@ func (ar *AlphaRestorer) Encode(ctx context.Context, nd format.Node) (format.Nod // get all links of the node nodes = append(nodes, nd) - ndps := format.GetDAG(ctx, ar.dag, pnd) + ndps := format.GetDAG(ctx, dag, nd) for _, ndp := range ndps { nd, err = ndp.Get(ctx) if err != nil { @@ -92,3 +60,39 @@ func (ar *AlphaRestorer) Encode(ctx context.Context, nd format.Node) (format.Nod return pnd, nil } + +// ValidateNode checks whenever the given IPLD Node can be applied with Reed-Solomon coding. +func ValidateNode(nd format.Node) error { + _, ok := nd.(*merkledag.ProtoNode) + if !ok { + return fmt.Errorf("reedsolomon: node must be proto") + } + + ls := nd.Links() + if len(ls) == 0 { + return fmt.Errorf("reedsolomon: node must have links") + } + + size := ls[0].Size + for _, l := range ls[1:] { + if l.Size != size { + return fmt.Errorf("reedsolomon: node's links must have equal size") + } + } + + return nil +} + +func xorByteSlice(a []byte, b []byte) ([]byte, error) { + if len(a) != len(b) { + return nil, fmt.Errorf("length of byte slices is not equivalent: %d != %d", len(a), len(b)) + } + + buf := make([]byte, len(a)) + + for i, _ := range a { + buf[i] = a[i] ^ b[i] + } + + return buf, nil +} diff --git a/entanglement/entanglement.go b/entanglement/entanglement.go new file mode 100644 index 0000000..62db7a7 --- /dev/null +++ b/entanglement/entanglement.go @@ -0,0 +1,50 @@ +package entanglement + +import ( + "context" + "fmt" + + recovery "github.com/Wondertan/go-ipfs-recovery" + "github.com/ipfs/go-cid" + format "github.com/ipfs/go-ipld-format" +) + +// Custom codec for Entanglement recovery Nodes. +const Codec = 0x701 // random number // TODO Register in IPFS codec table. + +func init() { + // register global decoder + format.Register(Codec, DecodeNode) + + // register codec + cid.Codecs["recovery-entanglement`"] = Codec + cid.CodecToStr[Codec] = "recovery-entanglement`" + +} + +type entangler struct { + dag format.DAGService +} + +// NewRestorer creates a new Entanglement Recoverer. +func NewRestorer(dag format.DAGService) recovery.Recoverer { + return &entangler{dag: dag} +} + +// NewEncoder creates new Entanglement Encoder. +func NewEncoder(dag format.DAGService) recovery.Encoder { + return &entangler{dag: dag} +} + +func (ent *entangler) Recover(ctx context.Context, nd recovery.Node, rids ...cid.Cid) ([]format.Node, error) { + pnd, ok := nd.(*Node) + if !ok { + return nil, fmt.Errorf("reedsolomon: wrong Node type") + } + + return Recover(ctx, ent.dag, pnd, rids...) +} + +func (ent *entangler) Encode(ctx context.Context, nd format.Node, r recovery.Recoverability) (recovery.Node, error) { + return Encode(ctx, ent.dag, nd, r) +} diff --git a/entanglement/alpha_entanglement_test.go b/entanglement/entanglement_test.go similarity index 100% rename from entanglement/alpha_entanglement_test.go rename to entanglement/entanglement_test.go diff --git a/entanglement/node.go b/entanglement/node.go index 4a381d3..1b31ac5 100644 --- a/entanglement/node.go +++ b/entanglement/node.go @@ -16,19 +16,23 @@ import ( // TODO Do not save sizes of all links independently as they are always the same. -// Node is a recovery Node based ob Reed-Solomon coding. +const ( + alpha = 3 + s = 5 + p = 5 +) + +// Node is a recovery Node based on Entanglement coding. type Node struct { *merkledag.ProtoNode - Left []*Block - Right []*Block - Position int - Data []byte - Class StrandClass - Identifier string - // recovery []*format.Link - cache []byte - cid cid.Cid + // represents position of node in entanglement lattice + Position int + + Inputs map[int]*format.Link // left->0, horizontal->1, right->2 + Outputs map[int]*format.Link // left->0, horizontal->1, right->2 + cache []byte + cid cid.Cid } func NewNode(proto *merkledag.ProtoNode) *Node { @@ -38,24 +42,44 @@ func NewNode(proto *merkledag.ProtoNode) *Node { } func (n *Node) Recoverability() recovery.Recoverability { - return len(n.recovery) + return alpha } func (n *Node) RecoveryLinks() []*format.Link { - return n.recovery + var rls []*format.Link + + for _, v := range n.Outputs { + rls = append(rls, v) + } + + return rls } +// Don't use func (n *Node) AddRedundantNode(nd format.Node) { if nd == nil { return } + if len(n.Outputs) == 3 { + return // error? + } + n.cache = nil - n.recovery = append(n.recovery, &format.Link{ - Name: strconv.Itoa(len(n.recovery)), - Size: uint64(len(nd.RawData())), - Cid: nd.Cid(), - }) +} + +func (n *Node) GetInputs() (ins []*format.Link) { + for _, v := range n.Inputs { + ins = append(ins, v) + } + return +} + +func (n *Node) GetOutputs() (ins []*format.Link) { + for _, v := range n.Inputs { + ins = append(ins, v) + } + return } func (n *Node) RemoveRedundantNode(id cid.Cid) { @@ -63,15 +87,20 @@ func (n *Node) RemoveRedundantNode(id cid.Cid) { return } - ref := n.recovery[:0] - for _, v := range n.recovery { + for k, v := range n.Inputs { + if v.Cid.Equals(id) { + delete(n.Inputs, k) + n.cache = nil + return + } + } + for k, v := range n.Outputs { if v.Cid.Equals(id) { + delete(n.Outputs, k) n.cache = nil - } else { - ref = append(ref, v) + return } } - n.recovery = ref } func (n *Node) RawData() []byte { @@ -103,17 +132,18 @@ func (n *Node) Cid() cid.Cid { } func (n *Node) String() string { - return n.Cid().String() + return "CID: " + n.Cid().String() + "; Position: " + strconv.FormatInt(int64(n.Position), 10) } func (n *Node) Copy() format.Node { nd := new(Node) nd.ProtoNode = n.ProtoNode.Copy().(*merkledag.ProtoNode) - l := len(n.recovery) - if l > 0 { - nd.recovery = make([]*format.Link, l) - for i, r := range n.recovery { - nd.recovery[i] = &format.Link{ + lIn := len(n.Inputs) + if lIn > 0 { + nd.Inputs = make(map[int]*format.Link, lIn) + for i := 0; i <= lIn; i++ { + r := n.Inputs[i] + nd.Inputs[i] = &format.Link{ Name: r.Name, Size: r.Size, Cid: r.Cid, @@ -121,6 +151,21 @@ func (n *Node) Copy() format.Node { } } + lOut := len(n.Outputs) + if lOut > 0 { + nd.Outputs = make(map[int]*format.Link, lOut) + for i := 0; i <= lOut; i++ { + r := n.Inputs[i] + nd.Outputs[i] = &format.Link{ + Name: r.Name, + Size: r.Size, + Cid: r.Cid, + } + } + } + + nd.Position = n.Position + return nd } @@ -145,7 +190,10 @@ func (n *Node) Size() (uint64, error) { for _, l := range n.Links() { s += l.Size } - for _, l := range n.recovery { + for _, l := range n.GetInputs() { + s += l.Size + } + for _, l := range n.GetOutputs() { s += l.Size } return s, nil @@ -159,11 +207,12 @@ func MarshalNode(n *Node) ([]byte, error) { return nil, err } - l := len(n.recovery) + l := len(n.Inputs) + len(n.Outputs) if l > 0 { - sort.Stable(merkledag.LinkSlice(n.recovery)) + recovery := append(n.GetInputs(), n.GetOutputs()...) + sort.Stable(merkledag.LinkSlice(recovery)) pb.Recovery = make([]*cpb.PBLink, l) - for i, r := range n.recovery { + for i, r := range recovery { pb.Recovery[i] = &cpb.PBLink{ Name: r.Name, Size_: r.Size, @@ -189,21 +238,21 @@ func UnmarshalNode(data []byte) (*Node, error) { } l := len(pb.Recovery) - if l > 0 { - nd.recovery = make([]*format.Link, l) - for i, r := range pb.Recovery { - nd.recovery[i] = &format.Link{ - Name: r.Name, - Size: r.Size_, - } - - nd.recovery[i].Cid, err = cid.Cast(r.Hash) - if err != nil { - return nil, err - } - } - sort.Stable(merkledag.LinkSlice(nd.recovery)) - } + // if l > 0 { + // nd.recovery = make([]*format.Link, l) + // for i, r := range pb.Recovery { + // nd.recovery[i] = &format.Link{ + // Name: r.Name, + // Size: r.Size_, + // } + + // nd.recovery[i].Cid, err = cid.Cast(r.Hash) + // if err != nil { + // return nil, err + // } + // } + // sort.Stable(merkledag.LinkSlice(nd.recovery)) + // } return nd, nil } diff --git a/go.sum b/go.sum index 9336840..afee6e0 100644 --- a/go.sum +++ b/go.sum @@ -336,6 +336,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a h1:+KkCgOMgnKSgenxTBoiwkMqTiouMIy/3o8RLdmSbGoY= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 529f42bb67e404f2d8a2833501c8330942f927ae Mon Sep 17 00:00:00 2001 From: gov Date: Wed, 5 Aug 2020 04:22:43 -0400 Subject: [PATCH 4/8] WIP --- entanglement/encode.go | 65 +-- entanglement/entanglement.go | 43 +- entanglement/entanglement_test.go | 23 +- entanglement/node.go | 13 +- entanglement/node_test.go | 108 ++--- entanglement/pb/readsolomon.pb.go | 779 ++++++++++++++++++++++++++++++ entanglement/pb/readsolomon.proto | 13 + entanglement/recover_test.go | 168 +++---- 8 files changed, 996 insertions(+), 216 deletions(-) create mode 100644 entanglement/pb/readsolomon.pb.go create mode 100644 entanglement/pb/readsolomon.proto diff --git a/entanglement/encode.go b/entanglement/encode.go index 1dfa38d..e7ab395 100644 --- a/entanglement/encode.go +++ b/entanglement/encode.go @@ -9,59 +9,16 @@ import ( "github.com/ipfs/go-merkledag" ) -func Encode(ctx context.Context, dag format.DAGService, nd format.Node, r recovery.Recoverability) (*Node, error) { - // get protonode +func Encode(ctx context.Context, dag format.DAGService, nd format.Node, r recovery.Recoverability, ent entangler) (*Node, error) { err := ValidateNode(nd) if err != nil { return nil, err } - var nodes []format.Node - var reds [][]byte - - // get all links of the node - nodes = append(nodes, nd) - ndps := format.GetDAG(ctx, dag, nd) - for _, ndp := range ndps { - nd, err = ndp.Get(ctx) - if err != nil { - return nil, err - } - - nodes = append(nodes, nd) - } - - // for all links, create an alpha entanglement - lastRedundancy := make([]byte, len(nodes[0].RawData())) - - // n.b. how does this work in terms of ordering? - for _, linkedNode := range nodes { - - red, err := xorByteSlice(lastRedundancy, linkedNode.RawData()) - lastRedundancy = red - if err != nil { - return nil, err - } - reds = append(reds, red) - - redNode := merkledag.NodeWithData(red) - - linkedNodeProto, err := rs.ValidateNode(linkedNode) - - rnode := restore.NewNode(linkedNodeProto) - if err != nil { - return nil, err - } - - rnode.AddRedundantNode(redNode) - - ar.dag.Add(ctx, redNode) - } - - return pnd, nil + return nil, nil } -// ValidateNode checks whenever the given IPLD Node can be applied with Reed-Solomon coding. +// ValidateNode checks whenever the given IPLD Node can be applied with Entanglement coding. func ValidateNode(nd format.Node) error { _, ok := nd.(*merkledag.ProtoNode) if !ok { @@ -69,14 +26,14 @@ func ValidateNode(nd format.Node) error { } ls := nd.Links() - if len(ls) == 0 { - return fmt.Errorf("reedsolomon: node must have links") - } - - size := ls[0].Size - for _, l := range ls[1:] { - if l.Size != size { - return fmt.Errorf("reedsolomon: node's links must have equal size") + // can relax need for links + if len(ls) != 0 { + + size := ls[0].Size + for _, l := range ls[1:] { + if l.Size != size { + return fmt.Errorf("reedsolomon: node's links must have equal size") + } } } diff --git a/entanglement/entanglement.go b/entanglement/entanglement.go index 62db7a7..66e1eee 100644 --- a/entanglement/entanglement.go +++ b/entanglement/entanglement.go @@ -7,6 +7,7 @@ import ( recovery "github.com/Wondertan/go-ipfs-recovery" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" ) // Custom codec for Entanglement recovery Nodes. @@ -23,7 +24,31 @@ func init() { } type entangler struct { - dag format.DAGService + dag format.DAGService + Length int // length of all nodes in the DAG + LTbl map[int]cid.Cid // Lookup Table for lattice position +} + +// orderDagNode gives a lattice position to all nodes in the DAG +func (ent *entangler) orderDagNode(nd format.Node) error { + err := ValidateNode(nd) + ctx := context.Background() + if err != nil { + return err + } + + for _, ndp := range format.GetDAG(ctx, ent.dag, nd) { + c, err := ndp.Get(ctx) + if err != nil { + return err + } + ent.orderDagNode(c) + } + + end := NewNode(nd.(*merkledag.ProtoNode), ent.Length) + ent.Length += 1 + ent.LTbl[ent.Length] = end.Cid() + return nil } // NewRestorer creates a new Entanglement Recoverer. @@ -32,8 +57,17 @@ func NewRestorer(dag format.DAGService) recovery.Recoverer { } // NewEncoder creates new Entanglement Encoder. -func NewEncoder(dag format.DAGService) recovery.Encoder { - return &entangler{dag: dag} +func NewEncoder(dag format.DAGService, nd format.Node) (recovery.Encoder, error) { + + ent := &entangler{dag: dag, Length: 0} + ent.LTbl = make(map[int]cid.Cid) + err := ent.orderDagNode(nd) + if err != nil { + return nil, err + } + + return ent, nil + } func (ent *entangler) Recover(ctx context.Context, nd recovery.Node, rids ...cid.Cid) ([]format.Node, error) { @@ -46,5 +80,6 @@ func (ent *entangler) Recover(ctx context.Context, nd recovery.Node, rids ...cid } func (ent *entangler) Encode(ctx context.Context, nd format.Node, r recovery.Recoverability) (recovery.Node, error) { - return Encode(ctx, ent.dag, nd, r) + // return Encode(ctx, ent.dag, nd, r, ent) + return nil, nil } diff --git a/entanglement/entanglement_test.go b/entanglement/entanglement_test.go index e4f7905..2886833 100644 --- a/entanglement/entanglement_test.go +++ b/entanglement/entanglement_test.go @@ -6,37 +6,32 @@ import ( "testing" format "github.com/ipfs/go-ipld-format" - ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" dstest "github.com/ipfs/go-merkledag/test" assert "github.com/stretchr/testify/assert" ) -func TestEncode(t *testing.T) { +func TestNewEncoder(t *testing.T) { // Arrange in := merkledag.NodeWithData([]byte("1234567890")) in2 := merkledag.NodeWithData([]byte("0987654321")) in3 := merkledag.NodeWithData([]byte("1234509876")) - ar := NewAlphaRestorer(dstest.Mock()) + dag := dstest.Mock() ctx := context.Background() in.AddNodeLink("link", in2) in.AddNodeLink("link", in3) - ar.dag.AddMany(ctx, []format.Node{in, in2, in3}) - - var ng ipld.NodeGetter = ar.dag - - ng = merkledag.NewSession(ctx, ng) + dag.AddMany(ctx, []format.Node{in, in2, in3}) // Act - pnd, err := ar.Encode(ctx, in) + enc, err := NewEncoder(dag, in) + ent := enc.(*entangler) // Assert assert.Nil(t, err) - assert.NotNil(t, pnd) + assert.NotNil(t, enc) - for _, link := range pnd.Links() { - nd, err := link.GetNode(ctx, ng) - assert.Nil(t, err) - fmt.Println(nd.RawData()) + assert.Equal(t, ent.Length, 3) + for i := 0; i < ent.Length; i++ { + assert.NotNil(t, ent.LTbl[i+1]) } } diff --git a/entanglement/node.go b/entanglement/node.go index 1b31ac5..676b376 100644 --- a/entanglement/node.go +++ b/entanglement/node.go @@ -26,18 +26,19 @@ const ( type Node struct { *merkledag.ProtoNode - // represents position of node in entanglement lattice - Position int + Position int // position of node in lattice Inputs map[int]*format.Link // left->0, horizontal->1, right->2 Outputs map[int]*format.Link // left->0, horizontal->1, right->2 - cache []byte - cid cid.Cid + + cache []byte + cid cid.Cid } -func NewNode(proto *merkledag.ProtoNode) *Node { +func NewNode(proto *merkledag.ProtoNode, pos int) *Node { nd := &Node{ProtoNode: proto.Copy().(*merkledag.ProtoNode)} nd.SetCidBuilder(nd.CidBuilder().WithCodec(Codec)) + nd.Position = pos return nd } @@ -237,7 +238,7 @@ func UnmarshalNode(data []byte) (*Node, error) { return nil, err } - l := len(pb.Recovery) + // l := len(pb.Recovery) // if l > 0 { // nd.recovery = make([]*format.Link, l) // for i, r := range pb.Recovery { diff --git a/entanglement/node_test.go b/entanglement/node_test.go index 5465965..0ba4e62 100644 --- a/entanglement/node_test.go +++ b/entanglement/node_test.go @@ -1,69 +1,69 @@ package entanglement -import ( - "testing" +// import ( +// "testing" - "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) +// "github.com/ipfs/go-ipld-format" +// "github.com/ipfs/go-merkledag" +// "github.com/stretchr/testify/assert" +// "github.com/stretchr/testify/require" +// ) -func TestNodeRedundant(t *testing.T) { - nd := NewNode(merkledag.NodeWithData([]byte("1234567890"))) - r := merkledag.NewRawNode([]byte("12345")) - nd.AddRedundantNode(r) - assert.Len(t, nd.RecoveryLinks(), 1) - nd.RemoveRedundantNode(r.Cid()) - assert.Len(t, nd.RecoveryLinks(), 0) -} +// func TestNodeRedundant(t *testing.T) { +// nd := NewNode(merkledag.NodeWithData([]byte("1234567890"))) +// r := merkledag.NewRawNode([]byte("12345")) +// nd.AddRedundantNode(r) +// assert.Len(t, nd.RecoveryLinks(), 1) +// nd.RemoveRedundantNode(r.Cid()) +// assert.Len(t, nd.RecoveryLinks(), 0) +// } -func TestNodeMarshalUnmarshal(t *testing.T) { - in := NewNode(merkledag.NodeWithData([]byte("1234567890"))) - red := merkledag.NewRawNode([]byte("12345")) - in.AddRedundantNode(red) +// func TestNodeMarshalUnmarshal(t *testing.T) { +// in := NewNode(merkledag.NodeWithData([]byte("1234567890"))) +// red := merkledag.NewRawNode([]byte("12345")) +// in.AddRedundantNode(red) - data, err := MarshalNode(in) - require.NoError(t, err) +// data, err := MarshalNode(in) +// require.NoError(t, err) - out, err := UnmarshalNode(data) - require.NoError(t, err) +// out, err := UnmarshalNode(data) +// require.NoError(t, err) - out.SetCidBuilder(in.CidBuilder()) - assert.True(t, in.Cid().Equals(out.Cid())) - assert.Equal(t, in.RecoveryLinks(), out.RecoveryLinks()) -} +// out.SetCidBuilder(in.CidBuilder()) +// assert.True(t, in.Cid().Equals(out.Cid())) +// assert.Equal(t, in.RecoveryLinks(), out.RecoveryLinks()) +// } -func TestNodeDecode(t *testing.T) { - in := NewNode(merkledag.NodeWithData([]byte("1234567890"))) - out, err := format.Decode(in) - require.NoError(t, err) - assert.True(t, in.Cid().Equals(out.Cid())) -} +// func TestNodeDecode(t *testing.T) { +// in := NewNode(merkledag.NodeWithData([]byte("1234567890"))) +// out, err := format.Decode(in) +// require.NoError(t, err) +// assert.True(t, in.Cid().Equals(out.Cid())) +// } -func TestNodeCache(t *testing.T) { - nd := NewNode(merkledag.NodeWithData([]byte("1234567890"))) - nd.AddRedundantNode(merkledag.NewRawNode([]byte("12345"))) - assert.Zero(t, nd.cid) - assert.Nil(t, nd.cache) +// func TestNodeCache(t *testing.T) { +// nd := NewNode(merkledag.NodeWithData([]byte("1234567890"))) +// nd.AddRedundantNode(merkledag.NewRawNode([]byte("12345"))) +// assert.Zero(t, nd.cid) +// assert.Nil(t, nd.cache) - nd.Cid() - assert.NotZero(t, nd.cid) - assert.NotNil(t, nd.cache) +// nd.Cid() +// assert.NotZero(t, nd.cid) +// assert.NotNil(t, nd.cache) - nd.AddNodeLink("", merkledag.NewRawNode([]byte("12345"))) - assert.Nil(t, nd.cache) -} +// nd.AddNodeLink("", merkledag.NewRawNode([]byte("12345"))) +// assert.Nil(t, nd.cache) +// } -func TestNode_Copy(t *testing.T) { - nd := NewNode(merkledag.NodeWithData([]byte("1234567890"))) - r := merkledag.NewRawNode([]byte("12345")) - nd.AddRedundantNode(r) +// func TestNode_Copy(t *testing.T) { +// nd := NewNode(merkledag.NodeWithData([]byte("1234567890"))) +// r := merkledag.NewRawNode([]byte("12345")) +// nd.AddRedundantNode(r) - cp := nd.Copy() - nd.SetData([]byte{}) - nd.RemoveRedundantNode(r.Cid()) +// cp := nd.Copy() +// nd.SetData([]byte{}) +// nd.RemoveRedundantNode(r.Cid()) - assert.NotNil(t, cp.(*Node).RecoveryLinks()) - assert.NotNil(t, cp.(*Node).Data()) -} +// assert.NotNil(t, cp.(*Node).RecoveryLinks()) +// assert.NotNil(t, cp.(*Node).Data()) +// } diff --git a/entanglement/pb/readsolomon.pb.go b/entanglement/pb/readsolomon.pb.go new file mode 100644 index 0000000..f1a1d14 --- /dev/null +++ b/entanglement/pb/readsolomon.pb.go @@ -0,0 +1,779 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: reedsolomon/pb/readsolomon.proto + +package recovery_pb + +import ( + bytes "bytes" + fmt "fmt" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" + reflect "reflect" + strings "strings" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +type PBNode struct { + Proto []byte `protobuf:"bytes,1,opt,name=proto,proto3" json:"proto,omitempty"` + Recovery []*PBLink `protobuf:"bytes,2,rep,name=recovery,proto3" json:"recovery,omitempty"` +} + +func (m *PBNode) Reset() { *m = PBNode{} } +func (*PBNode) ProtoMessage() {} +func (*PBNode) Descriptor() ([]byte, []int) { + return fileDescriptor_4d7ca2a520a34643, []int{0} +} +func (m *PBNode) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *PBNode) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_PBNode.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *PBNode) XXX_Merge(src proto.Message) { + xxx_messageInfo_PBNode.Merge(m, src) +} +func (m *PBNode) XXX_Size() int { + return m.Size() +} +func (m *PBNode) XXX_DiscardUnknown() { + xxx_messageInfo_PBNode.DiscardUnknown(m) +} + +var xxx_messageInfo_PBNode proto.InternalMessageInfo + +func (m *PBNode) GetProto() []byte { + if m != nil { + return m.Proto + } + return nil +} + +func (m *PBNode) GetRecovery() []*PBLink { + if m != nil { + return m.Recovery + } + return nil +} + +type PBLink struct { + Hash []byte `protobuf:"bytes,1,opt,name=Hash,proto3" json:"Hash,omitempty"` + Name string `protobuf:"bytes,2,opt,name=Name,proto3" json:"Name,omitempty"` + Size_ uint64 `protobuf:"varint,3,opt,name=Size,proto3" json:"Size,omitempty"` +} + +func (m *PBLink) Reset() { *m = PBLink{} } +func (*PBLink) ProtoMessage() {} +func (*PBLink) Descriptor() ([]byte, []int) { + return fileDescriptor_4d7ca2a520a34643, []int{1} +} +func (m *PBLink) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *PBLink) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_PBLink.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *PBLink) XXX_Merge(src proto.Message) { + xxx_messageInfo_PBLink.Merge(m, src) +} +func (m *PBLink) XXX_Size() int { + return m.Size() +} +func (m *PBLink) XXX_DiscardUnknown() { + xxx_messageInfo_PBLink.DiscardUnknown(m) +} + +var xxx_messageInfo_PBLink proto.InternalMessageInfo + +func (m *PBLink) GetHash() []byte { + if m != nil { + return m.Hash + } + return nil +} + +func (m *PBLink) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *PBLink) GetSize_() uint64 { + if m != nil { + return m.Size_ + } + return 0 +} + +func init() { + proto.RegisterType((*PBNode)(nil), "recovery.pb.PBNode") + proto.RegisterType((*PBLink)(nil), "recovery.pb.PBLink") +} + +func init() { proto.RegisterFile("reedsolomon/pb/readsolomon.proto", fileDescriptor_4d7ca2a520a34643) } + +var fileDescriptor_4d7ca2a520a34643 = []byte{ + // 226 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x28, 0x4a, 0x4d, 0x4d, + 0x29, 0xce, 0xcf, 0xc9, 0xcf, 0xcd, 0xcf, 0xd3, 0x2f, 0x48, 0xd2, 0x2f, 0x4a, 0x4d, 0x84, 0x71, + 0xf5, 0x0a, 0x8a, 0xf2, 0x4b, 0xf2, 0x85, 0xb8, 0x8b, 0x52, 0x93, 0xf3, 0xcb, 0x52, 0x8b, 0x2a, + 0xf5, 0x0a, 0x92, 0x94, 0xfc, 0xb9, 0xd8, 0x02, 0x9c, 0xfc, 0xf2, 0x53, 0x52, 0x85, 0x44, 0xb8, + 0x58, 0xc1, 0xf2, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x3c, 0x41, 0x10, 0x8e, 0x90, 0x3e, 0x17, 0x07, + 0x4c, 0xb9, 0x04, 0x93, 0x02, 0xb3, 0x06, 0xb7, 0x91, 0xb0, 0x1e, 0x92, 0x7e, 0xbd, 0x00, 0x27, + 0x9f, 0xcc, 0xbc, 0xec, 0x20, 0xb8, 0x22, 0x25, 0x17, 0x90, 0x81, 0x20, 0x31, 0x21, 0x21, 0x2e, + 0x16, 0x8f, 0xc4, 0xe2, 0x0c, 0xa8, 0x79, 0x60, 0x36, 0x48, 0xcc, 0x2f, 0x31, 0x37, 0x55, 0x82, + 0x49, 0x81, 0x51, 0x83, 0x33, 0x08, 0xcc, 0x06, 0x89, 0x05, 0x67, 0x56, 0xa5, 0x4a, 0x30, 0x2b, + 0x30, 0x6a, 0xb0, 0x04, 0x81, 0xd9, 0x4e, 0x26, 0x17, 0x1e, 0xca, 0x31, 0xdc, 0x78, 0x28, 0xc7, + 0xf0, 0xe1, 0xa1, 0x1c, 0x63, 0xc3, 0x23, 0x39, 0xc6, 0x15, 0x8f, 0xe4, 0x18, 0x4f, 0x3c, 0x92, + 0x63, 0xbc, 0xf0, 0x48, 0x8e, 0xf1, 0xc1, 0x23, 0x39, 0xc6, 0x17, 0x8f, 0xe4, 0x18, 0x3e, 0x3c, + 0x92, 0x63, 0x9c, 0xf0, 0x58, 0x8e, 0xe1, 0xc2, 0x63, 0x39, 0x86, 0x1b, 0x8f, 0xe5, 0x18, 0x92, + 0xd8, 0xc0, 0x6e, 0x36, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0xad, 0xa0, 0x5a, 0x12, 0x04, 0x01, + 0x00, 0x00, +} + +func (this *PBNode) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*PBNode) + if !ok { + that2, ok := that.(PBNode) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if !bytes.Equal(this.Proto, that1.Proto) { + return false + } + if len(this.Recovery) != len(that1.Recovery) { + return false + } + for i := range this.Recovery { + if !this.Recovery[i].Equal(that1.Recovery[i]) { + return false + } + } + return true +} +func (this *PBLink) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*PBLink) + if !ok { + that2, ok := that.(PBLink) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if !bytes.Equal(this.Hash, that1.Hash) { + return false + } + if this.Name != that1.Name { + return false + } + if this.Size_ != that1.Size_ { + return false + } + return true +} +func (this *PBNode) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 6) + s = append(s, "&recovery_pb.PBNode{") + s = append(s, "Proto: "+fmt.Sprintf("%#v", this.Proto)+",\n") + if this.Recovery != nil { + s = append(s, "Recovery: "+fmt.Sprintf("%#v", this.Recovery)+",\n") + } + s = append(s, "}") + return strings.Join(s, "") +} +func (this *PBLink) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 7) + s = append(s, "&recovery_pb.PBLink{") + s = append(s, "Hash: "+fmt.Sprintf("%#v", this.Hash)+",\n") + s = append(s, "Name: "+fmt.Sprintf("%#v", this.Name)+",\n") + s = append(s, "Size_: "+fmt.Sprintf("%#v", this.Size_)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func valueToGoStringReadsolomon(v interface{}, typ string) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) +} +func (m *PBNode) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PBNode) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PBNode) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Recovery) > 0 { + for iNdEx := len(m.Recovery) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Recovery[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintReadsolomon(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + } + if len(m.Proto) > 0 { + i -= len(m.Proto) + copy(dAtA[i:], m.Proto) + i = encodeVarintReadsolomon(dAtA, i, uint64(len(m.Proto))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *PBLink) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PBLink) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PBLink) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Size_ != 0 { + i = encodeVarintReadsolomon(dAtA, i, uint64(m.Size_)) + i-- + dAtA[i] = 0x18 + } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintReadsolomon(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0x12 + } + if len(m.Hash) > 0 { + i -= len(m.Hash) + copy(dAtA[i:], m.Hash) + i = encodeVarintReadsolomon(dAtA, i, uint64(len(m.Hash))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func encodeVarintReadsolomon(dAtA []byte, offset int, v uint64) int { + offset -= sovReadsolomon(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *PBNode) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Proto) + if l > 0 { + n += 1 + l + sovReadsolomon(uint64(l)) + } + if len(m.Recovery) > 0 { + for _, e := range m.Recovery { + l = e.Size() + n += 1 + l + sovReadsolomon(uint64(l)) + } + } + return n +} + +func (m *PBLink) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Hash) + if l > 0 { + n += 1 + l + sovReadsolomon(uint64(l)) + } + l = len(m.Name) + if l > 0 { + n += 1 + l + sovReadsolomon(uint64(l)) + } + if m.Size_ != 0 { + n += 1 + sovReadsolomon(uint64(m.Size_)) + } + return n +} + +func sovReadsolomon(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozReadsolomon(x uint64) (n int) { + return sovReadsolomon(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *PBNode) String() string { + if this == nil { + return "nil" + } + repeatedStringForRecovery := "[]*PBLink{" + for _, f := range this.Recovery { + repeatedStringForRecovery += strings.Replace(f.String(), "PBLink", "PBLink", 1) + "," + } + repeatedStringForRecovery += "}" + s := strings.Join([]string{`&PBNode{`, + `Proto:` + fmt.Sprintf("%v", this.Proto) + `,`, + `Recovery:` + repeatedStringForRecovery + `,`, + `}`, + }, "") + return s +} +func (this *PBLink) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&PBLink{`, + `Hash:` + fmt.Sprintf("%v", this.Hash) + `,`, + `Name:` + fmt.Sprintf("%v", this.Name) + `,`, + `Size_:` + fmt.Sprintf("%v", this.Size_) + `,`, + `}`, + }, "") + return s +} +func valueToStringReadsolomon(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} +func (m *PBNode) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowReadsolomon + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PBNode: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PBNode: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Proto", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowReadsolomon + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthReadsolomon + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthReadsolomon + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Proto = append(m.Proto[:0], dAtA[iNdEx:postIndex]...) + if m.Proto == nil { + m.Proto = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Recovery", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowReadsolomon + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthReadsolomon + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthReadsolomon + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Recovery = append(m.Recovery, &PBLink{}) + if err := m.Recovery[len(m.Recovery)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipReadsolomon(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthReadsolomon + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthReadsolomon + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PBLink) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowReadsolomon + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PBLink: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PBLink: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowReadsolomon + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthReadsolomon + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthReadsolomon + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Hash = append(m.Hash[:0], dAtA[iNdEx:postIndex]...) + if m.Hash == nil { + m.Hash = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowReadsolomon + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthReadsolomon + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthReadsolomon + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType) + } + m.Size_ = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowReadsolomon + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size_ |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipReadsolomon(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthReadsolomon + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthReadsolomon + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipReadsolomon(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowReadsolomon + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowReadsolomon + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowReadsolomon + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthReadsolomon + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupReadsolomon + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthReadsolomon + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthReadsolomon = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowReadsolomon = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupReadsolomon = fmt.Errorf("proto: unexpected end of group") +) diff --git a/entanglement/pb/readsolomon.proto b/entanglement/pb/readsolomon.proto new file mode 100644 index 0000000..b3f13e8 --- /dev/null +++ b/entanglement/pb/readsolomon.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; +package recovery.pb; + +message PBNode { + bytes proto = 1; + repeated PBLink recovery = 2; +} + +message PBLink { + bytes Hash = 1; + string Name = 2; + uint64 Size = 3; +} \ No newline at end of file diff --git a/entanglement/recover_test.go b/entanglement/recover_test.go index 284ca97..c333db3 100644 --- a/entanglement/recover_test.go +++ b/entanglement/recover_test.go @@ -1,121 +1,121 @@ package entanglement -import ( - "context" - "testing" +// import ( +// "context" +// "testing" - "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" - dstest "github.com/ipfs/go-merkledag/test" - "github.com/stretchr/testify/assert" -) +// "github.com/ipfs/go-ipld-format" +// "github.com/ipfs/go-merkledag" +// dstest "github.com/ipfs/go-merkledag/test" +// "github.com/stretchr/testify/assert" +// ) -func TestRecover(t *testing.T) { - // Arrange - ctx := context.Background() +// func TestRecover(t *testing.T) { +// // Arrange +// ctx := context.Background() - in := merkledag.NodeWithData([]byte("1234567890")) - in2 := merkledag.NodeWithData([]byte("0987654321")) - in3 := merkledag.NodeWithData([]byte("1234509876")) +// in := merkledag.NodeWithData([]byte("1234567890")) +// in2 := merkledag.NodeWithData([]byte("0987654321")) +// in3 := merkledag.NodeWithData([]byte("1234509876")) - dag := dstest.Mock() - dag.AddMany(ctx, []format.Node{in, in2, in3}) +// dag := dstest.Mock() +// dag.AddMany(ctx, []format.Node{in, in2, in3}) - e := NewEncoder(dag) +// e := NewEncoder(dag) - in.AddNodeLink("link", in2) - in.AddNodeLink("link", in3) +// in.AddNodeLink("link", in2) +// in.AddNodeLink("link", in3) - rnd, err := e.Encode(ctx, in, 2) +// rnd, err := e.Encode(ctx, in, 2) - nd, ok := rnd.(*Node) +// nd, ok := rnd.(*Node) - // Act - failures, err := Recover(ctx, dag, nd) +// // Act +// failures, err := Recover(ctx, dag, nd) - // Assert - assert.Nil(t, err) - assert.NotNil(t, failures) - assert.True(t, ok) -} +// // Assert +// assert.Nil(t, err) +// assert.NotNil(t, failures) +// assert.True(t, ok) +// } -func TestRecoverOneFailure(t *testing.T) { - // Arrange - ctx := context.Background() +// func TestRecoverOneFailure(t *testing.T) { +// // Arrange +// ctx := context.Background() - in := merkledag.NodeWithData([]byte("1234567890")) - in2 := merkledag.NodeWithData([]byte("0987654321")) - in3 := merkledag.NodeWithData([]byte("1234509876")) +// in := merkledag.NodeWithData([]byte("1234567890")) +// in2 := merkledag.NodeWithData([]byte("0987654321")) +// in3 := merkledag.NodeWithData([]byte("1234509876")) - dag := dstest.Mock() - dag.AddMany(ctx, []format.Node{in, in2, in3}) +// dag := dstest.Mock() +// dag.AddMany(ctx, []format.Node{in, in2, in3}) - e := NewEncoder(dag) +// e := NewEncoder(dag) - in.AddNodeLink("link", in2) - in.AddNodeLink("link", in3) +// in.AddNodeLink("link", in2) +// in.AddNodeLink("link", in3) - rnd, err := e.Encode(ctx, in, 2) +// rnd, err := e.Encode(ctx, in, 2) - nd, ok := rnd.(*Node) +// nd, ok := rnd.(*Node) - // Act - dag.Remove(ctx, in2.Cid()) - failures, err := Recover(ctx, dag, nd) +// // Act +// dag.Remove(ctx, in2.Cid()) +// failures, err := Recover(ctx, dag, nd) - // Assert - assert.Nil(t, err) - assert.NotNil(t, failures) - assert.True(t, ok) - assert.Empty(t, failures) +// // Assert +// assert.Nil(t, err) +// assert.NotNil(t, failures) +// assert.True(t, ok) +// assert.Empty(t, failures) - lnk := nd.Links()[0] - r, err := dag.Get(ctx, lnk.Cid) +// lnk := nd.Links()[0] +// r, err := dag.Get(ctx, lnk.Cid) - assert.Equal(t, r.RawData(), in2.RawData()) +// assert.Equal(t, r.RawData(), in2.RawData()) - assert.Nil(t, err) -} +// assert.Nil(t, err) +// } -func TestRecoverTwoFailures(t *testing.T) { - // Arrange - ctx := context.Background() +// func TestRecoverTwoFailures(t *testing.T) { +// // Arrange +// ctx := context.Background() - in := merkledag.NodeWithData([]byte("1234567890")) - in2 := merkledag.NodeWithData([]byte("0987654321")) - in3 := merkledag.NodeWithData([]byte("1234509876")) +// in := merkledag.NodeWithData([]byte("1234567890")) +// in2 := merkledag.NodeWithData([]byte("0987654321")) +// in3 := merkledag.NodeWithData([]byte("1234509876")) - dag := dstest.Mock() - dag.AddMany(ctx, []format.Node{in, in2, in3}) +// dag := dstest.Mock() +// dag.AddMany(ctx, []format.Node{in, in2, in3}) - e := NewEncoder(dag) +// e := NewEncoder(dag) - in.AddNodeLink("link", in2) - in.AddNodeLink("link", in3) +// in.AddNodeLink("link", in2) +// in.AddNodeLink("link", in3) - rnd, err := e.Encode(ctx, in, 2) +// rnd, err := e.Encode(ctx, in, 2) - nd, ok := rnd.(*Node) +// nd, ok := rnd.(*Node) - // Act - dag.Remove(ctx, in2.Cid()) - dag.Remove(ctx, in3.Cid()) - failures, err := Recover(ctx, dag, nd) +// // Act +// dag.Remove(ctx, in2.Cid()) +// dag.Remove(ctx, in3.Cid()) +// failures, err := Recover(ctx, dag, nd) - // Assert - assert.Nil(t, err) - assert.NotNil(t, failures) - assert.True(t, ok) - assert.Empty(t, failures) +// // Assert +// assert.Nil(t, err) +// assert.NotNil(t, failures) +// assert.True(t, ok) +// assert.Empty(t, failures) - lnk := nd.Links()[0] - r, err := dag.Get(ctx, lnk.Cid) +// lnk := nd.Links()[0] +// r, err := dag.Get(ctx, lnk.Cid) - lnk1 := nd.Links()[1] - r1, err := dag.Get(ctx, lnk1.Cid) +// lnk1 := nd.Links()[1] +// r1, err := dag.Get(ctx, lnk1.Cid) - assert.Equal(t, r.RawData(), in2.RawData()) - assert.Equal(t, r1.RawData(), in3.RawData()) +// assert.Equal(t, r.RawData(), in2.RawData()) +// assert.Equal(t, r1.RawData(), in3.RawData()) - assert.Nil(t, err) -} +// assert.Nil(t, err) +// } From 7cd5ea8ebd6cc406ae40caf65b63a22aa5af30ae Mon Sep 17 00:00:00 2001 From: gov Date: Wed, 5 Aug 2020 16:36:20 -0400 Subject: [PATCH 5/8] wip --- entanglement/encode.go | 55 --------- entanglement/encode_utils.go | 187 ++++++++++++++++++++++++++++++ entanglement/entanglement.go | 90 ++++++++++++-- entanglement/entanglement_test.go | 13 ++- entanglement/node.go | 33 +++++- 5 files changed, 307 insertions(+), 71 deletions(-) delete mode 100644 entanglement/encode.go create mode 100644 entanglement/encode_utils.go diff --git a/entanglement/encode.go b/entanglement/encode.go deleted file mode 100644 index e7ab395..0000000 --- a/entanglement/encode.go +++ /dev/null @@ -1,55 +0,0 @@ -package entanglement - -import ( - "context" - "fmt" - - recovery "github.com/Wondertan/go-ipfs-recovery" - format "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" -) - -func Encode(ctx context.Context, dag format.DAGService, nd format.Node, r recovery.Recoverability, ent entangler) (*Node, error) { - err := ValidateNode(nd) - if err != nil { - return nil, err - } - - return nil, nil -} - -// ValidateNode checks whenever the given IPLD Node can be applied with Entanglement coding. -func ValidateNode(nd format.Node) error { - _, ok := nd.(*merkledag.ProtoNode) - if !ok { - return fmt.Errorf("reedsolomon: node must be proto") - } - - ls := nd.Links() - // can relax need for links - if len(ls) != 0 { - - size := ls[0].Size - for _, l := range ls[1:] { - if l.Size != size { - return fmt.Errorf("reedsolomon: node's links must have equal size") - } - } - } - - return nil -} - -func xorByteSlice(a []byte, b []byte) ([]byte, error) { - if len(a) != len(b) { - return nil, fmt.Errorf("length of byte slices is not equivalent: %d != %d", len(a), len(b)) - } - - buf := make([]byte, len(a)) - - for i, _ := range a { - buf[i] = a[i] ^ b[i] - } - - return buf, nil -} diff --git a/entanglement/encode_utils.go b/entanglement/encode_utils.go new file mode 100644 index 0000000..8214f23 --- /dev/null +++ b/entanglement/encode_utils.go @@ -0,0 +1,187 @@ +package entanglement + +import ( + "fmt" + "math" + + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" +) + +const ( + LH = iota + H = iota + RH = iota + alpha = 3 + s = 5 + p = 5 +) + +// Alpha = 3 +func GetForwardNeighbours(i int) (r, h, l int) { + // Check is it top, center or bottom in the lattice + // 1 -> Top, 0 -> Bottom, else Center + var nodePos = i % s + + if nodePos == 1 { + r = i + s + 1 + h = i + s + l = i + (s * p) - int(math.Pow(float64(s-1), 2)) + } else if nodePos == 0 { + r = i + (s * p) - int(math.Pow(float64(s), 2)-1) + h = i + s + l = i + s - 1 + } else { + r = i + s + 1 + h = i + s + l = i + (s - 1) + } + return +} + +// TODO: Fix underflow naming errors on the nodes on the extreme of the lattice. +func GetBackwardNeighbours(i int) (r, h, l int) { + // Check is it top, center or bottom in the lattice + // 1 -> Top, 0 -> Bottom, else Center + var nodePos = i % s + + if nodePos == 1 { + r = i - (s * p) + int((math.Pow(float64(s), 2) - 1)) + h = i - s + l = i - (s - 1) + } else if nodePos == 0 { + r = i - (s + 1) + h = i - s + l = i - (s * p) + int(math.Pow(float64(s-1), 2)) + } else { + r = i - (s + 1) + h = i - s + l = i - (s - 1) + } + return +} + +func GetMemoryPosition(index int) (r, h, l int) { + // Get the position in the ParityMemory array where the parity is located + // For now this will recursively call the GetBackwardNeighbours function + + h = ((index - 1) % s) + s + r, l = index, index + + for ; r > s; r, _, _ = GetBackwardNeighbours(r) { + } + + switch r { + case 1: + r = 0 + case 2: + r = 4 + case 3: + r = 3 + case 4: + r = 2 + case 5: + r = 1 + } + + for ; l > s; _, _, l = GetBackwardNeighbours(l) { + } + + switch l { + case 1: + l = 11 + case 2: + l = 12 + case 3: + l = 13 + case 4: + l = 14 + case 5: + l = 10 + } + + return +} + +// NextEntangleNode specifies the rules for creating a helical structure +// with indexed nodes arranged by s, p +func nextEntangleNode(i, strand int) int { + nodePos := i % s + + switch strand { + case LH: + switch nodePos { + case -4: // special case for (3,5,5); Go doesn't like negative modulo + fallthrough + case 1: + return i + s*p - (s-1)*(s-1) + case 0: + return i + s - 1 + default: + return i + s - 1 + } + case H: + switch nodePos { + case -4: + fallthrough + case 1: + return i + s + case 0: + return i + s + default: + return i + s + + } + case RH: + switch nodePos { + case -4: + fallthrough + case 1: + return i + s + 1 + case 0: + return i + s*p - (s*s - 1) + default: + return i + s + 1 + } + default: + // this shouldn't be hit + return 0 + } +} + +// ValidateNode checks whenever the given IPLD Node can be applied with Entanglement coding. +func ValidateNode(nd format.Node) error { + _, ok := nd.(*merkledag.ProtoNode) + if !ok { + return fmt.Errorf("reedsolomon: node must be proto") + } + + ls := nd.Links() + // can relax need for links + if len(ls) != 0 { + + size := ls[0].Size + for _, l := range ls[1:] { + if l.Size != size { + return fmt.Errorf("reedsolomon: node's links must have equal size") + } + } + } + + return nil +} + +// XORByteSlice returns an XOR slice of 2 input slices +func XORByteSlice(a []byte, b []byte) ([]byte, error) { + if len(a) != len(b) { + return nil, fmt.Errorf("length of byte slices is not equivalent: %d != %d", len(a), len(b)) + } + + buf := make([]byte, len(a)) + + for i, _ := range a { + buf[i] = a[i] ^ b[i] + } + + return buf, nil +} diff --git a/entanglement/entanglement.go b/entanglement/entanglement.go index 66e1eee..803279c 100644 --- a/entanglement/entanglement.go +++ b/entanglement/entanglement.go @@ -24,21 +24,24 @@ func init() { } type entangler struct { - dag format.DAGService - Length int // length of all nodes in the DAG - LTbl map[int]cid.Cid // Lookup Table for lattice position + dag format.DAGService + Length int // length of all nodes in the DAG + LTbl map[int]cid.Cid // Lookup Table for lattice position + Parities map[[2]int]cid.Cid // Lookup table for parity [from pos, to pos] + ParityMemory [15][]byte // 5 left strands + 5 right + 5 horizontal } // orderDagNode gives a lattice position to all nodes in the DAG func (ent *entangler) orderDagNode(nd format.Node) error { - err := ValidateNode(nd) ctx := context.Background() + err := ValidateNode(nd) // Variable length sizes are passing!! if err != nil { return err } - for _, ndp := range format.GetDAG(ctx, ent.dag, nd) { - c, err := ndp.Get(ctx) + for _, l := range nd.Links() { + c, err := ent.dag.Get(ctx, l.Cid) + fmt.Println(l.Cid) if err != nil { return err } @@ -80,6 +83,79 @@ func (ent *entangler) Recover(ctx context.Context, nd recovery.Node, rids ...cid } func (ent *entangler) Encode(ctx context.Context, nd format.Node, r recovery.Recoverability) (recovery.Node, error) { - // return Encode(ctx, ent.dag, nd, r, ent) + return nil, nil } + +func (ent *entangler) updateParities(ctx context.Context, r, h, l *RedundantNode) { + ent.Parities[r.Position] = r.Cid() + ent.Parities[h.Position] = h.Cid() + ent.Parities[l.Position] = l.Cid() + ent.dag.AddMany(ctx, []format.Node{r, h, l}) +} + +func (ent *entangler) entangle(nd *Node, rnd *RedundantNode, i int) { + // r, h, l := GetMemoryPosition(i) + // rBack, hBack, lBack := GetBackwardNeighbours(i) + // rParity := ent.ParityMemory[r] + // hParity := ent.ParityMemory[h] + // lParity := ent.ParityMemory[l] + + // rnd.AddRedundantNode(nd,) + // WriteChunkToFile(rParity, rBack, index) + // WriteChunkToFile(hParity, hBack, index) + // WriteChunkToFile(lParity, lBack, index) + + // rNext, _ := XORByteSlice(datachunk, rParity) + // ent.ParityMemory[r] = rNext + + // hNext, _ := XORByteSlice(datachunk, hParity) + // ent.ParityMemory[h] = hNext + + // lNext, _ := XORByteSlice(datachunk, lParity) + // ent.ParityMemory[l] = lNext +} + +// EncodeDag creates an entangled lattice of redundancies +func (ent *entangler) EncodeDag(ctx context.Context) error { + nd, err := ent.dag.Get(ctx, ent.LTbl[1]) + if err != nil { + return err + } + + ent.Parities = make(map[[2]int]cid.Cid) + + rPrev := NewRedundantNode(merkledag.NodeWithData(make([]byte, len(nd.RawData())))) + rPrev.Position = [2]int{1, nextEntangleNode(1, RH)} + + hPrev := NewRedundantNode(merkledag.NodeWithData(make([]byte, len(nd.RawData())))) + hPrev.Position = [2]int{1, nextEntangleNode(1, H)} + + lPrev := NewRedundantNode(merkledag.NodeWithData(make([]byte, len(nd.RawData())))) + lPrev.Position = [2]int{1, nextEntangleNode(1, LH)} + + ent.updateParities(ctx, rPrev, hPrev, lPrev) + + for i := 1; i < ent.Length; i++ { + nd, err := ent.dag.Get(ctx, ent.LTbl[i+1]) + if err != nil { + return err + } + + r := NewRedundantNode(merkledag.NodeWithData([]byte{})) + r.AddRedundantNode(nd.(*Node), rPrev, RH) + + h := NewRedundantNode(merkledag.NodeWithData([]byte{})) + h.AddRedundantNode(nd.(*Node), hPrev, H) + + l := NewRedundantNode(merkledag.NodeWithData([]byte{})) + l.AddRedundantNode(nd.(*Node), lPrev, LH) + + ent.updateParities(ctx, r, h, l) + rPrev = r + hPrev = h + lPrev = l + } + + return nil +} diff --git a/entanglement/entanglement_test.go b/entanglement/entanglement_test.go index 2886833..6d26c23 100644 --- a/entanglement/entanglement_test.go +++ b/entanglement/entanglement_test.go @@ -2,7 +2,6 @@ package entanglement import ( "context" - "fmt" "testing" format "github.com/ipfs/go-ipld-format" @@ -16,21 +15,27 @@ func TestNewEncoder(t *testing.T) { in := merkledag.NodeWithData([]byte("1234567890")) in2 := merkledag.NodeWithData([]byte("0987654321")) in3 := merkledag.NodeWithData([]byte("1234509876")) + in4 := merkledag.NodeWithData([]byte("0192837465")) + in5 := merkledag.NodeWithData([]byte("0392817465")) dag := dstest.Mock() ctx := context.Background() + in3.AddNodeLink("link", in4) + in2.AddNodeLink("link", in5) in.AddNodeLink("link", in2) in.AddNodeLink("link", in3) - dag.AddMany(ctx, []format.Node{in, in2, in3}) + + dag.AddMany(ctx, []format.Node{in, in2, in3, in4, in5}) // Act enc, err := NewEncoder(dag, in) - ent := enc.(*entangler) // Assert assert.Nil(t, err) assert.NotNil(t, enc) - assert.Equal(t, ent.Length, 3) + ent := enc.(*entangler) + + assert.Equal(t, ent.Length, 5) for i := 0; i < ent.Length; i++ { assert.NotNil(t, ent.LTbl[i+1]) } diff --git a/entanglement/node.go b/entanglement/node.go index 676b376..011105c 100644 --- a/entanglement/node.go +++ b/entanglement/node.go @@ -1,6 +1,7 @@ package entanglement import ( + "errors" "fmt" "sort" "strconv" @@ -16,11 +17,33 @@ import ( // TODO Do not save sizes of all links independently as they are always the same. -const ( - alpha = 3 - s = 5 - p = 5 -) +type RedundantNode struct { + *merkledag.ProtoNode + + Position [2]int // from, to lattice pos +} + +func NewRedundantNode(n *merkledag.ProtoNode) *RedundantNode { + nd := &RedundantNode{ProtoNode: n.Copy().(*merkledag.ProtoNode)} + nd.SetCidBuilder(nd.CidBuilder().WithCodec(Codec)) + return nd +} + +func (r *RedundantNode) AddRedundantNode(n *Node, prev *RedundantNode, strand int) error { + data, _ := XORByteSlice(n.Data(), prev.Data()) + r.SetData(data) + switch strand { + case LH: + r.Position = [2]int{n.Position, nextEntangleNode(n.Position, LH)} + case RH: + r.Position = [2]int{n.Position, nextEntangleNode(n.Position, RH)} + case H: + r.Position = [2]int{n.Position, nextEntangleNode(n.Position, H)} + default: + return errors.New("Strand must be LH=0,H=1,RH=2") + } + return nil +} // Node is a recovery Node based on Entanglement coding. type Node struct { From 3fc3aa3939f072a2aa3847e438e856069b5a6f34 Mon Sep 17 00:00:00 2001 From: gov Date: Thu, 6 Aug 2020 17:33:23 -0400 Subject: [PATCH 6/8] encode parent complete --- entanglement/encoder.go | 87 +++++++++++++++++++ entanglement/entanglement.go | 119 ++------------------------ entanglement/node.go | 161 ++++++++++------------------------- 3 files changed, 137 insertions(+), 230 deletions(-) create mode 100644 entanglement/encoder.go diff --git a/entanglement/encoder.go b/entanglement/encoder.go new file mode 100644 index 0000000..13c3199 --- /dev/null +++ b/entanglement/encoder.go @@ -0,0 +1,87 @@ +package entanglement + +import ( + "context" + "fmt" + + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" + "github.com/templexxx/reedsolomon" + "golang.org/x/tools/godoc/redirect" + + recovery "github.com/Wondertan/go-ipfs-recovery" +) + +// Encode applies Reed-Solomon coding on the given IPLD Node promoting it to a recovery Node. +// Use `r` to specify needed amount of generated recovery Nodes. +func Encode(ctx context.Context, dag format.DAGService, nd format.Node, r recovery.Recoverability) (*Node, error) { + err := ValidateNode(nd) + if err != nil { + return nil, err + } + + ls := nd.Links() + rd := NewNode(nd.(*merkledag.ProtoNode)) + + nd1, err := ls[0].GetNode(ctx, dag) + if err != nil { + return nil, err + } + + // create 1st redundancy and add to dag, recovery + red := merkledag.NewRawNode(nd1.RawData()) + err = dag.Add(ctx, red) + if err != nil { + return nil, err + } + rd.AddRedundantNode(red) + + for i := 1; i < len(ls); i++ { + nd1, err = ls[i].GetNode(ctx, dag) + if err != nil { + return nil, err + } + bs, err := XORByteSlice(nd1.RawData(), red.RawData()) + if err != nil { + return nil, err + } + + // create 1st redundancy and add to dag, recovery + red = merkledag.NewRawNode(bs) + err = dag.Add(ctx, red) + if err != nil { + return nil, err + } + // add link in order of redundancy value + rd.AddRedundantNode(red) + } + + err = dag.Add(ctx, rd) + if err != nil { + return nil, err + } + + return rd, dag.Remove(ctx, nd.Cid()) +} + +// ValidateNode checks whenever the given IPLD Node can be applied with Reed-Solomon coding. +func ValidateNode(nd format.Node) error { + _, ok := nd.(*merkledag.ProtoNode) + if !ok { + return fmt.Errorf("entanglement: node must be proto") + } + + ls := nd.Links() + if len(ls) == 0 { + return fmt.Errorf("entanglement: node must have links") + } + + size := ls[0].Size + for _, l := range ls[1:] { + if l.Size != size { + return fmt.Errorf("entanglement: node's links must have equal size") + } + } + + return nil +} diff --git a/entanglement/entanglement.go b/entanglement/entanglement.go index 803279c..afe9bb9 100644 --- a/entanglement/entanglement.go +++ b/entanglement/entanglement.go @@ -11,7 +11,8 @@ import ( ) // Custom codec for Entanglement recovery Nodes. -const Codec = 0x701 // random number // TODO Register in IPFS codec table. +const Codec = 0x701 // random number // TODO Register in IPFS codec table. +const CodecRedundant = 0x702 // random number // TODO Register in IPFS codec table. func init() { // register global decoder @@ -24,34 +25,7 @@ func init() { } type entangler struct { - dag format.DAGService - Length int // length of all nodes in the DAG - LTbl map[int]cid.Cid // Lookup Table for lattice position - Parities map[[2]int]cid.Cid // Lookup table for parity [from pos, to pos] - ParityMemory [15][]byte // 5 left strands + 5 right + 5 horizontal -} - -// orderDagNode gives a lattice position to all nodes in the DAG -func (ent *entangler) orderDagNode(nd format.Node) error { - ctx := context.Background() - err := ValidateNode(nd) // Variable length sizes are passing!! - if err != nil { - return err - } - - for _, l := range nd.Links() { - c, err := ent.dag.Get(ctx, l.Cid) - fmt.Println(l.Cid) - if err != nil { - return err - } - ent.orderDagNode(c) - } - - end := NewNode(nd.(*merkledag.ProtoNode), ent.Length) - ent.Length += 1 - ent.LTbl[ent.Length] = end.Cid() - return nil + dag format.DAGService } // NewRestorer creates a new Entanglement Recoverer. @@ -60,23 +34,15 @@ func NewRestorer(dag format.DAGService) recovery.Recoverer { } // NewEncoder creates new Entanglement Encoder. -func NewEncoder(dag format.DAGService, nd format.Node) (recovery.Encoder, error) { - - ent := &entangler{dag: dag, Length: 0} - ent.LTbl = make(map[int]cid.Cid) - err := ent.orderDagNode(nd) - if err != nil { - return nil, err - } - - return ent, nil +func NewEncoder(dag format.DAGService, nd format.Node) recovery.Encoder { + return &entangler{dag: dag} } func (ent *entangler) Recover(ctx context.Context, nd recovery.Node, rids ...cid.Cid) ([]format.Node, error) { pnd, ok := nd.(*Node) if !ok { - return nil, fmt.Errorf("reedsolomon: wrong Node type") + return nil, fmt.Errorf("Entanglement: wrong Node type") } return Recover(ctx, ent.dag, pnd, rids...) @@ -86,76 +52,3 @@ func (ent *entangler) Encode(ctx context.Context, nd format.Node, r recovery.Rec return nil, nil } - -func (ent *entangler) updateParities(ctx context.Context, r, h, l *RedundantNode) { - ent.Parities[r.Position] = r.Cid() - ent.Parities[h.Position] = h.Cid() - ent.Parities[l.Position] = l.Cid() - ent.dag.AddMany(ctx, []format.Node{r, h, l}) -} - -func (ent *entangler) entangle(nd *Node, rnd *RedundantNode, i int) { - // r, h, l := GetMemoryPosition(i) - // rBack, hBack, lBack := GetBackwardNeighbours(i) - // rParity := ent.ParityMemory[r] - // hParity := ent.ParityMemory[h] - // lParity := ent.ParityMemory[l] - - // rnd.AddRedundantNode(nd,) - // WriteChunkToFile(rParity, rBack, index) - // WriteChunkToFile(hParity, hBack, index) - // WriteChunkToFile(lParity, lBack, index) - - // rNext, _ := XORByteSlice(datachunk, rParity) - // ent.ParityMemory[r] = rNext - - // hNext, _ := XORByteSlice(datachunk, hParity) - // ent.ParityMemory[h] = hNext - - // lNext, _ := XORByteSlice(datachunk, lParity) - // ent.ParityMemory[l] = lNext -} - -// EncodeDag creates an entangled lattice of redundancies -func (ent *entangler) EncodeDag(ctx context.Context) error { - nd, err := ent.dag.Get(ctx, ent.LTbl[1]) - if err != nil { - return err - } - - ent.Parities = make(map[[2]int]cid.Cid) - - rPrev := NewRedundantNode(merkledag.NodeWithData(make([]byte, len(nd.RawData())))) - rPrev.Position = [2]int{1, nextEntangleNode(1, RH)} - - hPrev := NewRedundantNode(merkledag.NodeWithData(make([]byte, len(nd.RawData())))) - hPrev.Position = [2]int{1, nextEntangleNode(1, H)} - - lPrev := NewRedundantNode(merkledag.NodeWithData(make([]byte, len(nd.RawData())))) - lPrev.Position = [2]int{1, nextEntangleNode(1, LH)} - - ent.updateParities(ctx, rPrev, hPrev, lPrev) - - for i := 1; i < ent.Length; i++ { - nd, err := ent.dag.Get(ctx, ent.LTbl[i+1]) - if err != nil { - return err - } - - r := NewRedundantNode(merkledag.NodeWithData([]byte{})) - r.AddRedundantNode(nd.(*Node), rPrev, RH) - - h := NewRedundantNode(merkledag.NodeWithData([]byte{})) - h.AddRedundantNode(nd.(*Node), hPrev, H) - - l := NewRedundantNode(merkledag.NodeWithData([]byte{})) - l.AddRedundantNode(nd.(*Node), lPrev, LH) - - ent.updateParities(ctx, r, h, l) - rPrev = r - hPrev = h - lPrev = l - } - - return nil -} diff --git a/entanglement/node.go b/entanglement/node.go index 011105c..a642a96 100644 --- a/entanglement/node.go +++ b/entanglement/node.go @@ -1,7 +1,6 @@ package entanglement import ( - "errors" "fmt" "sort" "strconv" @@ -17,51 +16,23 @@ import ( // TODO Do not save sizes of all links independently as they are always the same. -type RedundantNode struct { - *merkledag.ProtoNode - - Position [2]int // from, to lattice pos -} - -func NewRedundantNode(n *merkledag.ProtoNode) *RedundantNode { - nd := &RedundantNode{ProtoNode: n.Copy().(*merkledag.ProtoNode)} - nd.SetCidBuilder(nd.CidBuilder().WithCodec(Codec)) - return nd -} - -func (r *RedundantNode) AddRedundantNode(n *Node, prev *RedundantNode, strand int) error { - data, _ := XORByteSlice(n.Data(), prev.Data()) - r.SetData(data) - switch strand { - case LH: - r.Position = [2]int{n.Position, nextEntangleNode(n.Position, LH)} - case RH: - r.Position = [2]int{n.Position, nextEntangleNode(n.Position, RH)} - case H: - r.Position = [2]int{n.Position, nextEntangleNode(n.Position, H)} - default: - return errors.New("Strand must be LH=0,H=1,RH=2") - } - return nil -} - // Node is a recovery Node based on Entanglement coding. type Node struct { *merkledag.ProtoNode - Position int // position of node in lattice - - Inputs map[int]*format.Link // left->0, horizontal->1, right->2 - Outputs map[int]*format.Link // left->0, horizontal->1, right->2 + NextNode *format.Link + recovery []*format.Link // 0->previous; 1-> next + alpha int + s int + p int cache []byte cid cid.Cid } -func NewNode(proto *merkledag.ProtoNode, pos int) *Node { +func NewNode(proto *merkledag.ProtoNode) *Node { nd := &Node{ProtoNode: proto.Copy().(*merkledag.ProtoNode)} nd.SetCidBuilder(nd.CidBuilder().WithCodec(Codec)) - nd.Position = pos return nd } @@ -70,40 +41,20 @@ func (n *Node) Recoverability() recovery.Recoverability { } func (n *Node) RecoveryLinks() []*format.Link { - var rls []*format.Link - - for _, v := range n.Outputs { - rls = append(rls, v) - } - - return rls + return n.recovery } -// Don't use func (n *Node) AddRedundantNode(nd format.Node) { if nd == nil { return } - if len(n.Outputs) == 3 { - return // error? - } - n.cache = nil -} - -func (n *Node) GetInputs() (ins []*format.Link) { - for _, v := range n.Inputs { - ins = append(ins, v) - } - return -} - -func (n *Node) GetOutputs() (ins []*format.Link) { - for _, v := range n.Inputs { - ins = append(ins, v) - } - return + n.recovery = append(n.recovery, &format.Link{ + Name: strconv.Itoa(len(n.recovery)), + Size: uint64(len(nd.RawData())), + Cid: nd.Cid(), + }) } func (n *Node) RemoveRedundantNode(id cid.Cid) { @@ -111,20 +62,15 @@ func (n *Node) RemoveRedundantNode(id cid.Cid) { return } - for k, v := range n.Inputs { + ref := n.recovery[:0] + for _, v := range n.recovery { if v.Cid.Equals(id) { - delete(n.Inputs, k) n.cache = nil - return - } - } - for k, v := range n.Outputs { - if v.Cid.Equals(id) { - delete(n.Outputs, k) - n.cache = nil - return + } else { + ref = append(ref, v) } } + n.recovery = ref } func (n *Node) RawData() []byte { @@ -156,31 +102,18 @@ func (n *Node) Cid() cid.Cid { } func (n *Node) String() string { - return "CID: " + n.Cid().String() + "; Position: " + strconv.FormatInt(int64(n.Position), 10) + return n.Cid().String() } func (n *Node) Copy() format.Node { nd := new(Node) nd.ProtoNode = n.ProtoNode.Copy().(*merkledag.ProtoNode) - lIn := len(n.Inputs) - if lIn > 0 { - nd.Inputs = make(map[int]*format.Link, lIn) - for i := 0; i <= lIn; i++ { - r := n.Inputs[i] - nd.Inputs[i] = &format.Link{ - Name: r.Name, - Size: r.Size, - Cid: r.Cid, - } - } - } - - lOut := len(n.Outputs) - if lOut > 0 { - nd.Outputs = make(map[int]*format.Link, lOut) - for i := 0; i <= lOut; i++ { - r := n.Inputs[i] - nd.Outputs[i] = &format.Link{ + nd.ProtoNode = n.ProtoNode.Copy().(*merkledag.ProtoNode) + l := len(n.recovery) + if l > 0 { + nd.recovery = make([]*format.Link, l) + for i, r := range n.recovery { + nd.recovery[i] = &format.Link{ Name: r.Name, Size: r.Size, Cid: r.Cid, @@ -188,8 +121,6 @@ func (n *Node) Copy() format.Node { } } - nd.Position = n.Position - return nd } @@ -214,10 +145,7 @@ func (n *Node) Size() (uint64, error) { for _, l := range n.Links() { s += l.Size } - for _, l := range n.GetInputs() { - s += l.Size - } - for _, l := range n.GetOutputs() { + for _, l := range n.recovery { s += l.Size } return s, nil @@ -231,12 +159,11 @@ func MarshalNode(n *Node) ([]byte, error) { return nil, err } - l := len(n.Inputs) + len(n.Outputs) + l := len(n.recovery) if l > 0 { - recovery := append(n.GetInputs(), n.GetOutputs()...) - sort.Stable(merkledag.LinkSlice(recovery)) + sort.Stable(merkledag.LinkSlice(n.recovery)) pb.Recovery = make([]*cpb.PBLink, l) - for i, r := range recovery { + for i, r := range n.recovery { pb.Recovery[i] = &cpb.PBLink{ Name: r.Name, Size_: r.Size, @@ -261,22 +188,22 @@ func UnmarshalNode(data []byte) (*Node, error) { return nil, err } - // l := len(pb.Recovery) - // if l > 0 { - // nd.recovery = make([]*format.Link, l) - // for i, r := range pb.Recovery { - // nd.recovery[i] = &format.Link{ - // Name: r.Name, - // Size: r.Size_, - // } - - // nd.recovery[i].Cid, err = cid.Cast(r.Hash) - // if err != nil { - // return nil, err - // } - // } - // sort.Stable(merkledag.LinkSlice(nd.recovery)) - // } + l := len(pb.Recovery) + if l > 0 { + nd.recovery = make([]*format.Link, l) + for i, r := range pb.Recovery { + nd.recovery[i] = &format.Link{ + Name: r.Name, + Size: r.Size_, + } + + nd.recovery[i].Cid, err = cid.Cast(r.Hash) + if err != nil { + return nil, err + } + } + sort.Stable(merkledag.LinkSlice(nd.recovery)) + } return nd, nil } From 8429e82f581a9fbf4f9c6017d8676dc95418f72f Mon Sep 17 00:00:00 2001 From: gov Date: Thu, 6 Aug 2020 18:36:34 -0400 Subject: [PATCH 7/8] entanglement encoder working --- entanglement/encode_test.go | 30 +++++ entanglement/encode_utils.go | 187 ------------------------------ entanglement/encoder.go | 87 +++++++------- entanglement/entanglement.go | 4 +- entanglement/entanglement_test.go | 121 ++++++++++++------- entanglement/node.go | 6 +- 6 files changed, 160 insertions(+), 275 deletions(-) create mode 100644 entanglement/encode_test.go delete mode 100644 entanglement/encode_utils.go diff --git a/entanglement/encode_test.go b/entanglement/encode_test.go new file mode 100644 index 0000000..3f5f5db --- /dev/null +++ b/entanglement/encode_test.go @@ -0,0 +1,30 @@ +package entanglement + +import ( + "context" + "testing" + + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" + dstest "github.com/ipfs/go-merkledag/test" + "github.com/stretchr/testify/assert" +) + +func TestEncode(t *testing.T) { + ctx := context.Background() + dag := dstest.Mock() + + in := merkledag.NodeWithData([]byte("1234567890")) + in2 := merkledag.NodeWithData([]byte("0987654321")) + in3 := merkledag.NodeWithData([]byte("1234509876")) + in.AddNodeLink("link", in2) + in.AddNodeLink("link", in3) + dag.AddMany(ctx, []format.Node{in, in2, in3}) + + nd, err := Encode(ctx, dag, in, 3) + assert.NoError(t, err) + assert.Equal(t, 2, nd.Recoverability()) + for _, r := range nd.RecoveryLinks() { + assert.NotNil(t, r) + } +} diff --git a/entanglement/encode_utils.go b/entanglement/encode_utils.go deleted file mode 100644 index 8214f23..0000000 --- a/entanglement/encode_utils.go +++ /dev/null @@ -1,187 +0,0 @@ -package entanglement - -import ( - "fmt" - "math" - - format "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" -) - -const ( - LH = iota - H = iota - RH = iota - alpha = 3 - s = 5 - p = 5 -) - -// Alpha = 3 -func GetForwardNeighbours(i int) (r, h, l int) { - // Check is it top, center or bottom in the lattice - // 1 -> Top, 0 -> Bottom, else Center - var nodePos = i % s - - if nodePos == 1 { - r = i + s + 1 - h = i + s - l = i + (s * p) - int(math.Pow(float64(s-1), 2)) - } else if nodePos == 0 { - r = i + (s * p) - int(math.Pow(float64(s), 2)-1) - h = i + s - l = i + s - 1 - } else { - r = i + s + 1 - h = i + s - l = i + (s - 1) - } - return -} - -// TODO: Fix underflow naming errors on the nodes on the extreme of the lattice. -func GetBackwardNeighbours(i int) (r, h, l int) { - // Check is it top, center or bottom in the lattice - // 1 -> Top, 0 -> Bottom, else Center - var nodePos = i % s - - if nodePos == 1 { - r = i - (s * p) + int((math.Pow(float64(s), 2) - 1)) - h = i - s - l = i - (s - 1) - } else if nodePos == 0 { - r = i - (s + 1) - h = i - s - l = i - (s * p) + int(math.Pow(float64(s-1), 2)) - } else { - r = i - (s + 1) - h = i - s - l = i - (s - 1) - } - return -} - -func GetMemoryPosition(index int) (r, h, l int) { - // Get the position in the ParityMemory array where the parity is located - // For now this will recursively call the GetBackwardNeighbours function - - h = ((index - 1) % s) + s - r, l = index, index - - for ; r > s; r, _, _ = GetBackwardNeighbours(r) { - } - - switch r { - case 1: - r = 0 - case 2: - r = 4 - case 3: - r = 3 - case 4: - r = 2 - case 5: - r = 1 - } - - for ; l > s; _, _, l = GetBackwardNeighbours(l) { - } - - switch l { - case 1: - l = 11 - case 2: - l = 12 - case 3: - l = 13 - case 4: - l = 14 - case 5: - l = 10 - } - - return -} - -// NextEntangleNode specifies the rules for creating a helical structure -// with indexed nodes arranged by s, p -func nextEntangleNode(i, strand int) int { - nodePos := i % s - - switch strand { - case LH: - switch nodePos { - case -4: // special case for (3,5,5); Go doesn't like negative modulo - fallthrough - case 1: - return i + s*p - (s-1)*(s-1) - case 0: - return i + s - 1 - default: - return i + s - 1 - } - case H: - switch nodePos { - case -4: - fallthrough - case 1: - return i + s - case 0: - return i + s - default: - return i + s - - } - case RH: - switch nodePos { - case -4: - fallthrough - case 1: - return i + s + 1 - case 0: - return i + s*p - (s*s - 1) - default: - return i + s + 1 - } - default: - // this shouldn't be hit - return 0 - } -} - -// ValidateNode checks whenever the given IPLD Node can be applied with Entanglement coding. -func ValidateNode(nd format.Node) error { - _, ok := nd.(*merkledag.ProtoNode) - if !ok { - return fmt.Errorf("reedsolomon: node must be proto") - } - - ls := nd.Links() - // can relax need for links - if len(ls) != 0 { - - size := ls[0].Size - for _, l := range ls[1:] { - if l.Size != size { - return fmt.Errorf("reedsolomon: node's links must have equal size") - } - } - } - - return nil -} - -// XORByteSlice returns an XOR slice of 2 input slices -func XORByteSlice(a []byte, b []byte) ([]byte, error) { - if len(a) != len(b) { - return nil, fmt.Errorf("length of byte slices is not equivalent: %d != %d", len(a), len(b)) - } - - buf := make([]byte, len(a)) - - for i, _ := range a { - buf[i] = a[i] ^ b[i] - } - - return buf, nil -} diff --git a/entanglement/encoder.go b/entanglement/encoder.go index 13c3199..dde500f 100644 --- a/entanglement/encoder.go +++ b/entanglement/encoder.go @@ -6,8 +6,7 @@ import ( format "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" - "github.com/templexxx/reedsolomon" - "golang.org/x/tools/godoc/redirect" + "github.com/multiformats/go-varint" recovery "github.com/Wondertan/go-ipfs-recovery" ) @@ -15,45 +14,52 @@ import ( // Encode applies Reed-Solomon coding on the given IPLD Node promoting it to a recovery Node. // Use `r` to specify needed amount of generated recovery Nodes. func Encode(ctx context.Context, dag format.DAGService, nd format.Node, r recovery.Recoverability) (*Node, error) { - err := ValidateNode(nd) - if err != nil { - return nil, err - } - + var err error ls := nd.Links() rd := NewNode(nd.(*merkledag.ProtoNode)) - nd1, err := ls[0].GetNode(ctx, dag) - if err != nil { - return nil, err - } + nds, s := make([]format.Node, len(rd.Links())), 0 + for i, l := range rd.Links() { + nds[i], err = l.GetNode(ctx, dag) + if err != nil { + return nil, err + } - // create 1st redundancy and add to dag, recovery - red := merkledag.NewRawNode(nd1.RawData()) - err = dag.Add(ctx, red) - if err != nil { - return nil, err + if len(nds[i].RawData()) > s { // finding the largest child + s = len(nds[i].RawData()) + } } - rd.AddRedundantNode(red) - for i := 1; i < len(ls); i++ { - nd1, err = ls[i].GetNode(ctx, dag) - if err != nil { - return nil, err + // encode size of redundant data + s += varint.UvarintSize(uint64(s)) + ln := len(ls) + bs := make([][]byte, ln*2) + for i := range bs { + bs[i] = make([]byte, s) + if i < ln { + l := len(nds[i].RawData()) + n := varint.PutUvarint(bs[i], uint64(l)) + copy(bs[i][n:], nds[i].RawData()) } - bs, err := XORByteSlice(nd1.RawData(), red.RawData()) - if err != nil { - return nil, err + } + + for i, b := range bs[ln:] { + if i == 0 { + b = bs[0] + } else { + b, err = XORByteSlice(bs[ln+i], bs[i]) + if err != nil { + return nil, err + } } - // create 1st redundancy and add to dag, recovery - red = merkledag.NewRawNode(bs) - err = dag.Add(ctx, red) + rnd := merkledag.NewRawNode(b) + err = dag.Add(ctx, rnd) if err != nil { return nil, err } - // add link in order of redundancy value - rd.AddRedundantNode(red) + + rd.AddRedundantNode(rnd) } err = dag.Add(ctx, rd) @@ -64,24 +70,17 @@ func Encode(ctx context.Context, dag format.DAGService, nd format.Node, r recove return rd, dag.Remove(ctx, nd.Cid()) } -// ValidateNode checks whenever the given IPLD Node can be applied with Reed-Solomon coding. -func ValidateNode(nd format.Node) error { - _, ok := nd.(*merkledag.ProtoNode) - if !ok { - return fmt.Errorf("entanglement: node must be proto") +// XORByteSlice returns an XOR slice of 2 input slices +func XORByteSlice(a []byte, b []byte) ([]byte, error) { + if len(a) != len(b) { + return nil, fmt.Errorf("length of byte slices is not equivalent: %d != %d", len(a), len(b)) } - ls := nd.Links() - if len(ls) == 0 { - return fmt.Errorf("entanglement: node must have links") - } + buf := make([]byte, len(a)) - size := ls[0].Size - for _, l := range ls[1:] { - if l.Size != size { - return fmt.Errorf("entanglement: node's links must have equal size") - } + for i, _ := range a { + buf[i] = a[i] ^ b[i] } - return nil + return buf, nil } diff --git a/entanglement/entanglement.go b/entanglement/entanglement.go index afe9bb9..efac431 100644 --- a/entanglement/entanglement.go +++ b/entanglement/entanglement.go @@ -7,12 +7,10 @@ import ( recovery "github.com/Wondertan/go-ipfs-recovery" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" ) // Custom codec for Entanglement recovery Nodes. -const Codec = 0x701 // random number // TODO Register in IPFS codec table. -const CodecRedundant = 0x702 // random number // TODO Register in IPFS codec table. +const Codec = 0x701 // random number // TODO Register in IPFS codec table. func init() { // register global decoder diff --git a/entanglement/entanglement_test.go b/entanglement/entanglement_test.go index 6d26c23..578c426 100644 --- a/entanglement/entanglement_test.go +++ b/entanglement/entanglement_test.go @@ -1,42 +1,83 @@ package entanglement -import ( - "context" - "testing" - - format "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" - dstest "github.com/ipfs/go-merkledag/test" - assert "github.com/stretchr/testify/assert" -) - -func TestNewEncoder(t *testing.T) { - // Arrange - in := merkledag.NodeWithData([]byte("1234567890")) - in2 := merkledag.NodeWithData([]byte("0987654321")) - in3 := merkledag.NodeWithData([]byte("1234509876")) - in4 := merkledag.NodeWithData([]byte("0192837465")) - in5 := merkledag.NodeWithData([]byte("0392817465")) - dag := dstest.Mock() - ctx := context.Background() - in3.AddNodeLink("link", in4) - in2.AddNodeLink("link", in5) - in.AddNodeLink("link", in2) - in.AddNodeLink("link", in3) - - dag.AddMany(ctx, []format.Node{in, in2, in3, in4, in5}) - - // Act - enc, err := NewEncoder(dag, in) - - // Assert - assert.Nil(t, err) - assert.NotNil(t, enc) - - ent := enc.(*entangler) - - assert.Equal(t, ent.Length, 5) - for i := 0; i < ent.Length; i++ { - assert.NotNil(t, ent.LTbl[i+1]) - } -} +// import ( +// "context" +// "testing" + +// format "github.com/ipfs/go-ipld-format" +// "github.com/ipfs/go-merkledag" +// dstest "github.com/ipfs/go-merkledag/test" +// "github.com/stretchr/testify/assert" +// "github.com/stretchr/testify/require" + +// recovery "github.com/Wondertan/go-ipfs-recovery" +// "github.com/Wondertan/go-ipfs-recovery/test" +// ) + +// func TestEncodeRecover(t *testing.T) { +// ctx := context.Background() +// dag := dstest.Mock() + +// in := merkledag.NodeWithData([]byte("1234567890")) +// in2 := merkledag.NodeWithData([]byte("03243423423423")) +// in3 := merkledag.NodeWithData([]byte("123450")) +// in4 := merkledag.NodeWithData([]byte("1234509876")) +// in.AddNodeLink("link", in2) +// in.AddNodeLink("link", in3) +// in.AddNodeLink("link", in4) +// dag.AddMany(ctx, []format.Node{in, in2, in3, in4}) + +// enc, err := Encode(ctx, dag, in, 2) +// require.NoError(t, err) + +// dag.Remove(ctx, in2.Cid()) +// dag.Remove(ctx, in4.Cid()) + +// out, err := Recover(ctx, dag, enc, in2.Cid(), in4.Cid()) +// require.NoError(t, err) +// assert.Equal(t, in2.RawData(), out[0].RawData()) +// assert.Equal(t, in4.RawData(), out[1].RawData()) + +// out2, err := dag.Get(ctx, in2.Cid()) +// assert.NoError(t, err) +// assert.Equal(t, in2.RawData(), out2.RawData()) + +// out4, err := dag.Get(ctx, in4.Cid()) +// assert.NoError(t, err) +// assert.Equal(t, in4.RawData(), out4.RawData()) +// } + +// func TestUnixFS(t *testing.T) { +// ctx := context.Background() +// dag := dstest.Mock() +// dr := test.NewFSDagger(t, ctx, &merkledag.ComboService{ +// Read: recovery.NewNodeGetter(dag, NewRecoverer(dag)), +// Write: dag, +// }) +// dr.Morpher = func(nd format.Node) (format.Node, error) { +// if len(nd.Links()) == 0 { +// return nd, nil +// } + +// return Encode(ctx, dag, nd, 3) +// } + +// root := +// dr.NewDir("root", +// dr.RandNode("file1"), +// dr.RandNode("file2"), +// dr.NewDir("dir1", +// dr.RandNode("file3"), +// dr.RandNode("file4"), +// ), +// dr.NewDir("dir2", +// dr.RandNode("file5"), +// ), +// ) + +// dr.Remove("file1") +// dr.Remove("dir2") +// dr.Remove("file4") + +// root.Validate() +// } diff --git a/entanglement/node.go b/entanglement/node.go index a642a96..57aeb36 100644 --- a/entanglement/node.go +++ b/entanglement/node.go @@ -36,8 +36,12 @@ func NewNode(proto *merkledag.ProtoNode) *Node { return nd } +func (n *Node) Proto() *merkledag.ProtoNode { + return n.ProtoNode +} + func (n *Node) Recoverability() recovery.Recoverability { - return alpha + return len(n.recovery) } func (n *Node) RecoveryLinks() []*format.Link { From 854575cc29df3ef7dc2a1ee3ac1e79902513d8af Mon Sep 17 00:00:00 2001 From: gov Date: Thu, 6 Aug 2020 21:16:09 -0400 Subject: [PATCH 8/8] working entangle transcode --- entanglement/encode_test.go | 11 +- entanglement/encoder.go | 24 +- entanglement/entanglement.go | 20 +- entanglement/entanglement_test.go | 135 +++--- entanglement/node.go | 7 +- entanglement/pb/readsolomon.pb.go | 779 ------------------------------ entanglement/pb/readsolomon.proto | 13 - entanglement/recover.go | 103 ++-- entanglement/recover_test.go | 120 ----- 9 files changed, 174 insertions(+), 1038 deletions(-) delete mode 100644 entanglement/pb/readsolomon.pb.go delete mode 100644 entanglement/pb/readsolomon.proto diff --git a/entanglement/encode_test.go b/entanglement/encode_test.go index 3f5f5db..5610c46 100644 --- a/entanglement/encode_test.go +++ b/entanglement/encode_test.go @@ -15,8 +15,8 @@ func TestEncode(t *testing.T) { dag := dstest.Mock() in := merkledag.NodeWithData([]byte("1234567890")) - in2 := merkledag.NodeWithData([]byte("0987654321")) - in3 := merkledag.NodeWithData([]byte("1234509876")) + in2 := merkledag.NodeWithData([]byte("987654321")) + in3 := merkledag.NodeWithData([]byte("1khgk234509876")) in.AddNodeLink("link", in2) in.AddNodeLink("link", in3) dag.AddMany(ctx, []format.Node{in, in2, in3}) @@ -24,7 +24,14 @@ func TestEncode(t *testing.T) { nd, err := Encode(ctx, dag, in, 3) assert.NoError(t, err) assert.Equal(t, 2, nd.Recoverability()) + for _, r := range nd.RecoveryLinks() { assert.NotNil(t, r) } + r1, _ := nd.RecoveryLinks()[0].GetNode(ctx, dag) + r2, _ := nd.RecoveryLinks()[1].GetNode(ctx, dag) + rec, _ := XORByteSlice(r1.RawData(), r2.RawData()) + + assert.Equal(t, in2.RawData(), r1.RawData()[1:12]) + assert.Equal(t, in3.RawData(), rec[1:]) } diff --git a/entanglement/encoder.go b/entanglement/encoder.go index dde500f..39b2c1b 100644 --- a/entanglement/encoder.go +++ b/entanglement/encoder.go @@ -2,7 +2,6 @@ package entanglement import ( "context" - "fmt" format "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" @@ -43,17 +42,17 @@ func Encode(ctx context.Context, dag format.DAGService, nd format.Node, r recove } } - for i, b := range bs[ln:] { + for i := range bs[ln:] { if i == 0 { - b = bs[0] + bs[ln] = bs[0] } else { - b, err = XORByteSlice(bs[ln+i], bs[i]) + bs[ln+i], err = XORByteSlice(bs[ln+i-1], bs[i]) if err != nil { return nil, err } } - rnd := merkledag.NewRawNode(b) + rnd := merkledag.NewRawNode(bs[ln+i]) err = dag.Add(ctx, rnd) if err != nil { return nil, err @@ -69,18 +68,3 @@ func Encode(ctx context.Context, dag format.DAGService, nd format.Node, r recove return rd, dag.Remove(ctx, nd.Cid()) } - -// XORByteSlice returns an XOR slice of 2 input slices -func XORByteSlice(a []byte, b []byte) ([]byte, error) { - if len(a) != len(b) { - return nil, fmt.Errorf("length of byte slices is not equivalent: %d != %d", len(a), len(b)) - } - - buf := make([]byte, len(a)) - - for i, _ := range a { - buf[i] = a[i] ^ b[i] - } - - return buf, nil -} diff --git a/entanglement/entanglement.go b/entanglement/entanglement.go index efac431..9fbd21a 100644 --- a/entanglement/entanglement.go +++ b/entanglement/entanglement.go @@ -27,13 +27,12 @@ type entangler struct { } // NewRestorer creates a new Entanglement Recoverer. -func NewRestorer(dag format.DAGService) recovery.Recoverer { +func NewRecoverer(dag format.DAGService) recovery.Recoverer { return &entangler{dag: dag} } // NewEncoder creates new Entanglement Encoder. -func NewEncoder(dag format.DAGService, nd format.Node) recovery.Encoder { - +func NewEncoder(dag format.DAGService) recovery.Encoder { return &entangler{dag: dag} } @@ -50,3 +49,18 @@ func (ent *entangler) Encode(ctx context.Context, nd format.Node, r recovery.Rec return nil, nil } + +// XORByteSlice returns an XOR slice of 2 input slices +func XORByteSlice(a []byte, b []byte) ([]byte, error) { + if len(a) != len(b) { + return nil, fmt.Errorf("length of byte slices is not equivalent: %d != %d", len(a), len(b)) + } + + buf := make([]byte, len(a)) + + for i := range a { + buf[i] = a[i] ^ b[i] + } + + return buf, nil +} diff --git a/entanglement/entanglement_test.go b/entanglement/entanglement_test.go index 578c426..7773d15 100644 --- a/entanglement/entanglement_test.go +++ b/entanglement/entanglement_test.go @@ -1,83 +1,82 @@ package entanglement -// import ( -// "context" -// "testing" +import ( + "context" + "testing" -// format "github.com/ipfs/go-ipld-format" -// "github.com/ipfs/go-merkledag" -// dstest "github.com/ipfs/go-merkledag/test" -// "github.com/stretchr/testify/assert" -// "github.com/stretchr/testify/require" + recovery "github.com/Wondertan/go-ipfs-recovery" + "github.com/Wondertan/go-ipfs-recovery/test" + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" + dstest "github.com/ipfs/go-merkledag/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) -// recovery "github.com/Wondertan/go-ipfs-recovery" -// "github.com/Wondertan/go-ipfs-recovery/test" -// ) +func TestEncodeRecover(t *testing.T) { + ctx := context.Background() + dag := dstest.Mock() -// func TestEncodeRecover(t *testing.T) { -// ctx := context.Background() -// dag := dstest.Mock() + in := merkledag.NodeWithData([]byte("1234567890")) + in2 := merkledag.NodeWithData([]byte("03243423423423")) + in3 := merkledag.NodeWithData([]byte("123450")) + in4 := merkledag.NodeWithData([]byte("1234509876")) + in.AddNodeLink("link", in2) + in.AddNodeLink("link", in3) + in.AddNodeLink("link", in4) + dag.AddMany(ctx, []format.Node{in, in2, in3, in4}) -// in := merkledag.NodeWithData([]byte("1234567890")) -// in2 := merkledag.NodeWithData([]byte("03243423423423")) -// in3 := merkledag.NodeWithData([]byte("123450")) -// in4 := merkledag.NodeWithData([]byte("1234509876")) -// in.AddNodeLink("link", in2) -// in.AddNodeLink("link", in3) -// in.AddNodeLink("link", in4) -// dag.AddMany(ctx, []format.Node{in, in2, in3, in4}) + enc, err := Encode(ctx, dag, in, 1) + require.NoError(t, err) -// enc, err := Encode(ctx, dag, in, 2) -// require.NoError(t, err) + dag.Remove(ctx, in2.Cid()) + dag.Remove(ctx, in4.Cid()) -// dag.Remove(ctx, in2.Cid()) -// dag.Remove(ctx, in4.Cid()) + out, err := Recover(ctx, dag, enc, in2.Cid(), in4.Cid()) + require.NoError(t, err) + assert.Equal(t, in2.RawData(), out[0].RawData()) + assert.Equal(t, in4.RawData(), out[1].RawData()) -// out, err := Recover(ctx, dag, enc, in2.Cid(), in4.Cid()) -// require.NoError(t, err) -// assert.Equal(t, in2.RawData(), out[0].RawData()) -// assert.Equal(t, in4.RawData(), out[1].RawData()) + out2, err := dag.Get(ctx, in2.Cid()) + assert.NoError(t, err) + assert.Equal(t, in2.RawData(), out2.RawData()) -// out2, err := dag.Get(ctx, in2.Cid()) -// assert.NoError(t, err) -// assert.Equal(t, in2.RawData(), out2.RawData()) + out4, err := dag.Get(ctx, in4.Cid()) + assert.NoError(t, err) + assert.Equal(t, in4.RawData(), out4.RawData()) +} -// out4, err := dag.Get(ctx, in4.Cid()) -// assert.NoError(t, err) -// assert.Equal(t, in4.RawData(), out4.RawData()) -// } +func TestUnixFS(t *testing.T) { + ctx := context.Background() + dag := dstest.Mock() + dr := test.NewFSDagger(t, ctx, &merkledag.ComboService{ + Read: recovery.NewNodeGetter(dag, NewRecoverer(dag)), + Write: dag, + }) + dr.Morpher = func(nd format.Node) (format.Node, error) { + if len(nd.Links()) == 0 { + return nd, nil + } -// func TestUnixFS(t *testing.T) { -// ctx := context.Background() -// dag := dstest.Mock() -// dr := test.NewFSDagger(t, ctx, &merkledag.ComboService{ -// Read: recovery.NewNodeGetter(dag, NewRecoverer(dag)), -// Write: dag, -// }) -// dr.Morpher = func(nd format.Node) (format.Node, error) { -// if len(nd.Links()) == 0 { -// return nd, nil -// } + return Encode(ctx, dag, nd, 3) + } -// return Encode(ctx, dag, nd, 3) -// } + root := + dr.NewDir("root", + dr.RandNode("file1"), + dr.RandNode("file2"), + dr.NewDir("dir1", + dr.RandNode("file3"), + dr.RandNode("file4"), + ), + dr.NewDir("dir2", + dr.RandNode("file5"), + ), + ) -// root := -// dr.NewDir("root", -// dr.RandNode("file1"), -// dr.RandNode("file2"), -// dr.NewDir("dir1", -// dr.RandNode("file3"), -// dr.RandNode("file4"), -// ), -// dr.NewDir("dir2", -// dr.RandNode("file5"), -// ), -// ) + dr.Remove("file1") + dr.Remove("dir2") + dr.Remove("file4") -// dr.Remove("file1") -// dr.Remove("dir2") -// dr.Remove("file4") - -// root.Validate() -// } + root.Validate() +} diff --git a/entanglement/node.go b/entanglement/node.go index 57aeb36..56442e1 100644 --- a/entanglement/node.go +++ b/entanglement/node.go @@ -19,12 +19,7 @@ import ( // Node is a recovery Node based on Entanglement coding. type Node struct { *merkledag.ProtoNode - - NextNode *format.Link - recovery []*format.Link // 0->previous; 1-> next - alpha int - s int - p int + recovery []*format.Link cache []byte cid cid.Cid diff --git a/entanglement/pb/readsolomon.pb.go b/entanglement/pb/readsolomon.pb.go deleted file mode 100644 index f1a1d14..0000000 --- a/entanglement/pb/readsolomon.pb.go +++ /dev/null @@ -1,779 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: reedsolomon/pb/readsolomon.proto - -package recovery_pb - -import ( - bytes "bytes" - fmt "fmt" - proto "github.com/gogo/protobuf/proto" - io "io" - math "math" - math_bits "math/bits" - reflect "reflect" - strings "strings" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package - -type PBNode struct { - Proto []byte `protobuf:"bytes,1,opt,name=proto,proto3" json:"proto,omitempty"` - Recovery []*PBLink `protobuf:"bytes,2,rep,name=recovery,proto3" json:"recovery,omitempty"` -} - -func (m *PBNode) Reset() { *m = PBNode{} } -func (*PBNode) ProtoMessage() {} -func (*PBNode) Descriptor() ([]byte, []int) { - return fileDescriptor_4d7ca2a520a34643, []int{0} -} -func (m *PBNode) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *PBNode) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_PBNode.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *PBNode) XXX_Merge(src proto.Message) { - xxx_messageInfo_PBNode.Merge(m, src) -} -func (m *PBNode) XXX_Size() int { - return m.Size() -} -func (m *PBNode) XXX_DiscardUnknown() { - xxx_messageInfo_PBNode.DiscardUnknown(m) -} - -var xxx_messageInfo_PBNode proto.InternalMessageInfo - -func (m *PBNode) GetProto() []byte { - if m != nil { - return m.Proto - } - return nil -} - -func (m *PBNode) GetRecovery() []*PBLink { - if m != nil { - return m.Recovery - } - return nil -} - -type PBLink struct { - Hash []byte `protobuf:"bytes,1,opt,name=Hash,proto3" json:"Hash,omitempty"` - Name string `protobuf:"bytes,2,opt,name=Name,proto3" json:"Name,omitempty"` - Size_ uint64 `protobuf:"varint,3,opt,name=Size,proto3" json:"Size,omitempty"` -} - -func (m *PBLink) Reset() { *m = PBLink{} } -func (*PBLink) ProtoMessage() {} -func (*PBLink) Descriptor() ([]byte, []int) { - return fileDescriptor_4d7ca2a520a34643, []int{1} -} -func (m *PBLink) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *PBLink) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_PBLink.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *PBLink) XXX_Merge(src proto.Message) { - xxx_messageInfo_PBLink.Merge(m, src) -} -func (m *PBLink) XXX_Size() int { - return m.Size() -} -func (m *PBLink) XXX_DiscardUnknown() { - xxx_messageInfo_PBLink.DiscardUnknown(m) -} - -var xxx_messageInfo_PBLink proto.InternalMessageInfo - -func (m *PBLink) GetHash() []byte { - if m != nil { - return m.Hash - } - return nil -} - -func (m *PBLink) GetName() string { - if m != nil { - return m.Name - } - return "" -} - -func (m *PBLink) GetSize_() uint64 { - if m != nil { - return m.Size_ - } - return 0 -} - -func init() { - proto.RegisterType((*PBNode)(nil), "recovery.pb.PBNode") - proto.RegisterType((*PBLink)(nil), "recovery.pb.PBLink") -} - -func init() { proto.RegisterFile("reedsolomon/pb/readsolomon.proto", fileDescriptor_4d7ca2a520a34643) } - -var fileDescriptor_4d7ca2a520a34643 = []byte{ - // 226 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x28, 0x4a, 0x4d, 0x4d, - 0x29, 0xce, 0xcf, 0xc9, 0xcf, 0xcd, 0xcf, 0xd3, 0x2f, 0x48, 0xd2, 0x2f, 0x4a, 0x4d, 0x84, 0x71, - 0xf5, 0x0a, 0x8a, 0xf2, 0x4b, 0xf2, 0x85, 0xb8, 0x8b, 0x52, 0x93, 0xf3, 0xcb, 0x52, 0x8b, 0x2a, - 0xf5, 0x0a, 0x92, 0x94, 0xfc, 0xb9, 0xd8, 0x02, 0x9c, 0xfc, 0xf2, 0x53, 0x52, 0x85, 0x44, 0xb8, - 0x58, 0xc1, 0xf2, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x3c, 0x41, 0x10, 0x8e, 0x90, 0x3e, 0x17, 0x07, - 0x4c, 0xb9, 0x04, 0x93, 0x02, 0xb3, 0x06, 0xb7, 0x91, 0xb0, 0x1e, 0x92, 0x7e, 0xbd, 0x00, 0x27, - 0x9f, 0xcc, 0xbc, 0xec, 0x20, 0xb8, 0x22, 0x25, 0x17, 0x90, 0x81, 0x20, 0x31, 0x21, 0x21, 0x2e, - 0x16, 0x8f, 0xc4, 0xe2, 0x0c, 0xa8, 0x79, 0x60, 0x36, 0x48, 0xcc, 0x2f, 0x31, 0x37, 0x55, 0x82, - 0x49, 0x81, 0x51, 0x83, 0x33, 0x08, 0xcc, 0x06, 0x89, 0x05, 0x67, 0x56, 0xa5, 0x4a, 0x30, 0x2b, - 0x30, 0x6a, 0xb0, 0x04, 0x81, 0xd9, 0x4e, 0x26, 0x17, 0x1e, 0xca, 0x31, 0xdc, 0x78, 0x28, 0xc7, - 0xf0, 0xe1, 0xa1, 0x1c, 0x63, 0xc3, 0x23, 0x39, 0xc6, 0x15, 0x8f, 0xe4, 0x18, 0x4f, 0x3c, 0x92, - 0x63, 0xbc, 0xf0, 0x48, 0x8e, 0xf1, 0xc1, 0x23, 0x39, 0xc6, 0x17, 0x8f, 0xe4, 0x18, 0x3e, 0x3c, - 0x92, 0x63, 0x9c, 0xf0, 0x58, 0x8e, 0xe1, 0xc2, 0x63, 0x39, 0x86, 0x1b, 0x8f, 0xe5, 0x18, 0x92, - 0xd8, 0xc0, 0x6e, 0x36, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0xad, 0xa0, 0x5a, 0x12, 0x04, 0x01, - 0x00, 0x00, -} - -func (this *PBNode) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*PBNode) - if !ok { - that2, ok := that.(PBNode) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if !bytes.Equal(this.Proto, that1.Proto) { - return false - } - if len(this.Recovery) != len(that1.Recovery) { - return false - } - for i := range this.Recovery { - if !this.Recovery[i].Equal(that1.Recovery[i]) { - return false - } - } - return true -} -func (this *PBLink) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*PBLink) - if !ok { - that2, ok := that.(PBLink) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if !bytes.Equal(this.Hash, that1.Hash) { - return false - } - if this.Name != that1.Name { - return false - } - if this.Size_ != that1.Size_ { - return false - } - return true -} -func (this *PBNode) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 6) - s = append(s, "&recovery_pb.PBNode{") - s = append(s, "Proto: "+fmt.Sprintf("%#v", this.Proto)+",\n") - if this.Recovery != nil { - s = append(s, "Recovery: "+fmt.Sprintf("%#v", this.Recovery)+",\n") - } - s = append(s, "}") - return strings.Join(s, "") -} -func (this *PBLink) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 7) - s = append(s, "&recovery_pb.PBLink{") - s = append(s, "Hash: "+fmt.Sprintf("%#v", this.Hash)+",\n") - s = append(s, "Name: "+fmt.Sprintf("%#v", this.Name)+",\n") - s = append(s, "Size_: "+fmt.Sprintf("%#v", this.Size_)+",\n") - s = append(s, "}") - return strings.Join(s, "") -} -func valueToGoStringReadsolomon(v interface{}, typ string) string { - rv := reflect.ValueOf(v) - if rv.IsNil() { - return "nil" - } - pv := reflect.Indirect(rv).Interface() - return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) -} -func (m *PBNode) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *PBNode) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *PBNode) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Recovery) > 0 { - for iNdEx := len(m.Recovery) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Recovery[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintReadsolomon(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - } - if len(m.Proto) > 0 { - i -= len(m.Proto) - copy(dAtA[i:], m.Proto) - i = encodeVarintReadsolomon(dAtA, i, uint64(len(m.Proto))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *PBLink) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *PBLink) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *PBLink) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.Size_ != 0 { - i = encodeVarintReadsolomon(dAtA, i, uint64(m.Size_)) - i-- - dAtA[i] = 0x18 - } - if len(m.Name) > 0 { - i -= len(m.Name) - copy(dAtA[i:], m.Name) - i = encodeVarintReadsolomon(dAtA, i, uint64(len(m.Name))) - i-- - dAtA[i] = 0x12 - } - if len(m.Hash) > 0 { - i -= len(m.Hash) - copy(dAtA[i:], m.Hash) - i = encodeVarintReadsolomon(dAtA, i, uint64(len(m.Hash))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func encodeVarintReadsolomon(dAtA []byte, offset int, v uint64) int { - offset -= sovReadsolomon(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return base -} -func (m *PBNode) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.Proto) - if l > 0 { - n += 1 + l + sovReadsolomon(uint64(l)) - } - if len(m.Recovery) > 0 { - for _, e := range m.Recovery { - l = e.Size() - n += 1 + l + sovReadsolomon(uint64(l)) - } - } - return n -} - -func (m *PBLink) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.Hash) - if l > 0 { - n += 1 + l + sovReadsolomon(uint64(l)) - } - l = len(m.Name) - if l > 0 { - n += 1 + l + sovReadsolomon(uint64(l)) - } - if m.Size_ != 0 { - n += 1 + sovReadsolomon(uint64(m.Size_)) - } - return n -} - -func sovReadsolomon(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 -} -func sozReadsolomon(x uint64) (n int) { - return sovReadsolomon(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (this *PBNode) String() string { - if this == nil { - return "nil" - } - repeatedStringForRecovery := "[]*PBLink{" - for _, f := range this.Recovery { - repeatedStringForRecovery += strings.Replace(f.String(), "PBLink", "PBLink", 1) + "," - } - repeatedStringForRecovery += "}" - s := strings.Join([]string{`&PBNode{`, - `Proto:` + fmt.Sprintf("%v", this.Proto) + `,`, - `Recovery:` + repeatedStringForRecovery + `,`, - `}`, - }, "") - return s -} -func (this *PBLink) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&PBLink{`, - `Hash:` + fmt.Sprintf("%v", this.Hash) + `,`, - `Name:` + fmt.Sprintf("%v", this.Name) + `,`, - `Size_:` + fmt.Sprintf("%v", this.Size_) + `,`, - `}`, - }, "") - return s -} -func valueToStringReadsolomon(v interface{}) string { - rv := reflect.ValueOf(v) - if rv.IsNil() { - return "nil" - } - pv := reflect.Indirect(rv).Interface() - return fmt.Sprintf("*%v", pv) -} -func (m *PBNode) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowReadsolomon - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: PBNode: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: PBNode: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Proto", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowReadsolomon - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthReadsolomon - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthReadsolomon - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Proto = append(m.Proto[:0], dAtA[iNdEx:postIndex]...) - if m.Proto == nil { - m.Proto = []byte{} - } - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Recovery", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowReadsolomon - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthReadsolomon - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthReadsolomon - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Recovery = append(m.Recovery, &PBLink{}) - if err := m.Recovery[len(m.Recovery)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipReadsolomon(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthReadsolomon - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthReadsolomon - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *PBLink) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowReadsolomon - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: PBLink: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: PBLink: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowReadsolomon - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthReadsolomon - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthReadsolomon - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Hash = append(m.Hash[:0], dAtA[iNdEx:postIndex]...) - if m.Hash == nil { - m.Hash = []byte{} - } - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowReadsolomon - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthReadsolomon - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthReadsolomon - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Name = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType) - } - m.Size_ = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowReadsolomon - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Size_ |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := skipReadsolomon(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthReadsolomon - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthReadsolomon - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipReadsolomon(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - depth := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowReadsolomon - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowReadsolomon - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - case 1: - iNdEx += 8 - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowReadsolomon - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if length < 0 { - return 0, ErrInvalidLengthReadsolomon - } - iNdEx += length - case 3: - depth++ - case 4: - if depth == 0 { - return 0, ErrUnexpectedEndOfGroupReadsolomon - } - depth-- - case 5: - iNdEx += 4 - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - if iNdEx < 0 { - return 0, ErrInvalidLengthReadsolomon - } - if depth == 0 { - return iNdEx, nil - } - } - return 0, io.ErrUnexpectedEOF -} - -var ( - ErrInvalidLengthReadsolomon = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowReadsolomon = fmt.Errorf("proto: integer overflow") - ErrUnexpectedEndOfGroupReadsolomon = fmt.Errorf("proto: unexpected end of group") -) diff --git a/entanglement/pb/readsolomon.proto b/entanglement/pb/readsolomon.proto deleted file mode 100644 index b3f13e8..0000000 --- a/entanglement/pb/readsolomon.proto +++ /dev/null @@ -1,13 +0,0 @@ -syntax = "proto3"; -package recovery.pb; - -message PBNode { - bytes proto = 1; - repeated PBLink recovery = 2; -} - -message PBLink { - bytes Hash = 1; - string Name = 2; - uint64 Size = 3; -} \ No newline at end of file diff --git a/entanglement/recover.go b/entanglement/recover.go index 9a012b4..e356638 100644 --- a/entanglement/recover.go +++ b/entanglement/recover.go @@ -2,16 +2,18 @@ package entanglement import ( "context" + "fmt" "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" - "github.com/templexxx/reedsolomon" + "github.com/multiformats/go-multihash" + "github.com/multiformats/go-varint" "github.com/Wondertan/go-ipfs-recovery" ) -// Recover tries to recompute all lost IPLD Nodes using Reed-Solomon coded recovery Node. +// Recover tries to recompute all lost IPLD Nodes using Entanglement coded recovery Node. // Pass known lost ids explicitly to avoid re-requesting them and to return corresponding Nodes on success. func Recover(ctx context.Context, dag format.DAGService, pnd *Node, lost ...cid.Cid) ([]format.Node, error) { // collect ids of all linked nodes. @@ -24,8 +26,8 @@ outer: // exclude known lost ids. for _, id := range lost { if l.Cid.Equals(id) { - ids[i] = cid.Undef - break outer + ids[i] = wrongCid // this is needed to fail blockservice cid validation. + continue outer } } @@ -36,9 +38,10 @@ outer: ids[i+lpnd] = l.Cid } - // track `lost` or `not lost` blocks indexes with actual data. - var lst, nlst []int - bs := make([][]byte, lpnd+lrpnd) + // track `not lost` or `lost` blocks indexes with actual data. + var nlst, lst []int + var err error + bs, s := make([][]byte, lpnd+lrpnd), pnd.RecoveryLinks()[0].Size for i, ndp := range format.GetNodes(ctx, dag, ids) { nd, err := ndp.Get(ctx) @@ -46,37 +49,76 @@ outer: case context.DeadlineExceeded, context.Canceled: return nil, err case nil: - bs[i] = nd.RawData() - lst = append(lst, i) - default: - bs[i] = make([]byte, pnd.Links()[0].Size) // the size is always the same, validated by Encode. + bs[i] = make([]byte, s) + if i < lpnd { + n := varint.PutUvarint(bs[i], uint64(len(nd.RawData()))) + copy(bs[i][n:], nd.RawData()) + } else { + copy(bs[i], nd.RawData()) + } nlst = append(nlst, i) + default: + bs[i] = nil + lst = append(lst, i) } } - if lrpnd < len(nlst) { - return nil, recovery.ErrRecoveryExceeded - } - - rs, err := reedsolomon.New(lpnd, lrpnd) - if err != nil { - return nil, err - } - - err = rs.Reconst(bs, lst, nlst) - if err != nil { - return nil, err + for _, b := range lst { + if b < lpnd { + if b == 0 { + r := bs[lpnd] + if r == nil { + fmt.Println("1") + return nil, recovery.ErrRecoveryExceeded + } + bs[b] = r + continue + } + // get recovery slices + r1, r2 := bs[lpnd+b], bs[lpnd+b-1] + if r1 == nil || r2 == nil { + fmt.Println("2") + fmt.Println(r1, r2) + return nil, recovery.ErrRecoveryExceeded + } + bs[b], err = XORByteSlice(r1, r2) + if err != nil { + return nil, err + } + } else { + // get data and recovery slice + r1, r2 := bs[lpnd-b-1], bs[b-1] + if r1 == nil || r2 == nil { + fmt.Println("3") + return nil, recovery.ErrRecoveryExceeded + } + bs[b], err = XORByteSlice(r1, r2) + if err != nil { + return nil, err + } + } } // decode and save recomputed nodes, filter and return known to be lost. nds := make([]format.Node, 0, len(lost)) - for _, i := range nlst { + for _, i := range lst { id := ids[i] - if !id.Defined() { + if id.Equals(wrongCid) { id = pnd.Links()[i].Cid } - b, _ := blocks.NewBlockWithCid(bs[i], id) + var b blocks.Block + if i < lpnd { + s, n, err := varint.FromUvarint(bs[i]) + if err != nil { + return nil, err + } + + b, _ = blocks.NewBlockWithCid(bs[i][n:int(s)+n], id) + } else { + b, _ = blocks.NewBlockWithCid(bs[i], id) + } + nd, err := format.Decode(b) if err != nil { return nil, err @@ -87,10 +129,17 @@ outer: return nil, err } - if !ids[i].Defined() { + if ids[i].Equals(wrongCid) { nds = append(nds, nd) } } return nds, nil } + +var wrongCid, _ = cid.Prefix{ + Version: 1, + Codec: 1, + MhType: multihash.IDENTITY, + MhLength: 1, +}.Sum([]byte("f")) diff --git a/entanglement/recover_test.go b/entanglement/recover_test.go index c333db3..138c444 100644 --- a/entanglement/recover_test.go +++ b/entanglement/recover_test.go @@ -1,121 +1 @@ package entanglement - -// import ( -// "context" -// "testing" - -// "github.com/ipfs/go-ipld-format" -// "github.com/ipfs/go-merkledag" -// dstest "github.com/ipfs/go-merkledag/test" -// "github.com/stretchr/testify/assert" -// ) - -// func TestRecover(t *testing.T) { -// // Arrange -// ctx := context.Background() - -// in := merkledag.NodeWithData([]byte("1234567890")) -// in2 := merkledag.NodeWithData([]byte("0987654321")) -// in3 := merkledag.NodeWithData([]byte("1234509876")) - -// dag := dstest.Mock() -// dag.AddMany(ctx, []format.Node{in, in2, in3}) - -// e := NewEncoder(dag) - -// in.AddNodeLink("link", in2) -// in.AddNodeLink("link", in3) - -// rnd, err := e.Encode(ctx, in, 2) - -// nd, ok := rnd.(*Node) - -// // Act -// failures, err := Recover(ctx, dag, nd) - -// // Assert -// assert.Nil(t, err) -// assert.NotNil(t, failures) -// assert.True(t, ok) -// } - -// func TestRecoverOneFailure(t *testing.T) { -// // Arrange -// ctx := context.Background() - -// in := merkledag.NodeWithData([]byte("1234567890")) -// in2 := merkledag.NodeWithData([]byte("0987654321")) -// in3 := merkledag.NodeWithData([]byte("1234509876")) - -// dag := dstest.Mock() -// dag.AddMany(ctx, []format.Node{in, in2, in3}) - -// e := NewEncoder(dag) - -// in.AddNodeLink("link", in2) -// in.AddNodeLink("link", in3) - -// rnd, err := e.Encode(ctx, in, 2) - -// nd, ok := rnd.(*Node) - -// // Act -// dag.Remove(ctx, in2.Cid()) -// failures, err := Recover(ctx, dag, nd) - -// // Assert -// assert.Nil(t, err) -// assert.NotNil(t, failures) -// assert.True(t, ok) -// assert.Empty(t, failures) - -// lnk := nd.Links()[0] -// r, err := dag.Get(ctx, lnk.Cid) - -// assert.Equal(t, r.RawData(), in2.RawData()) - -// assert.Nil(t, err) -// } - -// func TestRecoverTwoFailures(t *testing.T) { -// // Arrange -// ctx := context.Background() - -// in := merkledag.NodeWithData([]byte("1234567890")) -// in2 := merkledag.NodeWithData([]byte("0987654321")) -// in3 := merkledag.NodeWithData([]byte("1234509876")) - -// dag := dstest.Mock() -// dag.AddMany(ctx, []format.Node{in, in2, in3}) - -// e := NewEncoder(dag) - -// in.AddNodeLink("link", in2) -// in.AddNodeLink("link", in3) - -// rnd, err := e.Encode(ctx, in, 2) - -// nd, ok := rnd.(*Node) - -// // Act -// dag.Remove(ctx, in2.Cid()) -// dag.Remove(ctx, in3.Cid()) -// failures, err := Recover(ctx, dag, nd) - -// // Assert -// assert.Nil(t, err) -// assert.NotNil(t, failures) -// assert.True(t, ok) -// assert.Empty(t, failures) - -// lnk := nd.Links()[0] -// r, err := dag.Get(ctx, lnk.Cid) - -// lnk1 := nd.Links()[1] -// r1, err := dag.Get(ctx, lnk1.Cid) - -// assert.Equal(t, r.RawData(), in2.RawData()) -// assert.Equal(t, r1.RawData(), in3.RawData()) - -// assert.Nil(t, err) -// }