Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GC feature #108

Merged
merged 7 commits into from
Mar 19, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add --enable-gc flag
tarassh committed Mar 16, 2024
commit 84882735f2b88c788897bafb3f49f67623bc69eb
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -66,6 +66,12 @@ NOTE: If you want more control over CLI params, you can run the server binary (a

NOTE: If you get some error when running this, check if the diagnostic is there in [known issues](#known-issues)

NOTE: If you want to run the server with garbage collection enabled, you can run the server binary with the `--enable-gc` flag:

```bash
./build/bin/server -w3-agent-key <AGENT_KEY> -w3-delegation-file <DELEGATION_PROOF_FILE_PATH> --enable-gc
```

ipfs-pinner can be run as a server and allows two functionalities currently - `/get` and `/upload`

### Upload a file
4 changes: 2 additions & 2 deletions core/ipfs.go
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@ import (
)

// returns a go-ipfs node backend CoreAPI instance
func CreateIpfsNode(cidComputeOnly bool) (*core.IpfsNode, error) {
func CreateIpfsNode(ctx context.Context, cidComputeOnly bool) (*core.IpfsNode, error) {
cfg := core.BuildCfg{
Online: !cidComputeOnly, // networking
Permanent: !cidComputeOnly, // data persists across restarts?
@@ -50,7 +50,7 @@ func CreateIpfsNode(cidComputeOnly bool) (*core.IpfsNode, error) {
}

var nnode *core.IpfsNode
if nnode, err = core.NewNode(context.Background(), &cfg); err != nil {
if nnode, err = core.NewNode(ctx, &cfg); err != nil {
return nil, err
}

1 change: 1 addition & 0 deletions coreapi/api.go
Original file line number Diff line number Diff line change
@@ -15,4 +15,5 @@ type CoreExtensionAPI interface {

type GarbageCollectAPI interface {
GarbageCollect(ctx context.Context)
InitPeriodicGC(ctx context.Context) <-chan error
}
9 changes: 9 additions & 0 deletions coreapi/impl.go
Original file line number Diff line number Diff line change
@@ -43,3 +43,12 @@ func (gci *garbageCollectorImpl) GarbageCollect(ctx context.Context) {
log.Println("error getting garbage collector: %w", err)
}
}

func (gci *garbageCollectorImpl) InitPeriodicGC(ctx context.Context) <-chan error {
errc := make(chan error)
go func() {
errc <- corerepo.PeriodicGC(ctx, gci.node)
close(errc)
}()
return errc
}
6 changes: 6 additions & 0 deletions pinclient/client_create_request.go
Original file line number Diff line number Diff line change
@@ -16,6 +16,7 @@ type ClientCreateRequest struct {
W3_AgentKey string
W3_AgentDid did.DID
W3_DelegationProofPath string
GC_Enable bool

httpClient *http.Client
}
@@ -56,3 +57,8 @@ func (r ClientCreateRequest) HttpClient(client http.Client) ClientCreateRequest
r.httpClient = &client
return r
}

func (r ClientCreateRequest) GcEnable(gcEnable bool) ClientCreateRequest {
r.GC_Enable = gcEnable
return r
}
9 changes: 7 additions & 2 deletions pinner.go
Original file line number Diff line number Diff line change
@@ -5,6 +5,7 @@
package pinner

import (
"context"
"log"

car "github.com/covalenthq/ipfs-pinner/car"
@@ -22,14 +23,18 @@ type pinnerNode struct {
pinApiClient pinclient.PinServiceAPI
}

func NewPinnerNode(req PinnerNodeCreateRequest) PinnerNode {
func NewPinnerNode(ctx context.Context, req PinnerNodeCreateRequest) PinnerNode {
node := pinnerNode{}
ipfsNode, err := core.CreateIpfsNode(req.cidComputeOnly)
ipfsNode, err := core.CreateIpfsNode(ctx, req.cidComputeOnly)
if err != nil {
log.Fatal("error initializing ipfs node: ", err)
}

node.ipfsCore = coreapi.NewCoreExtensionApi(ipfsNode)
if req.enableGC {
log.Print("enabling garbage collection....")
node.ipfsCore.GC().InitPeriodicGC(ctx)
}

//SETUP W3UP
log.Print("setting up w3up for uploads....")
4 changes: 3 additions & 1 deletion pinner_create_request.go
Original file line number Diff line number Diff line change
@@ -9,14 +9,16 @@ type PinnerNodeCreateRequest struct {
cidComputeOnly bool
cidVersion int
ipfsFetchUrls []string
enableGC bool
}

func NewNodeRequest(clientRequest pinclient.ClientCreateRequest, ipfsFetchUrls []string) *PinnerNodeCreateRequest {
func NewNodeRequest(clientRequest pinclient.ClientCreateRequest, ipfsFetchUrls []string, enableGC bool) *PinnerNodeCreateRequest {
request := new(PinnerNodeCreateRequest)
request.cidVersion = 0
request.cidComputeOnly = true
request.pinServiceRequest = clientRequest
request.ipfsFetchUrls = ipfsFetchUrls
request.enableGC = enableGC
return request
}

42 changes: 19 additions & 23 deletions server/main.go
Original file line number Diff line number Diff line change
@@ -45,6 +45,11 @@ type Config struct {
w3AgentDid did.DID
delegationProofPath string
ipfsGatewayUrls []string
enableGC bool
}

func NewConfig(portNumber int, w3AgentKey string, w3AgentDid did.DID, delegationProofPath string, ipfsGatewayUrls []string, enableGC bool) *Config {
return &Config{portNumber, w3AgentKey, w3AgentDid, delegationProofPath, ipfsGatewayUrls, enableGC}
}

var (
@@ -62,6 +67,8 @@ func main() {

ipfsGatewayUrls := flag.String("ipfs-gateway-urls", "https://w3s.link/ipfs/%s,https://dweb.link/ipfs/%s,https://ipfs.io/ipfs/%s", "comma separated list of ipfs gateway urls")

enableGC := flag.Bool("enable-gc", false, "enable garbage collection")

flag.Parse()
core.Version()

@@ -85,17 +92,25 @@ func main() {

log.Printf("agent did: %s", agentSigner.DID().DID().String())

setUpAndRunServer(Config{*portNumber, *w3AgentKey, agentSigner.DID().DID(), *w3DelegationFile, strings.Split(*ipfsGatewayUrls, ",")})
config := NewConfig(*portNumber, *w3AgentKey, agentSigner.DID().DID(), *w3DelegationFile, strings.Split(*ipfsGatewayUrls, ","), *enableGC)

setUpAndRunServer(*config)
}

func setUpAndRunServer(config Config) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
mux := http.NewServeMux()
httpState := NewState()

clientCreateReq := client.NewClientRequest(core.Web3Storage).W3AgentKey(config.w3AgentKey).W3AgentDid(config.w3AgentDid).DelegationProofPath(config.delegationProofPath)
clientCreateReq := client.NewClientRequest(core.Web3Storage).
W3AgentKey(config.w3AgentKey).
W3AgentDid(config.w3AgentDid).
DelegationProofPath(config.delegationProofPath).
GcEnable(config.enableGC)
// check if cid compute true works with car uploads
nodeCreateReq := pinner.NewNodeRequest(clientCreateReq, config.ipfsGatewayUrls).CidVersion(1).CidComputeOnly(false)
node := pinner.NewPinnerNode(*nodeCreateReq)
nodeCreateReq := pinner.NewNodeRequest(clientCreateReq, config.ipfsGatewayUrls, config.enableGC).CidVersion(1).CidComputeOnly(false)
node := pinner.NewPinnerNode(ctx, *nodeCreateReq)

mux.Handle("/upload", recoveryWrapper(uploadHttpHandler(node)))
mux.Handle("/get", recoveryWrapper(downloadHttpHandler(node)))
@@ -319,19 +334,6 @@ func uploadHandler(contents string, node pinner.PinnerNode) (cid.Cid, error) {
log.Printf("uploaded file has root cid: %s\n", ccid)

carf.Close()

assertEquals(fcid, ccid)
log.Printf("the two cids match: %s\n", ccid.String())

log.Printf("removing dag...")
curr := time.Now().UnixMilli()
err = node.UnixfsService().RemoveDag(ctx, ccid)
after := time.Now().UnixMilli()
log.Println("time taken:", after-curr)
if err != nil {
log.Fatalf("%v", err)
}

return ccid, nil
}

@@ -406,9 +408,3 @@ func timeoutHttpHandler(s *State) http.Handler {
}
return http.HandlerFunc(fn)
}

func assertEquals(obj1 interface{}, obj2 interface{}) {
if obj1 != obj2 {
log.Fatalf("fail %v and %v doesn't match", obj1, obj2)
}
}
Loading