Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: improvements & gateway-conformance support (light) #57

Merged
merged 27 commits into from
Dec 6, 2023
Merged
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
4a95ee3
ci: add gateway-conformance tests
aschmahmann Dec 4, 2023
db993fa
ci(gateway-conformance): use minimal specs for websites (no _redirects)
aschmahmann Dec 4, 2023
46c4ad9
feat: enable gateway conformance testing locally
SgtPooki Dec 4, 2023
a57d76c
docs: gwc info added to DEVELOPER-NOTES.md
SgtPooki Dec 4, 2023
98afa15
fix(test:gwc): test:gwc-setup split out from test:gwc
SgtPooki Dec 4, 2023
5549c41
fix: use proper env var and cleanup scripts
SgtPooki Dec 4, 2023
0778d65
chore(ci:gwc): enable only subdomain ipfs and ipns
SgtPooki Dec 4, 2023
daefd4f
fix: enable kubo routing-v1 for ipns
SgtPooki Dec 4, 2023
8722856
chore: kubo-init daemon starting line
SgtPooki Dec 4, 2023
13ed71a
fix(ci:gwc): set subdomain-url arg
SgtPooki Dec 4, 2023
4b00467
fix(ci:gwc): skip dir list and gateway cache tests
SgtPooki Dec 4, 2023
d39a3db
chore(ci:gcw): don't use bitswap
SgtPooki Dec 4, 2023
ad5be2e
docs(ci:gwc): add more details about gwc local testing
SgtPooki Dec 4, 2023
b137aac
docs(ci:gwc): more documentation of things
SgtPooki Dec 5, 2023
fc3e4f3
chore: add ECHO_HEADERS & notes
SgtPooki Dec 5, 2023
337bf7e
chore: some more things
SgtPooki Dec 5, 2023
25cd97a
Merge branch 'main' into ci/gateway-conformance
SgtPooki Dec 5, 2023
0e81d25
fix: webrtc errors
SgtPooki Dec 5, 2023
b047f52
feat: use custom libp2p node
SgtPooki Dec 5, 2023
1317ada
fix: custom libp2p & playwright config
SgtPooki Dec 6, 2023
a520039
chore: add fastify/cors patch
SgtPooki Dec 6, 2023
f3557a3
feat: use @helia/ipns
SgtPooki Dec 6, 2023
2fe9ce3
chore: remove RESOLVE_REDIRECTS
SgtPooki Dec 6, 2023
ad50a77
chore: various refactorings to support more gateway stuff
SgtPooki Dec 6, 2023
82a19d4
chore: some notes
SgtPooki Dec 6, 2023
ce971eb
fix: minor fix to using delegated routing
SgtPooki Dec 6, 2023
a2bd0dc
chore: some env var setup, patches, and notes
SgtPooki Dec 6, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix: minor fix to using delegated routing
SgtPooki committed Dec 6, 2023
commit ce971eb7acb61d2d38db62ce571bce89613a4026
13 changes: 13 additions & 0 deletions patches/@helia+delegated-routing-v1-http-api-client+1.1.2.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
diff --git a/node_modules/@helia/delegated-routing-v1-http-api-client/dist/src/client.js b/node_modules/@helia/delegated-routing-v1-http-api-client/dist/src/client.js
index 9461349..042b83e 100644
--- a/node_modules/@helia/delegated-routing-v1-http-api-client/dist/src/client.js
+++ b/node_modules/@helia/delegated-routing-v1-http-api-client/dist/src/client.js
@@ -208,7 +208,7 @@ export class DefaultDelegatedRoutingV1HttpApiClient {
if (record.Schema === 'peer') {
// Peer schema can have additional, user-defined, fields.
record.ID = peerIdFromString(record.ID);
- record.Addrs = record.Addrs.map(multiaddr);
+ record.Addrs = record.Addrs?.map(multiaddr) ?? [];
record.Protocols = record.Protocols ?? [];
return record;
}
5 changes: 5 additions & 0 deletions src/constants.ts
Original file line number Diff line number Diff line change
@@ -46,6 +46,11 @@ export const FILE_DATASTORE_PATH = process.env.FILE_DATASTORE_PATH ?? null
*/
export const FILE_BLOCKSTORE_PATH = process.env.FILE_BLOCKSTORE_PATH ?? null

/**
* Whether to use the delegated routing v1 API. Defaults to true.
*/
export const USE_DELEGATED_ROUTING = process.env.USE_DELEGATED_ROUTING !== 'false'

/**
* If not set, we will default delegated routing to `https://delegated-ipfs.dev`
*/
58 changes: 31 additions & 27 deletions src/getCustomLibp2p.ts
Original file line number Diff line number Diff line change
@@ -17,8 +17,8 @@ import { webSockets } from '@libp2p/websockets'
import { ipnsSelector } from 'ipns/selector'
import { ipnsValidator } from 'ipns/validator'
import { createLibp2p as create, type Libp2pOptions } from 'libp2p'
import { DELEGATED_ROUTING_V1_HOST, USE_LIBP2P } from './constants.js'
import type { Libp2p, PubSub } from '@libp2p/interface'
import { DELEGATED_ROUTING_V1_HOST, USE_DELEGATED_ROUTING, USE_LIBP2P } from './constants.js'
import type { Libp2p, PubSub, ServiceMap } from '@libp2p/interface'
import type { HeliaInit } from 'helia'

interface HeliaGatewayLibp2pServices extends Record<string, unknown> {
@@ -38,6 +38,33 @@ interface HeliaGatewayLibp2pOptions extends Pick<HeliaInit, 'datastore'> {
}

export async function getCustomLibp2p ({ datastore }: HeliaGatewayLibp2pOptions): Promise<Libp2p<HeliaGatewayLibp2pServices>> {
const libp2pServices: ServiceMap = {
identify: identifyService(),
autoNAT: autoNATService(),
upnp: uPnPNATService(),
pubsub: gossipsub(),
dcutr: dcutrService(),
dht: kadDHT({
// don't do DHT server work.
clientMode: true,
validators: {
ipns: ipnsValidator
},
selectors: {
ipns: ipnsSelector
}
}),
relay: circuitRelayServer({
// don't advertise as a circuitRelay server because we have one job, and that is to: listen for http requests, maybe fetch content, return http responses.
// advertise: true
}),
ping: pingService(),
delegatedRouting: undefined
}

if (USE_DELEGATED_ROUTING) {
libp2pServices.delegatedRouting = () => createDelegatedRoutingV1HttpApiClient(DELEGATED_ROUTING_V1_HOST)
}
const options: Libp2pOptions<HeliaGatewayLibp2pServices> = {
datastore,
addresses: {
@@ -91,37 +118,14 @@ export async function getCustomLibp2p ({ datastore }: HeliaGatewayLibp2pOptions)
]
})
],
services: {
identify: identifyService(),
autoNAT: autoNATService(),
upnp: uPnPNATService(),
pubsub: gossipsub(),
dcutr: dcutrService(),
delegatedRouting: () => createDelegatedRoutingV1HttpApiClient(DELEGATED_ROUTING_V1_HOST),
dht: kadDHT({
// don't do DHT server work.
clientMode: true,
validators: {
ipns: ipnsValidator
},
selectors: {
ipns: ipnsSelector
}
}),
relay: circuitRelayServer({
// don't advertise as a circuitRelay server because we have one job, and that is to: listen for http requests, maybe fetch content, return http responses.
// advertise: true
}),
ping: pingService()
}
// @ts-expect-error - types are borked
services: libp2pServices
}

if (!USE_LIBP2P) {
// we should not be running libp2p things
options.start = false
options.services = {}
options.peerDiscovery = []
options.transports = []
}

return create(options)