keystone deployment fixes

integration-tests/deployment/keystone/changeset/deploy_ocr3.go

@@ -5,36 +5,29 @@ import (
 
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
 	"github.com/smartcontractkit/chainlink/integration-tests/deployment"
+	"github.com/smartcontractkit/chainlink/integration-tests/deployment/clo/models"
 	kslib "github.com/smartcontractkit/chainlink/integration-tests/deployment/keystone"
 )
 
 func DeployOCR3(lggr logger.Logger, env deployment.Environment, ab deployment.AddressBook, registryChainSel uint64) (deployment.ChangesetOutput, error) {
-	// must have capabilities registry deployed
-	regAddrs, err := ab.AddressesForChain(registryChainSel)
-	if err != nil {
-		return deployment.ChangesetOutput{}, fmt.Errorf("no addresses found for chain %d: %w", registryChainSel, err)
-	}
-	found := false
-	for _, addr := range regAddrs {
-		if addr.Type == kslib.CapabilityRegistryTypeVersion.Type {
-			found = true
-			break
-		}
-	}
-	if !found {
-		return deployment.ChangesetOutput{}, fmt.Errorf("no capabilities registry found for changeset %s", "0001_deploy_registry")
-	}
-
 	// ocr3 only deployed on registry chain
 	c, ok := env.Chains[registryChainSel]
 	if !ok {
 		return deployment.ChangesetOutput{}, fmt.Errorf("chain not found in environment")
 	}
-	err = kslib.DeployOCR3(lggr, c, ab)
+	err := kslib.DeployOCR3(lggr, c, ab)
 	if err != nil {
 		return deployment.ChangesetOutput{}, fmt.Errorf("failed to deploy OCR3Capability: %w", err)
 	}
-
 	return deployment.ChangesetOutput{AddressBook: ab}, nil
+}
+
+func ConfigureOCR3Contract(lggr logger.Logger, env deployment.Environment, ab deployment.AddressBook, registryChainSel uint64, nodes []*models.Node, cfg kslib.OracleConfigSource) (deployment.ChangesetOutput, error) {
+
+	err := kslib.ConfigureOCR3ContractFromCLO(&env, registryChainSel, nodes, ab, &cfg)
+	if err != nil {
+		return deployment.ChangesetOutput{}, fmt.Errorf("failed to configure OCR3Capability: %w", err)
+	}
 
+	return deployment.ChangesetOutput{AddressBook: ab}, nil
 }

integration-tests/deployment/keystone/changeset/deploy_ocr3_test.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
 	"github.com/smartcontractkit/chainlink/integration-tests/deployment"
-	kslb "github.com/smartcontractkit/chainlink/integration-tests/deployment/keystone"
 	"github.com/smartcontractkit/chainlink/integration-tests/deployment/keystone/changeset"
 	"github.com/smartcontractkit/chainlink/integration-tests/deployment/memory"
 )
@@ -26,20 +25,14 @@ func TestDeployOCR3(t *testing.T) {
 	env := memory.NewMemoryEnvironment(t, lggr, zapcore.DebugLevel, cfg)
 
 	registrySel := env.AllChainSelectors()[0]
-	// err if no capabilities registry on chain 0
-	_, err := changeset.DeployOCR3(lggr, env, ab, registrySel)
-	require.Error(t, err)
 
-	// fake capabilities registry
-	err = ab.Save(registrySel, "0x0000000000000000000000000000000000000001", kslb.CapabilityRegistryTypeVersion)
-	require.NoError(t, err)
 	resp, err := changeset.DeployOCR3(lggr, env, ab, registrySel)
 	require.NoError(t, err)
 	require.NotNil(t, resp)
 	// OCR3 should be deployed on chain 0
 	addrs, err := resp.AddressBook.AddressesForChain(registrySel)
 	require.NoError(t, err)
-	require.Len(t, addrs, 2)
+	require.Len(t, addrs, 1)
 
 	// nothing on chain 1
 	require.NotEqual(t, registrySel, env.AllChainSelectors()[1])

integration-tests/deployment/keystone/deploy.go

@@ -15,6 +15,7 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 
 	"github.com/smartcontractkit/chainlink/integration-tests/deployment"
+	"github.com/smartcontractkit/chainlink/integration-tests/deployment/clo/models"
 
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/durationpb"
@@ -94,7 +95,7 @@ func ConfigureContracts(ctx context.Context, lggr logger.Logger, req ConfigureCo
 	}
 
 	// now we have the capability registry set up we need to configure the forwarder contracts and the OCR3 contract
-	dons, err := joinInfoAndNodes(cfgRegistryResp.DonInfos, req.Dons)
+	dons, err := joinInfoAndNodes(cfgRegistryResp.DonInfos, req.Dons, req.RegistryChainSel)
 	if err != nil {
 		return nil, fmt.Errorf("failed to assimilate registry to Dons: %w", err)
 	}
@@ -170,7 +171,7 @@ func ConfigureRegistry(ctx context.Context, lggr logger.Logger, req ConfigureCon
 
 	// all the subsequent calls to the registry are in terms of nodes
 	// compute the mapping of dons to their nodes for reuse in various registry calls
-	donToOcr2Nodes, err := mapDonsToNodes(req.Dons, true)
+	donToOcr2Nodes, err := mapDonsToNodes(req.Dons, true, req.RegistryChainSel)
 	if err != nil {
 		return nil, fmt.Errorf("failed to map dons to nodes: %w", err)
 	}
@@ -267,7 +268,7 @@ func ConfigureForwardContracts(env *deployment.Environment, dons []RegisteredDon
 			return fmt.Errorf("no forwarder contract found for chain %d", chain.Selector)
 		}
 
-		err := configureForwarder(chain, fwrd, dons)
+		err := configureForwarder(env.Logger, chain, fwrd, dons)
 		if err != nil {
 			return fmt.Errorf("failed to configure forwarder for chain selector %d: %w", chain.Selector, err)
 		}
@@ -301,14 +302,14 @@ func ConfigureOCR3Contract(env *deployment.Environment, chainSel uint64, dons []
 		}
 		contract := contracts.OCR3
 		if contract == nil {
-			return fmt.Errorf("no forwarder contract found for chain %d", chainSel)
+			return fmt.Errorf("no ocr3 contract found for chain %d", chainSel)
 		}
 
 		_, err := configureOCR3contract(configureOCR3Request{
 			cfg:      cfg,
 			chain:    registryChain,
 			contract: contract,
-			don:      don,
+			nodes:    don.Nodes,
 		})
 		if err != nil {
 			return fmt.Errorf("failed to configure OCR3 contract for don %s: %w", don.Name, err)
@@ -317,6 +318,43 @@ func ConfigureOCR3Contract(env *deployment.Environment, chainSel uint64, dons []
 	return nil
 }
 
+func ConfigureOCR3ContractFromCLO(env *deployment.Environment, chainSel uint64, nodes []*models.Node, addrBook deployment.AddressBook, cfg *OracleConfigSource) error {
+	registryChain, ok := env.Chains[chainSel]
+	if !ok {
+		return fmt.Errorf("chain %d not found in environment", chainSel)
+	}
+	contractSetsResp, err := GetContractSets(&GetContractSetsRequest{
+		Chains:      env.Chains,
+		AddressBook: addrBook,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to get contract sets: %w", err)
+	}
+	contracts, ok := contractSetsResp.ContractSets[chainSel]
+	if !ok {
+		return fmt.Errorf("failed to get contract set for chain %d", chainSel)
+	}
+	contract := contracts.OCR3
+	if contract == nil {
+		return fmt.Errorf("no ocr3 contract found for chain %d", chainSel)
+	}
+	var ocr2nodes []*ocr2Node
+	for _, node := range nodes {
+		n, err := newOcr2NodeFromClo(node, chainSel)
+		if err != nil {
+			return fmt.Errorf("failed to create ocr2 node from clo node: %w", err)
+		}
+		ocr2nodes = append(ocr2nodes, n)
+	}
+	_, err = configureOCR3contract(configureOCR3Request{
+		cfg:      cfg,
+		chain:    registryChain,
+		contract: contract,
+		nodes:    ocr2nodes,
+	})
+	return err
+}
+
 type registerCapabilitiesRequest struct {
 	chain             deployment.Chain
 	registry          *capabilities_registry.CapabilitiesRegistry
@@ -669,12 +707,13 @@ func sortedHash(p2pids [][32]byte) string {
 }
 
 func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsResponse, error) {
-	resp := &registerDonsResponse{
+	resp := registerDonsResponse{
 		donInfos: make(map[string]capabilities_registry.CapabilitiesRegistryDONInfo),
 	}
 	// track hash of sorted p2pids to don name because the registry return value does not include the don name
 	// and we need to map it back to the don name to access the other mapping data such as the don's capabilities & nodes
 	p2pIdsToDon := make(map[string]string)
+	var registeredDons = 0
 
 	for don, ocr2nodes := range req.donToOcr2Nodes {
 		var p2pIds [][32]byte
@@ -724,25 +763,47 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes
 			return nil, fmt.Errorf("failed to confirm AddDON transaction %s for don %s: %w", tx.Hash().String(), don, err)
 		}
 		lggr.Debugw("registered DON", "don", don, "p2p sorted hash", p2pSortedHash, "cgs", cfgs, "wfSupported", wfSupported, "f", f)
+		registeredDons++
+	}
+	lggr.Debugf("Registered all DONS %d, waiting for registry to update", registeredDons)
+
+	// occasionally the registry does not return the expected number of DONS immediately after the txns above
+	// so we retry a few times. while crude, it is effective
+	var donInfos []capabilities_registry.CapabilitiesRegistryDONInfo
+	var err error
+	for i := 0; i < 10; i++ {
+		lggr.Debug("attempting to get DONS from registry", i)
+		donInfos, err = req.registry.GetDONs(&bind.CallOpts{})
+		if len(donInfos) != registeredDons {
+			lggr.Debugw("expected dons not registered", "expected", registeredDons, "got", len(donInfos))
+			time.Sleep(2 * time.Second)
+		} else {
+			break
+		}
 	}
-	donInfos, err := req.registry.GetDONs(&bind.CallOpts{})
 	if err != nil {
 		err = DecodeErr(kcr.CapabilitiesRegistryABI, err)
 		return nil, fmt.Errorf("failed to call GetDONs: %w", err)
 	}
+
 	for i, donInfo := range donInfos {
 		donName, ok := p2pIdsToDon[sortedHash(donInfo.NodeP2PIds)]
 		if !ok {
 			return nil, fmt.Errorf("don not found for p2pids %s in %v", sortedHash(donInfo.NodeP2PIds), p2pIdsToDon)
 		}
+		lggr.Debugw("adding don info", "don", donName, "cnt", i)
 		resp.donInfos[donName] = donInfos[i]
 	}
-	return resp, nil
+	lggr.Debugw("found registered DONs", "count", len(resp.donInfos))
+	if len(resp.donInfos) != registeredDons {
+		return nil, fmt.Errorf("expected %d dons, got %d", registeredDons, len(resp.donInfos))
+	}
+	return &resp, nil
 }
 
 // configureForwarder sets the config for the forwarder contract on the chain for all Dons that accept workflows
 // dons that don't accept workflows are not registered with the forwarder
-func configureForwarder(chain deployment.Chain, fwdr *kf.KeystoneForwarder, dons []RegisteredDon) error {
+func configureForwarder(lggr logger.Logger, chain deployment.Chain, fwdr *kf.KeystoneForwarder, dons []RegisteredDon) error {
 	if fwdr == nil {
 		return errors.New("nil forwarder contract")
 	}
@@ -761,6 +822,7 @@ func configureForwarder(chain deployment.Chain, fwdr *kf.KeystoneForwarder, dons
 			err = DecodeErr(kf.KeystoneForwarderABI, err)
 			return fmt.Errorf("failed to confirm SetConfig for forwarder %s: %w", fwdr.Address().String(), err)
 		}
+		lggr.Debugw("configured forwarder", "forwarder", fwdr.Address().String(), "donId", dn.Info.Id, "version", ver, "f", dn.Info.F, "signers", dn.signers())
 	}
 	return nil
 }
@@ -769,7 +831,7 @@ type configureOCR3Request struct {
 	cfg      *OracleConfigSource
 	chain    deployment.Chain
 	contract *kocr3.OCR3Capability
-	don      RegisteredDon
+	nodes    []*ocr2Node
 }
 type configureOCR3Response struct {
 	ocrConfig Orc2drOracleConfig
@@ -779,7 +841,7 @@ func configureOCR3contract(req configureOCR3Request) (*configureOCR3Response, er
 	if req.contract == nil {
 		return nil, fmt.Errorf("OCR3 contract is nil")
 	}
-	nks := makeNodeKeysSlice(req.don.Nodes)
+	nks := makeNodeKeysSlice(req.nodes)
 	ocrConfig, err := GenerateOCR3Config(*req.cfg, nks)
 	if err != nil {
 		return nil, fmt.Errorf("failed to generate OCR3 config: %w", err)

integration-tests/deployment/keystone/types.go

@@ -72,24 +72,55 @@ type ocr2Node struct {
 }
 
 func (o *ocr2Node) signerAddress() common.Address {
-	return common.BytesToAddress(o.Signer[:])
+	// eth address is the first 20 bytes of the Signer
+	return common.BytesToAddress(o.Signer[:20])
 }
 
 func (o *ocr2Node) toNodeKeys() NodeKeys {
+	var aptosOcr2KeyBundleId string
+	var aptosOnchainPublicKey string
+	if o.aptosOcr2KeyBundle != nil {
+		aptosOcr2KeyBundleId = o.aptosOcr2KeyBundle.BundleId
+		aptosOnchainPublicKey = o.aptosOcr2KeyBundle.OnchainSigningAddress
+	}
 	return NodeKeys{
 		EthAddress:            o.accountAddress,
-		P2PPeerID:             o.p2pKeyBundle.PeerId,
+		P2PPeerID:             strings.TrimPrefix(o.p2pKeyBundle.PeerId, "p2p_"),
 		OCR2BundleID:          o.ethOcr2KeyBundle.BundleId,
 		OCR2OnchainPublicKey:  o.ethOcr2KeyBundle.OnchainSigningAddress,
 		OCR2OffchainPublicKey: o.ethOcr2KeyBundle.OffchainPublicKey,
 		OCR2ConfigPublicKey:   o.ethOcr2KeyBundle.ConfigPublicKey,
 		CSAPublicKey:          o.csaKey,
 		// default value of encryption public key is the CSA public key
 		// TODO: DEVSVCS-760
-		EncryptionPublicKey: o.csaKey,
+		EncryptionPublicKey: strings.TrimPrefix(o.csaKey, "csa_"),
 		// TODO Aptos support. How will that be modeled in clo data?
+		AptosBundleID:         aptosOcr2KeyBundleId,
+		AptosOnchainPublicKey: aptosOnchainPublicKey,
 	}
 }
+func newOcr2NodeFromClo(n *models.Node, registryChainSel uint64) (*ocr2Node, error) {
+	if n.PublicKey == nil {
+		return nil, errors.New("no public key")
+	}
+	// the chain configs are equivalent as far as the ocr2 config is concerned so take the first one
+	if len(n.ChainConfigs) == 0 {
+		return nil, errors.New("no chain configs")
+	}
+	// all nodes should have an evm chain config, specifically the registry chain
+	evmCC, err := registryChainConfig(n.ChainConfigs, chaintype.EVM, registryChainSel)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get registry chain config for sel %d: %w", registryChainSel, err)
+	}
+	cfgs := map[chaintype.ChainType]*v1.ChainConfig{
+		chaintype.EVM: evmCC,
+	}
+	aptosCC, exists := firstChainConfigByType(n.ChainConfigs, chaintype.Aptos)
+	if exists {
+		cfgs[chaintype.Aptos] = aptosCC
+	}
+	return newOcr2Node(n.ID, cfgs, *n.PublicKey)
+}
 
 func newOcr2Node(id string, ccfgs map[chaintype.ChainType]*v1.ChainConfig, csaPubKey string) (*ocr2Node, error) {
 	if ccfgs == nil {
@@ -227,35 +258,15 @@ func mapDonsToCaps(dons []DonCapabilities) map[string][]kcr.CapabilitiesRegistry
 
 // mapDonsToNodes returns a map of don name to simplified representation of their nodes
 // all nodes must have evm config and ocr3 capability nodes are must also have an aptos chain config
-func mapDonsToNodes(dons []DonCapabilities, excludeBootstraps bool) (map[string][]*ocr2Node, error) {
+func mapDonsToNodes(dons []DonCapabilities, excludeBootstraps bool, registryChainSel uint64) (map[string][]*ocr2Node, error) {
 	donToOcr2Nodes := make(map[string][]*ocr2Node)
 	// get the nodes for each don from the offchain client, get ocr2 config from one of the chain configs for the node b/c
 	// they are equivalent, and transform to ocr2node representation
 
 	for _, don := range dons {
 		for _, nop := range don.Nops {
 			for _, node := range nop.Nodes {
-				csaPubKey := node.PublicKey
-				if csaPubKey == nil {
-					return nil, fmt.Errorf("no public key for node %s", node.ID)
-				}
-				// the chain configs are equivalent as far as the ocr2 config is concerned so take the first one
-				if len(node.ChainConfigs) == 0 {
-					return nil, fmt.Errorf("no chain configs for node %s. cannot obtain keys", node.ID)
-				}
-				// all nodes should have an evm chain config, specifically the registry chain
-				evmCC, exists := firstChainConfigByType(node.ChainConfigs, chaintype.EVM)
-				if !exists {
-					return nil, fmt.Errorf("no evm chain config for node %s", node.ID)
-				}
-				cfgs := map[chaintype.ChainType]*v1.ChainConfig{
-					chaintype.EVM: evmCC,
-				}
-				aptosCC, exists := firstChainConfigByType(node.ChainConfigs, chaintype.Aptos)
-				if exists {
-					cfgs[chaintype.Aptos] = aptosCC
-				}
-				ocr2n, err := newOcr2Node(node.ID, cfgs, *csaPubKey)
+				ocr2n, err := newOcr2NodeFromClo(node, registryChainSel)
 				if err != nil {
 					return nil, fmt.Errorf("failed to create ocr2 node for node %s: %w", node.ID, err)
 				}
@@ -284,6 +295,21 @@ func firstChainConfigByType(ccfgs []*models.NodeChainConfig, t chaintype.ChainTy
 	return nil, false
 }
 
+func registryChainConfig(ccfgs []*models.NodeChainConfig, t chaintype.ChainType, sel uint64) (*v1.ChainConfig, error) {
+	chainId, err := chainsel.ChainIdFromSelector(sel)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get chain id from selector %d: %w", sel, err)
+	}
+	chainIdStr := strconv.FormatUint(chainId, 10)
+	for _, c := range ccfgs {
+		//nolint:staticcheck //ignore EqualFold it broke ci for some reason (go version skew btw local and ci?)
+		if strings.ToLower(c.Network.ChainType.String()) == strings.ToLower(string(t)) && c.Network.ChainID == chainIdStr {
+			return chainConfigFromClo(c), nil
+		}
+	}
+	return nil, fmt.Errorf("no chain config for chain %d", chainId)
+}
+
 // RegisteredDon is a representation of a don that exists in the in the capabilities registry all with the enriched node data
 type RegisteredDon struct {
 	Name  string
@@ -305,9 +331,9 @@ func (d RegisteredDon) signers() []common.Address {
 	return out
 }
 
-func joinInfoAndNodes(donInfos map[string]kcr.CapabilitiesRegistryDONInfo, dons []DonCapabilities) ([]RegisteredDon, error) {
+func joinInfoAndNodes(donInfos map[string]kcr.CapabilitiesRegistryDONInfo, dons []DonCapabilities, registryChainSel uint64) ([]RegisteredDon, error) {
 	// all maps should have the same keys
-	nodes, err := mapDonsToNodes(dons, true)
+	nodes, err := mapDonsToNodes(dons, true, registryChainSel)
 	if err != nil {
 		return nil, fmt.Errorf("failed to map dons to capabilities: %w", err)
 	}