Merge pull request #725 from oasisprotocol/ptrus/feature/proposals-me…

…tadata

nodeapi: introduce genesis document struct and include proposal metadata

.changelog/725.feature.md

@@ -0,0 +1 @@
+consensus/proposals: Add support for proposal metadata

analyzer/consensus/consensus.go

@@ -19,7 +19,6 @@ import (
 	"github.com/oasisprotocol/nexus/analyzer/consensus/static"
 	"github.com/oasisprotocol/nexus/analyzer/util/addresses"
 	"github.com/oasisprotocol/nexus/coreapi/v22.2.11/consensus/api/transaction"
-	genesis "github.com/oasisprotocol/nexus/coreapi/v22.2.11/genesis/api"
 	staking "github.com/oasisprotocol/nexus/coreapi/v22.2.11/staking/api"
 
 	"github.com/oasisprotocol/nexus/analyzer"
@@ -154,7 +153,7 @@ func (m *processor) FinalizeFastSync(ctx context.Context, lastFastSyncHeight int
 	if err != nil {
 		return fmt.Errorf("no history record for first slow-sync height %d: %w", firstSlowSyncHeight, err)
 	}
-	var genesisDoc *genesis.Document
+	var genesisDoc *nodeapi.GenesisDocument
 	var nodes []nodeapi.Node
 	if r.GenesisHeight == firstSlowSyncHeight {
 		m.logger.Info("fetching genesis document before starting with the first block of a chain", "chain_context", r.ChainContext, "genesis_height", r.GenesisHeight)
@@ -196,7 +195,7 @@ func (m *processor) aggregateFastSyncTables(ctx context.Context) error {
 }
 
 // Dumps the genesis document to a JSON file if instructed via env variables. For debug only.
-func (m *processor) debugDumpGenesisJSON(genesisDoc *genesis.Document, heightOrName string) {
+func (m *processor) debugDumpGenesisJSON(genesisDoc *nodeapi.GenesisDocument, heightOrName string) {
 	debugPath := os.Getenv("NEXUS_DUMP_GENESIS") // can be templatized with "{{height}}"
 	if debugPath == "" {
 		return
@@ -216,7 +215,7 @@ func (m *processor) debugDumpGenesisJSON(genesisDoc *genesis.Document, heightOrN
 
 // Executes SQL queries to index the contents of the genesis document.
 // If nodesOverride is non-nil, it is used instead of the nodes from the genesis document.
-func (m *processor) processGenesis(ctx context.Context, genesisDoc *genesis.Document, nodesOverride []nodeapi.Node) error {
+func (m *processor) processGenesis(ctx context.Context, genesisDoc *nodeapi.GenesisDocument, nodesOverride []nodeapi.Node) error {
 	m.logger.Info("processing genesis document")
 	gen := NewGenesisProcessor(m.logger.With("height", "genesis"))
 	batch, err := gen.Process(genesisDoc, nodesOverride)
@@ -1127,13 +1126,22 @@ func (m *processor) queueSubmissions(batch *storage.QueryBatch, data *governance
 	}
 
 	for _, submission := range data.ProposalSubmissions {
+		var title, description *string
+		if submission.Content.Metadata != nil {
+			metadata := submission.Content.Metadata
+			title = &metadata.Title
+			description = &metadata.Description
+		}
+
 		switch {
 		case submission.Content.Upgrade != nil:
 			batch.Queue(queries.ConsensusProposalSubmissionInsert,
 				submission.ID,
 				submission.Submitter.String(),
 				submission.State.String(),
 				submission.Deposit.String(),
+				title,
+				description,
 				submission.Content.Upgrade.Handler,
 				submission.Content.Upgrade.Target.ConsensusProtocol.String(),
 				submission.Content.Upgrade.Target.RuntimeHostProtocol.String(),
@@ -1149,6 +1157,8 @@ func (m *processor) queueSubmissions(batch *storage.QueryBatch, data *governance
 				submission.Submitter.String(),
 				submission.State.String(),
 				submission.Deposit.String(),
+				title,
+				description,
 				submission.Content.CancelUpgrade.ProposalID,
 				submission.CreatedAt,
 				submission.ClosesAt,
@@ -1160,6 +1170,8 @@ func (m *processor) queueSubmissions(batch *storage.QueryBatch, data *governance
 				submission.Submitter.String(),
 				submission.State.String(),
 				submission.Deposit.String(),
+				title,
+				description,
 				submission.Content.ChangeParameters.Module,
 				[]byte(submission.Content.ChangeParameters.Changes),
 				submission.CreatedAt,

analyzer/consensus/genesis.go

@@ -11,8 +11,6 @@ import (
 
 	"github.com/oasisprotocol/nexus/analyzer/queries"
 	"github.com/oasisprotocol/nexus/common"
-	beacon "github.com/oasisprotocol/nexus/coreapi/v22.2.11/beacon/api"
-	genesis "github.com/oasisprotocol/nexus/coreapi/v22.2.11/genesis/api"
 	registry "github.com/oasisprotocol/nexus/coreapi/v22.2.11/registry/api"
 	staking "github.com/oasisprotocol/nexus/coreapi/v22.2.11/staking/api"
 	"github.com/oasisprotocol/nexus/storage"
@@ -33,7 +31,7 @@ func NewGenesisProcessor(logger *log.Logger) *GenesisProcessor {
 
 // Process generates SQL statements for indexing the genesis state.
 // `nodesOverride` can be nil; if non-nil, the behavior is as if the genesis document contained that set of nodes instead of whatever it contains.
-func (mg *GenesisProcessor) Process(document *genesis.Document, nodesOverride []nodeapi.Node) (*storage.QueryBatch, error) {
+func (mg *GenesisProcessor) Process(document *nodeapi.GenesisDocument, nodesOverride []nodeapi.Node) (*storage.QueryBatch, error) {
 	batch := &storage.QueryBatch{}
 
 	if err := mg.addRegistryBackendMigrations(batch, document, nodesOverride); err != nil {
@@ -51,7 +49,7 @@ func (mg *GenesisProcessor) Process(document *genesis.Document, nodesOverride []
 	return batch, nil
 }
 
-func (mg *GenesisProcessor) addRegistryBackendMigrations(batch *storage.QueryBatch, document *genesis.Document, nodesOverride []nodeapi.Node) error {
+func (mg *GenesisProcessor) addRegistryBackendMigrations(batch *storage.QueryBatch, document *nodeapi.GenesisDocument, nodesOverride []nodeapi.Node) error {
 	// Populate entities.
 	for _, signedEntity := range document.Registry.Entities {
 		var entity entity.Entity
@@ -102,7 +100,7 @@ func (mg *GenesisProcessor) addRegistryBackendMigrations(batch *storage.QueryBat
 				//   Also, nexus performs internal lossy data conversions where signatures are lost.
 				return err
 			}
-			if beacon.EpochTime(node.Expiration) < document.Beacon.Base {
+			if node.Expiration < document.BaseEpoch {
 				// Node expired before the genesis epoch, skip.
 				continue
 			}
@@ -139,7 +137,7 @@ func (mg *GenesisProcessor) addRegistryBackendMigrations(batch *storage.QueryBat
 	return nil
 }
 
-func (mg *GenesisProcessor) addStakingBackendMigrations(batch *storage.QueryBatch, document *genesis.Document) error {
+func (mg *GenesisProcessor) addStakingBackendMigrations(batch *storage.QueryBatch, document *nodeapi.GenesisDocument) error {
 	// Populate accounts.
 
 	// Populate special accounts with reserved addresses.
@@ -259,18 +257,26 @@ func (mg *GenesisProcessor) addStakingBackendMigrations(batch *storage.QueryBatc
 	return nil
 }
 
-func (mg *GenesisProcessor) addGovernanceBackendMigrations(batch *storage.QueryBatch, document *genesis.Document) {
+func (mg *GenesisProcessor) addGovernanceBackendMigrations(batch *storage.QueryBatch, document *nodeapi.GenesisDocument) {
 	// Populate proposals.
 
 	// TODO: Extract `executed` for proposal.
 	for _, proposal := range document.Governance.Proposals {
+		var title, description *string
+		if proposal.Content.Metadata != nil {
+			title = &proposal.Content.Metadata.Title
+			description = &proposal.Content.Metadata.Description
+		}
+
 		switch {
 		case proposal.Content.Upgrade != nil:
 			batch.Queue(queries.ConsensusProposalSubmissionInsert,
 				proposal.ID,
 				proposal.Submitter.String(),
 				proposal.State.String(),
 				proposal.Deposit.ToBigInt(),
+				title,
+				description,
 				proposal.Content.Upgrade.Handler,
 				proposal.Content.Upgrade.Target.ConsensusProtocol.String(),
 				proposal.Content.Upgrade.Target.RuntimeHostProtocol.String(),
@@ -286,6 +292,8 @@ func (mg *GenesisProcessor) addGovernanceBackendMigrations(batch *storage.QueryB
 				proposal.Submitter.String(),
 				proposal.State.String(),
 				proposal.Deposit.ToBigInt(),
+				title,
+				description,
 				proposal.Content.CancelUpgrade.ProposalID,
 				proposal.CreatedAt,
 				proposal.ClosesAt,
@@ -297,6 +305,8 @@ func (mg *GenesisProcessor) addGovernanceBackendMigrations(batch *storage.QueryB
 				proposal.Submitter.String(),
 				proposal.State.String(),
 				proposal.Deposit.ToBigInt(),
+				title,
+				description,
 				proposal.Content.ChangeParameters.Module,
 				proposal.Content.ChangeParameters.Changes,
 				proposal.CreatedAt,

analyzer/queries/queries.go

@@ -457,16 +457,16 @@ var (
     TRUNCATE chain.committee_members`
 
 	ConsensusProposalSubmissionInsert = `
-    INSERT INTO chain.proposals (id, submitter, state, deposit, handler, cp_target_version, rhp_target_version, rcp_target_version, upgrade_epoch, created_at, closes_at, invalid_votes)
-      VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`
+    INSERT INTO chain.proposals (id, submitter, state, deposit, title, description, handler, cp_target_version, rhp_target_version, rcp_target_version, upgrade_epoch, created_at, closes_at, invalid_votes)
+      VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`
 
 	ConsensusProposalSubmissionCancelInsert = `
-    INSERT INTO chain.proposals (id, submitter, state, deposit, cancels, created_at, closes_at, invalid_votes)
-      VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`
+    INSERT INTO chain.proposals (id, submitter, state, deposit, title, description, cancels, created_at, closes_at, invalid_votes)
+      VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`
 
 	ConsensusProposalSubmissionChangeParametersInsert = `
-    INSERT INTO chain.proposals (id, submitter, state, deposit, parameters_change_module, parameters_change, created_at, closes_at, invalid_votes)
-      VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`
+    INSERT INTO chain.proposals (id, submitter, state, deposit, title, description, parameters_change_module, parameters_change, created_at, closes_at, invalid_votes)
+      VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`
 
 	ConsensusProposalExecutionsUpdate = `
     UPDATE chain.proposals

api/spec/v1.yaml

@@ -2286,6 +2286,12 @@ components:
         deposit:
           allOf: [$ref: '#/components/schemas/TextBigInt']
           description: The deposit attached to this proposal.
+        title:
+          type: string
+          description: The (optional) title of the proposal.
+        description:
+          type: string
+          description: The (optional) description of the proposal.
         handler:
           type: string
           description: The name of the upgrade handler.

storage/client/client.go

@@ -1118,6 +1118,8 @@ func (c *StorageClient) Proposals(ctx context.Context, p apiTypes.GetConsensusPr
 			&p.Submitter,
 			&p.State,
 			&p.Deposit,
+			&p.Title,
+			&p.Description,
 			&p.Handler,
 			&p.Target.ConsensusProtocol,
 			&p.Target.RuntimeHostProtocol,
@@ -1159,6 +1161,8 @@ func (c *StorageClient) Proposal(ctx context.Context, proposalID uint64) (*Propo
 		&p.Submitter,
 		&p.State,
 		&p.Deposit,
+		&p.Title,
+		&p.Description,
 		&p.Handler,
 		&p.Target.ConsensusProtocol,
 		&p.Target.RuntimeHostProtocol,

storage/client/queries/queries.go

@@ -254,7 +254,7 @@ const (
 		OFFSET $3::bigint`
 
 	Proposals = `
-		SELECT id, submitter, state, deposit, handler, cp_target_version, rhp_target_version, rcp_target_version,
+		SELECT id, submitter, state, deposit, title, description, handler, cp_target_version, rhp_target_version, rcp_target_version,
 				upgrade_epoch, cancels, parameters_change_module, parameters_change, created_at, closes_at, invalid_votes
 			FROM chain.proposals
 			WHERE ($1::text IS NULL OR submitter = $1::text) AND
@@ -264,7 +264,7 @@ const (
 		OFFSET $4::bigint`
 
 	Proposal = `
-		SELECT id, submitter, state, deposit, handler, cp_target_version, rhp_target_version, rcp_target_version,
+		SELECT id, submitter, state, deposit, title, description, handler, cp_target_version, rhp_target_version, rcp_target_version,
 				upgrade_epoch, cancels, parameters_change_module, parameters_change, created_at, closes_at, invalid_votes
 			FROM chain.proposals
 			WHERE id = $1::bigint`

storage/migrations/00_consensus.up.sql

@@ -282,6 +282,10 @@ CREATE TABLE chain.proposals
   executed      BOOLEAN NOT NULL DEFAULT false,
   deposit       UINT_NUMERIC NOT NULL,
 
+  -- Added in 27_proposals_metadata.up.sql.
+  -- title TEXT,
+  -- description TEXT,
+
   -- If this proposal is a new proposal.
   handler            TEXT,
   cp_target_version  TEXT,

storage/migrations/27_proposals_metadata.up.sql

@@ -0,0 +1,9 @@
+BEGIN;
+
+ALTER TABLE chain.proposals
+	ADD COLUMN title TEXT;
+
+ALTER TABLE chain.proposals
+	ADD COLUMN description TEXT;
+
+COMMIT;

storage/oasis/nodeapi/api.go

@@ -1,3 +1,15 @@
+// Due to the (sometimes breaking) changes in the types used by different
+// evolutions of oasis-core over the Cobalt, Damask, and Eden upgrades,
+// Nexus defines its own set of internal types here.
+//
+// The types that it exposes are mostly simplified versions of the types exposed by
+// oasis-core Damask: The top-level type structs are defined in this file, and
+// the types of their fields are almost universally directly the types exposed
+// by oasis-core Damask. The reason is that as oasis-core evolves, Damask types
+// are mostly able to represent all the information from Cobalt, plus some. However,
+// there are a few exceptions where we use oasis-core Eden types instead, e.g. in the
+// `governance` module. Over time, types should be migrated to newer versions, e.g. Eden
+// or later, as needed.
 package nodeapi
 
 import (
@@ -16,13 +28,12 @@ import (
 	consensus "github.com/oasisprotocol/nexus/coreapi/v22.2.11/consensus/api"
 	consensusTransaction "github.com/oasisprotocol/nexus/coreapi/v22.2.11/consensus/api/transaction"
 	consensusResults "github.com/oasisprotocol/nexus/coreapi/v22.2.11/consensus/api/transaction/results"
-	genesis "github.com/oasisprotocol/nexus/coreapi/v22.2.11/genesis/api"
-	governance "github.com/oasisprotocol/nexus/coreapi/v22.2.11/governance/api"
 	registry "github.com/oasisprotocol/nexus/coreapi/v22.2.11/registry/api"
 	roothash "github.com/oasisprotocol/nexus/coreapi/v22.2.11/roothash/api"
 	"github.com/oasisprotocol/nexus/coreapi/v22.2.11/roothash/api/message"
 	scheduler "github.com/oasisprotocol/nexus/coreapi/v22.2.11/scheduler/api"
 	staking "github.com/oasisprotocol/nexus/coreapi/v22.2.11/staking/api"
+	governance "github.com/oasisprotocol/nexus/coreapi/v24.0/governance/api"
 	"github.com/oasisprotocol/nexus/storage/oasis/connections"
 
 	sdkClient "github.com/oasisprotocol/oasis-sdk/client-sdk/go/client"
@@ -49,9 +60,8 @@ import (
 // Since the types are simpler and fewer, their structure is, in
 // places, flattened compared to their counterparts in oasis-core.
 type ConsensusApiLite interface {
-	// TODO: Introduce internal, stripped-down version of `genesis.Document`.
-	GetGenesisDocument(ctx context.Context, chainContext string) (*genesis.Document, error)
-	StateToGenesis(ctx context.Context, height int64) (*genesis.Document, error)
+	GetGenesisDocument(ctx context.Context, chainContext string) (*GenesisDocument, error)
+	StateToGenesis(ctx context.Context, height int64) (*GenesisDocument, error)
 	GetConsensusParameters(ctx context.Context, height int64) (*ConsensusParameters, error)
 	GetBlock(ctx context.Context, height int64) (*consensus.Block, error)
 	GetTransactionsWithResults(ctx context.Context, height int64) ([]TransactionWithResults, error)
@@ -293,3 +303,21 @@ type RuntimeBlockHeader struct { //nolint: maligned
 	MessagesHash   hash.Hash
 	InMessagesHash hash.Hash // NOTE: Available starting in Damask.
 }
+
+// GenesisDocument is a stripped-down version of `genesis.Document`.
+type GenesisDocument struct {
+	// Height is the block height at which the document was generated.
+	Height int64 `json:"height"`
+	// Time is the time the genesis block was constructed.
+	Time time.Time `json:"genesis_time"`
+	// ChainID is the ID of the chain.
+	ChainID string `json:"chain_id"`
+	// BaseEpoch is the base epoch for the chain.
+	BaseEpoch uint64 `json:"base_epoch"`
+	// Registry is the registry genesis state.
+	Registry registry.Genesis `json:"registry"`
+	// RootHash is the roothash genesis state.
+	Staking staking.Genesis `json:"staking"`
+	// Governance is the governance genesis state.
+	Governance governance.Genesis `json:"governance"`
+}

storage/oasis/nodeapi/cobalt/convert.go

@@ -12,13 +12,12 @@ import (
 	coreCommon "github.com/oasisprotocol/oasis-core/go/common"
 
 	consensus "github.com/oasisprotocol/nexus/coreapi/v22.2.11/consensus/api"
-	genesis "github.com/oasisprotocol/nexus/coreapi/v22.2.11/genesis/api"
-	governance "github.com/oasisprotocol/nexus/coreapi/v22.2.11/governance/api"
 	registry "github.com/oasisprotocol/nexus/coreapi/v22.2.11/registry/api"
 	"github.com/oasisprotocol/nexus/coreapi/v22.2.11/roothash/api/message"
 	scheduler "github.com/oasisprotocol/nexus/coreapi/v22.2.11/scheduler/api"
 	staking "github.com/oasisprotocol/nexus/coreapi/v22.2.11/staking/api"
-	upgrade "github.com/oasisprotocol/nexus/coreapi/v22.2.11/upgrade/api"
+	governance "github.com/oasisprotocol/nexus/coreapi/v24.0/governance/api"
+	upgrade "github.com/oasisprotocol/nexus/coreapi/v24.0/upgrade/api"
 
 	apiTypes "github.com/oasisprotocol/nexus/api/v1/types"
 	"github.com/oasisprotocol/nexus/common"
@@ -48,6 +47,7 @@ func convertProposal(p *governanceCobalt.Proposal) *governance.Proposal {
 		State:     governance.ProposalState(p.State),
 		Deposit:   p.Deposit,
 		Content: governance.ProposalContent{
+			Metadata: nil, // Not present in Damask.
 			Upgrade: &governance.UpgradeProposal{
 				Descriptor: upgrade.Descriptor{
 					Versioned: p.Content.Upgrade.Descriptor.Versioned,
@@ -102,7 +102,7 @@ func convertRuntime(r *registryCobalt.Runtime) *registry.Runtime {
 // nexus-internal (= current oasis-core) format.
 // WARNING: This is a partial conversion, only the fields that are used by
 // Nexus are filled in the output document.
-func ConvertGenesis(d genesisCobalt.Document) *genesis.Document {
+func ConvertGenesis(d genesisCobalt.Document) *nodeapi.GenesisDocument {
 	proposals := make([]*governance.Proposal, len(d.Governance.Proposals))
 	for i, p := range d.Governance.Proposals {
 		proposals[i] = convertProposal(p)
@@ -153,18 +153,24 @@ func ConvertGenesis(d genesisCobalt.Document) *genesis.Document {
 		runtimes[i] = convertRuntime(r)
 	}
 
-	return &genesis.Document{
-		Height:  d.Height,
-		Time:    d.Time,
-		ChainID: d.ChainID,
+	suspendedRuntimes := make([]*registry.Runtime, len(d.Registry.SuspendedRuntimes))
+	for i, r := range d.Registry.SuspendedRuntimes {
+		suspendedRuntimes[i] = convertRuntime(r)
+	}
+
+	return &nodeapi.GenesisDocument{
+		Height:    d.Height,
+		Time:      d.Time,
+		ChainID:   d.ChainID,
+		BaseEpoch: uint64(d.Beacon.Base),
 		Governance: governance.Genesis{
 			Proposals:   proposals,
 			VoteEntries: voteEntries,
 		},
 		Registry: registry.Genesis{
 			Entities:          d.Registry.Entities,
-			Runtimes:          []*registry.Runtime{},
-			SuspendedRuntimes: []*registry.Runtime{},
+			Runtimes:          runtimes,
+			SuspendedRuntimes: suspendedRuntimes,
 			Nodes:             d.Registry.Nodes,
 		},
 		Staking: staking.Genesis{