feat(x/metrics): add module for emiting custom chain metrics (backport #1668) (#1677)

* feat(x/metrics): add module for emiting custom chain metrics (#1668)

* initialize x/metrics with metrics collection

* include global labels in x/metrics metrics

* add x/metrics spec

* add x/metrics test coverage

* update changelog

(cherry picked from commit 9a0aed7626)

# Conflicts:
#	CHANGELOG.md

* fix changlog conflicts

---------

Co-authored-by: Robert Pirtle <Astropirtle@gmail.com>

CHANGELOG.md

@@ -36,6 +36,11 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 ## [Unreleased]
 
+### Features
+- (metrics) [#1668] Adds non-state breaking x/metrics module for custom telemetry.
+
+## [v0.24.0](https://github.com/Kava-Labs/kava/releases/tag/v0.24.0)
+
 ### Features
 - (evmutil) [#1590] & [#1596] Add allow list param of sdk native denoms that can be transferred to evm
 - (evmutil) [#1591] & [#1596] Configure module to support deploying ERC20KavaWrappedCosmosCoin contracts
@@ -268,6 +273,7 @@ the [changelog](https://github.com/cosmos/cosmos-sdk/blob/v0.38.4/CHANGELOG.md).
 - [#257](https://github.com/Kava-Labs/kava/pulls/257) Include scripts to run
   large-scale simulations remotely using aws-batch
 
+[#1668]: https://github.com/Kava-Labs/kava/pull/1668
 [#1624]: https://github.com/Kava-Labs/kava/pull/1624
 [#1622]: https://github.com/Kava-Labs/kava/pull/1622
 [#1614]: https://github.com/Kava-Labs/kava/pull/1614

app/app.go

@@ -139,6 +139,8 @@ import (
 	"github.com/kava-labs/kava/x/liquid"
 	liquidkeeper "github.com/kava-labs/kava/x/liquid/keeper"
 	liquidtypes "github.com/kava-labs/kava/x/liquid/types"
+	metrics "github.com/kava-labs/kava/x/metrics"
+	metricstypes "github.com/kava-labs/kava/x/metrics/types"
 	pricefeed "github.com/kava-labs/kava/x/pricefeed"
 	pricefeedkeeper "github.com/kava-labs/kava/x/pricefeed/keeper"
 	pricefeedtypes "github.com/kava-labs/kava/x/pricefeed/types"
@@ -216,6 +218,7 @@ var (
 		router.AppModuleBasic{},
 		mint.AppModuleBasic{},
 		community.AppModuleBasic{},
+		metrics.AppModuleBasic{},
 	)
 
 	// module account permissions
@@ -261,6 +264,7 @@ type Options struct {
 	MempoolAuthAddresses  []sdk.AccAddress
 	EVMTrace              string
 	EVMMaxGasWanted       uint64
+	TelemetryOptions      metricstypes.TelemetryOptions
 }
 
 // DefaultOptions is a sensible default Options value.
@@ -790,10 +794,12 @@ func NewApp(
 		// nil InflationCalculationFn, use SDK's default inflation function
 		mint.NewAppModule(appCodec, app.mintKeeper, app.accountKeeper, nil),
 		community.NewAppModule(app.communityKeeper, app.accountKeeper),
+		metrics.NewAppModule(options.TelemetryOptions),
 	)
 
 	// Warning: Some begin blockers must run before others. Ensure the dependencies are understood before modifying this list.
 	app.mm.SetOrderBeginBlockers(
+		metricstypes.ModuleName,
 		// Upgrade begin blocker runs migrations on the first block after an upgrade. It should run before any other module.
 		upgradetypes.ModuleName,
 		// Capability begin blocker runs non state changing initialization.
@@ -882,6 +888,7 @@ func NewApp(
 		routertypes.ModuleName,
 		minttypes.ModuleName,
 		communitytypes.ModuleName,
+		metricstypes.ModuleName,
 	)
 
 	// Warning: Some init genesis methods must run before others. Ensure the dependencies are understood before modifying this list
@@ -923,6 +930,7 @@ func NewApp(
 		validatorvestingtypes.ModuleName,
 		liquidtypes.ModuleName,
 		routertypes.ModuleName,
+		metricstypes.ModuleName,
 	)
 
 	app.mm.RegisterInvariants(&app.crisisKeeper)

cmd/kava/cmd/app.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/kava-labs/kava/app"
 	"github.com/kava-labs/kava/app/params"
+	metricstypes "github.com/kava-labs/kava/x/metrics/types"
 )
 
 const (
@@ -99,6 +100,7 @@ func (ac appCreator) newApp(
 			MempoolAuthAddresses:  mempoolAuthAddresses,
 			EVMTrace:              cast.ToString(appOpts.Get(ethermintflags.EVMTracer)),
 			EVMMaxGasWanted:       cast.ToUint64(appOpts.Get(ethermintflags.EVMMaxTxGasWanted)),
+			TelemetryOptions:      metricstypes.TelemetryOptionsFromAppOpts(appOpts),
 		},
 		baseapp.SetPruning(pruningOpts),
 		baseapp.SetMinGasPrices(strings.Replace(cast.ToString(appOpts.Get(server.FlagMinGasPrices)), ";", ",", -1)),

go.mod

@@ -12,11 +12,13 @@ require (
 	github.com/cosmos/ibc-go/v6 v6.1.1
 	github.com/ethereum/go-ethereum v1.10.26
 	github.com/evmos/ethermint v0.21.0
+	github.com/go-kit/kit v0.12.0
 	github.com/gogo/protobuf v1.3.3
 	github.com/golang/protobuf v1.5.3
 	github.com/gorilla/mux v1.8.0
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0
 	github.com/pelletier/go-toml/v2 v2.0.6
+	github.com/prometheus/client_golang v1.14.0
 	github.com/spf13/cast v1.5.0
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/viper v1.15.0
@@ -84,7 +86,6 @@ require (
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
 	github.com/gin-gonic/gin v1.8.1 // indirect
-	github.com/go-kit/kit v0.12.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
@@ -148,7 +149,6 @@ require (
 	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.40.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect

x/metrics/abci.go

@@ -0,0 +1,12 @@
+package metrics
+
+import (
+	"github.com/kava-labs/kava/x/metrics/types"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+)
+
+// BeginBlocker publishes metrics at the start of each block.
+func BeginBlocker(ctx sdk.Context, metrics *types.Metrics) {
+	metrics.LatestBlockHeight.Set(float64(ctx.BlockHeight()))
+}

x/metrics/abci_test.go

@@ -0,0 +1,45 @@
+package metrics_test
+
+import (
+	"testing"
+
+	kitmetrics "github.com/go-kit/kit/metrics"
+	"github.com/stretchr/testify/require"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+
+	"github.com/kava-labs/kava/app"
+	"github.com/kava-labs/kava/x/metrics"
+	"github.com/kava-labs/kava/x/metrics/types"
+)
+
+type MockGauge struct {
+	value float64
+}
+
+func (mg *MockGauge) With(labelValues ...string) kitmetrics.Gauge { return mg }
+func (mg *MockGauge) Set(value float64)                           { mg.value = value }
+func (*MockGauge) Add(_ float64)                                  {}
+
+func ctxWithHeight(height int64) sdk.Context {
+	tApp := app.NewTestApp()
+	tApp.InitializeFromGenesisStates()
+	return tApp.NewContext(false, tmproto.Header{Height: height})
+}
+
+func TestBeginBlockEmitsLatestHeight(t *testing.T) {
+	gauge := MockGauge{}
+	myMetrics := &types.Metrics{
+		LatestBlockHeight: &gauge,
+	}
+
+	metrics.BeginBlocker(ctxWithHeight(1), myMetrics)
+	require.EqualValues(t, 1, gauge.value)
+
+	metrics.BeginBlocker(ctxWithHeight(100), myMetrics)
+	require.EqualValues(t, 100, gauge.value)
+
+	metrics.BeginBlocker(ctxWithHeight(17e6), myMetrics)
+	require.EqualValues(t, 17e6, gauge.value)
+}

x/metrics/module.go

@@ -0,0 +1,125 @@
+package metrics
+
+import (
+	"encoding/json"
+
+	"github.com/grpc-ecosystem/grpc-gateway/runtime"
+	"github.com/spf13/cobra"
+
+	"github.com/cosmos/cosmos-sdk/client"
+	"github.com/cosmos/cosmos-sdk/codec"
+	codectypes "github.com/cosmos/cosmos-sdk/codec/types"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/cosmos/cosmos-sdk/types/module"
+
+	abci "github.com/tendermint/tendermint/abci/types"
+
+	"github.com/kava-labs/kava/x/metrics/types"
+)
+
+var (
+	_ module.AppModule      = AppModule{}
+	_ module.AppModuleBasic = AppModuleBasic{}
+)
+
+// AppModuleBasic app module basics object
+type AppModuleBasic struct{}
+
+// Name returns the module name
+func (AppModuleBasic) Name() string {
+	return types.ModuleName
+}
+
+// RegisterLegacyAminoCodec register module codec
+// Deprecated: unused but necessary to fulfill AppModuleBasic interface
+func (AppModuleBasic) RegisterLegacyAminoCodec(cdc *codec.LegacyAmino) {}
+
+// DefaultGenesis default genesis state
+func (AppModuleBasic) DefaultGenesis(_ codec.JSONCodec) json.RawMessage {
+	return []byte("{}")
+}
+
+// ValidateGenesis module validate genesis
+func (AppModuleBasic) ValidateGenesis(_ codec.JSONCodec, _ client.TxEncodingConfig, _ json.RawMessage) error {
+	return nil
+}
+
+// RegisterInterfaces implements InterfaceModule.RegisterInterfaces
+func (a AppModuleBasic) RegisterInterfaces(registry codectypes.InterfaceRegistry) {}
+
+// RegisterGRPCGatewayRoutes registers the gRPC Gateway routes for the module.
+func (a AppModuleBasic) RegisterGRPCGatewayRoutes(clientCtx client.Context, mux *runtime.ServeMux) {}
+
+// GetTxCmd returns the root tx command for the module.
+func (AppModuleBasic) GetTxCmd() *cobra.Command {
+	return nil
+}
+
+// GetQueryCmd returns no root query command for the module.
+func (AppModuleBasic) GetQueryCmd() *cobra.Command {
+	return nil
+}
+
+//____________________________________________________________________________
+
+// AppModule app module type
+type AppModule struct {
+	AppModuleBasic
+	metrics *types.Metrics
+}
+
+// NewAppModule creates a new AppModule object
+func NewAppModule(telemetryOpts types.TelemetryOptions) AppModule {
+	return AppModule{
+		AppModuleBasic: AppModuleBasic{},
+		metrics:        types.NewMetrics(telemetryOpts),
+	}
+}
+
+// Name module name
+func (am AppModule) Name() string {
+	return am.AppModuleBasic.Name()
+}
+
+// RegisterInvariants register module invariants
+func (am AppModule) RegisterInvariants(_ sdk.InvariantRegistry) {}
+
+// Route module message route name
+// Deprecated: unused but necessary to fulfill AppModule interface
+func (am AppModule) Route() sdk.Route { return sdk.Route{} }
+
+// QuerierRoute module querier route name
+// Deprecated: unused but necessary to fulfill AppModule interface
+func (AppModule) QuerierRoute() string { return types.ModuleName }
+
+// LegacyQuerierHandler returns no sdk.Querier.
+// Deprecated: unused but necessary to fulfill AppModule interface
+func (am AppModule) LegacyQuerierHandler(_ *codec.LegacyAmino) sdk.Querier {
+	return nil
+}
+
+// ConsensusVersion implements AppModule/ConsensusVersion.
+func (AppModule) ConsensusVersion() uint64 { return 1 }
+
+// RegisterServices registers module services.
+func (am AppModule) RegisterServices(cfg module.Configurator) {}
+
+// InitGenesis module init-genesis
+func (am AppModule) InitGenesis(ctx sdk.Context, _ codec.JSONCodec, _ json.RawMessage) []abci.ValidatorUpdate {
+	return []abci.ValidatorUpdate{}
+}
+
+// ExportGenesis module export genesis
+func (am AppModule) ExportGenesis(_ sdk.Context, cdc codec.JSONCodec) json.RawMessage {
+	return nil
+}
+
+// BeginBlock module begin-block
+func (am AppModule) BeginBlock(ctx sdk.Context, req abci.RequestBeginBlock) {
+	BeginBlocker(ctx, am.metrics)
+}
+
+// EndBlock module end-block
+func (am AppModule) EndBlock(_ sdk.Context, _ abci.RequestEndBlock) []abci.ValidatorUpdate {
+	return []abci.ValidatorUpdate{}
+}

x/metrics/spec/README.md

@@ -0,0 +1,36 @@
+<!--
+order: 0
+title: "Metrics Overview"
+parent:
+  title: "metrics"
+-->
+
+# `metrics`
+
+
+## Abstract
+
+`x/metrics` is a stateless module that does not affect consensus. It captures chain metrics and emits them when the `instrumentation.prometheus` option is enabled in `config.toml`.
+
+## Precision
+
+The metrics emitted by `x/metrics` are `float64`s. They use `github.com/go-kit/kit/metrics` Prometheus gauges. Cosmos-sdk's `telemetry` package was not used because, at the time of writing, it only supports `float32`s and so does not maintain accurate representations of ints larger than ~16.8M. With `float64`s, integers may be accurately represented up to ~9e15.
+
+## Metrics
+
+The following metrics are defined:
+* `cometbft_blocksync_latest_block_height` - this emulates the blocksync `latest_block_height` metric in CometBFT v0.38+. The `cometbft` namespace comes from the `instrumentation.namespace` config.toml value.
+
+## Metric Labels
+
+All metrics emitted have the labels defined in app.toml's `telemetry.global-labels` field. This is the same field used by cosmos-sdk's `telemetry` package.
+
+example:
+```toml
+# app.toml
+[telemetry]
+global-labels = [
+  ["chain_id", "kava_2222-10"],
+  ["my_label", "my_value"],
+]
+```

x/metrics/types/keys.go

@@ -0,0 +1,6 @@
+package types
+
+const (
+	// Name of the module
+	ModuleName = "metrics"
+)

x/metrics/types/metrics.go

@@ -0,0 +1,89 @@
+package types
+
+import (
+	"github.com/go-kit/kit/metrics"
+	"github.com/go-kit/kit/metrics/discard"
+	prometheus "github.com/go-kit/kit/metrics/prometheus"
+	stdprometheus "github.com/prometheus/client_golang/prometheus"
+	"github.com/spf13/cast"
+
+	servertypes "github.com/cosmos/cosmos-sdk/server/types"
+)
+
+// TelemetryOptions defines the app configurations for the x/metrics module
+type TelemetryOptions struct {
+	// CometBFT config value for instrumentation.prometheus (config.toml)
+	PrometheusEnabled bool
+	// Namespace for CometBFT metrics. Used to emulate CometBFT metrics.
+	CometBFTMetricsNamespace string
+	// A list of keys and values used as labels on all metrics
+	GlobalLabelsAndValues []string
+}
+
+// TelemetryOptionsFromAppOpts creates the TelemetryOptions from server AppOptions
+func TelemetryOptionsFromAppOpts(appOpts servertypes.AppOptions) TelemetryOptions {
+	prometheusEnabled := cast.ToBool(appOpts.Get("instrumentation.prometheus"))
+	if !prometheusEnabled {
+		return TelemetryOptions{
+			GlobalLabelsAndValues: []string{},
+		}
+	}
+
+	// parse the app.toml global-labels into a slice of alternating label & value strings
+	// the value is expected to be a list of [label, value] tuples.
+	rawLabels := cast.ToSlice(appOpts.Get("telemetry.global-labels"))
+	globalLabelsAndValues := make([]string, 0, len(rawLabels)*2)
+	for _, gl := range rawLabels {
+		l := cast.ToStringSlice(gl)
+		globalLabelsAndValues = append(globalLabelsAndValues, l[0], l[1])
+	}
+
+	return TelemetryOptions{
+		PrometheusEnabled:        true,
+		CometBFTMetricsNamespace: cast.ToString(appOpts.Get("instrumentation.namespace")),
+		GlobalLabelsAndValues:    globalLabelsAndValues,
+	}
+}
+
+// Metrics contains metrics exposed by this module.
+// They use go-kit metrics like CometBFT as opposed to using cosmos-sdk telemetry
+// because the sdk's telemetry only supports float32s, whereas go-kit prometheus
+// metrics correctly handle float64s (and thus a larger number of int64s)
+type Metrics struct {
+	// The height of the latest block.
+	// This gauges exactly emulates the default blocksync metric in CometBFT v0.38+
+	// It should be removed when kava has been updated to CometBFT v0.38+.
+	// see https://github.com/cometbft/cometbft/blob/v0.38.0-rc3/blocksync/metrics.gen.go
+	LatestBlockHeight metrics.Gauge
+}
+
+// NewMetrics creates a new Metrics object based on whether or not prometheus instrumentation is enabled.
+func NewMetrics(opts TelemetryOptions) *Metrics {
+	if opts.PrometheusEnabled {
+		return PrometheusMetrics(opts)
+	}
+	return NoopMetrics()
+}
+
+// PrometheusMetrics returns the gauges for when prometheus instrumentation is enabled.
+func PrometheusMetrics(opts TelemetryOptions) *Metrics {
+	labels := []string{}
+	for i := 0; i < len(opts.GlobalLabelsAndValues); i += 2 {
+		labels = append(labels, opts.GlobalLabelsAndValues[i])
+	}
+	return &Metrics{
+		LatestBlockHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: opts.CometBFTMetricsNamespace,
+			Subsystem: "blocksync",
+			Name:      "latest_block_height",
+			Help:      "The height of the latest block.",
+		}, labels).With(opts.GlobalLabelsAndValues...),
+	}
+}
+
+// NoopMetrics are a do-nothing placeholder used when prometheus instrumentation is not enabled.
+func NoopMetrics() *Metrics {
+	return &Metrics{
+		LatestBlockHeight: discard.NewGauge(),
+	}
+}

x/metrics/types/metrics_test.go

@@ -0,0 +1,72 @@
+package types_test
+
+import (
+	"testing"
+
+	"github.com/go-kit/kit/metrics"
+	"github.com/go-kit/kit/metrics/prometheus"
+	"github.com/kava-labs/kava/x/metrics/types"
+	"github.com/stretchr/testify/require"
+)
+
+func isPrometheusGauge(g metrics.Gauge) bool {
+	_, ok := g.(*prometheus.Gauge)
+	return ok
+}
+
+var (
+	disabledOpts = types.TelemetryOptions{
+		PrometheusEnabled: false,
+	}
+	enabledOpts = types.TelemetryOptions{
+		PrometheusEnabled:        true,
+		CometBFTMetricsNamespace: "cometbft",
+		GlobalLabelsAndValues:    []string{"label1", "value1", "label2", "value2"},
+	}
+)
+
+func TestNewMetrics_DisabledVsEnabled(t *testing.T) {
+	myMetrics := types.NewMetrics(disabledOpts)
+	require.False(t, isPrometheusGauge(myMetrics.LatestBlockHeight))
+
+	myMetrics = types.NewMetrics(enabledOpts)
+	require.True(t, isPrometheusGauge(myMetrics.LatestBlockHeight))
+}
+
+type MockAppOpts struct {
+	store map[string]interface{}
+}
+
+func (mao *MockAppOpts) Get(key string) interface{} {
+	return mao.store[key]
+}
+
+func TestTelemetryOptionsFromAppOpts(t *testing.T) {
+	appOpts := MockAppOpts{store: make(map[string]interface{})}
+
+	// test disabled functionality
+	appOpts.store["instrumentation.prometheus"] = false
+
+	opts := types.TelemetryOptionsFromAppOpts(&appOpts)
+	require.False(t, opts.PrometheusEnabled)
+
+	// test enabled functionality
+	appOpts.store["instrumentation.prometheus"] = true
+	appOpts.store["instrumentation.namespace"] = "magic"
+	appOpts.store["telemetry.global-labels"] = []interface{}{}
+
+	opts = types.TelemetryOptionsFromAppOpts(&appOpts)
+	require.True(t, opts.PrometheusEnabled)
+	require.Equal(t, "magic", opts.CometBFTMetricsNamespace)
+	require.Len(t, opts.GlobalLabelsAndValues, 0)
+
+	appOpts.store["telemetry.global-labels"] = []interface{}{
+		[]interface{}{"label1", "value1"},
+		[]interface{}{"label2", "value2"},
+	}
+	opts = types.TelemetryOptionsFromAppOpts(&appOpts)
+	require.True(t, opts.PrometheusEnabled)
+	require.Equal(t, "magic", opts.CometBFTMetricsNamespace)
+	require.Len(t, opts.GlobalLabelsAndValues, 4)
+	require.Equal(t, enabledOpts.GlobalLabelsAndValues, opts.GlobalLabelsAndValues)
+}