From 9a0aed7626582a50ac08bd69d8f43b98de8ae196 Mon Sep 17 00:00:00 2001 From: Robert Pirtle Date: Fri, 25 Aug 2023 12:23:53 -0700 Subject: [PATCH] feat(x/metrics): add module for emiting custom chain metrics (#1668) * initialize x/metrics with metrics collection * include global labels in x/metrics metrics * add x/metrics spec * add x/metrics test coverage * update changelog --- CHANGELOG.md | 4 + app/app.go | 8 ++ cmd/kava/cmd/app.go | 2 + go.mod | 4 +- x/metrics/abci.go | 12 +++ x/metrics/abci_test.go | 45 ++++++++++++ x/metrics/module.go | 125 ++++++++++++++++++++++++++++++++ x/metrics/spec/README.md | 36 +++++++++ x/metrics/types/keys.go | 6 ++ x/metrics/types/metrics.go | 89 +++++++++++++++++++++++ x/metrics/types/metrics_test.go | 72 ++++++++++++++++++ 11 files changed, 401 insertions(+), 2 deletions(-) create mode 100644 x/metrics/abci.go create mode 100644 x/metrics/abci_test.go create mode 100644 x/metrics/module.go create mode 100644 x/metrics/spec/README.md create mode 100644 x/metrics/types/keys.go create mode 100644 x/metrics/types/metrics.go create mode 100644 x/metrics/types/metrics_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index ed7c3854..6f9f98c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,9 @@ Ref: https://keepachangelog.com/en/1.0.0/ ## [Unreleased] +### Features +- (metrics) [#1668] Adds non-state breaking x/metrics module for custom telemetry. + ### Bug Fixes - (evmutil) [#1655] Initialize x/evmutil module account in InitGenesis @@ -276,6 +279,7 @@ the [changelog](https://github.com/cosmos/cosmos-sdk/blob/v0.38.4/CHANGELOG.md). - [#257](https://github.com/Kava-Labs/kava/pulls/257) Include scripts to run large-scale simulations remotely using aws-batch +[#1668]: https://github.com/Kava-Labs/kava/pull/1668 [#1655]: https://github.com/Kava-Labs/kava/pull/1655 [#1624]: https://github.com/Kava-Labs/kava/pull/1624 [#1631]: https://github.com/Kava-Labs/kava/pull/1631 diff --git a/app/app.go b/app/app.go index b43ccf1e..c8636ab6 100644 --- a/app/app.go +++ b/app/app.go @@ -139,6 +139,8 @@ import ( "github.com/kava-labs/kava/x/liquid" liquidkeeper "github.com/kava-labs/kava/x/liquid/keeper" liquidtypes "github.com/kava-labs/kava/x/liquid/types" + metrics "github.com/kava-labs/kava/x/metrics" + metricstypes "github.com/kava-labs/kava/x/metrics/types" pricefeed "github.com/kava-labs/kava/x/pricefeed" pricefeedkeeper "github.com/kava-labs/kava/x/pricefeed/keeper" pricefeedtypes "github.com/kava-labs/kava/x/pricefeed/types" @@ -216,6 +218,7 @@ var ( router.AppModuleBasic{}, mint.AppModuleBasic{}, community.AppModuleBasic{}, + metrics.AppModuleBasic{}, ) // module account permissions @@ -261,6 +264,7 @@ type Options struct { MempoolAuthAddresses []sdk.AccAddress EVMTrace string EVMMaxGasWanted uint64 + TelemetryOptions metricstypes.TelemetryOptions } // DefaultOptions is a sensible default Options value. @@ -790,10 +794,12 @@ func NewApp( // nil InflationCalculationFn, use SDK's default inflation function mint.NewAppModule(appCodec, app.mintKeeper, app.accountKeeper, nil), community.NewAppModule(app.communityKeeper, app.accountKeeper), + metrics.NewAppModule(options.TelemetryOptions), ) // Warning: Some begin blockers must run before others. Ensure the dependencies are understood before modifying this list. app.mm.SetOrderBeginBlockers( + metricstypes.ModuleName, // Upgrade begin blocker runs migrations on the first block after an upgrade. It should run before any other module. upgradetypes.ModuleName, // Capability begin blocker runs non state changing initialization. @@ -882,6 +888,7 @@ func NewApp( routertypes.ModuleName, minttypes.ModuleName, communitytypes.ModuleName, + metricstypes.ModuleName, ) // Warning: Some init genesis methods must run before others. Ensure the dependencies are understood before modifying this list @@ -923,6 +930,7 @@ func NewApp( validatorvestingtypes.ModuleName, liquidtypes.ModuleName, routertypes.ModuleName, + metricstypes.ModuleName, ) app.mm.RegisterInvariants(&app.crisisKeeper) diff --git a/cmd/kava/cmd/app.go b/cmd/kava/cmd/app.go index ac238430..b6c57042 100644 --- a/cmd/kava/cmd/app.go +++ b/cmd/kava/cmd/app.go @@ -24,6 +24,7 @@ import ( "github.com/kava-labs/kava/app" "github.com/kava-labs/kava/app/params" + metricstypes "github.com/kava-labs/kava/x/metrics/types" ) const ( @@ -99,6 +100,7 @@ func (ac appCreator) newApp( MempoolAuthAddresses: mempoolAuthAddresses, EVMTrace: cast.ToString(appOpts.Get(ethermintflags.EVMTracer)), EVMMaxGasWanted: cast.ToUint64(appOpts.Get(ethermintflags.EVMMaxTxGasWanted)), + TelemetryOptions: metricstypes.TelemetryOptionsFromAppOpts(appOpts), }, baseapp.SetPruning(pruningOpts), baseapp.SetMinGasPrices(strings.Replace(cast.ToString(appOpts.Get(server.FlagMinGasPrices)), ";", ",", -1)), diff --git a/go.mod b/go.mod index e9d430f7..a872a2e7 100644 --- a/go.mod +++ b/go.mod @@ -12,12 +12,14 @@ require ( github.com/cosmos/ibc-go/v6 v6.1.1 github.com/ethereum/go-ethereum v1.10.26 github.com/evmos/ethermint v0.21.0 + github.com/go-kit/kit v0.12.0 github.com/gogo/protobuf v1.3.3 github.com/golang/protobuf v1.5.3 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/linxGnu/grocksdb v1.8.0 github.com/pelletier/go-toml/v2 v2.0.6 + github.com/prometheus/client_golang v1.14.0 github.com/spf13/cast v1.5.0 github.com/spf13/cobra v1.6.1 github.com/spf13/viper v1.15.0 @@ -85,7 +87,6 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect github.com/gin-gonic/gin v1.8.1 // indirect - github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -148,7 +149,6 @@ require ( github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.40.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect diff --git a/x/metrics/abci.go b/x/metrics/abci.go new file mode 100644 index 00000000..14a77d71 --- /dev/null +++ b/x/metrics/abci.go @@ -0,0 +1,12 @@ +package metrics + +import ( + "github.com/kava-labs/kava/x/metrics/types" + + sdk "github.com/cosmos/cosmos-sdk/types" +) + +// BeginBlocker publishes metrics at the start of each block. +func BeginBlocker(ctx sdk.Context, metrics *types.Metrics) { + metrics.LatestBlockHeight.Set(float64(ctx.BlockHeight())) +} diff --git a/x/metrics/abci_test.go b/x/metrics/abci_test.go new file mode 100644 index 00000000..5ab542ee --- /dev/null +++ b/x/metrics/abci_test.go @@ -0,0 +1,45 @@ +package metrics_test + +import ( + "testing" + + kitmetrics "github.com/go-kit/kit/metrics" + "github.com/stretchr/testify/require" + + sdk "github.com/cosmos/cosmos-sdk/types" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + + "github.com/kava-labs/kava/app" + "github.com/kava-labs/kava/x/metrics" + "github.com/kava-labs/kava/x/metrics/types" +) + +type MockGauge struct { + value float64 +} + +func (mg *MockGauge) With(labelValues ...string) kitmetrics.Gauge { return mg } +func (mg *MockGauge) Set(value float64) { mg.value = value } +func (*MockGauge) Add(_ float64) {} + +func ctxWithHeight(height int64) sdk.Context { + tApp := app.NewTestApp() + tApp.InitializeFromGenesisStates() + return tApp.NewContext(false, tmproto.Header{Height: height}) +} + +func TestBeginBlockEmitsLatestHeight(t *testing.T) { + gauge := MockGauge{} + myMetrics := &types.Metrics{ + LatestBlockHeight: &gauge, + } + + metrics.BeginBlocker(ctxWithHeight(1), myMetrics) + require.EqualValues(t, 1, gauge.value) + + metrics.BeginBlocker(ctxWithHeight(100), myMetrics) + require.EqualValues(t, 100, gauge.value) + + metrics.BeginBlocker(ctxWithHeight(17e6), myMetrics) + require.EqualValues(t, 17e6, gauge.value) +} diff --git a/x/metrics/module.go b/x/metrics/module.go new file mode 100644 index 00000000..3b74fad5 --- /dev/null +++ b/x/metrics/module.go @@ -0,0 +1,125 @@ +package metrics + +import ( + "encoding/json" + + "github.com/grpc-ecosystem/grpc-gateway/runtime" + "github.com/spf13/cobra" + + "github.com/cosmos/cosmos-sdk/client" + "github.com/cosmos/cosmos-sdk/codec" + codectypes "github.com/cosmos/cosmos-sdk/codec/types" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/types/module" + + abci "github.com/tendermint/tendermint/abci/types" + + "github.com/kava-labs/kava/x/metrics/types" +) + +var ( + _ module.AppModule = AppModule{} + _ module.AppModuleBasic = AppModuleBasic{} +) + +// AppModuleBasic app module basics object +type AppModuleBasic struct{} + +// Name returns the module name +func (AppModuleBasic) Name() string { + return types.ModuleName +} + +// RegisterLegacyAminoCodec register module codec +// Deprecated: unused but necessary to fulfill AppModuleBasic interface +func (AppModuleBasic) RegisterLegacyAminoCodec(cdc *codec.LegacyAmino) {} + +// DefaultGenesis default genesis state +func (AppModuleBasic) DefaultGenesis(_ codec.JSONCodec) json.RawMessage { + return []byte("{}") +} + +// ValidateGenesis module validate genesis +func (AppModuleBasic) ValidateGenesis(_ codec.JSONCodec, _ client.TxEncodingConfig, _ json.RawMessage) error { + return nil +} + +// RegisterInterfaces implements InterfaceModule.RegisterInterfaces +func (a AppModuleBasic) RegisterInterfaces(registry codectypes.InterfaceRegistry) {} + +// RegisterGRPCGatewayRoutes registers the gRPC Gateway routes for the module. +func (a AppModuleBasic) RegisterGRPCGatewayRoutes(clientCtx client.Context, mux *runtime.ServeMux) {} + +// GetTxCmd returns the root tx command for the module. +func (AppModuleBasic) GetTxCmd() *cobra.Command { + return nil +} + +// GetQueryCmd returns no root query command for the module. +func (AppModuleBasic) GetQueryCmd() *cobra.Command { + return nil +} + +//____________________________________________________________________________ + +// AppModule app module type +type AppModule struct { + AppModuleBasic + metrics *types.Metrics +} + +// NewAppModule creates a new AppModule object +func NewAppModule(telemetryOpts types.TelemetryOptions) AppModule { + return AppModule{ + AppModuleBasic: AppModuleBasic{}, + metrics: types.NewMetrics(telemetryOpts), + } +} + +// Name module name +func (am AppModule) Name() string { + return am.AppModuleBasic.Name() +} + +// RegisterInvariants register module invariants +func (am AppModule) RegisterInvariants(_ sdk.InvariantRegistry) {} + +// Route module message route name +// Deprecated: unused but necessary to fulfill AppModule interface +func (am AppModule) Route() sdk.Route { return sdk.Route{} } + +// QuerierRoute module querier route name +// Deprecated: unused but necessary to fulfill AppModule interface +func (AppModule) QuerierRoute() string { return types.ModuleName } + +// LegacyQuerierHandler returns no sdk.Querier. +// Deprecated: unused but necessary to fulfill AppModule interface +func (am AppModule) LegacyQuerierHandler(_ *codec.LegacyAmino) sdk.Querier { + return nil +} + +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } + +// RegisterServices registers module services. +func (am AppModule) RegisterServices(cfg module.Configurator) {} + +// InitGenesis module init-genesis +func (am AppModule) InitGenesis(ctx sdk.Context, _ codec.JSONCodec, _ json.RawMessage) []abci.ValidatorUpdate { + return []abci.ValidatorUpdate{} +} + +// ExportGenesis module export genesis +func (am AppModule) ExportGenesis(_ sdk.Context, cdc codec.JSONCodec) json.RawMessage { + return nil +} + +// BeginBlock module begin-block +func (am AppModule) BeginBlock(ctx sdk.Context, req abci.RequestBeginBlock) { + BeginBlocker(ctx, am.metrics) +} + +// EndBlock module end-block +func (am AppModule) EndBlock(_ sdk.Context, _ abci.RequestEndBlock) []abci.ValidatorUpdate { + return []abci.ValidatorUpdate{} +} diff --git a/x/metrics/spec/README.md b/x/metrics/spec/README.md new file mode 100644 index 00000000..648ce38a --- /dev/null +++ b/x/metrics/spec/README.md @@ -0,0 +1,36 @@ + + +# `metrics` + + +## Abstract + +`x/metrics` is a stateless module that does not affect consensus. It captures chain metrics and emits them when the `instrumentation.prometheus` option is enabled in `config.toml`. + +## Precision + +The metrics emitted by `x/metrics` are `float64`s. They use `github.com/go-kit/kit/metrics` Prometheus gauges. Cosmos-sdk's `telemetry` package was not used because, at the time of writing, it only supports `float32`s and so does not maintain accurate representations of ints larger than ~16.8M. With `float64`s, integers may be accurately represented up to ~9e15. + +## Metrics + +The following metrics are defined: +* `cometbft_blocksync_latest_block_height` - this emulates the blocksync `latest_block_height` metric in CometBFT v0.38+. The `cometbft` namespace comes from the `instrumentation.namespace` config.toml value. + +## Metric Labels + +All metrics emitted have the labels defined in app.toml's `telemetry.global-labels` field. This is the same field used by cosmos-sdk's `telemetry` package. + +example: +```toml +# app.toml +[telemetry] +global-labels = [ + ["chain_id", "kava_2222-10"], + ["my_label", "my_value"], +] +``` diff --git a/x/metrics/types/keys.go b/x/metrics/types/keys.go new file mode 100644 index 00000000..c7a9577a --- /dev/null +++ b/x/metrics/types/keys.go @@ -0,0 +1,6 @@ +package types + +const ( + // Name of the module + ModuleName = "metrics" +) diff --git a/x/metrics/types/metrics.go b/x/metrics/types/metrics.go new file mode 100644 index 00000000..7c01e474 --- /dev/null +++ b/x/metrics/types/metrics.go @@ -0,0 +1,89 @@ +package types + +import ( + "github.com/go-kit/kit/metrics" + "github.com/go-kit/kit/metrics/discard" + prometheus "github.com/go-kit/kit/metrics/prometheus" + stdprometheus "github.com/prometheus/client_golang/prometheus" + "github.com/spf13/cast" + + servertypes "github.com/cosmos/cosmos-sdk/server/types" +) + +// TelemetryOptions defines the app configurations for the x/metrics module +type TelemetryOptions struct { + // CometBFT config value for instrumentation.prometheus (config.toml) + PrometheusEnabled bool + // Namespace for CometBFT metrics. Used to emulate CometBFT metrics. + CometBFTMetricsNamespace string + // A list of keys and values used as labels on all metrics + GlobalLabelsAndValues []string +} + +// TelemetryOptionsFromAppOpts creates the TelemetryOptions from server AppOptions +func TelemetryOptionsFromAppOpts(appOpts servertypes.AppOptions) TelemetryOptions { + prometheusEnabled := cast.ToBool(appOpts.Get("instrumentation.prometheus")) + if !prometheusEnabled { + return TelemetryOptions{ + GlobalLabelsAndValues: []string{}, + } + } + + // parse the app.toml global-labels into a slice of alternating label & value strings + // the value is expected to be a list of [label, value] tuples. + rawLabels := cast.ToSlice(appOpts.Get("telemetry.global-labels")) + globalLabelsAndValues := make([]string, 0, len(rawLabels)*2) + for _, gl := range rawLabels { + l := cast.ToStringSlice(gl) + globalLabelsAndValues = append(globalLabelsAndValues, l[0], l[1]) + } + + return TelemetryOptions{ + PrometheusEnabled: true, + CometBFTMetricsNamespace: cast.ToString(appOpts.Get("instrumentation.namespace")), + GlobalLabelsAndValues: globalLabelsAndValues, + } +} + +// Metrics contains metrics exposed by this module. +// They use go-kit metrics like CometBFT as opposed to using cosmos-sdk telemetry +// because the sdk's telemetry only supports float32s, whereas go-kit prometheus +// metrics correctly handle float64s (and thus a larger number of int64s) +type Metrics struct { + // The height of the latest block. + // This gauges exactly emulates the default blocksync metric in CometBFT v0.38+ + // It should be removed when kava has been updated to CometBFT v0.38+. + // see https://github.com/cometbft/cometbft/blob/v0.38.0-rc3/blocksync/metrics.gen.go + LatestBlockHeight metrics.Gauge +} + +// NewMetrics creates a new Metrics object based on whether or not prometheus instrumentation is enabled. +func NewMetrics(opts TelemetryOptions) *Metrics { + if opts.PrometheusEnabled { + return PrometheusMetrics(opts) + } + return NoopMetrics() +} + +// PrometheusMetrics returns the gauges for when prometheus instrumentation is enabled. +func PrometheusMetrics(opts TelemetryOptions) *Metrics { + labels := []string{} + for i := 0; i < len(opts.GlobalLabelsAndValues); i += 2 { + labels = append(labels, opts.GlobalLabelsAndValues[i]) + } + return &Metrics{ + LatestBlockHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Namespace: opts.CometBFTMetricsNamespace, + Subsystem: "blocksync", + Name: "latest_block_height", + Help: "The height of the latest block.", + }, labels).With(opts.GlobalLabelsAndValues...), + } +} + +// NoopMetrics are a do-nothing placeholder used when prometheus instrumentation is not enabled. +func NoopMetrics() *Metrics { + return &Metrics{ + LatestBlockHeight: discard.NewGauge(), + } +} diff --git a/x/metrics/types/metrics_test.go b/x/metrics/types/metrics_test.go new file mode 100644 index 00000000..2af68d25 --- /dev/null +++ b/x/metrics/types/metrics_test.go @@ -0,0 +1,72 @@ +package types_test + +import ( + "testing" + + "github.com/go-kit/kit/metrics" + "github.com/go-kit/kit/metrics/prometheus" + "github.com/kava-labs/kava/x/metrics/types" + "github.com/stretchr/testify/require" +) + +func isPrometheusGauge(g metrics.Gauge) bool { + _, ok := g.(*prometheus.Gauge) + return ok +} + +var ( + disabledOpts = types.TelemetryOptions{ + PrometheusEnabled: false, + } + enabledOpts = types.TelemetryOptions{ + PrometheusEnabled: true, + CometBFTMetricsNamespace: "cometbft", + GlobalLabelsAndValues: []string{"label1", "value1", "label2", "value2"}, + } +) + +func TestNewMetrics_DisabledVsEnabled(t *testing.T) { + myMetrics := types.NewMetrics(disabledOpts) + require.False(t, isPrometheusGauge(myMetrics.LatestBlockHeight)) + + myMetrics = types.NewMetrics(enabledOpts) + require.True(t, isPrometheusGauge(myMetrics.LatestBlockHeight)) +} + +type MockAppOpts struct { + store map[string]interface{} +} + +func (mao *MockAppOpts) Get(key string) interface{} { + return mao.store[key] +} + +func TestTelemetryOptionsFromAppOpts(t *testing.T) { + appOpts := MockAppOpts{store: make(map[string]interface{})} + + // test disabled functionality + appOpts.store["instrumentation.prometheus"] = false + + opts := types.TelemetryOptionsFromAppOpts(&appOpts) + require.False(t, opts.PrometheusEnabled) + + // test enabled functionality + appOpts.store["instrumentation.prometheus"] = true + appOpts.store["instrumentation.namespace"] = "magic" + appOpts.store["telemetry.global-labels"] = []interface{}{} + + opts = types.TelemetryOptionsFromAppOpts(&appOpts) + require.True(t, opts.PrometheusEnabled) + require.Equal(t, "magic", opts.CometBFTMetricsNamespace) + require.Len(t, opts.GlobalLabelsAndValues, 0) + + appOpts.store["telemetry.global-labels"] = []interface{}{ + []interface{}{"label1", "value1"}, + []interface{}{"label2", "value2"}, + } + opts = types.TelemetryOptionsFromAppOpts(&appOpts) + require.True(t, opts.PrometheusEnabled) + require.Equal(t, "magic", opts.CometBFTMetricsNamespace) + require.Len(t, opts.GlobalLabelsAndValues, 4) + require.Equal(t, enabledOpts.GlobalLabelsAndValues, opts.GlobalLabelsAndValues) +}