Add /cdp/totalPrincipal and /cdp/totalCollateral Endpoints (#1027)

* Add cdp/totals rest endpoint for aggregated principal and fees

* Add total collateral to aggregated cdp

* Add pagination

* Update totalPrincipal endpoint to use keeper GetTotalPrincipal

* Update documentation for queryGetTotalPrincipal

Co-authored-by: Kevin Davis <karzak@users.noreply.github.com>

* Remove unused AggregatedCDP types

* Make CDP type optional, return all if not provided

* add total collateral endpoint that efficient fetches collateral
for a types, only iterating cdps for types b, etc

* correctly and efficienlty filter for a single collateral type

Co-authored-by: Kevin Davis <karzak@users.noreply.github.com>
Co-authored-by: Nick DeLuca <nickdeluca08@gmail.com>

x/cdp/client/rest/query.go

@@ -18,8 +18,10 @@ import (
 func registerQueryRoutes(cliCtx context.CLIContext, r *mux.Router) {
 	r.HandleFunc("/cdp/accounts", getAccountsHandlerFn(cliCtx)).Methods("GET")
 	r.HandleFunc("/cdp/parameters", getParamsHandlerFn(cliCtx)).Methods("GET")
+	r.HandleFunc("/cdp/totalPrincipal", getTotalPrincipal(cliCtx)).Methods("GET")
+	r.HandleFunc("/cdp/totalCollateral", getTotalCollateral(cliCtx)).Methods("GET")
+	r.HandleFunc("/cdp/cdps", queryCdpsHandlerFn(cliCtx)).Methods("GET")
 	r.HandleFunc(fmt.Sprintf("/cdp/cdps/cdp/{%s}/{%s}", types.RestOwner, types.RestCollateralType), queryCdpHandlerFn(cliCtx)).Methods("GET")
-	r.HandleFunc(fmt.Sprintf("/cdp/cdps"), queryCdpsHandlerFn(cliCtx)).Methods("GET")
 	r.HandleFunc(fmt.Sprintf("/cdp/cdps/collateralType/{%s}", types.RestCollateralType), queryCdpsByCollateralTypeHandlerFn(cliCtx)).Methods("GET")     // legacy
 	r.HandleFunc(fmt.Sprintf("/cdp/cdps/ratio/{%s}/{%s}", types.RestCollateralType, types.RestRatio), queryCdpsByRatioHandlerFn(cliCtx)).Methods("GET") // legacy
 	r.HandleFunc(fmt.Sprintf("/cdp/cdps/cdp/deposits/{%s}/{%s}", types.RestOwner, types.RestCollateralType), queryCdpDepositsHandlerFn(cliCtx)).Methods("GET")
@@ -266,3 +268,69 @@ func queryCdpsHandlerFn(cliCtx context.CLIContext) http.HandlerFunc {
 		rest.PostProcessResponse(w, cliCtx, res)
 	}
 }
+
+func getTotalPrincipal(cliCtx context.CLIContext) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		// Parse the query height
+		cliCtx, ok := rest.ParseQueryHeightOrReturnBadRequest(w, cliCtx, r)
+		if !ok {
+			return
+		}
+
+		var cdpCollateralType string
+
+		if x := r.URL.Query().Get(RestCollateralType); len(x) != 0 {
+			cdpCollateralType = strings.TrimSpace(x)
+		}
+
+		params := types.NewQueryGetTotalPrincipalParams(cdpCollateralType)
+		bz, err := cliCtx.Codec.MarshalJSON(params)
+		if err != nil {
+			rest.WriteErrorResponse(w, http.StatusBadRequest, err.Error())
+			return
+		}
+
+		route := fmt.Sprintf("custom/%s/%s", types.ModuleName, types.QueryGetTotalPrincipal)
+		res, height, err := cliCtx.QueryWithData(route, bz)
+		cliCtx = cliCtx.WithHeight(height)
+		if err != nil {
+			rest.WriteErrorResponse(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+
+		rest.PostProcessResponse(w, cliCtx, res)
+	}
+}
+
+func getTotalCollateral(cliCtx context.CLIContext) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		// Parse the query height
+		cliCtx, ok := rest.ParseQueryHeightOrReturnBadRequest(w, cliCtx, r)
+		if !ok {
+			return
+		}
+
+		var cdpCollateralType string
+
+		if x := r.URL.Query().Get(RestCollateralType); len(x) != 0 {
+			cdpCollateralType = strings.TrimSpace(x)
+		}
+
+		params := types.NewQueryGetTotalCollateralParams(cdpCollateralType)
+		bz, err := cliCtx.Codec.MarshalJSON(params)
+		if err != nil {
+			rest.WriteErrorResponse(w, http.StatusBadRequest, err.Error())
+			return
+		}
+
+		route := fmt.Sprintf("custom/%s/%s", types.ModuleName, types.QueryGetTotalCollateral)
+		res, height, err := cliCtx.QueryWithData(route, bz)
+		cliCtx = cliCtx.WithHeight(height)
+		if err != nil {
+			rest.WriteErrorResponse(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+
+		rest.PostProcessResponse(w, cliCtx, res)
+	}
+}

x/cdp/keeper/querier.go

@@ -1,6 +1,8 @@
 package keeper
 
 import (
+	"sort"
+
 	abci "github.com/tendermint/tendermint/abci/types"
 
 	"github.com/cosmos/cosmos-sdk/client"
@@ -30,6 +32,10 @@ func NewQuerier(keeper Keeper) sdk.Querier {
 			return queryGetParams(ctx, req, keeper)
 		case types.QueryGetAccounts:
 			return queryGetAccounts(ctx, req, keeper)
+		case types.QueryGetTotalPrincipal:
+			return queryGetTotalPrincipal(ctx, req, keeper)
+		case types.QueryGetTotalCollateral:
+			return queryGetTotalCollateral(ctx, req, keeper)
 		default:
 			return nil, sdkerrors.Wrapf(sdkerrors.ErrUnknownRequest, "unknown %s query endpoint %s", types.ModuleName, path[0])
 		}
@@ -199,6 +205,138 @@ func queryGetCdps(ctx sdk.Context, req abci.RequestQuery, keeper Keeper) ([]byte
 	return bz, nil
 }
 
+// query total amount of principal (ie. usdx) that has been minted with a particular collateral type
+func queryGetTotalPrincipal(ctx sdk.Context, req abci.RequestQuery, keeper Keeper) ([]byte, error) {
+	var params types.QueryGetTotalPrincipalParams
+	err := types.ModuleCdc.UnmarshalJSON(req.Data, &params)
+	if err != nil {
+		return nil, sdkerrors.Wrap(sdkerrors.ErrJSONUnmarshal, err.Error())
+	}
+
+	var queryCollateralTypes []string
+
+	if params.CollateralType != "" {
+		// Single collateralType provided
+		queryCollateralTypes = append(queryCollateralTypes, params.CollateralType)
+	} else {
+		// No collateralType provided, respond with all of them
+		keeperParams := keeper.GetParams(ctx)
+
+		for _, collateral := range keeperParams.CollateralParams {
+			queryCollateralTypes = append(queryCollateralTypes, collateral.Type)
+		}
+	}
+
+	var collateralPrincipals []types.TotalCDPPrincipal
+
+	for _, queryType := range queryCollateralTypes {
+		// Hardcoded to default USDX
+		principalAmount := keeper.GetTotalPrincipal(ctx, queryType, types.DefaultStableDenom)
+		// Wrap it in an sdk.Coin
+		totalAmountCoin := sdk.NewCoin(types.DefaultStableDenom, principalAmount)
+
+		totalPrincipal := types.NewTotalCDPPrincipal(queryType, totalAmountCoin)
+		collateralPrincipals = append(collateralPrincipals, totalPrincipal)
+	}
+
+	bz, err := codec.MarshalJSONIndent(keeper.cdc, collateralPrincipals)
+	if err != nil {
+		return nil, sdkerrors.Wrap(sdkerrors.ErrJSONMarshal, err.Error())
+	}
+
+	return bz, nil
+}
+
+// query total amount of collateral (ie. btcb) that has been deposited with a particular collateral type
+func queryGetTotalCollateral(ctx sdk.Context, req abci.RequestQuery, keeper Keeper) ([]byte, error) {
+	var request types.QueryGetTotalCollateralParams
+	err := types.ModuleCdc.UnmarshalJSON(req.Data, &request)
+	if err != nil {
+		return nil, sdkerrors.Wrap(sdkerrors.ErrJSONUnmarshal, err.Error())
+	}
+
+	params := keeper.GetParams(ctx)
+	denomCollateralTypes := make(map[string][]string)
+
+	// collect collateral types for each denom
+	for _, collateralParam := range params.CollateralParams {
+		denomCollateralTypes[collateralParam.Denom] =
+			append(denomCollateralTypes[collateralParam.Denom], collateralParam.Type)
+	}
+
+	// sort collateral types alphabetically
+	for _, collateralTypes := range denomCollateralTypes {
+		sort.Slice(collateralTypes, func(i int, j int) bool {
+			return collateralTypes[i] < collateralTypes[j]
+		})
+	}
+
+	// get total collateral in all cdps
+	cdpAccount := keeper.supplyKeeper.GetModuleAccount(ctx, types.ModuleName)
+	totalCdpCollateral := cdpAccount.GetCoins()
+
+	var response []types.TotalCDPCollateral
+
+	for denom, collateralTypes := range denomCollateralTypes {
+		// skip any denoms that do not match the requested collateral type
+		if request.CollateralType != "" {
+			match := false
+			for _, ctype := range collateralTypes {
+				if ctype == request.CollateralType {
+					match = true
+				}
+			}
+
+			if !match {
+				continue
+			}
+		}
+
+		totalCollateral := totalCdpCollateral.AmountOf(denom)
+
+		// we need to query individual cdps for denoms with more than one collateral type
+		for i := len(collateralTypes) - 1; i > 0; i-- {
+			cdps := keeper.GetAllCdpsByCollateralType(ctx, collateralTypes[i])
+
+			collateral := sdk.ZeroInt()
+
+			for _, cdp := range cdps {
+				collateral = collateral.Add(cdp.Collateral.Amount)
+			}
+
+			totalCollateral = totalCollateral.Sub(collateral)
+
+			// if we have no collateralType filter, or the filter matches, include it in the response
+			if request.CollateralType == "" || collateralTypes[i] == request.CollateralType {
+				response = append(response, types.NewTotalCDPCollateral(collateralTypes[i], sdk.NewCoin(denom, collateral)))
+			}
+
+			// skip the rest of the cdp queries if we have a matching filter
+			if collateralTypes[i] == request.CollateralType {
+				break
+			}
+		}
+
+		if request.CollateralType == "" || collateralTypes[0] == request.CollateralType {
+			// all leftover total collateral belongs to the first collateral type
+			response = append(response, types.NewTotalCDPCollateral(collateralTypes[0], sdk.NewCoin(denom, totalCollateral)))
+		}
+	}
+
+	// sort to ensure deterministic response
+	sort.Slice(response, func(i int, j int) bool {
+		return response[i].CollateralType < response[j].CollateralType
+	})
+
+	// encode response
+	bz, err := codec.MarshalJSONIndent(keeper.cdc, response)
+	if err != nil {
+		return nil, sdkerrors.Wrap(sdkerrors.ErrJSONMarshal, err.Error())
+	}
+
+	return bz, nil
+}
+
 // FilterCDPs queries the store for all CDPs that match query params
 func FilterCDPs(ctx sdk.Context, k Keeper, params types.QueryCdpsParams) types.AugmentedCDPs {
 	var matchCollateralType, matchOwner, matchID, matchRatio types.CDPs

x/cdp/keeper/querier_test.go

@@ -1,6 +1,7 @@
 package keeper_test
 
 import (
+	"fmt"
 	"math/rand"
 	"sort"
 	"strings"
@@ -374,6 +375,96 @@ func (suite *QuerierTestSuite) TestQueryCdps() {
 	suite.Equal(50, len(output))
 }
 
+func (suite *QuerierTestSuite) TestQueryTotalPrincipal() {
+	ctx := suite.ctx.WithIsCheckTx(false)
+	params := types.NewQueryGetTotalPrincipalParams("btc-a")
+
+	query := abci.RequestQuery{
+		Path: strings.Join([]string{custom, types.QuerierRoute, types.QueryGetTotalPrincipal}, "/"),
+		Data: types.ModuleCdc.MustMarshalJSON(params),
+	}
+
+	bz, err := suite.querier(ctx, []string{types.QueryGetTotalPrincipal}, query)
+	suite.Nil(err)
+	suite.NotNil(bz)
+
+	output := []types.TotalCDPPrincipal{}
+	suite.Nil(types.ModuleCdc.UnmarshalJSON(bz, &output))
+	fmt.Printf("%s", output)
+	suite.Equal(1, len(output))
+	suite.Equal("btc-a", output[0].CollateralType)
+}
+
+func (suite *QuerierTestSuite) TestQueryTotalPrincipalAll() {
+	ctx := suite.ctx.WithIsCheckTx(false)
+	params := types.NewQueryGetTotalPrincipalParams("")
+
+	query := abci.RequestQuery{
+		Path: strings.Join([]string{custom, types.QuerierRoute, types.QueryGetTotalPrincipal}, "/"),
+		Data: types.ModuleCdc.MustMarshalJSON(params),
+	}
+
+	bz, err := suite.querier(ctx, []string{types.QueryGetTotalPrincipal}, query)
+	suite.Nil(err)
+	suite.NotNil(bz)
+
+	output := []types.TotalCDPPrincipal{}
+	suite.Nil(types.ModuleCdc.UnmarshalJSON(bz, &output))
+
+	var outputTypes []string
+	for _, c := range output {
+		outputTypes = append(outputTypes, c.CollateralType)
+	}
+
+	suite.Greater(len(output), 0)
+	suite.Subset(outputTypes, []string{"btc-a", "xrp-a"})
+}
+
+func (suite *QuerierTestSuite) TestQueryTotalCollateral() {
+	ctx := suite.ctx.WithIsCheckTx(false)
+	params := types.NewQueryGetTotalCollateralParams("btc-a")
+
+	query := abci.RequestQuery{
+		Path: strings.Join([]string{custom, types.QuerierRoute, types.QueryGetTotalCollateral}, "/"),
+		Data: types.ModuleCdc.MustMarshalJSON(params),
+	}
+
+	bz, err := suite.querier(ctx, []string{types.QueryGetTotalCollateral}, query)
+	suite.Nil(err)
+	suite.NotNil(bz)
+
+	output := []types.TotalCDPCollateral{}
+	suite.Nil(types.ModuleCdc.UnmarshalJSON(bz, &output))
+	fmt.Printf("%s", output)
+	suite.Equal(1, len(output))
+	suite.Equal("btc-a", output[0].CollateralType)
+}
+
+func (suite *QuerierTestSuite) TestQueryTotalCollateralAll() {
+	ctx := suite.ctx.WithIsCheckTx(false)
+	params := types.NewQueryGetTotalCollateralParams("")
+
+	query := abci.RequestQuery{
+		Path: strings.Join([]string{custom, types.QuerierRoute, types.QueryGetTotalCollateral}, "/"),
+		Data: types.ModuleCdc.MustMarshalJSON(params),
+	}
+
+	bz, err := suite.querier(ctx, []string{types.QueryGetTotalCollateral}, query)
+	suite.Nil(err)
+	suite.NotNil(bz)
+
+	output := []types.TotalCDPCollateral{}
+	suite.Nil(types.ModuleCdc.UnmarshalJSON(bz, &output))
+
+	var outputTypes []string
+	for _, c := range output {
+		outputTypes = append(outputTypes, c.CollateralType)
+	}
+
+	suite.Greater(len(output), 0)
+	suite.Subset(outputTypes, []string{"btc-a", "xrp-a"})
+}
+
 func TestQuerierTestSuite(t *testing.T) {
 	suite.Run(t, new(QuerierTestSuite))
 }

x/cdp/types/cdp.go

@@ -203,3 +203,31 @@ func (augcdps AugmentedCDPs) String() string {
 	}
 	return out
 }
+
+// TotalCDPPrincipal is a total principal of a given collateral type
+type TotalCDPPrincipal struct {
+	CollateralType string   `json:"collateral_type" yaml:"collateral_type"` // string representing the unique collateral type of the CDP
+	Amount         sdk.Coin `json:"amount" yaml:"amount"`                   // Amount of principal stored in this CDP
+}
+
+// TotalCDPPrincipal returns a new TotalCDPPrincipal
+func NewTotalCDPPrincipal(collateralType string, amount sdk.Coin) TotalCDPPrincipal {
+	return TotalCDPPrincipal{
+		CollateralType: collateralType,
+		Amount:         amount,
+	}
+}
+
+// TotalCDPCollateral is a total principal of a given collateral type
+type TotalCDPCollateral struct {
+	CollateralType string   `json:"collateral_type" yaml:"collateral_type"` // string representing the unique collateral type of the CDP
+	Amount         sdk.Coin `json:"amount" yaml:"amount"`                   // Amount of collateral stored in this CDP
+}
+
+// TotalCDPCollateral returns a new TotalCDPCollateral
+func NewTotalCDPCollateral(collateralType string, amount sdk.Coin) TotalCDPCollateral {
+	return TotalCDPCollateral{
+		CollateralType: collateralType,
+		Amount:         amount,
+	}
+}

x/cdp/types/querier.go

@@ -13,6 +13,8 @@ const (
 	QueryGetCdpsByCollateralType    = "collateralType" // legacy query, maintained for REST API
 	QueryGetParams                  = "params"
 	QueryGetAccounts                = "accounts"
+	QueryGetTotalPrincipal          = "totalPrincipal"
+	QueryGetTotalCollateral         = "totalCollateral"
 	RestOwner                       = "owner"
 	RestCollateralType              = "collateral-type"
 	RestRatio                       = "ratio"
@@ -93,3 +95,27 @@ func NewQueryCdpsByRatioParams(collateralType string, ratio sdk.Dec) QueryCdpsBy
 		Ratio:          ratio,
 	}
 }
+
+// QueryGetTotalPrincipalParams params for query /cdp/totalPrincipal
+type QueryGetTotalPrincipalParams struct {
+	CollateralType string
+}
+
+// NewQueryGetTotalPrincipalParams returns QueryGetTotalPrincipalParams
+func NewQueryGetTotalPrincipalParams(collateralType string) QueryGetTotalPrincipalParams {
+	return QueryGetTotalPrincipalParams{
+		CollateralType: collateralType,
+	}
+}
+
+// QueryGetTotalCollateralParams params for query /cdp/totalCollateral
+type QueryGetTotalCollateralParams struct {
+	CollateralType string
+}
+
+// NewQueryGetTotalCollateralParams returns QueryGetTotalCollateralParams
+func NewQueryGetTotalCollateralParams(collateralType string) QueryGetTotalCollateralParams {
+	return QueryGetTotalCollateralParams{
+		CollateralType: collateralType,
+	}
+}