stake-pool: Add borsh support and specify size on creation (#1505)

* lending: Update JS tests to solana-test-validator

* Add solana tools install

* Fix oopsie on the path

* Move where deployed programs go

* stake-pool: Add borsh support and size on creation

We can't specify the size in the instruction unfortunately, since we'd
only have 10kb max for the validator list. At roughly 50 bytes per
validator, that only gives us 200 validators.

On the flip side, using Borsh means we can allow the validator stake list
to be any size!

* Add AccountType enum

* Remove V1 everywhere

* Add max validators as parameter and get_instance_packed_len

* Add test for adding too many validators

* Clippy

Author: joncinque

Cargo.lock

@@ -9,6 +9,7 @@ repository = "https://github.com/solana-labs/solana-program-library"
 version = "2.0.1"
 
 [dependencies]
+borsh = "0.8"
 clap = "2.33.3"
 serde_json = "1.0.62"
 solana-account-decoder = "1.6.1"

stake-pool/cli/Cargo.toml

@@ -1,52 +1,58 @@
 #[macro_use]
 extern crate lazy_static;
-use bincode::deserialize;
-use clap::{
-    crate_description, crate_name, crate_version, value_t, value_t_or_exit, App, AppSettings, Arg,
-    ArgGroup, SubCommand,
-};
-use solana_account_decoder::UiAccountEncoding;
-use solana_clap_utils::{
-    input_parsers::pubkey_of,
-    input_validators::{is_amount, is_keypair, is_parsable, is_pubkey, is_url},
-    keypair::signer_from_path,
-};
-use solana_client::{
-    rpc_client::RpcClient,
-    rpc_config::RpcAccountInfoConfig,
-    rpc_config::RpcProgramAccountsConfig,
-    rpc_filter::{Memcmp, MemcmpEncodedBytes, RpcFilterType},
-};
-use solana_program::{instruction::Instruction, program_pack::Pack, pubkey::Pubkey};
-use solana_sdk::{
-    account::Account,
-    commitment_config::CommitmentConfig,
-    native_token::{self, Sol},
-    signature::{Keypair, Signer},
-    system_instruction,
-    transaction::Transaction,
-};
-use spl_stake_pool::{
-    instruction::{
-        add_validator_stake_account, create_validator_stake_account, deposit,
-        initialize as initialize_pool, remove_validator_stake_account, set_owner,
-        update_list_balance, update_pool_balance, withdraw, Fee as PoolFee,
-        InitArgs as PoolInitArgs,
+
+use {
+    bincode::deserialize,
+    borsh::BorshDeserialize,
+    clap::{
+        crate_description, crate_name, crate_version, value_t, value_t_or_exit, App, AppSettings,
+        Arg, ArgGroup, SubCommand,
     },
-    processor::Processor as PoolProcessor,
-    stake::authorize as authorize_stake,
-    stake::id as stake_program_id,
-    stake::StakeAuthorize,
-    stake::StakeState,
-    state::StakePool,
-    state::ValidatorStakeList,
-};
-use spl_token::{
-    self, instruction::approve as approve_token, instruction::initialize_account,
-    instruction::initialize_mint, native_mint, state::Account as TokenAccount,
-    state::Mint as TokenMint,
+    solana_account_decoder::UiAccountEncoding,
+    solana_clap_utils::{
+        input_parsers::pubkey_of,
+        input_validators::{is_amount, is_keypair, is_parsable, is_pubkey, is_url},
+        keypair::signer_from_path,
+    },
+    solana_client::{
+        rpc_client::RpcClient,
+        rpc_config::RpcAccountInfoConfig,
+        rpc_config::RpcProgramAccountsConfig,
+        rpc_filter::{Memcmp, MemcmpEncodedBytes, RpcFilterType},
+    },
+    solana_program::{
+        borsh::get_packed_len, instruction::Instruction, program_pack::Pack, pubkey::Pubkey,
+    },
+    solana_sdk::{
+        account::Account,
+        commitment_config::CommitmentConfig,
+        native_token::{self, Sol},
+        signature::{Keypair, Signer},
+        system_instruction,
+        transaction::Transaction,
+    },
+    spl_stake_pool::{
+        borsh::{get_instance_packed_len, try_from_slice_unchecked},
+        instruction::{
+            add_validator_stake_account, create_validator_stake_account, deposit,
+            initialize as initialize_pool, remove_validator_stake_account, set_owner,
+            update_list_balance, update_pool_balance, withdraw, Fee as PoolFee,
+        },
+        processor::Processor as PoolProcessor,
+        stake::authorize as authorize_stake,
+        stake::id as stake_program_id,
+        stake::StakeAuthorize,
+        stake::StakeState,
+        state::StakePool,
+        state::ValidatorStakeList,
+    },
+    spl_token::{
+        self, instruction::approve as approve_token, instruction::initialize_account,
+        instruction::initialize_mint, native_mint, state::Account as TokenAccount,
+        state::Mint as TokenMint,
+    },
+    std::process::exit,
 };
-use std::process::exit;
 
 struct Config {
     rpc_client: RpcClient,
@@ -109,7 +115,7 @@ fn get_authority_accounts(config: &Config, authority: &Pubkey) -> Vec<(Pubkey, A
         .unwrap()
 }
 
-fn command_create_pool(config: &Config, fee: PoolFee) -> CommandResult {
+fn command_create_pool(config: &Config, fee: PoolFee, max_validators: u32) -> CommandResult {
     let mint_account = Keypair::new();
     println!("Creating mint {}", mint_account.pubkey());
 
@@ -132,10 +138,12 @@ fn command_create_pool(config: &Config, fee: PoolFee) -> CommandResult {
         .get_minimum_balance_for_rent_exemption(TokenAccount::LEN)?;
     let pool_account_balance = config
         .rpc_client
-        .get_minimum_balance_for_rent_exemption(StakePool::LEN)?;
+        .get_minimum_balance_for_rent_exemption(get_packed_len::<StakePool>())?;
+    let empty_validator_list = ValidatorStakeList::new_with_max_validators(max_validators);
+    let validator_stake_list_size = get_instance_packed_len(&empty_validator_list)?;
     let validator_stake_list_balance = config
         .rpc_client
-        .get_minimum_balance_for_rent_exemption(ValidatorStakeList::LEN)?;
+        .get_minimum_balance_for_rent_exemption(validator_stake_list_size)?;
     let total_rent_free_balances = mint_account_balance
         + pool_fee_account_balance
         + pool_account_balance
@@ -177,15 +185,15 @@ fn command_create_pool(config: &Config, fee: PoolFee) -> CommandResult {
                 &config.fee_payer.pubkey(),
                 &pool_account.pubkey(),
                 pool_account_balance,
-                StakePool::LEN as u64,
+                get_packed_len::<StakePool>() as u64,
                 &spl_stake_pool::id(),
             ),
             // Validator stake account list storage
             system_instruction::create_account(
                 &config.fee_payer.pubkey(),
                 &validator_stake_list.pubkey(),
                 validator_stake_list_balance,
-                ValidatorStakeList::LEN as u64,
+                validator_stake_list_size as u64,
                 &spl_stake_pool::id(),
             ),
             // Initialize pool token mint account
@@ -212,7 +220,8 @@ fn command_create_pool(config: &Config, fee: PoolFee) -> CommandResult {
                 &mint_account.pubkey(),
                 &pool_fee_account.pubkey(),
                 &spl_token::id(),
-                PoolInitArgs { fee },
+                fee,
+                max_validators,
             )?,
         ],
         Some(&config.fee_payer.pubkey()),
@@ -274,7 +283,7 @@ fn command_vsa_add(
 ) -> CommandResult {
     // Get stake pool state
     let pool_data = config.rpc_client.get_account_data(&pool)?;
-    let pool_data: StakePool = StakePool::deserialize(pool_data.as_slice()).unwrap();
+    let pool_data = StakePool::try_from_slice(pool_data.as_slice()).unwrap();
 
     let mut total_rent_free_balances: u64 = 0;
 
@@ -365,7 +374,7 @@ fn command_vsa_remove(
 ) -> CommandResult {
     // Get stake pool state
     let pool_data = config.rpc_client.get_account_data(&pool)?;
-    let pool_data: StakePool = StakePool::deserialize(pool_data.as_slice()).unwrap();
+    let pool_data: StakePool = StakePool::try_from_slice(pool_data.as_slice()).unwrap();
 
     let pool_withdraw_authority: Pubkey = PoolProcessor::authority_id(
         &spl_stake_pool::id(),
@@ -495,7 +504,7 @@ fn command_deposit(
 ) -> CommandResult {
     // Get stake pool state
     let pool_data = config.rpc_client.get_account_data(&pool)?;
-    let pool_data: StakePool = StakePool::deserialize(pool_data.as_slice()).unwrap();
+    let pool_data: StakePool = StakePool::try_from_slice(pool_data.as_slice()).unwrap();
 
     // Get stake account data
     let stake_data = config.rpc_client.get_account_data(&stake)?;
@@ -514,7 +523,7 @@ fn command_deposit(
         .rpc_client
         .get_account_data(&pool_data.validator_stake_list)?;
     let validator_stake_list_data =
-        ValidatorStakeList::deserialize(&validator_stake_list_data.as_slice())?;
+        try_from_slice_unchecked::<ValidatorStakeList>(&validator_stake_list_data.as_slice())?;
     if !validator_stake_list_data.contains(&validator) {
         return Err("Stake account for this validator does not exist in the pool.".into());
     }
@@ -617,14 +626,14 @@ fn command_deposit(
 fn command_list(config: &Config, pool: &Pubkey) -> CommandResult {
     // Get stake pool state
     let pool_data = config.rpc_client.get_account_data(&pool)?;
-    let pool_data = StakePool::deserialize(pool_data.as_slice()).unwrap();
+    let pool_data = StakePool::try_from_slice(pool_data.as_slice()).unwrap();
 
     if config.verbose {
         let validator_list = config
             .rpc_client
             .get_account_data(&pool_data.validator_stake_list)?;
         let validator_stake_list_data =
-            ValidatorStakeList::deserialize(&validator_list.as_slice())?;
+            try_from_slice_unchecked::<ValidatorStakeList>(&validator_list.as_slice())?;
         println!("Current validator list");
         for validator in validator_stake_list_data.validators {
             println!(
@@ -669,12 +678,12 @@ fn command_list(config: &Config, pool: &Pubkey) -> CommandResult {
 fn command_update(config: &Config, pool: &Pubkey) -> CommandResult {
     // Get stake pool state
     let pool_data = config.rpc_client.get_account_data(&pool)?;
-    let pool_data = StakePool::deserialize(pool_data.as_slice()).unwrap();
+    let pool_data = StakePool::try_from_slice(pool_data.as_slice()).unwrap();
     let validator_stake_list_data = config
         .rpc_client
         .get_account_data(&pool_data.validator_stake_list)?;
     let validator_stake_list_data =
-        ValidatorStakeList::deserialize(&validator_stake_list_data.as_slice())?;
+        try_from_slice_unchecked::<ValidatorStakeList>(&validator_stake_list_data.as_slice())?;
 
     let epoch_info = config.rpc_client.get_epoch_info()?;
 
@@ -815,7 +824,7 @@ fn command_withdraw(
 ) -> CommandResult {
     // Get stake pool state
     let pool_data = config.rpc_client.get_account_data(&pool)?;
-    let pool_data = StakePool::deserialize(pool_data.as_slice()).unwrap();
+    let pool_data = StakePool::try_from_slice(pool_data.as_slice()).unwrap();
 
     let pool_mint_data = config.rpc_client.get_account_data(&pool_data.pool_mint)?;
     let pool_mint = TokenMint::unpack_from_slice(pool_mint_data.as_slice()).unwrap();
@@ -954,7 +963,7 @@ fn command_set_owner(
     new_fee_receiver: &Option<Pubkey>,
 ) -> CommandResult {
     let pool_data = config.rpc_client.get_account_data(&pool)?;
-    let pool_data: StakePool = StakePool::deserialize(pool_data.as_slice()).unwrap();
+    let pool_data: StakePool = StakePool::try_from_slice(pool_data.as_slice()).unwrap();
 
     // If new accounts are missing in the arguments use the old ones
     let new_owner: Pubkey = match new_owner {
@@ -1088,6 +1097,16 @@ fn main() {
                     .required(true)
                     .help("Fee denominator, fee amount is numerator divided by denominator."),
             )
+            .arg(
+                Arg::with_name("max_validators")
+                    .long("max-validators")
+                    .short("m")
+                    .validator(is_parsable::<u32>)
+                    .value_name("NUMBER")
+                    .takes_value(true)
+                    .required(true)
+                    .help("Max number of validators included in the stake pool"),
+            )
         )
         .subcommand(SubCommand::with_name("create-validator-stake").about("Create a new validator stake account to use with the pool")
             .arg(
@@ -1344,12 +1363,14 @@ fn main() {
         ("create-pool", Some(arg_matches)) => {
             let numerator = value_t_or_exit!(arg_matches, "fee_numerator", u64);
             let denominator = value_t_or_exit!(arg_matches, "fee_denominator", u64);
+            let max_validators = value_t_or_exit!(arg_matches, "max_validators", u32);
             command_create_pool(
                 &config,
                 PoolFee {
                     denominator,
                     numerator,
                 },
+                max_validators,
             )
         }
         ("create-validator-stake", Some(arg_matches)) => {

stake-pool/cli/src/main.rs

@@ -13,6 +13,7 @@ test-bpf = []
 
 [dependencies]
 arrayref = "0.3.6"
+borsh = "0.8"
 num-derive = "0.3"
 num-traits = "0.2"
 num_enum = "0.5.1"
@@ -25,9 +26,10 @@ thiserror = "1.0"
 bincode = "1.3.1"
 
 [dev-dependencies]
+proptest = "0.10"
 solana-program-test = "1.6.1"
 solana-sdk = "1.6.1"
-solana-vote-program = "1.5.11"
+solana-vote-program = "1.6.1"
 
 [lib]
 crate-type = ["cdylib", "lib"]

stake-pool/program/Cargo.toml

@@ -0,0 +1,39 @@
+//! Extra borsh utils
+//! TODO delete once try_from_slice_unchecked has been published
+
+use {
+    borsh::{maybestd::io::Error, BorshDeserialize, BorshSerialize},
+    std::io::{Result as IoResult, Write},
+};
+
+/// Deserializes something and allows for incomplete reading
+pub fn try_from_slice_unchecked<T: BorshDeserialize>(data: &[u8]) -> Result<T, Error> {
+    let mut data_mut = data;
+    let result = T::deserialize(&mut data_mut)?;
+    Ok(result)
+}
+
+/// Helper struct which to count how much data would be written during serialization
+#[derive(Default)]
+struct WriteCounter {
+    count: usize,
+}
+
+impl Write for WriteCounter {
+    fn write(&mut self, data: &[u8]) -> IoResult<usize> {
+        let amount = data.len();
+        self.count += amount;
+        Ok(amount)
+    }
+
+    fn flush(&mut self) -> IoResult<()> {
+        Ok(())
+    }
+}
+
+/// Get the worst-case packed length for the given BorshSchema
+pub fn get_instance_packed_len<T: BorshSerialize>(instance: &T) -> Result<usize, Error> {
+    let mut counter = WriteCounter::default();
+    instance.serialize(&mut counter)?;
+    Ok(counter.count)
+}

stake-pool/program/src/borsh.rs

@@ -37,8 +37,6 @@ pub enum StakePoolError {
     /// Invalid validator stake list account.
     #[error("InvalidValidatorStakeList")]
     InvalidValidatorStakeList,
-
-    // 10.
     /// Invalid owner fee account.
     #[error("InvalidFeeAccount")]
     InvalidFeeAccount,
@@ -56,8 +54,6 @@ pub enum StakePoolError {
     /// Stake account voting for this validator already exists in the pool.
     #[error("ValidatorAlreadyAdded")]
     ValidatorAlreadyAdded,
-
-    // 15.
     /// Stake account for this validator not found in the pool.
     #[error("ValidatorNotFound")]
     ValidatorNotFound,
@@ -75,16 +71,14 @@ pub enum StakePoolError {
     /// Validator stake account is not found in the list storage.
     #[error("UnknownValidatorStakeAccount")]
     UnknownValidatorStakeAccount,
-
-    // 20.
     /// Wrong minting authority set for mint pool account
     #[error("WrongMintingAuthority")]
     WrongMintingAuthority,
 
     // 20.
-    /// Account is not rent-exempt
-    #[error("AccountNotRentExempt")]
-    AccountNotRentExempt,
+    /// The size of the given validator stake list does match the expected amount
+    #[error("UnexpectedValidatorListAccountSize")]
+    UnexpectedValidatorListAccountSize,
 }
 impl From<StakePoolError> for ProgramError {
     fn from(e: StakePoolError) -> Self {