From d2161aa93ad85eef086bccacfc8c12b2ae550924 Mon Sep 17 00:00:00 2001 From: Ayaz Abbas Date: Wed, 4 Sep 2024 13:20:24 +0100 Subject: [PATCH 1/7] create new mapping accounts when existing are full --- program_admin/__init__.py | 34 +++++++++++++++++++--------------- tests/test_sync.py | 3 ++- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/program_admin/__init__.py b/program_admin/__init__.py index 4561252..036caf7 100644 --- a/program_admin/__init__.py +++ b/program_admin/__init__.py @@ -1,5 +1,6 @@ import asyncio import json +import math import os from pathlib import Path from typing import Dict, List, Literal, Optional, Tuple @@ -246,8 +247,10 @@ async def sync( await self.send_transaction(authority_instructions, authority_signers) # Sync mapping accounts + + num_desired_mapping_accounts = math.ceil(len(ref_products) / 640) mapping_instructions, mapping_keypairs = await self.sync_mapping_instructions( - generate_keys + generate_keys, num_desired_mapping_accounts ) if mapping_instructions: @@ -257,10 +260,6 @@ async def sync( await self.refresh_program_accounts() - # FIXME: We should check if the mapping account has enough space to - # add/remove new products. That is not urgent because we are around 10% - # of the first mapping account capacity. - # Sync product/price accounts product_transactions: List[asyncio.Task[None]] = [] @@ -336,26 +335,31 @@ async def sync( async def sync_mapping_instructions( self, generate_keys: bool, + num_desired_accounts: int = 1, ) -> Tuple[List[TransactionInstruction], List[Keypair]]: - mapping_chain = sort_mapping_account_keys(list(self._mapping_accounts.values())) funding_keypair = load_keypair("funding", key_dir=self.key_dir) - mapping_0_keypair = load_keypair( - "mapping_0", key_dir=self.key_dir, generate=generate_keys - ) + + mapping_keypairs: List[Keypair] = [] + for n in range(0, num_desired_accounts): + mapping_keypairs.append( + load_keypair( + f"mapping_{n}", key_dir=self.key_dir, generate=generate_keys + ) + ) + instructions: List[TransactionInstruction] = [] - if not mapping_chain: - logger.info("Creating new mapping account") + for mapping_keypair in mapping_keypairs: if not ( - await account_exists(self.rpc_endpoint, mapping_0_keypair.public_key) + await account_exists(self.rpc_endpoint, mapping_keypair.public_key) ): logger.debug("Building system.program.create_account instruction") instructions.append( system_program.create_account( system_program.CreateAccountParams( from_pubkey=funding_keypair.public_key, - new_account_pubkey=mapping_0_keypair.public_key, + new_account_pubkey=mapping_keypair.public_key, # FIXME: Change to minimum rent-exempt amount lamports=await self.fetch_minimum_balance( MAPPING_ACCOUNT_SIZE @@ -371,11 +375,11 @@ async def sync_mapping_instructions( pyth_program.init_mapping( self.program_key, funding_keypair.public_key, - mapping_0_keypair.public_key, + mapping_keypair.public_key, ) ) - return (instructions, [funding_keypair, mapping_0_keypair]) + return (instructions, [funding_keypair] + mapping_keypairs) async def sync_product_instructions( self, diff --git a/tests/test_sync.py b/tests/test_sync.py index 4a0370c..d235065 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -21,6 +21,7 @@ from program_admin.util import apply_overrides LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.getLevelNamesMapping()[os.getenv("LOG_LEVEL", "INFO")]) BTC_USD = { "account": "", @@ -221,7 +222,7 @@ def localhost_overrides_json(): @pytest.fixture async def validator(): process = await asyncio.create_subprocess_shell( - "solana-test-validator", + "solana-test-validator --reset", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) From ee5a73ade6d0a8b6c64cc2512da3d1746dfc2db5 Mon Sep 17 00:00:00 2001 From: Ayaz Abbas Date: Thu, 5 Sep 2024 11:54:12 +0100 Subject: [PATCH 2/7] implement add_mapping instruction --- Makefile | 2 +- program_admin/__init__.py | 65 +++++++++++++++++++++++++++-------- program_admin/instructions.py | 36 +++++++++++++++++-- program_admin/util.py | 20 +++++++++++ tests/test_sync.py | 2 +- 5 files changed, 106 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 82728d1..31839ce 100644 --- a/Makefile +++ b/Makefile @@ -8,4 +8,4 @@ install: poetry install test: - poetry run pytest + poetry run pytest -rx diff --git a/program_admin/__init__.py b/program_admin/__init__.py index 036caf7..b2be0bc 100644 --- a/program_admin/__init__.py +++ b/program_admin/__init__.py @@ -30,12 +30,14 @@ ) from program_admin.util import ( MAPPING_ACCOUNT_SIZE, + MAPPING_ACCOUNT_PRODUCT_LIMIT, PRICE_ACCOUNT_V1_SIZE, PRICE_ACCOUNT_V2_SIZE, PRODUCT_ACCOUNT_SIZE, account_exists, compute_transaction_size, get_actual_signers, + get_available_mapping_account_key, recent_blockhash, sort_mapping_account_keys, ) @@ -248,9 +250,10 @@ async def sync( # Sync mapping accounts - num_desired_mapping_accounts = math.ceil(len(ref_products) / 640) + # Create all the mapping accounts we need for the the number of product accounts + num_mapping_accounts = math.ceil(len(ref_products) / MAPPING_ACCOUNT_PRODUCT_LIMIT) mapping_instructions, mapping_keypairs = await self.sync_mapping_instructions( - generate_keys, num_desired_mapping_accounts + generate_keys, num_mapping_accounts ) if mapping_instructions: @@ -333,24 +336,51 @@ async def sync( return instructions async def sync_mapping_instructions( - self, - generate_keys: bool, - num_desired_accounts: int = 1, + self, generate_keys: bool, num_mapping_accounts: int = 1 ) -> Tuple[List[TransactionInstruction], List[Keypair]]: funding_keypair = load_keypair("funding", key_dir=self.key_dir) + mapping_keypair_0 = load_keypair( + "mapping_0", key_dir=self.key_dir, generate=generate_keys + ) + logger.debug(f"mapping_0 public key: {mapping_keypair_0.public_key}") + + instructions: List[TransactionInstruction] = [] + + if not await account_exists(self.rpc_endpoint, mapping_keypair_0.public_key): + logger.debug("Building system.program.create_account instruction") + instructions.append( + system_program.create_account( + system_program.CreateAccountParams( + from_pubkey=funding_keypair.public_key, + new_account_pubkey=mapping_keypair_0.public_key, + # FIXME: Change to minimum rent-exempt amount + lamports=await self.fetch_minimum_balance(MAPPING_ACCOUNT_SIZE), + space=MAPPING_ACCOUNT_SIZE, + program_id=self.program_key, + ) + ) + ) + + logger.debug("Building pyth_program.init_mapping instruction") + instructions.append( + pyth_program.init_mapping( + self.program_key, + funding_keypair.public_key, + mapping_keypair_0.public_key, + ) + ) mapping_keypairs: List[Keypair] = [] - for n in range(0, num_desired_accounts): - mapping_keypairs.append( + if num_mapping_accounts > 1: + mapping_keypairs: List[Keypair] = [ load_keypair( f"mapping_{n}", key_dir=self.key_dir, generate=generate_keys ) - ) - - instructions: List[TransactionInstruction] = [] + for n in range(1, num_mapping_accounts) + ] + tail_mapping_keypair = mapping_keypair_0 for mapping_keypair in mapping_keypairs: - if not ( await account_exists(self.rpc_endpoint, mapping_keypair.public_key) ): @@ -372,14 +402,17 @@ async def sync_mapping_instructions( logger.debug("Building pyth_program.init_mapping instruction") instructions.append( - pyth_program.init_mapping( + pyth_program.add_mapping( self.program_key, funding_keypair.public_key, + tail_mapping_keypair.public_key, mapping_keypair.public_key, ) ) - return (instructions, [funding_keypair] + mapping_keypairs) + tail_mapping_keypair = mapping_keypair + + return (instructions, [funding_keypair, mapping_keypair_0] + mapping_keypairs) async def sync_product_instructions( self, @@ -389,8 +422,10 @@ async def sync_product_instructions( ) -> Tuple[List[TransactionInstruction], List[Keypair]]: instructions: List[TransactionInstruction] = [] funding_keypair = load_keypair("funding", key_dir=self.key_dir) - mapping_chain = sort_mapping_account_keys(list(self._mapping_accounts.values())) - mapping_keypair = load_keypair(mapping_chain[-1], key_dir=self.key_dir) + mapping_keypair = load_keypair( + get_available_mapping_account_key(list(self._mapping_accounts.values())), + key_dir=self.key_dir, + ) product_keypair = load_keypair( f"product_{product['jump_symbol']}", key_dir=self.key_dir, diff --git a/program_admin/instructions.py b/program_admin/instructions.py index 0f1d59c..a849a34 100644 --- a/program_admin/instructions.py +++ b/program_admin/instructions.py @@ -8,9 +8,8 @@ from program_admin.types import ReferenceAuthorityPermissions from program_admin.util import encode_product_metadata, get_permissions_account -# TODO: Implement add_mapping instruction - COMMAND_INIT_MAPPING = 0 +COMMAND_ADD_MAPPING = 1 COMMAND_ADD_PRODUCT = 2 COMMAND_UPD_PRODUCT = 3 COMMAND_ADD_PRICE = 4 @@ -59,6 +58,39 @@ def init_mapping( ) +def add_mapping( + program_key: PublicKey, + funding_key: PublicKey, + tail_mapping_key: PublicKey, + mapping_key: PublicKey, +) -> TransactionInstruction: + """ + Pyth program init_mapping instruction + + accounts: + - funding account (signer, writable) + - tail mapping account (signer, writable) + - mapping account (signer, writable) + """ + layout = Struct("version" / Int32ul, "command" / Int32sl) + data = layout.build(dict(version=PROGRAM_VERSION, command=COMMAND_INIT_MAPPING)) + + permissions_account = get_permissions_account( + program_key, AUTHORITY_PERMISSIONS_PDA_SEED + ) + + return TransactionInstruction( + data=data, + keys=[ + AccountMeta(pubkey=funding_key, is_signer=True, is_writable=True), + AccountMeta(pubkey=tail_mapping_key, is_signer=True, is_writable=True), + AccountMeta(pubkey=mapping_key, is_signer=True, is_writable=True), + AccountMeta(pubkey=permissions_account, is_signer=False, is_writable=True), + ], + program_id=program_key, + ) + + def add_product( program_key: PublicKey, funding_key: PublicKey, diff --git a/program_admin/util.py b/program_admin/util.py index 3b7b515..fac6359 100644 --- a/program_admin/util.py +++ b/program_admin/util.py @@ -103,6 +103,26 @@ def sort_mapping_account_keys(accounts: List[PythMappingAccount]) -> List[Public return sorted_keys +def get_available_mapping_account_key( + mapping_accounts: List[PythMappingAccount], +) -> PublicKey: + """ + Returns the first non-full mapping account. + """ + # Create a dictionary with account key as key and product count as value + num_products_by_key = { + account.public_key: account.data.product_count for account in mapping_accounts + } + + sorted_keys = sort_mapping_account_keys(mapping_accounts) + + for key in sorted_keys: + if num_products_by_key[key] < MAPPING_ACCOUNT_PRODUCT_LIMIT: + return key + + raise RuntimeError("All mapping accounts are full") + + def apply_overrides( ref_permissions: ReferencePermissions, ref_overrides: ReferenceOverrides, diff --git a/tests/test_sync.py b/tests/test_sync.py index d235065..3145a04 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -21,7 +21,7 @@ from program_admin.util import apply_overrides LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.getLevelNamesMapping()[os.getenv("LOG_LEVEL", "INFO")]) +LOGGER.setLevel(logging.getLevelName(os.getenv("LOG_LEVEL", "INFO").upper())) BTC_USD = { "account": "", From a687f8ab899e31bd1859977532ef1b7318591625 Mon Sep 17 00:00:00 2001 From: Ayaz Abbas Date: Thu, 5 Sep 2024 12:10:42 +0100 Subject: [PATCH 3/7] fix - move init_mapping instruction outside of if block --- program_admin/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/program_admin/__init__.py b/program_admin/__init__.py index b2be0bc..b27ff5d 100644 --- a/program_admin/__init__.py +++ b/program_admin/__init__.py @@ -361,14 +361,14 @@ async def sync_mapping_instructions( ) ) - logger.debug("Building pyth_program.init_mapping instruction") - instructions.append( - pyth_program.init_mapping( - self.program_key, - funding_keypair.public_key, - mapping_keypair_0.public_key, - ) + logger.debug("Building pyth_program.init_mapping instruction") + instructions.append( + pyth_program.init_mapping( + self.program_key, + funding_keypair.public_key, + mapping_keypair_0.public_key, ) + ) mapping_keypairs: List[Keypair] = [] if num_mapping_accounts > 1: From 2c8a35fe6d7c6c7391cf6b1d717d091194c6520d Mon Sep 17 00:00:00 2001 From: Ayaz Abbas Date: Thu, 5 Sep 2024 12:50:00 +0100 Subject: [PATCH 4/7] correct log --- program_admin/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program_admin/__init__.py b/program_admin/__init__.py index b27ff5d..37f6291 100644 --- a/program_admin/__init__.py +++ b/program_admin/__init__.py @@ -400,7 +400,7 @@ async def sync_mapping_instructions( ) ) - logger.debug("Building pyth_program.init_mapping instruction") + logger.debug("Building pyth_program.add_mapping instruction") instructions.append( pyth_program.add_mapping( self.program_key, From dec6bb0bae50346c5dcbe72229bd4a9e95b0d797 Mon Sep 17 00:00:00 2001 From: Ayaz Abbas Date: Thu, 5 Sep 2024 13:51:19 +0100 Subject: [PATCH 5/7] fix --- program_admin/instructions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program_admin/instructions.py b/program_admin/instructions.py index a849a34..5b8532b 100644 --- a/program_admin/instructions.py +++ b/program_admin/instructions.py @@ -73,7 +73,7 @@ def add_mapping( - mapping account (signer, writable) """ layout = Struct("version" / Int32ul, "command" / Int32sl) - data = layout.build(dict(version=PROGRAM_VERSION, command=COMMAND_INIT_MAPPING)) + data = layout.build(dict(version=PROGRAM_VERSION, command=COMMAND_ADD_MAPPING)) permissions_account = get_permissions_account( program_key, AUTHORITY_PERMISSIONS_PDA_SEED From 1503f4eb3ac065dffb3d301d935bac3084e0bbe4 Mon Sep 17 00:00:00 2001 From: Ayaz Abbas Date: Thu, 5 Sep 2024 14:04:53 +0100 Subject: [PATCH 6/7] fix - only create mapping accounts if they don't exist yet --- program_admin/__init__.py | 99 ++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 48 deletions(-) diff --git a/program_admin/__init__.py b/program_admin/__init__.py index 37f6291..5511e16 100644 --- a/program_admin/__init__.py +++ b/program_admin/__init__.py @@ -342,75 +342,78 @@ async def sync_mapping_instructions( mapping_keypair_0 = load_keypair( "mapping_0", key_dir=self.key_dir, generate=generate_keys ) - logger.debug(f"mapping_0 public key: {mapping_keypair_0.public_key}") instructions: List[TransactionInstruction] = [] - if not await account_exists(self.rpc_endpoint, mapping_keypair_0.public_key): - logger.debug("Building system.program.create_account instruction") - instructions.append( - system_program.create_account( - system_program.CreateAccountParams( - from_pubkey=funding_keypair.public_key, - new_account_pubkey=mapping_keypair_0.public_key, - # FIXME: Change to minimum rent-exempt amount - lamports=await self.fetch_minimum_balance(MAPPING_ACCOUNT_SIZE), - space=MAPPING_ACCOUNT_SIZE, - program_id=self.program_key, - ) - ) - ) - - logger.debug("Building pyth_program.init_mapping instruction") - instructions.append( - pyth_program.init_mapping( - self.program_key, - funding_keypair.public_key, - mapping_keypair_0.public_key, - ) - ) - - mapping_keypairs: List[Keypair] = [] - if num_mapping_accounts > 1: - mapping_keypairs: List[Keypair] = [ - load_keypair( - f"mapping_{n}", key_dir=self.key_dir, generate=generate_keys - ) - for n in range(1, num_mapping_accounts) - ] - - tail_mapping_keypair = mapping_keypair_0 - for mapping_keypair in mapping_keypairs: - if not ( - await account_exists(self.rpc_endpoint, mapping_keypair.public_key) - ): + # Create initial mapping account + if len(self._mapping_accounts) < 1: + if not await account_exists(self.rpc_endpoint, mapping_keypair_0.public_key): logger.debug("Building system.program.create_account instruction") instructions.append( system_program.create_account( system_program.CreateAccountParams( from_pubkey=funding_keypair.public_key, - new_account_pubkey=mapping_keypair.public_key, + new_account_pubkey=mapping_keypair_0.public_key, # FIXME: Change to minimum rent-exempt amount - lamports=await self.fetch_minimum_balance( - MAPPING_ACCOUNT_SIZE - ), + lamports=await self.fetch_minimum_balance(MAPPING_ACCOUNT_SIZE), space=MAPPING_ACCOUNT_SIZE, program_id=self.program_key, ) ) ) - logger.debug("Building pyth_program.add_mapping instruction") + logger.debug("Building pyth_program.init_mapping instruction") instructions.append( - pyth_program.add_mapping( + pyth_program.init_mapping( self.program_key, funding_keypair.public_key, - tail_mapping_keypair.public_key, - mapping_keypair.public_key, + mapping_keypair_0.public_key, ) ) - tail_mapping_keypair = mapping_keypair + # Add extra mapping accounts + if len(self._mapping_accounts) < num_mapping_accounts: + mapping_keypairs: List[Keypair] = [] + if num_mapping_accounts > 1: + mapping_keypairs: List[Keypair] = [ + load_keypair( + f"mapping_{n}", key_dir=self.key_dir, generate=generate_keys + ) + for n in range(1, num_mapping_accounts) + ] + + tail_mapping_keypair = mapping_keypair_0 + for mapping_keypair in mapping_keypairs: + if not ( + await account_exists(self.rpc_endpoint, mapping_keypair.public_key) + ): + logger.debug("Building system.program.create_account instruction") + instructions.append( + system_program.create_account( + system_program.CreateAccountParams( + from_pubkey=funding_keypair.public_key, + new_account_pubkey=mapping_keypair.public_key, + # FIXME: Change to minimum rent-exempt amount + lamports=await self.fetch_minimum_balance( + MAPPING_ACCOUNT_SIZE + ), + space=MAPPING_ACCOUNT_SIZE, + program_id=self.program_key, + ) + ) + ) + + logger.debug("Building pyth_program.add_mapping instruction") + instructions.append( + pyth_program.add_mapping( + self.program_key, + funding_keypair.public_key, + tail_mapping_keypair.public_key, + mapping_keypair.public_key, + ) + ) + + tail_mapping_keypair = mapping_keypair return (instructions, [funding_keypair, mapping_keypair_0] + mapping_keypairs) From 80cf47cdfa65f6598941f7e142250142f7cf2e4b Mon Sep 17 00:00:00 2001 From: Ayaz Abbas Date: Thu, 5 Sep 2024 14:13:15 +0100 Subject: [PATCH 7/7] fix variable referenced before assignment --- program_admin/__init__.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/program_admin/__init__.py b/program_admin/__init__.py index 5511e16..9db8d68 100644 --- a/program_admin/__init__.py +++ b/program_admin/__init__.py @@ -29,8 +29,8 @@ ReferencePublishers, ) from program_admin.util import ( - MAPPING_ACCOUNT_SIZE, MAPPING_ACCOUNT_PRODUCT_LIMIT, + MAPPING_ACCOUNT_SIZE, PRICE_ACCOUNT_V1_SIZE, PRICE_ACCOUNT_V2_SIZE, PRODUCT_ACCOUNT_SIZE, @@ -251,7 +251,9 @@ async def sync( # Sync mapping accounts # Create all the mapping accounts we need for the the number of product accounts - num_mapping_accounts = math.ceil(len(ref_products) / MAPPING_ACCOUNT_PRODUCT_LIMIT) + num_mapping_accounts = math.ceil( + len(ref_products) / MAPPING_ACCOUNT_PRODUCT_LIMIT + ) mapping_instructions, mapping_keypairs = await self.sync_mapping_instructions( generate_keys, num_mapping_accounts ) @@ -347,7 +349,9 @@ async def sync_mapping_instructions( # Create initial mapping account if len(self._mapping_accounts) < 1: - if not await account_exists(self.rpc_endpoint, mapping_keypair_0.public_key): + if not await account_exists( + self.rpc_endpoint, mapping_keypair_0.public_key + ): logger.debug("Building system.program.create_account instruction") instructions.append( system_program.create_account( @@ -355,7 +359,9 @@ async def sync_mapping_instructions( from_pubkey=funding_keypair.public_key, new_account_pubkey=mapping_keypair_0.public_key, # FIXME: Change to minimum rent-exempt amount - lamports=await self.fetch_minimum_balance(MAPPING_ACCOUNT_SIZE), + lamports=await self.fetch_minimum_balance( + MAPPING_ACCOUNT_SIZE + ), space=MAPPING_ACCOUNT_SIZE, program_id=self.program_key, ) @@ -372,8 +378,8 @@ async def sync_mapping_instructions( ) # Add extra mapping accounts + mapping_keypairs: List[Keypair] = [] if len(self._mapping_accounts) < num_mapping_accounts: - mapping_keypairs: List[Keypair] = [] if num_mapping_accounts > 1: mapping_keypairs: List[Keypair] = [ load_keypair(