Add resize_price_account support; create .dockerignore (#41)

* Add resize_price_account support; create .dockerignore

* parsing.py: Use num_components, .*ignore: add dist/

* parsing.py: Fix unused import items

Author: drozdziak1

.dockerignore

@@ -0,0 +1,13 @@
+.envrc
+.mypy_cache
+__pycache__
+dist
+keys
+permissions.json
+products.json
+publishers.json
+test-ledger
+
+# IntelliJ files
+.idea
+*.iml

.gitignore

@@ -1,6 +1,7 @@
 .envrc
 .mypy_cache
 __pycache__
+dist
 keys
 permissions.json
 products.json

program_admin/__init__.py

@@ -36,7 +36,8 @@
 )
 from program_admin.util import (
     MAPPING_ACCOUNT_SIZE,
-    PRICE_ACCOUNT_SIZE,
+    PRICE_ACCOUNT_V1_SIZE,
+    PRICE_ACCOUNT_V2_SIZE,
     PRODUCT_ACCOUNT_SIZE,
     account_exists,
     compute_transaction_size,
@@ -228,6 +229,7 @@ async def sync(
         ref_authority_permissions: Optional[ReferenceAuthorityPermissions],
         send_transactions: bool = True,
         generate_keys: bool = False,
+        allocate_price_v2: bool = True,
     ) -> List[TransactionInstruction]:
         instructions: List[TransactionInstruction] = []
 
@@ -261,7 +263,9 @@ async def sync(
             (
                 product_instructions,
                 product_keypairs,
-            ) = await self.sync_product_instructions(ref_product, generate_keys)
+            ) = await self.sync_product_instructions(
+                ref_product, generate_keys, allocate_price_v2
+            )
 
             if product_instructions:
                 product_updates = True
@@ -361,6 +365,7 @@ async def sync_product_instructions(
         self,
         product: ReferenceProduct,
         generate_keys: bool,
+        allocate_price_v2: bool,
     ) -> Tuple[List[TransactionInstruction], List[Keypair]]:
         instructions: List[TransactionInstruction] = []
         funding_keypair = load_keypair("funding", key_dir=self.key_dir)
@@ -423,16 +428,23 @@ async def sync_product_instructions(
             logger.info(f"Creating new price account for {product['jump_symbol']}")
 
             if not await account_exists(self.rpc_endpoint, price_keypair.public_key):
-                logger.debug("Building system_program.create_account instruction")
+                price_alloc_size = (
+                    PRICE_ACCOUNT_V2_SIZE
+                    if allocate_price_v2
+                    else PRICE_ACCOUNT_V1_SIZE
+                )
+
+                logger.debug(
+                    f"Building system_program.create_account instruction (allocate_price_v2: {allocate_price_v2}, {price_alloc_size} bytes)"
+                )
+
                 instructions.append(
                     system_program.create_account(
                         system_program.CreateAccountParams(
                             from_pubkey=funding_keypair.public_key,
                             new_account_pubkey=price_keypair.public_key,
-                            lamports=await self.fetch_minimum_balance(
-                                PRICE_ACCOUNT_SIZE
-                            ),
-                            space=PRICE_ACCOUNT_SIZE,
+                            lamports=await self.fetch_minimum_balance(price_alloc_size),
+                            space=price_alloc_size,
                             program_id=self.program_key,
                         )
                     )
@@ -574,3 +586,59 @@ async def sync_authority_permissions_instructions(
             logger.debug("Existing authority permissions OK, not updating")
 
         return (instructions, signers)
+
+    async def resize_price_accounts_v2(
+        self,
+        security_authority_path: Path,
+        send_transactions: bool,
+    ):
+
+        security_authority: Optional[Keypair] = None
+        with open(security_authority_path, encoding="utf8") as file:
+            data = bytes(json.load(file))
+
+            security_authority = Keypair.from_secret_key(data)
+
+        if not security_authority:
+            raise RuntimeError("Could not load security authority keypair")
+
+        await self.refresh_program_accounts()
+
+        v1_prices = {}
+        v2_prices = {}
+        odd_size_prices = {}
+
+        for (pubkey, account) in self._price_accounts.items():
+            # IMPORTANT: sizes must be checked in descending order
+            if account.data.used_size >= PRICE_ACCOUNT_V2_SIZE:
+                logger.debug(f"Price account {pubkey} is v2")
+                v2_prices[pubkey] = account
+            elif account.data.used_size >= PRICE_ACCOUNT_V1_SIZE:
+                logger.debug(f"Price account {pubkey} is v1")
+                v1_prices[pubkey] = account
+            else:
+                logger.warning(
+                    f"Price account {pubkey} of {account.data.used_size} bytes does not match any known used size"
+                )
+                odd_size_prices[pubkey] = account
+
+        if len(v1_prices) > 0:
+            logger.info(f"Found {len(v1_prices)} v1 price account(s)")
+        if len(v2_prices) > 0:
+            logger.info(f"Found {len(v2_prices)} v2 price account(s)")
+        if len(odd_size_prices) > 0:
+            logger.info(f"Found {len(odd_size_prices)} unrecognized price accounts")
+
+        instructions = []
+        signers = [security_authority]
+
+        for (pubkey, account) in v1_prices.items():
+            logger.debug("Building pyth_program.resize_price_account instruction")
+
+            instruction = pyth_program.resize_price_account_v2(
+                self.program_key, security_authority.public_key, pubkey
+            )
+            instructions.append(instruction)
+
+        if send_transactions:
+            await self.send_transaction(instructions, signers)

program_admin/cli.py

@@ -414,6 +414,12 @@ def restore_links(network, rpc_endpoint, program_key, keys, products, commitment
     envvar="GENERATE_KEYS",
     default="false",
 )
+@click.option(
+    "--allocate-price-v2",
+    help="Whether to use price account v2 byte size when allocating new price accounts. Used for resize_price_account_v2 testing",
+    envvar="ALLOCATE_PRICE_V2",
+    default="true",
+)
 def sync(
     network,
     rpc_endpoint,
@@ -427,6 +433,7 @@ def sync(
     commitment,
     send_transactions,
     generate_keys,
+    allocate_price_v2,
 ):
     program_admin = ProgramAdmin(
         network=network,
@@ -456,6 +463,7 @@ def sync(
             ref_authority_permissions=ref_authority_permissions,
             send_transactions=(send_transactions == "true"),
             generate_keys=(generate_keys == "true"),
+            allocate_price_v2=(allocate_price_v2 == "true"),
         )
     )
 
@@ -500,6 +508,52 @@ def migrate_upgrade_authority(
     asyncio.run(program_admin.send_transaction([instruction], [funding_keypair]))
 
 
+@click.command(help="Resize price accounts to the PriceAccountV2 format")
+@click.option("--network", help="Solana network", envvar="NETWORK")
+@click.option("--rpc-endpoint", help="Solana RPC endpoint", envvar="RPC_ENDPOINT")
+@click.option("--program-key", help="Pyth program key", envvar="PROGRAM_KEY")
+@click.option(
+    "--security-authority",
+    help="Path to security authority keypair for executing resize instruction",
+    envvar="SECURITY_AUTHORITY",
+)
+@click.option("--keys", help="Path to keys directory", envvar="KEYS")
+@click.option(
+    "--commitment",
+    help="Confirmation level to use",
+    envvar="COMMITMENT",
+    default="finalized",
+)
+@click.option(
+    "--send-transactions",
+    help="Whether to send transactions or just print instructions (set to 'true' or 'false')",
+    envvar="SEND_TRANSACTIONS",
+    default="true",
+)
+def resize_price_accounts_v2(
+    network,
+    rpc_endpoint,
+    keys,
+    program_key,
+    security_authority,
+    commitment,
+    send_transactions,
+):
+    program_admin = ProgramAdmin(
+        network=network,
+        rpc_endpoint=rpc_endpoint,
+        key_dir=keys,
+        program_key=program_key,
+        commitment=commitment,
+    )
+
+    asyncio.run(
+        program_admin.resize_price_accounts_v2(
+            Path(security_authority), (send_transactions == "true")
+        )
+    )
+
+
 cli.add_command(delete_price)
 cli.add_command(delete_product)
 cli.add_command(list_accounts)
@@ -509,5 +563,6 @@ def migrate_upgrade_authority(
 cli.add_command(toggle_publisher)
 cli.add_command(update_product_metadata)
 cli.add_command(migrate_upgrade_authority)
+cli.add_command(resize_price_accounts_v2)
 logger.remove()
 logger.add(sys.stdout, serialize=(not os.environ.get("DEV_MODE")))

program_admin/instructions.py

@@ -17,6 +17,7 @@
 COMMAND_ADD_PUBLISHER = 5
 COMMAND_DEL_PUBLISHER = 6
 COMMAND_MIN_PUBLISHERS = 12
+COMMAND_RESIZE_PRICE_ACCOUNT = 14
 COMMAND_DEL_PRICE = 15
 COMMAND_DEL_PRODUCT = 16
 COMMAND_UPD_PERMISSIONS = 17
@@ -330,3 +331,46 @@ def upd_permissions(
         ],
         program_id=program_key,
     )
+
+
+def resize_price_account_v2(
+    program_key: PublicKey, security_authority: PublicKey, price_account: PublicKey
+) -> TransactionInstruction:
+    """
+    Pyth program resize_price_account instruction. It migrates the
+    specified price account to a new v2 format. The new format
+    includes more price component slots, allowing more publishers per
+    price account. Additionally, PriceCumulative is included
+
+    The authority pubkeys are passed in instruction data.
+
+    Accounts:
+    - security authority  (signer, writable)     - must be the pubkey set as security_authority in permission account.
+    - price account       (signer, writable)     - The price account to resize
+    - system program      (non-signer, readonly) - Allows the resize_account() call
+    - permissions account (non-signer, readonly) - PDA of the oracle program, generated automatically, stores the permission information
+    """
+    ix_data_layout = Struct(
+        "version" / Int32ul,
+        "command" / Int32sl,
+    )
+
+    ix_data = ix_data_layout.build(
+        dict(version=PROGRAM_VERSION, command=COMMAND_RESIZE_PRICE_ACCOUNT)
+    )
+
+    [permissions_account, _bump] = PublicKey.find_program_address(
+        [AUTHORITY_PERMISSIONS_PDA_SEED],
+        program_key,
+    )
+
+    return TransactionInstruction(
+        data=ix_data,
+        keys=[
+            AccountMeta(pubkey=security_authority, is_signer=True, is_writable=True),
+            AccountMeta(pubkey=price_account, is_signer=False, is_writable=True),
+            AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False),
+            AccountMeta(pubkey=permissions_account, is_signer=False, is_writable=False),
+        ],
+        program_id=program_key,
+    )

program_admin/parsing.py

@@ -90,6 +90,9 @@ def parse_price_info(data: bytes) -> PriceInfo:
     return PriceInfo(price, confidence, status, corporate_action, publish_slot)
 
 
+# NOTE(2023-07-31): For v2 prices the parsed data does not include
+# price_cumulative values. This value is currently out-of-scope for
+# program-admin.
 def parse_price_data(data: bytes) -> PriceData:
     used_size = Int32ul.parse(data[12:])
     price_type = Int32ul.parse(data[16:])
@@ -113,25 +116,26 @@ def parse_price_data(data: bytes) -> PriceData:
     previous_timestamp = Int64sl.parse(data[200:])
     aggregate = parse_price_info(data[208:240])
     offset = 240
-    parse_next_component = True
+
     price_components = []
 
-    while offset < len(data) and parse_next_component:
+    for _ in range(components_count):
+
         publisher_key = PublicKey(data[offset : offset + 32])
         offset += 32
 
-        if publisher_key == PublicKey(bytes(32)):
-            parse_next_component = False
-        else:
-            aggregate_price = parse_price_info(data[offset : offset + 32])
-            offset += 32
+        aggregate_price = parse_price_info(data[offset : offset + 32])
+        offset += 32
+
+        latest_price = parse_price_info(data[offset : offset + 32])
+        offset += 32
 
-            latest_price = parse_price_info(data[offset : offset + 32])
-            offset += 32
+        price_components.append(
+            PriceComponent(publisher_key, aggregate_price, latest_price)
+        )
 
-            price_components.append(
-                PriceComponent(publisher_key, aggregate_price, latest_price)
-            )
+    # TODO(2023-07-31): Parse price_cumulative here if necessary;
+    # remember to re-check that this price account is v2 and adjust offset
 
     return PriceData(
         used_size,

program_admin/util.py

@@ -16,7 +16,10 @@
 
 MAPPING_ACCOUNT_SIZE = 20536  # https://github.com/pyth-network/pyth-client/blob/b49f73afe32ce8685a3d05e32d8f3bb51909b061/program/src/oracle/oracle.h#L88
 MAPPING_ACCOUNT_PRODUCT_LIMIT = 640
-PRICE_ACCOUNT_SIZE = 12576
+PRICE_ACCOUNT_V1_SIZE = 3312
+PRICE_ACCOUNT_V2_SIZE = 12576
+PRICE_V1_COMP_COUNT = 32
+PRICE_V2_COMP_COUNT = 128
 PRODUCT_ACCOUNT_SIZE = 512
 SOL_LAMPORTS = pow(10, 9)
 

pyproject.toml

@@ -9,7 +9,7 @@ known_local_folder = ["program_admin"]
 authors = ["Thomaz <[email protected]>"]
 description = "Syncs products and publishers of the Pyth program"
 name = "program-admin"
-version = "0.1.1"
+version = "0.1.2"
 
 [tool.poetry.dependencies]
 click = "^8.1.0"