From 056fbed184bedde40ac7ffbb71400cba5b0c9322 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 26 Nov 2024 18:16:54 -0500 Subject: [PATCH 01/15] add batch set weights async and sync; needs tests --- bittensor/core/extrinsics/async_weights.py | 150 ++++++++++++++++++++ bittensor/core/extrinsics/set_weights.py | 152 +++++++++++++++++++++ 2 files changed, 302 insertions(+) diff --git a/bittensor/core/extrinsics/async_weights.py b/bittensor/core/extrinsics/async_weights.py index fdac280646..af020884da 100644 --- a/bittensor/core/extrinsics/async_weights.py +++ b/bittensor/core/extrinsics/async_weights.py @@ -82,6 +82,74 @@ async def _do_set_weights( ) +async def _do_batch_set_weights( + subtensor: "AsyncSubtensor", + wallet: "Wallet", + uidss: list[list[int]], + valss: list[list[int]], + netuids: list[int], + version_keys: list[int] = [], + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, +) -> list[tuple[bool, Optional[str]]]: # (success, error_message) + """ + Internal method to send a transaction to the Bittensor blockchain, setting weights + for specified neurons. This method constructs and submits the transaction, handling + retries and blockchain communication. + + Args: + subtensor (subtensor.core.async_subtensor.AsyncSubtensor): Async Subtensor instance. + wallet (bittensor.wallet): The wallet associated with the neuron setting the weights. + uidss (list[list[int]]): List of neuron UIDs for which weights are being set. + valss (list[list[int]]): List of weight values corresponding to each UID. + netuids (list[int]): Unique identifier for the network. + version_keys (list[int], optional): Version key for compatibility with the network. + wait_for_inclusion (bool, optional): Waits for the transaction to be included in a block. + wait_for_finalization (bool, optional): Waits for the transaction to be finalized on the blockchain. + + Returns: + Tuple[bool, Optional[str]]: A tuple containing a success flag and an optional error message. + + This method is vital for the dynamic weighting mechanism in Bittensor, where neurons adjust their + trust in other neurons based on observed performance and contributions. + """ + packed_weights = [ + [(uid, val) for uid, val in zip(uids, vals)] for uids, vals in zip(uidss, valss) + ] + + call = await subtensor.substrate.compose_call( + call_module="SubtensorModule", + call_function="batch_set_weights", + call_params={ + "netuids": netuids, + "weights": packed_weights, + "version_keys": version_keys, + }, + ) + # Period dictates how long the extrinsic will stay as part of waiting pool + extrinsic = await subtensor.substrate.create_signed_extrinsic( + call=call, + keypair=wallet.hotkey, + era={"period": 5}, + ) + response = await subtensor.substrate.submit_extrinsic( + extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + # We only wait here if we expect finalization. + if not wait_for_finalization and not wait_for_inclusion: + return True, "Not waiting for finalization or inclusion." + + await response.process_events() + if await response.is_success: + return True, "Successfully set weights." + else: + return False, format_error_message( + response.error_message, substrate=subtensor.substrate + ) + + async def set_weights_extrinsic( subtensor: "AsyncSubtensor", wallet: "Wallet", @@ -155,6 +223,88 @@ async def set_weights_extrinsic( return False, str(error) +async def batch_set_weights_extrinsic( + subtensor: "AsyncSubtensor", + wallet: "Wallet", + netuids: list[int], + uids: list[Union[NDArray[np.int64], "torch.LongTensor", list]], + weights: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], + version_keys: list[int] = [], + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, +) -> tuple[bool, str]: + """Sets the given weights and values for multiple netuids as a batch on chain for wallet hotkey account. + + Args: + subtensor (bittensor.subtensor): Bittensor subtensor object. + wallet (bittensor.wallet): Bittensor wallet object. + netuids (list[int]): The ``netuid`` of the subnet to set weights for. + uids (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The ``uint64`` uids of destination neurons. + weights (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The weights to set. These must be ``float`` s and correspond to the passed ``uid`` s. + version_keys (list[int]): The version key of the validator. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``true``, or returns ``false`` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. + + Returns: + success (bool): Flag is ``true`` if extrinsic was finalized or included in the block. If we did not wait for finalization / inclusion, the response is ``true``. + """ + uids_to_set: list[Union[NDArray[np.int64], "torch.LongTensor", list]] = [] + weights_to_set: list[Union[NDArray[np.float32], "torch.FloatTensor", list]] = [] + if len(version_keys) == 0: + version_keys = [0] * len(netuids) # Default to version 0 if not provided + + for uids_, weights_ in zip(uids, weights): + # First convert types. + if use_torch(): + if isinstance(uids, list): + uids_ = torch.tensor(uids, dtype=torch.int64) + if isinstance(weights, list): + weights_ = torch.tensor(weights, dtype=torch.float32) + else: + if isinstance(uids, list): + uids_ = np.array(uids, dtype=np.int64) + if isinstance(weights, list): + weights_ = np.array(weights, dtype=np.float32) + + # Reformat and normalize. + weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit( + uids_, weights_ + ) + + uids_to_set.append(weight_uids) + weights_to_set.append(weight_vals) + + logging.info( + ":satellite: [magenta]Setting batch weights on [/magenta][blue]{subtensor.network}[/blue] [magenta]...[/magenta]" + ) + try: + success, error_message = await _do_batch_set_weights( + subtensor=subtensor, + wallet=wallet, + netuids=netuids, + uidss=uids_to_set, + valss=weights_to_set, + version_keys=version_keys, + wait_for_finalization=wait_for_finalization, + wait_for_inclusion=wait_for_inclusion, + ) + + if not wait_for_finalization and not wait_for_inclusion: + return True, "Not waiting for finalization or inclusion." + + if success is True: + message = "Successfully set weights and Finalized." + logging.success(f":white_heavy_check_mark: [green]{message}[/green]") + return True, message + else: + logging.error(f"[red]Failed[/red] set weights. Error: {error_message}") + return False, error_message + + except Exception as error: + logging.error(f":cross_mark: [red]Failed[/red] set weights. Error: {error}") + return False, str(error) + + async def _do_commit_weights( subtensor: "AsyncSubtensor", wallet: "Wallet", diff --git a/bittensor/core/extrinsics/set_weights.py b/bittensor/core/extrinsics/set_weights.py index 880480e998..c4321b6d5b 100644 --- a/bittensor/core/extrinsics/set_weights.py +++ b/bittensor/core/extrinsics/set_weights.py @@ -101,6 +101,76 @@ def do_set_weights( ) +# Chain call for `do_batch_set_weights` +@ensure_connected +def do_batch_set_weights( + self: "Subtensor", + wallet: "Wallet", + uidss: list[list[int]], + valss: list[list[int]], + netuids: list[int], + version_keys: list[int] = [], + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, + period: int = 5, +) -> tuple[bool, Optional[str]]: # (success, error_message) + """ + Internal method to send a transaction to the Bittensor blockchain, setting weights for specified neurons. This method constructs and submits the transaction, handling retries and blockchain communication. + + Args: + self (bittensor.core.subtensor.Subtensor): Subtensor interface + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron setting the weights. + uidss (list[list[int]]): List of neuron UIDs for which weights are being set. + valss (list[list[int]]): List of weight values corresponding to each UID. + netuids (list[int]): Unique identifier for the network. + version_keys (list[int]): Version key for compatibility with the network. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. + period (int): Period dictates how long the extrinsic will stay as part of waiting pool. + + Returns: + tuple[bool, Optional[str]]: A tuple containing a success flag and an optional response message. + + This method is vital for the dynamic weighting mechanism in Bittensor, where neurons adjust their trust in other neurons based on observed performance and contributions. + """ + packed_weights = [ + [(uid, val) for uid, val in zip(uids, vals)] for uids, vals in zip(uidss, valss) + ] + + call = self.substrate.compose_call( + call_module="SubtensorModule", + call_function="batch_set_weights", + call_params={ + "netuids": netuids, + "weights": packed_weights, + "version_keys": version_keys, + }, + ) + # Period dictates how long the extrinsic will stay as part of waiting pool + extrinsic = self.substrate.create_signed_extrinsic( + call=call, + keypair=wallet.hotkey, + era={"period": period}, + ) + response = submit_extrinsic( + substrate=self.substrate, + extrinsic=extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + # We only wait here if we expect finalization. + if not wait_for_finalization and not wait_for_inclusion: + return True, "Not waiting for finalization or inclusion." + + response.process_events() + if response.is_success: + return True, "Successfully set weights." + else: + return False, format_error_message( + response.error_message, substrate=self.substrate + ) + + # Community uses this extrinsic directly and via `subtensor.set_weights` def set_weights_extrinsic( subtensor: "Subtensor", @@ -175,3 +245,85 @@ def set_weights_extrinsic( logging.error(f":cross_mark: [red]Failed.[/red]: Error: {e}") logging.debug(str(e)) return False, str(e) + + +def batch_set_weights_extrinsic( + subtensor: "Subtensor", + wallet: "Wallet", + netuids: list[int], + uids: list[Union[NDArray[np.int64], "torch.LongTensor", list]], + weights: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], + version_keys: list[int] = [], + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, +) -> tuple[bool, str]: + """Sets the given weights and values for multiple netuids as a batch on chain for wallet hotkey account. + + Args: + subtensor (bittensor.subtensor): Bittensor subtensor object. + wallet (bittensor.wallet): Bittensor wallet object. + netuids (list[int]): The ``netuid`` of the subnet to set weights for. + uids (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The ``uint64`` uids of destination neurons. + weights (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The weights to set. These must be ``float`` s and correspond to the passed ``uid`` s. + version_keys (list[int]): The version key of the validator. + wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``true``, or returns ``false`` if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. + + Returns: + success (bool): Flag is ``true`` if extrinsic was finalized or included in the block. If we did not wait for finalization / inclusion, the response is ``true``. + """ + uids_to_set: list[Union[NDArray[np.int64], "torch.LongTensor", list]] = [] + weights_to_set: list[Union[NDArray[np.float32], "torch.FloatTensor", list]] = [] + if len(version_keys) == 0: + version_keys = [0] * len(netuids) # Default to version 0 if not provided + + for uids_, weights_ in zip(uids, weights): + # First convert types. + if use_torch(): + if isinstance(uids, list): + uids_ = torch.tensor(uids, dtype=torch.int64) + if isinstance(weights, list): + weights_ = torch.tensor(weights, dtype=torch.float32) + else: + if isinstance(uids, list): + uids_ = np.array(uids, dtype=np.int64) + if isinstance(weights, list): + weights_ = np.array(weights, dtype=np.float32) + + # Reformat and normalize. + weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit( + uids_, weights_ + ) + + uids_to_set.append(weight_uids) + weights_to_set.append(weight_vals) + + logging.info( + ":satellite: [magenta]Setting batch weights on [/magenta][blue]{subtensor.network}[/blue] [magenta]...[/magenta]" + ) + try: + success, error_message = do_batch_set_weights( + subtensor=subtensor, + wallet=wallet, + netuids=netuids, + uidss=uids_to_set, + valss=weights_to_set, + version_keys=version_keys, + wait_for_finalization=wait_for_finalization, + wait_for_inclusion=wait_for_inclusion, + ) + + if not wait_for_finalization and not wait_for_inclusion: + return True, "Not waiting for finalization or inclusion." + + if success is True: + message = "Successfully set weights and Finalized." + logging.success(f":white_heavy_check_mark: [green]{message}[/green]") + return True, message + else: + logging.error(f"[red]Failed[/red] set weights. Error: {error_message}") + return False, error_message + + except Exception as error: + logging.error(f":cross_mark: [red]Failed[/red] set weights. Error: {error}") + return False, str(error) From e9a78a9eece9da65116c715bff9888fb2b52f343 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 26 Nov 2024 18:25:17 -0500 Subject: [PATCH 02/15] add impl for batch commit weights; async and sync --- bittensor/core/extrinsics/async_weights.py | 101 +++++++++++++++++++ bittensor/core/extrinsics/commit_weights.py | 104 ++++++++++++++++++++ 2 files changed, 205 insertions(+) diff --git a/bittensor/core/extrinsics/async_weights.py b/bittensor/core/extrinsics/async_weights.py index af020884da..a3deb7584e 100644 --- a/bittensor/core/extrinsics/async_weights.py +++ b/bittensor/core/extrinsics/async_weights.py @@ -361,6 +361,62 @@ async def _do_commit_weights( ) +async def _do_batch_commit_weights( + subtensor: "AsyncSubtensor", + wallet: "Wallet", + netuids: list[int], + commit_hashes: list[str], + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, +) -> tuple[bool, Optional[str]]: + """ + Internal method to send a transaction to the Bittensor blockchain, committing the hash of a neuron's weights. + This method constructs and submits the transaction, handling retries and blockchain communication. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): The subtensor instance used for blockchain interaction. + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron committing the weights. + netuids (list[int]): The unique identifier of the subnet. + commit_hashes (list[str]): The hash of the neuron's weights to be committed. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. + + Returns: + tuple[bool, Optional[str]]: A tuple containing a success flag and an optional error message. + + This method ensures that the weight commitment is securely recorded on the Bittensor blockchain, providing a verifiable record of the neuron's weight distribution at a specific point in time. + """ + call = await subtensor.substrate.compose_call( + call_module="SubtensorModule", + call_function="commit_weights", + call_params={ + "netuids": netuids, + "commit_hashes": commit_hashes, + }, + ) + extrinsic = await subtensor.substrate.create_signed_extrinsic( + call=call, + keypair=wallet.hotkey, + ) + response = await subtensor.substrate.submit_extrinsic( + substrate=subtensor.substrate, + extrinsic=extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + if not wait_for_finalization and not wait_for_inclusion: + return True, None + + await response.process_events() + if await response.is_success: + return True, None + else: + return False, format_error_message( + response.error_message, substrate=subtensor.substrate + ) + + async def commit_weights_extrinsic( subtensor: "AsyncSubtensor", wallet: "Wallet", @@ -404,3 +460,48 @@ async def commit_weights_extrinsic( else: logging.error(f"Failed to commit weights: {error_message}") return False, error_message + + +async def batch_commit_weights_extrinsic( + subtensor: "AsyncSubtensor", + wallet: "Wallet", + netuids: list[int], + commit_hashes: list[str], + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, +) -> tuple[bool, str]: + """ + Commits a hash of the neuron's weights to the Bittensor blockchain using the provided wallet. + This function is a wrapper around the `do_batch_commit_weights` method. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): The subtensor instance used for blockchain interaction. + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron committing the weights. + netuids (list[int]): The unique identifier of the subnet. + commit_hashes (list[str]): The hash of the neuron's weights to be committed. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. + + Returns: + tuple[bool, str]: ``True`` if the weight commitment is successful, False otherwise. And `msg`, a string + value describing the success or potential error. + + This function provides a user-friendly interface for committing weights to the Bittensor blockchain, ensuring proper error handling and user interaction when required. + """ + + success, error_message = await _do_batch_commit_weights( + subtensor=subtensor, + wallet=wallet, + netuids=netuids, + commit_hashes=commit_hashes, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + if success: + success_message = "Successfully batch committed weights." + logging.info(success_message) + return True, success_message + else: + logging.error(f"Failed to batch commit weights: {error_message}") + return False, error_message diff --git a/bittensor/core/extrinsics/commit_weights.py b/bittensor/core/extrinsics/commit_weights.py index ef93a15d3e..32d79332a2 100644 --- a/bittensor/core/extrinsics/commit_weights.py +++ b/bittensor/core/extrinsics/commit_weights.py @@ -87,6 +87,63 @@ def do_commit_weights( return False, response.error_message +# Chain call for `batch_commit_weights_extrinsic` +@ensure_connected +def do_batch_commit_weights( + self: "Subtensor", + wallet: "Wallet", + netuids: list[int], + commit_hashes: list[str], + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, +) -> tuple[bool, Optional[dict]]: + """ + Internal method to send a transaction to the Bittensor blockchain, committing multiple hashes for multiple subnets. + This method constructs and submits the transaction, handling retries and blockchain communication. + + Args: + self (bittensor.core.subtensor.Subtensor): The subtensor instance used for blockchain interaction. + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron committing the weights. + netuids (list[int]): The unique identifiers of the subnets. + commit_hashes (list[str]): The hashes of the neuron's weights to be committed. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. + + Returns: + tuple[bool, Optional[str]]: A tuple containing a success flag and an optional error message. + + This method ensures that the weight commitment is securely recorded on the Bittensor blockchain, providing a verifiable record of the neuron's weight distribution at a specific point in time. + """ + + call = self.substrate.compose_call( + call_module="SubtensorModule", + call_function="batch_commit_weights", + call_params={ + "netuids": netuids, + "commit_hashes": commit_hashes, + }, + ) + extrinsic = self.substrate.create_signed_extrinsic( + call=call, + keypair=wallet.hotkey, + ) + response = submit_extrinsic( + substrate=self.substrate, + extrinsic=extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + if not wait_for_finalization and not wait_for_inclusion: + return True, None + + response.process_events() + if response.is_success: + return True, None + else: + return False, response.error_message + + def commit_weights_extrinsic( subtensor: "Subtensor", wallet: "Wallet", @@ -134,6 +191,53 @@ def commit_weights_extrinsic( return False, error_message +def batch_commit_weights_extrinsic( + subtensor: "Subtensor", + wallet: "Wallet", + netuids: list[int], + commit_hashes: list[str], + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, +) -> tuple[bool, str]: + """ + Commits multiple hashes of the neuron's weights to the Bittensor blockchain using the provided wallet. + This function is a wrapper around the `do_batch_commit_weights` method. + + Args: + subtensor (bittensor.core.subtensor.Subtensor): The subtensor instance used for blockchain interaction. + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron committing the weights. + netuids (list[int]): The unique identifiers of the subnets. + commit_hashes (list[str]): The hashes of the neuron's weights to be committed. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. + + Returns: + tuple[bool, str]: ``True`` if the weight commitment is successful, False otherwise. And `msg`, a string value describing the success or potential error. + + This function provides a user-friendly interface for committing weights to the Bittensor blockchain, ensuring proper error handling and user interaction when required. + """ + + success, error_message = do_batch_commit_weights( + self=subtensor, + wallet=wallet, + netuids=netuids, + commit_hashes=commit_hashes, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + if success: + success_message = "Successfully batch committed weights." + logging.info(success_message) + return True, success_message + else: + error_message = format_error_message( + error_message, substrate=subtensor.substrate + ) + logging.error(f"Failed to batch commit weights: {error_message}") + return False, error_message + + # Chain call for `reveal_weights_extrinsic` @ensure_connected def do_reveal_weights( From 9849b48c5f4294e738c636882415a5992ed06c8e Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 26 Nov 2024 18:41:38 -0500 Subject: [PATCH 03/15] add methods to subtensor --- bittensor/core/async_subtensor.py | 78 ++++++++++++++++++++++++++++++ bittensor/core/subtensor.py | 80 +++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) diff --git a/bittensor/core/async_subtensor.py b/bittensor/core/async_subtensor.py index 4c456ac83a..a43510194d 100644 --- a/bittensor/core/async_subtensor.py +++ b/bittensor/core/async_subtensor.py @@ -29,6 +29,7 @@ ) from bittensor.core.extrinsics.async_transfer import transfer_extrinsic from bittensor.core.extrinsics.async_weights import ( + batch_commit_weights_extrinsic, commit_weights_extrinsic, set_weights_extrinsic, ) @@ -1629,3 +1630,80 @@ async def commit_weights( retries += 1 return success, message + + async def batch_commit_weights( + self, + wallet: "Wallet", + netuids: list[int], + salts: list[list[int]], + uids: list[Union[NDArray[np.int64], list]], + weights: list[Union[NDArray[np.int64], list]], + version_keys: list[int] = [], + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, + max_retries: int = 5, + ) -> tuple[bool, str]: + """ + Commits a batch of hashes of weights to the Bittensor blockchain using the provided wallet. + This allows for multiple subnets to be committed to at once in a single extrinsic. + + Args: + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron committing the weights. + netuids (list[int]): The list of subnet uids. + salts (list[list[int]]): The list of salts to generate weight hashes. + uids (list[np.ndarray]): The list of NumPy arrays of neuron UIDs for which weights are being committed. + weights (list[np.ndarray]): The list of NumPy arrays of weight values corresponding to each UID. + version_keys (list[int]): The list of version keys for compatibility with the network. Default is ``int representation of Bittensor version.``. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is ``False``. + max_retries (int): The number of maximum attempts to commit weights. Default is ``5``. + + Returns: + tuple[bool, str]: ``True`` if the weight commitment is successful, False otherwise. And `msg`, a string value describing the success or potential error. + + This function allows commitments to be made for multiple subnets at once. + """ + retries = 0 + success = False + message = "No attempt made. Perhaps it is too soon to commit weights!" + + logging.info( + f"Committing a batch of weights with params: netuids={netuids}, salts={salts}, uids={uids}, weights={weights}, version_keys={version_keys}" + ) + + if len(version_keys) == 0: + version_keys = [version_as_int] * len(netuids) + + # Generate the hash of the weights + commit_hashes = [ + generate_weight_hash( + address=wallet.hotkey.ss58_address, + netuid=netuid, + uids=list(uids), + values=list(weights), + salt=salt, + version_key=version_key, + ) + for netuid, salt, uids, weights, version_key in zip( + netuids, salts, uids, weights, version_keys + ) + ] + + while retries < max_retries: + try: + success, message = await batch_commit_weights_extrinsic( + subtensor=self, + wallet=wallet, + netuids=netuids, + commit_hashes=commit_hashes, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + if success: + break + except Exception as e: + logging.error(f"Error batch committing weights: {e}") + finally: + retries += 1 + + return success, message diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 6c17cc87b0..8ebea24187 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -32,6 +32,7 @@ ) from bittensor.core.config import Config from bittensor.core.extrinsics.commit_weights import ( + batch_commit_weights_extrinsic, commit_weights_extrinsic, reveal_weights_extrinsic, ) @@ -2008,6 +2009,85 @@ def commit_weights( return success, message + def batch_commit_weights( + self, + wallet: "Wallet", + netuids: list[int], + salts: list[list[int]], + uids: list[Union[NDArray[np.int64], list]], + weights: list[Union[NDArray[np.int64], list]], + version_keys: list[int] = [], + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, + max_retries: int = 5, + ) -> tuple[bool, str]: + """ + Commits a batch of hashes of weights to the Bittensor blockchain using the provided wallet. + This allows for multiple subnets to be committed to at once in a single extrinsic. + + Args: + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron committing the weights. + netuids (list[int]): The list of subnet uids. + salts (list[list[int]]): The list of salts to generate weight hashes. + uids (list[np.ndarray]): The list of NumPy arrays of neuron UIDs for which weights are being committed. + weights (list[np.ndarray]): The list of NumPy arrays of weight values corresponding to each UID. + version_keys (list[int]): The list of version keys for compatibility with the network. Default is ``int representation of Bittensor version.``. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is ``False``. + max_retries (int): The number of maximum attempts to commit weights. Default is ``5``. + + Returns: + tuple[bool, str]: ``True`` if the weight commitment is successful, False otherwise. And `msg`, a string value describing the success or potential error. + + This function allows for multiple subnets to be committed to at once in a single extrinsic. + """ + retries = 0 + success = False + message = "No attempt made. Perhaps it is too soon to commit weights!" + + logging.info( + f"Committing a batch of weights with params: netuids={netuids}, salts={salts}, uids={uids}, weights={weights}, version_keys={version_keys}" + ) + + if len(version_keys) == 0: + version_keys = [settings.version_as_int] * len(netuids) + + # Generate the hash of the weights + commit_hashes = [ + generate_weight_hash( + address=wallet.hotkey.ss58_address, + netuid=netuid, + uids=list(uids), + values=list(weights), + salt=salt, + version_key=version_key, + ) + for netuid, salt, uids, weights, version_key in zip( + netuids, salts, uids, weights, version_keys + ) + ] + + logging.info(f"Commit Hashes: {commit_hashes}") + + while retries < max_retries: + try: + success, message = batch_commit_weights_extrinsic( + subtensor=self, + wallet=wallet, + netuids=netuids, + commit_hashes=commit_hashes, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + if success: + break + except Exception as e: + logging.error(f"Error batch committing weights: {e}") + finally: + retries += 1 + + return success, message + def reveal_weights( self, wallet: "Wallet", From 0b3e0de1697c6dae23008ed73f3e3670f4ac35eb Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 26 Nov 2024 19:45:16 -0500 Subject: [PATCH 04/15] add e2e tests relying on subtensor feat branch --- tests/e2e_tests/test_commit_weights.py | 259 ++++++++++++++++++++ tests/e2e_tests/test_set_weights.py | 325 +++++++++++++++++++++++++ 2 files changed, 584 insertions(+) create mode 100644 tests/e2e_tests/test_set_weights.py diff --git a/tests/e2e_tests/test_commit_weights.py b/tests/e2e_tests/test_commit_weights.py index 7596004603..2fdedc4970 100644 --- a/tests/e2e_tests/test_commit_weights.py +++ b/tests/e2e_tests/test_commit_weights.py @@ -170,3 +170,262 @@ async def test_commit_and_reveal_weights(local_chain): weight_vals[0] == revealed_weights.value[0][1] ), f"Incorrect revealed weights. Expected: {weights[0]}, Actual: {revealed_weights.value[0][1]}" print("✅ Passed test_commit_and_reveal_weights") + + +@pytest.mark.asyncio +async def test_batch_commit_weights(local_chain): + """ + Tests the batch commit weights mechanism with subprocess disabled (CR1.0) + + Steps: + 1. Register two subnets through Alice + 2. Register Alice's neuron and add stake + 3. Enable commit-reveal mechanism on the subnets + 4. Lower the commit_reveal interval and rate limit + 5. Commit weights using batch and verify on both subnets + Raises: + AssertionError: If any of the checks or verifications fail + """ + netuid_1 = 1 + netuid_2 = 2 + print("Testing test_batch_commit_weights") + # Register root as Alice + keypair, alice_wallet = setup_wallet("//Alice") + assert register_subnet( + local_chain, alice_wallet + ), "Unable to register the first subnet" + assert register_subnet( + local_chain, alice_wallet + ), "Unable to register the second subnet" + + # Verify subnet 1 created successfully + assert local_chain.query( + "SubtensorModule", "NetworksAdded", [netuid_1] + ).serialize(), "Subnet 1 wasn't created successfully" + assert local_chain.query( + "SubtensorModule", "NetworksAdded", [netuid_2] + ).serialize(), "Subnet 2 wasn't created successfully" + + subtensor = Subtensor(network="ws://localhost:9945") + + # Register Alice to both subnets + assert subtensor.burned_register( + alice_wallet, netuid_1 + ), "Unable to register Alice as a neuron" + assert subtensor.burned_register( + alice_wallet, netuid_2 + ), "Unable to register Alice as a neuron" + + # Stake to become to top neuron after the first epoch + add_stake(local_chain, alice_wallet, Balance.from_tao(100_000)) + + # Enable commit_reveal on both subnets + assert sudo_set_hyperparameter_bool( + local_chain, + alice_wallet, + "sudo_set_commit_reveal_weights_enabled", + True, + netuid_1, + ), "Unable to enable commit reveal on the subnet" + assert sudo_set_hyperparameter_bool( + local_chain, + alice_wallet, + "sudo_set_commit_reveal_weights_enabled", + True, + netuid_2, + ), "Unable to enable commit reveal on the subnet" + + assert subtensor.get_subnet_hyperparameters( + netuid=netuid_1, + ).commit_reveal_weights_enabled, "Failed to enable commit/reveal" + assert subtensor.get_subnet_hyperparameters( + netuid=netuid_2, + ).commit_reveal_weights_enabled, "Failed to enable commit/reveal" + + # Lower the commit_reveal interval + assert sudo_set_hyperparameter_values( + local_chain, + alice_wallet, + call_function="sudo_set_commit_reveal_weights_interval", + call_params={"netuid": netuid_1, "interval": "1"}, + return_error_message=True, + ) + assert sudo_set_hyperparameter_values( + local_chain, + alice_wallet, + call_function="sudo_set_commit_reveal_weights_interval", + call_params={"netuid": netuid_2, "interval": "1"}, + return_error_message=True, + ) + + # Verify commit/reveal periods are set correctly + assert ( + subtensor.get_subnet_hyperparameters( + netuid=netuid_1 + ).commit_reveal_weights_interval + == 1 + ), "Failed to set commit/reveal periods" + assert ( + subtensor.get_subnet_hyperparameters( + netuid=netuid_2 + ).commit_reveal_weights_interval + == 1 + ), "Failed to set commit/reveal periods" + + assert ( + subtensor.weights_rate_limit(netuid=netuid_1) > 0 + ), "Weights rate limit is below 0" + assert ( + subtensor.weights_rate_limit(netuid=netuid_2) > 0 + ), "Weights rate limit is below 0" + + # Lower the rate limit + assert sudo_set_hyperparameter_values( + local_chain, + alice_wallet, + call_function="sudo_set_weights_set_rate_limit", + call_params={"netuid": netuid_1, "weights_set_rate_limit": "0"}, + return_error_message=True, + ) + assert sudo_set_hyperparameter_values( + local_chain, + alice_wallet, + call_function="sudo_set_weights_set_rate_limit", + call_params={"netuid": netuid_2, "weights_set_rate_limit": "0"}, + return_error_message=True, + ) + + assert ( + subtensor.get_subnet_hyperparameters(netuid=netuid_1).weights_rate_limit == 0 + ), "Failed to set weights_rate_limit" + assert ( + subtensor.get_subnet_hyperparameters(netuid=netuid_2).weights_rate_limit == 0 + ), "Failed to set weights_rate_limit" + assert subtensor.weights_rate_limit(netuid=netuid_1) == 0 + assert subtensor.weights_rate_limit(netuid=netuid_2) == 0 + + # Commit-reveal values + uids = np.array([0], dtype=np.int64) + weights = np.array([0.1], dtype=np.float32) + salt = [18, 179, 107, 0, 165, 211, 141, 197] + weight_uids, weight_vals = convert_weights_and_uids_for_emit( + uids=uids, weights=weights + ) + + # Commit weights + success, message = subtensor.batch_commit_weights( + alice_wallet, + [netuid_1, netuid_2], + salts=[salt, salt], + uids=[weight_uids, weight_uids], + weights=[weight_vals, weight_vals], + wait_for_inclusion=True, + wait_for_finalization=True, + ) + + assert success is True + + weight_commits = subtensor.query_module( + module="SubtensorModule", + name="WeightCommits", + params=[netuid_1, alice_wallet.hotkey.ss58_address], + ) + # Assert that the committed weights are set correctly + assert weight_commits.value is not None, "Weight commit not found in storage" + commit_hash, commit_block, reveal_block, expire_block = weight_commits.value[0] + assert commit_block > 0, f"Invalid block number: {commit_block}" + + weight_commits = subtensor.query_module( + module="SubtensorModule", + name="WeightCommits", + params=[netuid_2, alice_wallet.hotkey.ss58_address], + ) + # Assert that the committed weights are set correctly + assert weight_commits.value is not None, "Weight commit not found in storage" + + ## Reveal for subnet 1 + + # Query the WeightCommitRevealInterval storage map + reveal_periods = subtensor.query_module( + module="SubtensorModule", name="RevealPeriodEpochs", params=[netuid_1] + ) + periods = reveal_periods.value + assert periods > 0, "Invalid RevealPeriodEpochs" + + # Wait until the reveal block range + await wait_interval( + subtensor.get_subnet_hyperparameters(netuid=netuid_1).tempo, subtensor + ) + + # Reveal weights + success, message = subtensor.reveal_weights( + alice_wallet, + netuid_1, + uids=weight_uids, + weights=weight_vals, + salt=salt, + wait_for_inclusion=True, + wait_for_finalization=True, + ) + + assert success is True + + ## Reveal for subnet 2 + + # Query the WeightCommitRevealInterval storage map + reveal_periods = subtensor.query_module( + module="SubtensorModule", name="RevealPeriodEpochs", params=[netuid_2] + ) + periods = reveal_periods.value + assert periods > 0, "Invalid RevealPeriodEpochs" + + # Wait until the reveal block range + await wait_interval( + subtensor.get_subnet_hyperparameters(netuid=netuid_2).tempo, subtensor + ) + + # Reveal weights + success, message = subtensor.reveal_weights( + alice_wallet, + netuid_2, + uids=weight_uids, + weights=weight_vals, + salt=salt, + wait_for_inclusion=True, + wait_for_finalization=True, + ) + + assert success is True + + time.sleep(10) + + ## Check subnet 1 weights are revealed correctly + + # Query the Weights storage map + revealed_weights = subtensor.query_module( + module="SubtensorModule", + name="Weights", + params=[netuid_1, 0], # netuid and uid + ) + + # Assert that the revealed weights are set correctly + assert revealed_weights.value is not None, "Weight reveal not found in storage" + + assert ( + weight_vals[0] == revealed_weights.value[0][1] + ), f"Incorrect revealed weights. Expected: {weights[0]}, Actual: {revealed_weights.value[0][1]}" + + ## Also check subnet 2 weights are revealed correctly + revealed_weights = subtensor.query_module( + module="SubtensorModule", + name="Weights", + params=[netuid_2, 0], # netuid and uid + ) + + assert revealed_weights.value is not None, "Weight reveal not found in storage" + + assert ( + weight_vals[0] == revealed_weights.value[0][1] + ), f"Incorrect revealed weights. Expected: {weights[0]}, Actual: {revealed_weights.value[0][1]}" + + print("✅ Passed test_batch_commit_weights") diff --git a/tests/e2e_tests/test_set_weights.py b/tests/e2e_tests/test_set_weights.py new file mode 100644 index 0000000000..0c5d7743a0 --- /dev/null +++ b/tests/e2e_tests/test_set_weights.py @@ -0,0 +1,325 @@ +import time + +import numpy as np +import pytest + +from bittensor.core.subtensor import Subtensor +from bittensor.utils.balance import Balance +from bittensor.utils.weight_utils import convert_weights_and_uids_for_emit +from tests.e2e_tests.utils.chain_interactions import ( + add_stake, + register_subnet, + sudo_set_hyperparameter_bool, + sudo_set_hyperparameter_values, + wait_interval, +) +from tests.e2e_tests.utils.e2e_test_utils import setup_wallet + + +@pytest.mark.asyncio +async def test_set_weights(local_chain): + """ + Tests the set weights mechanism + + Steps: + 1. Register a subnet through Alice + 2. Register Alice's neuron and add stake + 3. Register Bob's neuron + 4. Disable commit_reveal on the subnet + 5. Set min stake low enough for us to set weights + 6. Set weights rate limit + 7. Set weights and verify + Raises: + AssertionError: If any of the checks or verifications fail + """ + netuid = 1 + print("Testing test_set_weights") + # Register root as Alice + keypair, alice_wallet = setup_wallet("//Alice") + assert register_subnet(local_chain, alice_wallet), "Unable to register the subnet" + + # Verify subnet 1 created successfully + assert local_chain.query( + "SubtensorModule", "NetworksAdded", [netuid] + ).serialize(), "Subnet wasn't created successfully" + + subtensor = Subtensor(network="ws://localhost:9945") + + # Register Alice to the subnet + assert subtensor.burned_register( + alice_wallet, netuid + ), "Unable to register Alice as a neuron" + + # Register a second neuron + keypair, bob_wallet = setup_wallet("//Bob") + assert subtensor.burned_register( + bob_wallet, netuid + ), "Unable to register Bob as a neuron" + + # Stake to become to top neuron after the first epoch + add_stake(local_chain, alice_wallet, Balance.from_tao(100_000)) + + # Disable commit_reveal on the subnet + assert sudo_set_hyperparameter_bool( + local_chain, + alice_wallet, + "sudo_set_commit_reveal_weights_enabled", + False, + netuid, + ), "Unable to disable commit reveal on the subnet" + + assert ( + subtensor.get_subnet_hyperparameters( + netuid=netuid, + ).commit_reveal_weights_enabled + is False + ), "Failed to disable commit/reveal" + + # Set min stake low enough for us to set weights + assert sudo_set_hyperparameter_values( + local_chain, + alice_wallet, + call_function="sudo_set_weights_min_stake", + call_params={"min_stake": 100}, + return_error_message=True, + ) + + assert ( + subtensor.weights_rate_limit(netuid=netuid) > 0 + ), "Weights rate limit is below 0" + # Lower the rate limit + assert sudo_set_hyperparameter_values( + local_chain, + alice_wallet, + call_function="sudo_set_weights_set_rate_limit", + call_params={"netuid": netuid, "weights_set_rate_limit": "0"}, + return_error_message=True, + ) + + assert ( + subtensor.get_subnet_hyperparameters(netuid=netuid).weights_rate_limit == 0 + ), "Failed to set weights_rate_limit" + assert subtensor.weights_rate_limit(netuid=netuid) == 0 + + # Weight values + uids = np.array([0], dtype=np.int64) + weights = np.array([0.1], dtype=np.float32) + weight_uids, weight_vals = convert_weights_and_uids_for_emit( + uids=uids, weights=weights + ) + + # Set weights + success, message = subtensor.set_weights( + alice_wallet, + netuid, + uids=weight_uids, + weights=weight_vals, + wait_for_inclusion=True, + wait_for_finalization=True, + ) + + assert success is True + + time.sleep(10) + + # Query the Weights storage map + chain_weights = subtensor.query_module( + module="SubtensorModule", + name="Weights", + params=[netuid, 0], # netuid and uid + ) + + # Assert that the revealed weights are set correctly + assert chain_weights.value is not None, "Weight set not found in storage" + + assert ( + weight_vals[0] == chain_weights.value[0][1] + ), f"Incorrect weights. Expected: {weights[0]}, Actual: {chain_weights.value[0][1]}" + print("✅ Passed test_set_weights") + + +@pytest.mark.asyncio +async def test_batch_set_weights(local_chain): + """ + Tests the batch set weights mechanism + + Steps: + 1. Register multiple subnets through Alice + 2. Register Alice's neurons and add stake + 3. Register Bob's neurons + 4. Disable commit_reveal on the subnets + 5. Set min stake low enough for us to set weights + 6. Set weights rate limit + 7. Set weights and verify + Raises: + AssertionError: If any of the checks or verifications fail + """ + netuid_1 = 1 + netuid_2 = 2 + print("Testing test_batch_set_weights") + # Register root as Alice + keypair, alice_wallet = setup_wallet("//Alice") + assert register_subnet(local_chain, alice_wallet), "Unable to register the subnet" + assert register_subnet( + local_chain, alice_wallet + ), "Unable to register the second subnet" + + # Verify subnet 1 created successfully + assert local_chain.query( + "SubtensorModule", "NetworksAdded", [netuid_1] + ).serialize(), "Subnet wasn't created successfully" + + # Verify subnet 2 created successfully + assert local_chain.query( + "SubtensorModule", "NetworksAdded", [netuid_2] + ).serialize(), "Subnet wasn't created successfully" + + subtensor = Subtensor(network="ws://localhost:9945") + + # Register Alice to the subnet + assert subtensor.burned_register( + alice_wallet, netuid_1 + ), "Unable to register Alice as a neuron" + + # Register Alice to the second ubnet + assert subtensor.burned_register( + alice_wallet, netuid_2 + ), "Unable to register Alice as a neuron to the second subnet" + + # Register a second neuron + keypair, bob_wallet = setup_wallet("//Bob") + assert subtensor.burned_register( + bob_wallet, netuid_1 + ), "Unable to register Bob as a neuron" + + # Register a second neuron to the second subnet + assert subtensor.burned_register( + bob_wallet, netuid_2 + ), "Unable to register Bob as a neuron to the second subnet" + + # Stake to become to top neuron after the first epoch + add_stake(local_chain, alice_wallet, Balance.from_tao(100_000)) + + # Disable commit_reveal on both subnets + assert sudo_set_hyperparameter_bool( + local_chain, + alice_wallet, + "sudo_set_commit_reveal_weights_enabled", + False, + netuid_1, + ), "Unable to disable commit reveal on the first subnet" + + assert sudo_set_hyperparameter_bool( + local_chain, + alice_wallet, + "sudo_set_commit_reveal_weights_enabled", + False, + netuid_2, + ), "Unable to disable commit reveal on the second subnet" + + assert ( + subtensor.get_subnet_hyperparameters( + netuid=netuid_1, + ).commit_reveal_weights_enabled + is False + ), "Failed to disable commit/reveal on the first subnet" + + assert ( + subtensor.get_subnet_hyperparameters( + netuid=netuid_2, + ).commit_reveal_weights_enabled + is False + ), "Failed to disable commit/reveal on the second subnet" + + # Set min stake low enough for us to set weights + assert sudo_set_hyperparameter_values( + local_chain, + alice_wallet, + call_function="sudo_set_weights_min_stake", + call_params={"min_stake": 100}, + return_error_message=True, + ) + + assert ( + subtensor.weights_rate_limit(netuid=netuid_1) > 0 + ), "Weights rate limit is below 0" + + assert ( + subtensor.weights_rate_limit(netuid=netuid_2) > 0 + ), "Weights rate limit is below 0" + + # Lower the rate limit + assert sudo_set_hyperparameter_values( + local_chain, + alice_wallet, + call_function="sudo_set_weights_set_rate_limit", + call_params={"netuid": netuid_1, "weights_set_rate_limit": "0"}, + return_error_message=True, + ) + + assert sudo_set_hyperparameter_values( + local_chain, + alice_wallet, + call_function="sudo_set_weights_set_rate_limit", + call_params={"netuid": netuid_2, "weights_set_rate_limit": "0"}, + return_error_message=True, + ) + + assert ( + subtensor.get_subnet_hyperparameters(netuid=netuid_1).weights_rate_limit == 0 + ), "Failed to set weights_rate_limit on the first subnet" + + assert ( + subtensor.get_subnet_hyperparameters(netuid=netuid_2).weights_rate_limit == 0 + ), "Failed to set weights_rate_limit on the second subnet" + + assert subtensor.weights_rate_limit(netuid=netuid_1) == 0 + assert subtensor.weights_rate_limit(netuid=netuid_2) == 0 + + # Weight values + uids = np.array([0], dtype=np.int64) + weights = np.array([0.1], dtype=np.float32) + weight_uids, weight_vals = convert_weights_and_uids_for_emit( + uids=uids, weights=weights + ) + + # Set weights in a batch + success, message = subtensor.batch_set_weights( + alice_wallet, + [netuid_1, netuid_2], + uids=[weight_uids, weight_uids], + weights=[weight_vals, weight_vals], + wait_for_inclusion=True, + wait_for_finalization=True, + ) + + assert success is True + + time.sleep(10) + + # Query the Weights storage map + chain_weights_1 = subtensor.query_module( + module="SubtensorModule", + name="Weights", + params=[netuid_1, 0], # netuid and uid + ) + + chain_weights_2 = subtensor.query_module( + module="SubtensorModule", + name="Weights", + params=[netuid_2, 0], # netuid and uid + ) + + # Assert that the revealed weights are set correctly + assert chain_weights_1.value is not None, "Weight set not found in storage" + assert chain_weights_2.value is not None, "Weight set not found in storage" + + assert ( + weight_vals[0] == chain_weights_1.value[0][1] + ), f"Incorrect weights. Expected: {weights[0]}, Actual: {chain_weights_1.value[0][1]}" + + assert ( + weight_vals[0] == chain_weights_2.value[0][1] + ), f"Incorrect weights. Expected: {weights[0]}, Actual: {chain_weights_2.value[0][1]}" + + print("✅ Passed test_batch_set_weights") From f990cf0387c567060b4665cabbf97dfb877dd764 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 26 Nov 2024 21:06:29 -0500 Subject: [PATCH 05/15] add test helpers --- tests/e2e_tests/utils/chain_interactions.py | 36 +++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/e2e_tests/utils/chain_interactions.py b/tests/e2e_tests/utils/chain_interactions.py index 9c0d9100e8..3ad9ad8569 100644 --- a/tests/e2e_tests/utils/chain_interactions.py +++ b/tests/e2e_tests/utils/chain_interactions.py @@ -22,6 +22,7 @@ def sudo_set_hyperparameter_bool( call_function: str, value: bool, netuid: int, + requires_sudo: bool = False, ) -> bool: """ Sets boolean hyperparameter value through AdminUtils. Mimics setting hyperparams @@ -31,6 +32,13 @@ def sudo_set_hyperparameter_bool( call_function=call_function, call_params={"netuid": netuid, "enabled": value}, ) + if requires_sudo: # Only sudo can set these values + call = substrate.compose_call( + call_module="Sudo", + call_function="sudo", + call_params={"call": call}, + ) + extrinsic = substrate.create_signed_extrinsic(call=call, keypair=wallet.coldkey) response = substrate.submit_extrinsic( extrinsic, @@ -47,6 +55,7 @@ def sudo_set_hyperparameter_values( call_function: str, call_params: dict, return_error_message: bool = False, + requires_sudo: bool = False, ) -> Union[bool, tuple[bool, Optional[str]]]: """ Sets liquid alpha values using AdminUtils. Mimics setting hyperparams @@ -56,6 +65,13 @@ def sudo_set_hyperparameter_values( call_function=call_function, call_params=call_params, ) + if requires_sudo: # Only sudo can set these values + call = substrate.compose_call( + call_module="Sudo", + call_function="sudo", + call_params={"call": call}, + ) + extrinsic = substrate.create_signed_extrinsic(call=call, keypair=wallet.coldkey) response = substrate.submit_extrinsic( extrinsic, @@ -160,3 +176,23 @@ async def wait_interval(tempo: int, subtensor: "Subtensor", netuid: int = 1): logging.info( f"Current Block: {current_block} Next tempo for netuid {netuid} at: {next_tempo_block_start}" ) + + +async def wait_until_block( + block: int, subtensor: "Subtensor", block_time: int = 250e-3 +): + """ + Waits until a specific block is reached. + + Will return immediately if the block has already been reached. + """ + current_block = subtensor.get_current_block() + + while current_block < block: + wait_time = (block - current_block) * block_time + print(f"Waiting for {wait_time} seconds until block {block}") + logging.info(f"Waiting for {wait_time} seconds until block {block}") + await asyncio.sleep( + wait_time + ) # Wait for 1 second before checking the block number again + current_block = subtensor.get_current_block() From 5f9e9f38b30fcac990e0d3029c60f4fb40fa6fee Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 26 Nov 2024 21:06:47 -0500 Subject: [PATCH 06/15] fix e2e test for commit weights --- tests/e2e_tests/test_commit_weights.py | 60 +++++++++++++++----------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/tests/e2e_tests/test_commit_weights.py b/tests/e2e_tests/test_commit_weights.py index 2fdedc4970..df49d817e5 100644 --- a/tests/e2e_tests/test_commit_weights.py +++ b/tests/e2e_tests/test_commit_weights.py @@ -12,6 +12,7 @@ sudo_set_hyperparameter_bool, sudo_set_hyperparameter_values, wait_interval, + wait_until_block, ) from tests.e2e_tests.utils.e2e_test_utils import setup_wallet @@ -247,17 +248,40 @@ async def test_batch_commit_weights(local_chain): local_chain, alice_wallet, call_function="sudo_set_commit_reveal_weights_interval", - call_params={"netuid": netuid_1, "interval": "1"}, + call_params={"netuid": netuid_1, "interval": 1}, return_error_message=True, ) assert sudo_set_hyperparameter_values( local_chain, alice_wallet, call_function="sudo_set_commit_reveal_weights_interval", - call_params={"netuid": netuid_2, "interval": "1"}, + call_params={"netuid": netuid_2, "interval": 1}, return_error_message=True, ) + # Set the tempos + shorter_tempo = 75 + assert sudo_set_hyperparameter_values( + local_chain, + alice_wallet, + call_function="sudo_set_tempo", + call_params={"netuid": netuid_1, "tempo": shorter_tempo}, + return_error_message=True, + requires_sudo=True, + ) + assert sudo_set_hyperparameter_values( + local_chain, + alice_wallet, + call_function="sudo_set_tempo", + call_params={"netuid": netuid_2, "tempo": shorter_tempo}, + return_error_message=True, + requires_sudo=True, + ) + + # Verify the tempos are set correctly + assert subtensor.get_subnet_hyperparameters(netuid=netuid_1).tempo == shorter_tempo + assert subtensor.get_subnet_hyperparameters(netuid=netuid_2).tempo == shorter_tempo + # Verify commit/reveal periods are set correctly assert ( subtensor.get_subnet_hyperparameters( @@ -332,7 +356,7 @@ async def test_batch_commit_weights(local_chain): ) # Assert that the committed weights are set correctly assert weight_commits.value is not None, "Weight commit not found in storage" - commit_hash, commit_block, reveal_block, expire_block = weight_commits.value[0] + commit_hash, commit_block, reveal_block_1, expire_block_1 = weight_commits.value[0] assert commit_block > 0, f"Invalid block number: {commit_block}" weight_commits = subtensor.query_module( @@ -342,20 +366,13 @@ async def test_batch_commit_weights(local_chain): ) # Assert that the committed weights are set correctly assert weight_commits.value is not None, "Weight commit not found in storage" + commit_hash, commit_block, reveal_block_2, expire_block_2 = weight_commits.value[0] + assert commit_block > 0, f"Invalid block number: {commit_block}" ## Reveal for subnet 1 - # Query the WeightCommitRevealInterval storage map - reveal_periods = subtensor.query_module( - module="SubtensorModule", name="RevealPeriodEpochs", params=[netuid_1] - ) - periods = reveal_periods.value - assert periods > 0, "Invalid RevealPeriodEpochs" - # Wait until the reveal block range - await wait_interval( - subtensor.get_subnet_hyperparameters(netuid=netuid_1).tempo, subtensor - ) + await wait_until_block(reveal_block_1, subtensor) # Reveal weights success, message = subtensor.reveal_weights( @@ -368,21 +385,12 @@ async def test_batch_commit_weights(local_chain): wait_for_finalization=True, ) - assert success is True + assert success is True, "Failed to reveal weights for the first subnet" ## Reveal for subnet 2 - # Query the WeightCommitRevealInterval storage map - reveal_periods = subtensor.query_module( - module="SubtensorModule", name="RevealPeriodEpochs", params=[netuid_2] - ) - periods = reveal_periods.value - assert periods > 0, "Invalid RevealPeriodEpochs" - # Wait until the reveal block range - await wait_interval( - subtensor.get_subnet_hyperparameters(netuid=netuid_2).tempo, subtensor - ) + await wait_until_block(reveal_block_2, subtensor) # Reveal weights success, message = subtensor.reveal_weights( @@ -395,9 +403,9 @@ async def test_batch_commit_weights(local_chain): wait_for_finalization=True, ) - assert success is True + assert success is True, "Failed to reveal weights for the second subnet" - time.sleep(10) + time.sleep(6) ## Check subnet 1 weights are revealed correctly From 900905cc019d6b27fcf6e79d57b590074aebf03a Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 26 Nov 2024 21:46:01 -0500 Subject: [PATCH 07/15] fix batch set and commit weights methods --- bittensor/core/async_subtensor.py | 81 +++++++++++++++++++ bittensor/core/extrinsics/async_weights.py | 20 ++--- bittensor/core/extrinsics/set_weights.py | 22 +++--- bittensor/core/subtensor.py | 92 ++++++++++++++++++++-- tests/e2e_tests/test_commit_weights.py | 4 +- tests/e2e_tests/test_set_weights.py | 6 +- 6 files changed, 193 insertions(+), 32 deletions(-) diff --git a/bittensor/core/async_subtensor.py b/bittensor/core/async_subtensor.py index a43510194d..31fe85fbd7 100644 --- a/bittensor/core/async_subtensor.py +++ b/bittensor/core/async_subtensor.py @@ -30,6 +30,7 @@ from bittensor.core.extrinsics.async_transfer import transfer_extrinsic from bittensor.core.extrinsics.async_weights import ( batch_commit_weights_extrinsic, + batch_set_weights_extrinsic, commit_weights_extrinsic, set_weights_extrinsic, ) @@ -1531,6 +1532,86 @@ async def set_weights( return success, message + async def batch_set_weights( + self, + wallet: "Wallet", + netuids: list[int], + uidss: list[Union[NDArray[np.int64], "torch.LongTensor", list]], + weightss: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], + version_keys: list[int] = [], + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, + max_retries: int = 5, + ): + """ + Batch set weights for multiple subnets. + + Args: + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron setting the weights. + netuids (list[int]): The list of subnet uids. + uidss (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The list of neuron UIDs that the weights are being set for. + weightss (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The corresponding weights to be set for each UID. + version_keys (list[int]): Version keys for compatibility with the network. Default is ``int representation of Bittensor version.``. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is ``False``. + max_retries (int): The number of maximum attempts to set weights. Default is ``5``. + + Returns: + tuple[bool, str]: ``True`` if the setting of weights is successful, False otherwise. And `msg`, a string value describing the success or potential error. + + This function is crucial in shaping the network's collective intelligence, where each neuron's learning and contribution are influenced by the weights it sets towards others【81†source】. + """ + netuids_to_set = [] + uidss_to_set = [] + weightss_to_set = [] + version_keys_to_set = [] + + if len(version_keys) == 0: + version_keys = [version_as_int] * len(netuids) + + for i, netuid in enumerate(netuids): + uid = await self.get_uid_for_hotkey_on_subnet( + wallet.hotkey.ss58_address, netuid + ) + retries = 0 + success = False + message = "No attempt made. Perhaps it is too soon to set weights!" + + if await self.blocks_since_last_update( + netuid, uid + ) <= await self.weights_rate_limit(netuid): + logging.info( + f"Skipping subnet #{netuid} as it has not reached the weights rate limit." + ) + continue + + netuids_to_set.append(netuid) + uidss_to_set.append(uidss[i]) + weightss_to_set.append(weightss[i]) + version_keys_to_set.append(version_keys[i]) + + while retries < max_retries: + try: + logging.info( + f"Setting batch of weights for subnets #[blue]{netuids_to_set}[/blue]. Attempt [blue]{retries + 1} of {max_retries}[/blue]." + ) + success, message = await batch_set_weights_extrinsic( + subtensor=self, + wallet=wallet, + netuids=netuids_to_set, + uidss=uidss_to_set, + weightss=weightss_to_set, + version_keys=version_keys_to_set, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + except Exception as e: + logging.error(f"Error setting batch of weights: {e}") + finally: + retries += 1 + + return success, message + async def root_set_weights( self, wallet: "Wallet", diff --git a/bittensor/core/extrinsics/async_weights.py b/bittensor/core/extrinsics/async_weights.py index a3deb7584e..1b2a150d90 100644 --- a/bittensor/core/extrinsics/async_weights.py +++ b/bittensor/core/extrinsics/async_weights.py @@ -227,8 +227,8 @@ async def batch_set_weights_extrinsic( subtensor: "AsyncSubtensor", wallet: "Wallet", netuids: list[int], - uids: list[Union[NDArray[np.int64], "torch.LongTensor", list]], - weights: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], + uidss: list[Union[NDArray[np.int64], "torch.LongTensor", list]], + weightss: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], version_keys: list[int] = [], wait_for_inclusion: bool = False, wait_for_finalization: bool = False, @@ -239,8 +239,8 @@ async def batch_set_weights_extrinsic( subtensor (bittensor.subtensor): Bittensor subtensor object. wallet (bittensor.wallet): Bittensor wallet object. netuids (list[int]): The ``netuid`` of the subnet to set weights for. - uids (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The ``uint64`` uids of destination neurons. - weights (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The weights to set. These must be ``float`` s and correspond to the passed ``uid`` s. + uidss (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The ``uint64`` uids of destination neurons. + weightss (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The weights to set. These must be ``float`` s and correspond to the passed ``uid`` s. version_keys (list[int]): The version key of the validator. wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``true``, or returns ``false`` if the extrinsic fails to enter the block within the timeout. wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. @@ -253,22 +253,22 @@ async def batch_set_weights_extrinsic( if len(version_keys) == 0: version_keys = [0] * len(netuids) # Default to version 0 if not provided - for uids_, weights_ in zip(uids, weights): + for uids, weights in zip(uidss, weightss): # First convert types. if use_torch(): if isinstance(uids, list): - uids_ = torch.tensor(uids, dtype=torch.int64) + uids = torch.tensor(uids, dtype=torch.int64) if isinstance(weights, list): - weights_ = torch.tensor(weights, dtype=torch.float32) + weights = torch.tensor(weights, dtype=torch.float32) else: if isinstance(uids, list): - uids_ = np.array(uids, dtype=np.int64) + uids = np.array(uids, dtype=np.int64) if isinstance(weights, list): - weights_ = np.array(weights, dtype=np.float32) + weights = np.array(weights, dtype=np.float32) # Reformat and normalize. weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit( - uids_, weights_ + uids, weights ) uids_to_set.append(weight_uids) diff --git a/bittensor/core/extrinsics/set_weights.py b/bittensor/core/extrinsics/set_weights.py index c4321b6d5b..deb95effe3 100644 --- a/bittensor/core/extrinsics/set_weights.py +++ b/bittensor/core/extrinsics/set_weights.py @@ -251,8 +251,8 @@ def batch_set_weights_extrinsic( subtensor: "Subtensor", wallet: "Wallet", netuids: list[int], - uids: list[Union[NDArray[np.int64], "torch.LongTensor", list]], - weights: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], + uidss: list[Union[NDArray[np.int64], "torch.LongTensor", list]], + weightss: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], version_keys: list[int] = [], wait_for_inclusion: bool = False, wait_for_finalization: bool = False, @@ -263,8 +263,8 @@ def batch_set_weights_extrinsic( subtensor (bittensor.subtensor): Bittensor subtensor object. wallet (bittensor.wallet): Bittensor wallet object. netuids (list[int]): The ``netuid`` of the subnet to set weights for. - uids (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The ``uint64`` uids of destination neurons. - weights (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The weights to set. These must be ``float`` s and correspond to the passed ``uid`` s. + uidss (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The ``uint64`` uids of destination neurons. + weightss (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The weights to set. These must be ``float`` s and correspond to the passed ``uid`` s. version_keys (list[int]): The version key of the validator. wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``true``, or returns ``false`` if the extrinsic fails to enter the block within the timeout. wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. @@ -277,22 +277,22 @@ def batch_set_weights_extrinsic( if len(version_keys) == 0: version_keys = [0] * len(netuids) # Default to version 0 if not provided - for uids_, weights_ in zip(uids, weights): + for uids, weights in zip(uidss, weightss): # First convert types. if use_torch(): if isinstance(uids, list): - uids_ = torch.tensor(uids, dtype=torch.int64) + uids = torch.tensor(uids, dtype=torch.int64) if isinstance(weights, list): - weights_ = torch.tensor(weights, dtype=torch.float32) + weights = torch.tensor(weights, dtype=torch.float32) else: if isinstance(uids, list): - uids_ = np.array(uids, dtype=np.int64) + uids = np.array(uids, dtype=np.int64) if isinstance(weights, list): - weights_ = np.array(weights, dtype=np.float32) + weights = np.array(weights, dtype=np.float32) # Reformat and normalize. weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit( - uids_, weights_ + uids, weights ) uids_to_set.append(weight_uids) @@ -303,7 +303,7 @@ def batch_set_weights_extrinsic( ) try: success, error_message = do_batch_set_weights( - subtensor=subtensor, + self=subtensor, wallet=wallet, netuids=netuids, uidss=uids_to_set, diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 8ebea24187..70d2d02560 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -50,7 +50,10 @@ publish_metadata, get_metadata, ) -from bittensor.core.extrinsics.set_weights import set_weights_extrinsic +from bittensor.core.extrinsics.set_weights import ( + set_weights_extrinsic, + batch_set_weights_extrinsic, +) from bittensor.core.extrinsics.transfer import ( transfer_extrinsic, ) @@ -1733,6 +1736,83 @@ def set_weights( return success, message + def batch_set_weights( + self, + wallet: "Wallet", + netuids: list[int], + uidss: list[Union[NDArray[np.int64], "torch.LongTensor", list]], + weightss: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], + version_keys: list[int] = [], + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, + max_retries: int = 5, + ) -> tuple[bool, str]: + """ + Sets a batch of weights for multiple subnets. + + Args: + wallet (bittensor_wallet.Wallet): The wallet associated with the neuron setting the weights. + netuids (list[int]): The list of subnet netuids that the weights are being set for. + uidss (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The lists of neuron UIDs that the weights are being set for. + weightss (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The lists of corresponding weights to be set for each UID. + version_keys (list[int]): Version keys for compatibility with each subnet. + wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. + wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is ``False``. + max_retries (int): The number of maximum attempts to set weights. Default is ``5``. + + Returns: + tuple[bool, str]: ``True`` if the setting of weights is successful, False otherwise. And `msg`, a string value describing the success or potential error. + + This function is crucial in shaping the network's collective intelligence, where each neuron's learning and contribution are influenced by the weights it sets towards others【81†source】. + """ + netuids_to_set = [] + uidss_to_set = [] + weightss_to_set = [] + version_keys_to_set = [] + + if len(version_keys) == 0: + version_keys = [settings.version_as_int] * len(netuids) + + for i, netuid in enumerate(netuids): + uid = self.get_uid_for_hotkey_on_subnet(wallet.hotkey.ss58_address, netuid) + retries = 0 + success = False + message = "No attempt made. Perhaps it is too soon to set weights!" + if self.blocks_since_last_update(netuid, uid) <= self.weights_rate_limit( + netuid + ): + logging.info( + f"Skipping subnet #{netuid} as it has not reached the weights rate limit." + ) + continue + + netuids_to_set.append(netuid) + uidss_to_set.append(uidss[i]) + weightss_to_set.append(weightss[i]) + version_keys_to_set.append(version_keys[i]) + + while retries < max_retries: + try: + logging.info( + f"Setting batch of weights for subnets #{netuids_to_set}. Attempt {retries + 1} of {max_retries}." + ) + success, message = batch_set_weights_extrinsic( + subtensor=self, + wallet=wallet, + netuids=netuids_to_set, + uidss=uidss_to_set, + weightss=weightss_to_set, + version_keys=version_keys_to_set, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + except Exception as e: + logging.error(f"Error setting batch of weights: {e}") + finally: + retries += 1 + + return success, message + @legacy_torch_api_compat def root_set_weights( self, @@ -2014,8 +2094,8 @@ def batch_commit_weights( wallet: "Wallet", netuids: list[int], salts: list[list[int]], - uids: list[Union[NDArray[np.int64], list]], - weights: list[Union[NDArray[np.int64], list]], + uidss: list[Union[NDArray[np.int64], list]], + weightss: list[Union[NDArray[np.int64], list]], version_keys: list[int] = [], wait_for_inclusion: bool = False, wait_for_finalization: bool = False, @@ -2029,8 +2109,8 @@ def batch_commit_weights( wallet (bittensor_wallet.Wallet): The wallet associated with the neuron committing the weights. netuids (list[int]): The list of subnet uids. salts (list[list[int]]): The list of salts to generate weight hashes. - uids (list[np.ndarray]): The list of NumPy arrays of neuron UIDs for which weights are being committed. - weights (list[np.ndarray]): The list of NumPy arrays of weight values corresponding to each UID. + uidss (list[np.ndarray]): The list of NumPy arrays of neuron UIDs for which weights are being committed. + weightss (list[np.ndarray]): The list of NumPy arrays of weight values corresponding to each UID. version_keys (list[int]): The list of version keys for compatibility with the network. Default is ``int representation of Bittensor version.``. wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is ``False``. @@ -2063,7 +2143,7 @@ def batch_commit_weights( version_key=version_key, ) for netuid, salt, uids, weights, version_key in zip( - netuids, salts, uids, weights, version_keys + netuids, salts, uidss, weightss, version_keys ) ] diff --git a/tests/e2e_tests/test_commit_weights.py b/tests/e2e_tests/test_commit_weights.py index df49d817e5..5945de36b4 100644 --- a/tests/e2e_tests/test_commit_weights.py +++ b/tests/e2e_tests/test_commit_weights.py @@ -341,8 +341,8 @@ async def test_batch_commit_weights(local_chain): alice_wallet, [netuid_1, netuid_2], salts=[salt, salt], - uids=[weight_uids, weight_uids], - weights=[weight_vals, weight_vals], + uidss=[weight_uids, weight_uids], + weightss=[weight_vals, weight_vals], wait_for_inclusion=True, wait_for_finalization=True, ) diff --git a/tests/e2e_tests/test_set_weights.py b/tests/e2e_tests/test_set_weights.py index 0c5d7743a0..223c0c7e2f 100644 --- a/tests/e2e_tests/test_set_weights.py +++ b/tests/e2e_tests/test_set_weights.py @@ -286,9 +286,9 @@ async def test_batch_set_weights(local_chain): # Set weights in a batch success, message = subtensor.batch_set_weights( alice_wallet, - [netuid_1, netuid_2], - uids=[weight_uids, weight_uids], - weights=[weight_vals, weight_vals], + netuids=[netuid_1, netuid_2], + uidss=[weight_uids, weight_uids], + weightss=[weight_vals, weight_vals], wait_for_inclusion=True, wait_for_finalization=True, ) From ffc9ef33b085c6ac1382bed88cbe6a3f3e4b47df Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 27 Nov 2024 15:59:10 -0500 Subject: [PATCH 08/15] use optional = None instead of [] for ver_keys --- bittensor/core/async_subtensor.py | 16 ++++++++-------- bittensor/core/extrinsics/async_weights.py | 12 ++++++++---- bittensor/core/extrinsics/set_weights.py | 13 ++++++++----- bittensor/core/subtensor.py | 12 ++++++------ 4 files changed, 30 insertions(+), 23 deletions(-) diff --git a/bittensor/core/async_subtensor.py b/bittensor/core/async_subtensor.py index 148b32e626..cfa77033d5 100644 --- a/bittensor/core/async_subtensor.py +++ b/bittensor/core/async_subtensor.py @@ -1619,7 +1619,7 @@ async def batch_set_weights( netuids: list[int], uidss: list[Union[NDArray[np.int64], "torch.LongTensor", list]], weightss: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], - version_keys: list[int] = [], + version_keys: Optional[list[int]] = None, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, max_retries: int = 5, @@ -1632,7 +1632,7 @@ async def batch_set_weights( netuids (list[int]): The list of subnet uids. uidss (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The list of neuron UIDs that the weights are being set for. weightss (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The corresponding weights to be set for each UID. - version_keys (list[int]): Version keys for compatibility with the network. Default is ``int representation of Bittensor version.``. + version_keys (Optional[list[int]]): Version keys for compatibility with the network. Default is ``int representation of Bittensor version.``. wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is ``False``. max_retries (int): The number of maximum attempts to set weights. Default is ``5``. @@ -1647,7 +1647,7 @@ async def batch_set_weights( weightss_to_set = [] version_keys_to_set = [] - if len(version_keys) == 0: + if version_keys is None or len(version_keys) == 0: version_keys = [version_as_int] * len(netuids) for i, netuid in enumerate(netuids): @@ -1798,9 +1798,9 @@ async def batch_commit_weights( wallet: "Wallet", netuids: list[int], salts: list[list[int]], - uids: list[Union[NDArray[np.int64], list]], - weights: list[Union[NDArray[np.int64], list]], - version_keys: list[int] = [], + uidss: list[Union[NDArray[np.int64], list]], + weightss: list[Union[NDArray[np.int64], list]], + version_keys: Optional[list[int]] = None, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, max_retries: int = 5, @@ -1815,7 +1815,7 @@ async def batch_commit_weights( salts (list[list[int]]): The list of salts to generate weight hashes. uids (list[np.ndarray]): The list of NumPy arrays of neuron UIDs for which weights are being committed. weights (list[np.ndarray]): The list of NumPy arrays of weight values corresponding to each UID. - version_keys (list[int]): The list of version keys for compatibility with the network. Default is ``int representation of Bittensor version.``. + version_keys (Optional[list[int]]): The list of version keys for compatibility with the network. Default is ``int representation of Bittensor version.``. wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is ``False``. max_retries (int): The number of maximum attempts to commit weights. Default is ``5``. @@ -1833,7 +1833,7 @@ async def batch_commit_weights( f"Committing a batch of weights with params: netuids={netuids}, salts={salts}, uids={uids}, weights={weights}, version_keys={version_keys}" ) - if len(version_keys) == 0: + if version_keys is None or len(version_keys) == 0: version_keys = [version_as_int] * len(netuids) # Generate the hash of the weights diff --git a/bittensor/core/extrinsics/async_weights.py b/bittensor/core/extrinsics/async_weights.py index 1b2a150d90..69bb415349 100644 --- a/bittensor/core/extrinsics/async_weights.py +++ b/bittensor/core/extrinsics/async_weights.py @@ -88,7 +88,7 @@ async def _do_batch_set_weights( uidss: list[list[int]], valss: list[list[int]], netuids: list[int], - version_keys: list[int] = [], + version_keys: Optional[list[int]] = None, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, ) -> list[tuple[bool, Optional[str]]]: # (success, error_message) @@ -113,6 +113,10 @@ async def _do_batch_set_weights( This method is vital for the dynamic weighting mechanism in Bittensor, where neurons adjust their trust in other neurons based on observed performance and contributions. """ + + if version_keys is None or len(version_keys) == 0: + version_keys = [version_as_int] * len(netuids) + packed_weights = [ [(uid, val) for uid, val in zip(uids, vals)] for uids, vals in zip(uidss, valss) ] @@ -229,7 +233,7 @@ async def batch_set_weights_extrinsic( netuids: list[int], uidss: list[Union[NDArray[np.int64], "torch.LongTensor", list]], weightss: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], - version_keys: list[int] = [], + version_keys: Optional[list[int]] = None, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, ) -> tuple[bool, str]: @@ -241,7 +245,7 @@ async def batch_set_weights_extrinsic( netuids (list[int]): The ``netuid`` of the subnet to set weights for. uidss (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The ``uint64`` uids of destination neurons. weightss (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The weights to set. These must be ``float`` s and correspond to the passed ``uid`` s. - version_keys (list[int]): The version key of the validator. + version_keys (Optional[list[int]]): The version key of the validator. wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``true``, or returns ``false`` if the extrinsic fails to enter the block within the timeout. wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. @@ -250,7 +254,7 @@ async def batch_set_weights_extrinsic( """ uids_to_set: list[Union[NDArray[np.int64], "torch.LongTensor", list]] = [] weights_to_set: list[Union[NDArray[np.float32], "torch.FloatTensor", list]] = [] - if len(version_keys) == 0: + if version_keys is None or len(version_keys) == 0: version_keys = [0] * len(netuids) # Default to version 0 if not provided for uids, weights in zip(uidss, weightss): diff --git a/bittensor/core/extrinsics/set_weights.py b/bittensor/core/extrinsics/set_weights.py index deb95effe3..dd113d5135 100644 --- a/bittensor/core/extrinsics/set_weights.py +++ b/bittensor/core/extrinsics/set_weights.py @@ -109,7 +109,7 @@ def do_batch_set_weights( uidss: list[list[int]], valss: list[list[int]], netuids: list[int], - version_keys: list[int] = [], + version_keys: Optional[list[int]] = None, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, period: int = 5, @@ -123,7 +123,7 @@ def do_batch_set_weights( uidss (list[list[int]]): List of neuron UIDs for which weights are being set. valss (list[list[int]]): List of weight values corresponding to each UID. netuids (list[int]): Unique identifier for the network. - version_keys (list[int]): Version key for compatibility with the network. + version_keys (Optional[list[int]]): Version key for compatibility with the network. wait_for_inclusion (bool): Waits for the transaction to be included in a block. wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. period (int): Period dictates how long the extrinsic will stay as part of waiting pool. @@ -133,6 +133,9 @@ def do_batch_set_weights( This method is vital for the dynamic weighting mechanism in Bittensor, where neurons adjust their trust in other neurons based on observed performance and contributions. """ + if version_keys is None or len(version_keys) == 0: + version_keys = [version_as_int] * len(netuids) + packed_weights = [ [(uid, val) for uid, val in zip(uids, vals)] for uids, vals in zip(uidss, valss) ] @@ -253,7 +256,7 @@ def batch_set_weights_extrinsic( netuids: list[int], uidss: list[Union[NDArray[np.int64], "torch.LongTensor", list]], weightss: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], - version_keys: list[int] = [], + version_keys: Optional[list[int]] = None, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, ) -> tuple[bool, str]: @@ -265,7 +268,7 @@ def batch_set_weights_extrinsic( netuids (list[int]): The ``netuid`` of the subnet to set weights for. uidss (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The ``uint64`` uids of destination neurons. weightss (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The weights to set. These must be ``float`` s and correspond to the passed ``uid`` s. - version_keys (list[int]): The version key of the validator. + version_keys (Optional[list[int]]): The version key of the validator. wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``true``, or returns ``false`` if the extrinsic fails to enter the block within the timeout. wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. @@ -274,7 +277,7 @@ def batch_set_weights_extrinsic( """ uids_to_set: list[Union[NDArray[np.int64], "torch.LongTensor", list]] = [] weights_to_set: list[Union[NDArray[np.float32], "torch.FloatTensor", list]] = [] - if len(version_keys) == 0: + if version_keys is None or len(version_keys) == 0: version_keys = [0] * len(netuids) # Default to version 0 if not provided for uids, weights in zip(uidss, weightss): diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 70d2d02560..63c45cf392 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -1742,7 +1742,7 @@ def batch_set_weights( netuids: list[int], uidss: list[Union[NDArray[np.int64], "torch.LongTensor", list]], weightss: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], - version_keys: list[int] = [], + version_keys: Optional[list[int]] = None, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, max_retries: int = 5, @@ -1755,7 +1755,7 @@ def batch_set_weights( netuids (list[int]): The list of subnet netuids that the weights are being set for. uidss (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The lists of neuron UIDs that the weights are being set for. weightss (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The lists of corresponding weights to be set for each UID. - version_keys (list[int]): Version keys for compatibility with each subnet. + version_keys (Optional[list[int]]): Version keys for compatibility with each subnet. wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is ``False``. max_retries (int): The number of maximum attempts to set weights. Default is ``5``. @@ -1770,7 +1770,7 @@ def batch_set_weights( weightss_to_set = [] version_keys_to_set = [] - if len(version_keys) == 0: + if version_keys is None or len(version_keys) == 0: version_keys = [settings.version_as_int] * len(netuids) for i, netuid in enumerate(netuids): @@ -2096,7 +2096,7 @@ def batch_commit_weights( salts: list[list[int]], uidss: list[Union[NDArray[np.int64], list]], weightss: list[Union[NDArray[np.int64], list]], - version_keys: list[int] = [], + version_keys: Optional[list[int]] = None, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, max_retries: int = 5, @@ -2111,7 +2111,7 @@ def batch_commit_weights( salts (list[list[int]]): The list of salts to generate weight hashes. uidss (list[np.ndarray]): The list of NumPy arrays of neuron UIDs for which weights are being committed. weightss (list[np.ndarray]): The list of NumPy arrays of weight values corresponding to each UID. - version_keys (list[int]): The list of version keys for compatibility with the network. Default is ``int representation of Bittensor version.``. + version_keys (Optional[list[int]]): The list of version keys for compatibility with the network. Default is ``int representation of Bittensor version.``. wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is ``False``. max_retries (int): The number of maximum attempts to commit weights. Default is ``5``. @@ -2129,7 +2129,7 @@ def batch_commit_weights( f"Committing a batch of weights with params: netuids={netuids}, salts={salts}, uids={uids}, weights={weights}, version_keys={version_keys}" ) - if len(version_keys) == 0: + if version_keys is None or len(version_keys) == 0: version_keys = [settings.version_as_int] * len(netuids) # Generate the hash of the weights From d5255723d4879c05c8d0d30ffa1207424838bbc0 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 27 Nov 2024 16:04:35 -0500 Subject: [PATCH 09/15] oops: undeclared var --- bittensor/core/async_subtensor.py | 2 +- bittensor/core/subtensor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bittensor/core/async_subtensor.py b/bittensor/core/async_subtensor.py index cfa77033d5..b9016e2fc4 100644 --- a/bittensor/core/async_subtensor.py +++ b/bittensor/core/async_subtensor.py @@ -1830,7 +1830,7 @@ async def batch_commit_weights( message = "No attempt made. Perhaps it is too soon to commit weights!" logging.info( - f"Committing a batch of weights with params: netuids={netuids}, salts={salts}, uids={uids}, weights={weights}, version_keys={version_keys}" + f"Committing a batch of weights with params: netuids={netuids}, salts={salts}, uids={uidss}, weights={weightss}, version_keys={version_keys}" ) if version_keys is None or len(version_keys) == 0: diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 63c45cf392..2e7aa10054 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -2126,7 +2126,7 @@ def batch_commit_weights( message = "No attempt made. Perhaps it is too soon to commit weights!" logging.info( - f"Committing a batch of weights with params: netuids={netuids}, salts={salts}, uids={uids}, weights={weights}, version_keys={version_keys}" + f"Committing a batch of weights with params: netuids={netuids}, salts={salts}, uids={uidss}, weights={weightss}, version_keys={version_keys}" ) if version_keys is None or len(version_keys) == 0: From 23a49659e9181a4876a11c893b453707114fab87 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 27 Nov 2024 16:04:48 -0500 Subject: [PATCH 10/15] use torch legacy dec --- bittensor/core/extrinsics/async_weights.py | 32 ++++++++-------------- bittensor/core/extrinsics/set_weights.py | 32 ++++++++-------------- 2 files changed, 22 insertions(+), 42 deletions(-) diff --git a/bittensor/core/extrinsics/async_weights.py b/bittensor/core/extrinsics/async_weights.py index 69bb415349..ef49a3f616 100644 --- a/bittensor/core/extrinsics/async_weights.py +++ b/bittensor/core/extrinsics/async_weights.py @@ -9,7 +9,7 @@ from bittensor.core.settings import version_as_int from bittensor.utils import format_error_message from bittensor.utils.btlogging import logging -from bittensor.utils.registration import torch, use_torch +from bittensor.utils.registration import torch, use_torch, legacy_torch_api_compat if TYPE_CHECKING: from bittensor_wallet import Wallet @@ -154,6 +154,7 @@ async def _do_batch_set_weights( ) +@legacy_torch_api_compat async def set_weights_extrinsic( subtensor: "AsyncSubtensor", wallet: "Wallet", @@ -180,16 +181,10 @@ async def set_weights_extrinsic( success (bool): Flag is ``true`` if extrinsic was finalized or included in the block. If we did not wait for finalization / inclusion, the response is ``true``. """ # First convert types. - if use_torch(): - if isinstance(uids, list): - uids = torch.tensor(uids, dtype=torch.int64) - if isinstance(weights, list): - weights = torch.tensor(weights, dtype=torch.float32) - else: - if isinstance(uids, list): - uids = np.array(uids, dtype=np.int64) - if isinstance(weights, list): - weights = np.array(weights, dtype=np.float32) + if isinstance(uids, list): + uids = np.array(uids, dtype=np.int64) + if isinstance(weights, list): + weights = np.array(weights, dtype=np.float32) # Reformat and normalize. weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit( @@ -227,6 +222,7 @@ async def set_weights_extrinsic( return False, str(error) +@legacy_torch_api_compat async def batch_set_weights_extrinsic( subtensor: "AsyncSubtensor", wallet: "Wallet", @@ -259,16 +255,10 @@ async def batch_set_weights_extrinsic( for uids, weights in zip(uidss, weightss): # First convert types. - if use_torch(): - if isinstance(uids, list): - uids = torch.tensor(uids, dtype=torch.int64) - if isinstance(weights, list): - weights = torch.tensor(weights, dtype=torch.float32) - else: - if isinstance(uids, list): - uids = np.array(uids, dtype=np.int64) - if isinstance(weights, list): - weights = np.array(weights, dtype=np.float32) + if isinstance(uids, list): + uids = np.array(uids, dtype=np.int64) + if isinstance(weights, list): + weights = np.array(weights, dtype=np.float32) # Reformat and normalize. weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit( diff --git a/bittensor/core/extrinsics/set_weights.py b/bittensor/core/extrinsics/set_weights.py index dd113d5135..e1c34d639b 100644 --- a/bittensor/core/extrinsics/set_weights.py +++ b/bittensor/core/extrinsics/set_weights.py @@ -25,7 +25,7 @@ from bittensor.utils import format_error_message, weight_utils from bittensor.utils.btlogging import logging from bittensor.utils.networking import ensure_connected -from bittensor.utils.registration import torch, use_torch +from bittensor.utils.registration import torch, use_torch, legacy_torch_api_compat # For annotation purposes if TYPE_CHECKING: @@ -175,6 +175,7 @@ def do_batch_set_weights( # Community uses this extrinsic directly and via `subtensor.set_weights` +@legacy_torch_api_compat def set_weights_extrinsic( subtensor: "Subtensor", wallet: "Wallet", @@ -201,16 +202,10 @@ def set_weights_extrinsic( tuple[bool, str]: A tuple containing a success flag and an optional response message. """ # First convert types. - if use_torch(): - if isinstance(uids, list): - uids = torch.tensor(uids, dtype=torch.int64) - if isinstance(weights, list): - weights = torch.tensor(weights, dtype=torch.float32) - else: - if isinstance(uids, list): - uids = np.array(uids, dtype=np.int64) - if isinstance(weights, list): - weights = np.array(weights, dtype=np.float32) + if isinstance(uids, list): + uids = np.array(uids, dtype=np.int64) + if isinstance(weights, list): + weights = np.array(weights, dtype=np.float32) # Reformat and normalize. weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit( @@ -250,6 +245,7 @@ def set_weights_extrinsic( return False, str(e) +@legacy_torch_api_compat def batch_set_weights_extrinsic( subtensor: "Subtensor", wallet: "Wallet", @@ -282,16 +278,10 @@ def batch_set_weights_extrinsic( for uids, weights in zip(uidss, weightss): # First convert types. - if use_torch(): - if isinstance(uids, list): - uids = torch.tensor(uids, dtype=torch.int64) - if isinstance(weights, list): - weights = torch.tensor(weights, dtype=torch.float32) - else: - if isinstance(uids, list): - uids = np.array(uids, dtype=np.int64) - if isinstance(weights, list): - weights = np.array(weights, dtype=np.float32) + if isinstance(uids, list): + uids = np.array(uids, dtype=np.int64) + if isinstance(weights, list): + weights = np.array(weights, dtype=np.float32) # Reformat and normalize. weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit( From 2f068e30901b6f6e6c6537c63b13dd3b744fe3a6 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 27 Nov 2024 16:08:10 -0500 Subject: [PATCH 11/15] rename uidss/weightss -> nested_* --- bittensor/core/async_subtensor.py | 22 ++++++++--------- bittensor/core/extrinsics/async_weights.py | 19 ++++++++------- bittensor/core/extrinsics/set_weights.py | 19 ++++++++------- bittensor/core/subtensor.py | 28 +++++++++++----------- tests/e2e_tests/test_commit_weights.py | 4 ++-- tests/e2e_tests/test_set_weights.py | 4 ++-- 6 files changed, 49 insertions(+), 47 deletions(-) diff --git a/bittensor/core/async_subtensor.py b/bittensor/core/async_subtensor.py index b9016e2fc4..1bbe34df2b 100644 --- a/bittensor/core/async_subtensor.py +++ b/bittensor/core/async_subtensor.py @@ -1617,8 +1617,8 @@ async def batch_set_weights( self, wallet: "Wallet", netuids: list[int], - uidss: list[Union[NDArray[np.int64], "torch.LongTensor", list]], - weightss: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], + nested_uids: list[Union[NDArray[np.int64], "torch.LongTensor", list]], + nested_weights: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], version_keys: Optional[list[int]] = None, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, @@ -1630,8 +1630,8 @@ async def batch_set_weights( Args: wallet (bittensor_wallet.Wallet): The wallet associated with the neuron setting the weights. netuids (list[int]): The list of subnet uids. - uidss (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The list of neuron UIDs that the weights are being set for. - weightss (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The corresponding weights to be set for each UID. + nested_uids (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The list of neuron UIDs that the weights are being set for. + nested_weights (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The corresponding weights to be set for each UID. version_keys (Optional[list[int]]): Version keys for compatibility with the network. Default is ``int representation of Bittensor version.``. wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is ``False``. @@ -1667,8 +1667,8 @@ async def batch_set_weights( continue netuids_to_set.append(netuid) - uidss_to_set.append(uidss[i]) - weightss_to_set.append(weightss[i]) + uidss_to_set.append(nested_uids[i]) + weightss_to_set.append(nested_weights[i]) version_keys_to_set.append(version_keys[i]) while retries < max_retries: @@ -1680,8 +1680,8 @@ async def batch_set_weights( subtensor=self, wallet=wallet, netuids=netuids_to_set, - uidss=uidss_to_set, - weightss=weightss_to_set, + nested_uids=uidss_to_set, + nested_weights=weightss_to_set, version_keys=version_keys_to_set, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, @@ -1798,8 +1798,8 @@ async def batch_commit_weights( wallet: "Wallet", netuids: list[int], salts: list[list[int]], - uidss: list[Union[NDArray[np.int64], list]], - weightss: list[Union[NDArray[np.int64], list]], + nested_uids: list[Union[NDArray[np.int64], list]], + nested_weights: list[Union[NDArray[np.int64], list]], version_keys: Optional[list[int]] = None, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, @@ -1830,7 +1830,7 @@ async def batch_commit_weights( message = "No attempt made. Perhaps it is too soon to commit weights!" logging.info( - f"Committing a batch of weights with params: netuids={netuids}, salts={salts}, uids={uidss}, weights={weightss}, version_keys={version_keys}" + f"Committing a batch of weights with params: netuids={netuids}, salts={salts}, uids={nested_uids}, weights={nested_weights}, version_keys={version_keys}" ) if version_keys is None or len(version_keys) == 0: diff --git a/bittensor/core/extrinsics/async_weights.py b/bittensor/core/extrinsics/async_weights.py index ef49a3f616..261d02e6ee 100644 --- a/bittensor/core/extrinsics/async_weights.py +++ b/bittensor/core/extrinsics/async_weights.py @@ -85,7 +85,7 @@ async def _do_set_weights( async def _do_batch_set_weights( subtensor: "AsyncSubtensor", wallet: "Wallet", - uidss: list[list[int]], + nested_uids: list[list[int]], valss: list[list[int]], netuids: list[int], version_keys: Optional[list[int]] = None, @@ -100,7 +100,7 @@ async def _do_batch_set_weights( Args: subtensor (subtensor.core.async_subtensor.AsyncSubtensor): Async Subtensor instance. wallet (bittensor.wallet): The wallet associated with the neuron setting the weights. - uidss (list[list[int]]): List of neuron UIDs for which weights are being set. + nested_uids (list[list[int]]): List of neuron UIDs for which weights are being set. valss (list[list[int]]): List of weight values corresponding to each UID. netuids (list[int]): Unique identifier for the network. version_keys (list[int], optional): Version key for compatibility with the network. @@ -118,7 +118,8 @@ async def _do_batch_set_weights( version_keys = [version_as_int] * len(netuids) packed_weights = [ - [(uid, val) for uid, val in zip(uids, vals)] for uids, vals in zip(uidss, valss) + [(uid, val) for uid, val in zip(uids, vals)] + for uids, vals in zip(nested_uids, valss) ] call = await subtensor.substrate.compose_call( @@ -227,8 +228,8 @@ async def batch_set_weights_extrinsic( subtensor: "AsyncSubtensor", wallet: "Wallet", netuids: list[int], - uidss: list[Union[NDArray[np.int64], "torch.LongTensor", list]], - weightss: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], + nested_uids: list[Union[NDArray[np.int64], "torch.LongTensor", list]], + nested_weights: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], version_keys: Optional[list[int]] = None, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, @@ -239,8 +240,8 @@ async def batch_set_weights_extrinsic( subtensor (bittensor.subtensor): Bittensor subtensor object. wallet (bittensor.wallet): Bittensor wallet object. netuids (list[int]): The ``netuid`` of the subnet to set weights for. - uidss (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The ``uint64`` uids of destination neurons. - weightss (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The weights to set. These must be ``float`` s and correspond to the passed ``uid`` s. + nested_uids (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The ``uint64`` uids of destination neurons. + nested_weights (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The weights to set. These must be ``float`` s and correspond to the passed ``uid`` s. version_keys (Optional[list[int]]): The version key of the validator. wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``true``, or returns ``false`` if the extrinsic fails to enter the block within the timeout. wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. @@ -253,7 +254,7 @@ async def batch_set_weights_extrinsic( if version_keys is None or len(version_keys) == 0: version_keys = [0] * len(netuids) # Default to version 0 if not provided - for uids, weights in zip(uidss, weightss): + for uids, weights in zip(nested_uids, nested_weights): # First convert types. if isinstance(uids, list): uids = np.array(uids, dtype=np.int64) @@ -276,7 +277,7 @@ async def batch_set_weights_extrinsic( subtensor=subtensor, wallet=wallet, netuids=netuids, - uidss=uids_to_set, + nested_uids=uids_to_set, valss=weights_to_set, version_keys=version_keys, wait_for_finalization=wait_for_finalization, diff --git a/bittensor/core/extrinsics/set_weights.py b/bittensor/core/extrinsics/set_weights.py index e1c34d639b..bc8214d6c4 100644 --- a/bittensor/core/extrinsics/set_weights.py +++ b/bittensor/core/extrinsics/set_weights.py @@ -106,7 +106,7 @@ def do_set_weights( def do_batch_set_weights( self: "Subtensor", wallet: "Wallet", - uidss: list[list[int]], + nested_uids: list[list[int]], valss: list[list[int]], netuids: list[int], version_keys: Optional[list[int]] = None, @@ -120,7 +120,7 @@ def do_batch_set_weights( Args: self (bittensor.core.subtensor.Subtensor): Subtensor interface wallet (bittensor_wallet.Wallet): The wallet associated with the neuron setting the weights. - uidss (list[list[int]]): List of neuron UIDs for which weights are being set. + nested_uids (list[list[int]]): List of neuron UIDs for which weights are being set. valss (list[list[int]]): List of weight values corresponding to each UID. netuids (list[int]): Unique identifier for the network. version_keys (Optional[list[int]]): Version key for compatibility with the network. @@ -137,7 +137,8 @@ def do_batch_set_weights( version_keys = [version_as_int] * len(netuids) packed_weights = [ - [(uid, val) for uid, val in zip(uids, vals)] for uids, vals in zip(uidss, valss) + [(uid, val) for uid, val in zip(uids, vals)] + for uids, vals in zip(nested_uids, valss) ] call = self.substrate.compose_call( @@ -250,8 +251,8 @@ def batch_set_weights_extrinsic( subtensor: "Subtensor", wallet: "Wallet", netuids: list[int], - uidss: list[Union[NDArray[np.int64], "torch.LongTensor", list]], - weightss: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], + nested_uids: list[Union[NDArray[np.int64], "torch.LongTensor", list]], + nested_weights: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], version_keys: Optional[list[int]] = None, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, @@ -262,8 +263,8 @@ def batch_set_weights_extrinsic( subtensor (bittensor.subtensor): Bittensor subtensor object. wallet (bittensor.wallet): Bittensor wallet object. netuids (list[int]): The ``netuid`` of the subnet to set weights for. - uidss (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The ``uint64`` uids of destination neurons. - weightss (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The weights to set. These must be ``float`` s and correspond to the passed ``uid`` s. + nested_uids (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The ``uint64`` uids of destination neurons. + nested_weights (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The weights to set. These must be ``float`` s and correspond to the passed ``uid`` s. version_keys (Optional[list[int]]): The version key of the validator. wait_for_inclusion (bool): If set, waits for the extrinsic to enter a block before returning ``true``, or returns ``false`` if the extrinsic fails to enter the block within the timeout. wait_for_finalization (bool): If set, waits for the extrinsic to be finalized on the chain before returning ``true``, or returns ``false`` if the extrinsic fails to be finalized within the timeout. @@ -276,7 +277,7 @@ def batch_set_weights_extrinsic( if version_keys is None or len(version_keys) == 0: version_keys = [0] * len(netuids) # Default to version 0 if not provided - for uids, weights in zip(uidss, weightss): + for uids, weights in zip(nested_uids, nested_weights): # First convert types. if isinstance(uids, list): uids = np.array(uids, dtype=np.int64) @@ -299,7 +300,7 @@ def batch_set_weights_extrinsic( self=subtensor, wallet=wallet, netuids=netuids, - uidss=uids_to_set, + nested_uids=uids_to_set, valss=weights_to_set, version_keys=version_keys, wait_for_finalization=wait_for_finalization, diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index 2e7aa10054..b8320d5a71 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -1740,8 +1740,8 @@ def batch_set_weights( self, wallet: "Wallet", netuids: list[int], - uidss: list[Union[NDArray[np.int64], "torch.LongTensor", list]], - weightss: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], + nested_uids: list[Union[NDArray[np.int64], "torch.LongTensor", list]], + nested_weights: list[Union[NDArray[np.float32], "torch.FloatTensor", list]], version_keys: Optional[list[int]] = None, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, @@ -1753,8 +1753,8 @@ def batch_set_weights( Args: wallet (bittensor_wallet.Wallet): The wallet associated with the neuron setting the weights. netuids (list[int]): The list of subnet netuids that the weights are being set for. - uidss (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The lists of neuron UIDs that the weights are being set for. - weightss (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The lists of corresponding weights to be set for each UID. + nested_uids (list[Union[NDArray[np.int64], torch.LongTensor, list]]): The lists of neuron UIDs that the weights are being set for. + nested_weights (list[Union[NDArray[np.float32], torch.FloatTensor, list]]): The lists of corresponding weights to be set for each UID. version_keys (Optional[list[int]]): Version keys for compatibility with each subnet. wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is ``False``. @@ -1787,8 +1787,8 @@ def batch_set_weights( continue netuids_to_set.append(netuid) - uidss_to_set.append(uidss[i]) - weightss_to_set.append(weightss[i]) + uidss_to_set.append(nested_uids[i]) + weightss_to_set.append(nested_weights[i]) version_keys_to_set.append(version_keys[i]) while retries < max_retries: @@ -1800,8 +1800,8 @@ def batch_set_weights( subtensor=self, wallet=wallet, netuids=netuids_to_set, - uidss=uidss_to_set, - weightss=weightss_to_set, + nested_uids=uidss_to_set, + nested_weights=weightss_to_set, version_keys=version_keys_to_set, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, @@ -2094,8 +2094,8 @@ def batch_commit_weights( wallet: "Wallet", netuids: list[int], salts: list[list[int]], - uidss: list[Union[NDArray[np.int64], list]], - weightss: list[Union[NDArray[np.int64], list]], + nested_uids: list[Union[NDArray[np.int64], list]], + nested_weights: list[Union[NDArray[np.int64], list]], version_keys: Optional[list[int]] = None, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, @@ -2109,8 +2109,8 @@ def batch_commit_weights( wallet (bittensor_wallet.Wallet): The wallet associated with the neuron committing the weights. netuids (list[int]): The list of subnet uids. salts (list[list[int]]): The list of salts to generate weight hashes. - uidss (list[np.ndarray]): The list of NumPy arrays of neuron UIDs for which weights are being committed. - weightss (list[np.ndarray]): The list of NumPy arrays of weight values corresponding to each UID. + nested_uids (list[np.ndarray]): The list of NumPy arrays of neuron UIDs for which weights are being committed. + nested_weights (list[np.ndarray]): The list of NumPy arrays of weight values corresponding to each UID. version_keys (Optional[list[int]]): The list of version keys for compatibility with the network. Default is ``int representation of Bittensor version.``. wait_for_inclusion (bool): Waits for the transaction to be included in a block. Default is ``False``. wait_for_finalization (bool): Waits for the transaction to be finalized on the blockchain. Default is ``False``. @@ -2126,7 +2126,7 @@ def batch_commit_weights( message = "No attempt made. Perhaps it is too soon to commit weights!" logging.info( - f"Committing a batch of weights with params: netuids={netuids}, salts={salts}, uids={uidss}, weights={weightss}, version_keys={version_keys}" + f"Committing a batch of weights with params: netuids={netuids}, salts={salts}, uids={nested_uids}, weights={nested_weights}, version_keys={version_keys}" ) if version_keys is None or len(version_keys) == 0: @@ -2143,7 +2143,7 @@ def batch_commit_weights( version_key=version_key, ) for netuid, salt, uids, weights, version_key in zip( - netuids, salts, uidss, weightss, version_keys + netuids, salts, nested_uids, nested_weights, version_keys ) ] diff --git a/tests/e2e_tests/test_commit_weights.py b/tests/e2e_tests/test_commit_weights.py index 5945de36b4..2d0a2c25b5 100644 --- a/tests/e2e_tests/test_commit_weights.py +++ b/tests/e2e_tests/test_commit_weights.py @@ -341,8 +341,8 @@ async def test_batch_commit_weights(local_chain): alice_wallet, [netuid_1, netuid_2], salts=[salt, salt], - uidss=[weight_uids, weight_uids], - weightss=[weight_vals, weight_vals], + nested_uids=[weight_uids, weight_uids], + nested_weights=[weight_vals, weight_vals], wait_for_inclusion=True, wait_for_finalization=True, ) diff --git a/tests/e2e_tests/test_set_weights.py b/tests/e2e_tests/test_set_weights.py index 223c0c7e2f..4926e116a5 100644 --- a/tests/e2e_tests/test_set_weights.py +++ b/tests/e2e_tests/test_set_weights.py @@ -287,8 +287,8 @@ async def test_batch_set_weights(local_chain): success, message = subtensor.batch_set_weights( alice_wallet, netuids=[netuid_1, netuid_2], - uidss=[weight_uids, weight_uids], - weightss=[weight_vals, weight_vals], + nested_uids=[weight_uids, weight_uids], + nested_weights=[weight_vals, weight_vals], wait_for_inclusion=True, wait_for_finalization=True, ) From b93ecd8f9dc6f35d3cbc3491b64f2de688359343 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 27 Nov 2024 16:10:09 -0500 Subject: [PATCH 12/15] typo --- bittensor/core/async_subtensor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/core/async_subtensor.py b/bittensor/core/async_subtensor.py index 1bbe34df2b..3fc03fea94 100644 --- a/bittensor/core/async_subtensor.py +++ b/bittensor/core/async_subtensor.py @@ -1847,7 +1847,7 @@ async def batch_commit_weights( version_key=version_key, ) for netuid, salt, uids, weights, version_key in zip( - netuids, salts, uids, weights, version_keys + netuids, salts, nested_uids, nested_weights, version_keys ) ] From 0ca543da224d8d9f8f16bb41960d8afc52d5e095 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 27 Nov 2024 16:17:16 -0500 Subject: [PATCH 13/15] chore: remove unused imports --- bittensor/core/extrinsics/async_weights.py | 2 +- bittensor/core/extrinsics/set_weights.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bittensor/core/extrinsics/async_weights.py b/bittensor/core/extrinsics/async_weights.py index 261d02e6ee..3d932f5cc7 100644 --- a/bittensor/core/extrinsics/async_weights.py +++ b/bittensor/core/extrinsics/async_weights.py @@ -9,7 +9,7 @@ from bittensor.core.settings import version_as_int from bittensor.utils import format_error_message from bittensor.utils.btlogging import logging -from bittensor.utils.registration import torch, use_torch, legacy_torch_api_compat +from bittensor.utils.registration import torch, legacy_torch_api_compat if TYPE_CHECKING: from bittensor_wallet import Wallet diff --git a/bittensor/core/extrinsics/set_weights.py b/bittensor/core/extrinsics/set_weights.py index bc8214d6c4..612d44dcc3 100644 --- a/bittensor/core/extrinsics/set_weights.py +++ b/bittensor/core/extrinsics/set_weights.py @@ -25,7 +25,7 @@ from bittensor.utils import format_error_message, weight_utils from bittensor.utils.btlogging import logging from bittensor.utils.networking import ensure_connected -from bittensor.utils.registration import torch, use_torch, legacy_torch_api_compat +from bittensor.utils.registration import torch, legacy_torch_api_compat # For annotation purposes if TYPE_CHECKING: From 70c7cd183795c4fa7b634c2a2aa2304fec7b2741 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 27 Nov 2024 16:35:58 -0500 Subject: [PATCH 14/15] missed type ignore --- bittensor/core/subtensor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index d70ab0cbbe..f86662a6a3 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -1778,7 +1778,7 @@ def batch_set_weights( retries = 0 success = False message = "No attempt made. Perhaps it is too soon to set weights!" - if self.blocks_since_last_update(netuid, uid) <= self.weights_rate_limit( + if self.blocks_since_last_update(netuid, uid) <= self.weights_rate_limit( # type: ignore netuid ): logging.info( From 93fb3c0ba2b95a600c73966ca765cf8afb9b56c2 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 27 Nov 2024 16:36:38 -0500 Subject: [PATCH 15/15] chore: ruff --- bittensor/core/subtensor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py index f86662a6a3..a80cb56990 100644 --- a/bittensor/core/subtensor.py +++ b/bittensor/core/subtensor.py @@ -1778,7 +1778,7 @@ def batch_set_weights( retries = 0 success = False message = "No attempt made. Perhaps it is too soon to set weights!" - if self.blocks_since_last_update(netuid, uid) <= self.weights_rate_limit( # type: ignore + if self.blocks_since_last_update(netuid, uid) <= self.weights_rate_limit( # type: ignore netuid ): logging.info(