Improve Azure quota integration

autoscaler/azure.py

@@ -203,6 +203,9 @@ def set_desired_capacity(self, new_desired_capacity):
             if scale_out == 0 or remaining_instances == 0:
                 break
 
+        if remaining_instances == 0:
+            logger.warning("Out of quota for {}!".format(self.instance_type))
+
         if scale_out > 0:
             logger.error("Not enough scale sets to reach desired capacity {} for {}".format(new_desired_capacity, self))
 

autoscaler/azure_api.py

@@ -206,6 +206,11 @@ def __init__(self, delegate: AzureApi) -> None:
         self._scale_set_cache: MutableMapping[str, List[AzureScaleSet]] = {}
         self._remaining_instances_cache: MutableMapping[str, MutableMapping[str, int]] = {}
 
+    def invalidate_quota_cache(self, resource_group_name: str) -> None:
+        with self._lock:
+            if resource_group_name in self._remaining_instances_cache:
+                del self._remaining_instances_cache[resource_group_name]
+
     def list_scale_sets(self, resource_group_name: str, force_refresh=False) -> List[AzureScaleSet]:
         if not force_refresh:
             with self._lock:

autoscaler/cluster.py

@@ -215,6 +215,8 @@ def scale_loop(self):
                     # Force a refresh of the cache to pick up any new Scale Sets that have been created
                     # or modified externally.
                     self.azure_client.list_scale_sets(resource_group, force_refresh=True)
+                    # Force a refresh of the cache in case our quota was adjusted
+                    self.azure_client.invalidate_quota_cache(resource_group)
             asgs = self.autoscaling_groups.get_all_groups(all_nodes)
             azure_groups = self.azure_groups.get_all_groups(all_nodes)
             scaling_groups = asgs + azure_groups