Clean up logs

src/accelerate/logging.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
 import logging
 import os
 
@@ -67,6 +68,17 @@ def log(self, level, msg, *args, **kwargs):
                         self.logger.log(level, msg, *args, **kwargs)
                     state.wait_for_everyone()
 
+    @functools.lru_cache(None)
+    def warning_once(self, *args, **kwargs):
+        """
+        This method is identical to `logger.warning()`, but will emit the warning with the same message only once
+
+        Note: The cache is for the function arguments, so 2 different callers using the same arguments will hit the
+        cache. The assumption here is that all warning messages are unique across the code. If they aren't then need to
+        switch to another type of cache that includes the caller frame information in the hashing function.
+        """
+        self.warning(*args, **kwargs)
+
 
 def get_logger(name: str, log_level: str = None):
     """

src/accelerate/tracking.py

@@ -640,8 +640,8 @@ def store_init_configuration(self, values: dict):
         for name, value in list(values.items()):
             # internally, all values are converted to str in MLflow
             if len(str(value)) > mlflow.utils.validation.MAX_PARAM_VAL_LENGTH:
-                logger.warning(
-                    f'Trainer is attempting to log a value of "{value}" for key "{name}" as a parameter. MLflow\'s'
+                logger.warning_once(
+                    f'Accelerate is attempting to log a value of "{value}" for key "{name}" as a parameter. MLflow\'s'
                     f" log_param() only accepts values no longer than {mlflow.utils.validation.MAX_PARAM_VAL_LENGTH} characters so we dropped this attribute."
                 )
                 del values[name]
@@ -670,7 +670,7 @@ def log(self, values: dict, step: Optional[int]):
             if isinstance(v, (int, float)):
                 metrics[k] = v
             else:
-                logger.warning(
+                logger.warning_once(
                     f'MLflowTracker is attempting to log a value of "{v}" of type {type(v)} for key "{k}" as a metric. '
                     "MLflow's log_metric() only accepts float and int types so we dropped this attribute."
                 )
@@ -755,7 +755,7 @@ def log(self, values: Dict[str, Union[int, float]], step: Optional[int] = None,
         clearml_logger = self.task.get_logger()
         for k, v in values.items():
             if not isinstance(v, (int, float)):
-                logger.warning(
+                logger.warning_once(
                     "Accelerator is attempting to log a value of "
                     f'"{v}" of type {type(v)} for key "{k}" as a scalar. '
                     "This invocation of ClearML logger's  report_scalar() "
@@ -902,13 +902,14 @@ def log(self, values: dict, step: Optional[int] = None, **kwargs):
                 Additional key word arguments passed along to `dvclive.Live.log_metric()`.
         """
         from dvclive.plots import Metric
+
         if step is not None:
             self.live.step = step
         for k, v in values.items():
             if Metric.could_log(v):
                 self.live.log_metric(k, v, **kwargs)
             else:
-                logger.warning(
+                logger.warning_once(
                     "Accelerator attempted to log a value of "
                     f'"{v}" of type {type(v)} for key "{k} as a scalar. '
                     "This invocation of DVCLive's Live.log_metric() "