Merge pull request #1403 from girder/logger-config

Add a utility function for getting loggers

.gitignore

@@ -74,3 +74,5 @@ docs/source/*
 *.ipynb
 !docs/*.ipynb
 NOTES.md
+
+monkeytype.sqlite3

CHANGELOG.md

@@ -7,6 +7,7 @@
 - Harden OME tiff reader ([#1398](../../pull/1398))
 - Improve checks for formats we shouldn't read ([#1399](../../pull/1399))
 - Support negative x, y in addTile ([#1401](../../pull/1401))
+- Add a utility function for getting loggers ([#1403](../../pull/1403))
 
 ### Changes
 - Use an enum for priority constants ([#1400](../../pull/1400))

large_image/cache_util/cache.py

@@ -106,7 +106,7 @@ def wrapper(self, *args, **kwargs):
                 pass  # value too large
             except (KeyError, RuntimeError):
                 # the key was refused for some reason
-                config.getConfig('logger').debug(
+                config.getLogger().debug(
                     'Had a cache KeyError while trying to store a value to key %r' % (k))
             return v
         return wrapper
@@ -154,7 +154,7 @@ def __new__(metacls, name, bases, namespace, **kwargs):
                 inProcess=True,
             )
             LruCacheMetaclass.namedCaches[cacheName] = (cache, cacheLock)
-            config.getConfig('logger').debug(
+            config.getLogger().debug(
                 'Created LRU Cache for %r with %d maximum size' % (cacheName, cache.maxsize))
         else:
             (cache, cacheLock) = LruCacheMetaclass.namedCaches[cacheName]

large_image/cache_util/cachefactory.py

@@ -57,9 +57,9 @@ def loadCaches(entryPointName='large_image.cache', sourceDict=_availableCaches):
         try:
             cacheClass = entryPoint.load()
             sourceDict[entryPoint.name.lower()] = cacheClass
-            config.getConfig('logprint').debug(f'Loaded cache {entryPoint.name}')
+            config.getLogger('logprint').debug(f'Loaded cache {entryPoint.name}')
         except Exception:
-            config.getConfig('logprint').exception(
+            config.getLogger('logprint').exception(
                 f'Failed to load cache {entryPoint.name}',
             )
     # Load memcached last for now
@@ -114,7 +114,7 @@ def getFirstAvailableCache():
         except TileCacheError:
             continue
     if cache is not None:
-        config.getConfig('logprint').debug(
+        config.getLogger('logprint').debug(
             f'Automatically setting `{cacheBackend}` as cache_backend from availableCaches',
         )
         config.setConfig('cache_backend', cacheBackend)
@@ -166,7 +166,7 @@ def getCache(self, numItems=None, cacheName=None, inProcess=False):
             cacheLock = threading.Lock()
 
         if not inProcess and not CacheFactory.logged:
-            config.getConfig('logprint').debug(f'Using {cacheBackend} for large_image caching')
+            config.getLogger('logprint').debug(f'Using {cacheBackend} for large_image caching')
             CacheFactory.logged = True
 
         return cache, cacheLock

large_image/cache_util/memcache.py

@@ -82,12 +82,12 @@ def __getitem__(self, key):
         except KeyError:
             return self.__missing__(key)
         except pylibmc.ServerDown:
-            self.logError(pylibmc.ServerDown, config.getConfig('logprint').info,
+            self.logError(pylibmc.ServerDown, config.getLogger('logprint').info,
                           'Memcached ServerDown')
             self._reconnect()
             return self.__missing__(key)
         except pylibmc.Error:
-            self.logError(pylibmc.Error, config.getConfig('logprint').exception,
+            self.logError(pylibmc.Error, config.getLogger('logprint').exception,
                           'pylibmc exception')
             return self.__missing__(key)
 
@@ -103,11 +103,11 @@ def __setitem__(self, key, value):
             if len(valueRepr) > 500:
                 valueRepr = valueRepr[:500] + '...'
             self.logError(
-                exc.__class__, config.getConfig('logprint').error,
+                exc.__class__, config.getLogger('logprint').error,
                 '%s: Failed to save value (size %r) with key %s' % (
                     exc.__class__.__name__, valueSize, hashedKey))
         except pylibmc.ServerDown:
-            self.logError(pylibmc.ServerDown, config.getConfig('logprint').info,
+            self.logError(pylibmc.ServerDown, config.getLogger('logprint').info,
                           'Memcached ServerDown')
             self._reconnect()
         except pylibmc.TooBig:
@@ -116,7 +116,7 @@ def __setitem__(self, key, value):
             # memcached won't cache items larger than 1 Mb (or a configured
             # size), but this returns a 'SUCCESS' error.  Raise other errors.
             if 'SUCCESS' not in repr(exc.args):
-                self.logError(pylibmc.Error, config.getConfig('logprint').exception,
+                self.logError(pylibmc.Error, config.getLogger('logprint').exception,
                               'pylibmc exception')
 
     @property
@@ -135,7 +135,7 @@ def _reconnect(self):
         try:
             self._lastReconnectBackoff = getattr(self, '_lastReconnectBackoff', 2)
             if time.time() - getattr(self, '_lastReconnect', 0) > self._lastReconnectBackoff:
-                config.getConfig('logprint').info('Trying to reconnect to memcached server')
+                config.getLogger('logprint').info('Trying to reconnect to memcached server')
                 self._client = pylibmc.Client(self._clientParams[0], **self._clientParams[1])
                 self._lastReconnectBackoff = min(self._lastReconnectBackoff + 1, 30)
                 self._lastReconnect = time.time()
@@ -178,6 +178,6 @@ def getCache() -> Tuple['MemCache', threading.Lock]:
             cache = MemCache(url, memcachedUsername, memcachedPassword,
                              mustBeAvailable=True)
         except Exception:
-            config.getConfig('logger').info('Cannot use memcached for caching.')
+            config.getLogger().info('Cannot use memcached for caching.')
             cache = None
         return cache, cacheLock

large_image/config.py

@@ -1,9 +1,11 @@
 import logging
+from typing import cast
 
 try:
     import psutil
+    HAS_PSUTIL = True
 except ImportError:
-    psutil = None
+    HAS_PSUTIL = False
 
 # Default logger
 fallbackLogger = logging.getLogger('large_image')
@@ -47,7 +49,7 @@
 
     # The maximum size of an annotation file that will be ingested into girder
     # via direct load
-    'max_annotation_input_file_length': 1 * 1024 ** 3 if not psutil else max(
+    'max_annotation_input_file_length': 1 * 1024 ** 3 if not HAS_PSUTIL else max(
         1 * 1024 ** 3, psutil.virtual_memory().total // 16),
 }
 
@@ -66,6 +68,20 @@ def getConfig(key=None, default=None):
     return ConfigValues.get(key, default)
 
 
+def getLogger(key=None, default=None):
+    """
+    Get a logger from the config.  Ensure that it is a valid logger.
+
+    :param key: if None, return the 'logger'.
+    :param default: a value to return if a key is requested and not set.
+    :returns: a logger.
+    """
+    logger = cast(logging.Logger, getConfig(key or 'logger', default))
+    if not isinstance(logger, logging.Logger):
+        logger = fallbackLogger
+    return logger
+
+
 def setConfig(key, value):
     """
     Set a value in the config settings.

large_image/tilesource/__init__.py

@@ -55,9 +55,9 @@ def loadTileSources(entryPointName='large_image.source', sourceDict=AvailableTil
             sourceClass = entryPoint.load()
             if sourceClass.name and None in sourceClass.extensions:
                 sourceDict[entryPoint.name] = sourceClass
-                config.getConfig('logprint').debug('Loaded tile source %s' % entryPoint.name)
+                config.getLogger('logprint').debug('Loaded tile source %s' % entryPoint.name)
         except Exception:
-            config.getConfig('logprint').exception(
+            config.getLogger('logprint').exception(
                 'Failed to loaded tile source %s' % entryPoint.name)
 
 

large_image/tilesource/base.py

@@ -144,7 +144,7 @@ def __init__(self, encoding='JPEG', jpegQuality=95, jpegSubsampling=0,
             benefits, such as when first cataloging images that can be read.
         """
         super().__init__(**kwargs)
-        self.logger = config.getConfig('logger')
+        self.logger = config.getLogger()
         self.cache, self.cache_lock = getTileCache()
 
         self.tileWidth = None

sources/dicom/large_image_source_dicom/__init__.py

@@ -7,7 +7,6 @@
 
 import numpy as np
 
-from large_image import config
 from large_image.cache_util import LruCacheMetaclass, methodcache
 from large_image.constants import TILE_FORMAT_PIL, SourcePriority
 from large_image.exceptions import TileSourceError, TileSourceFileNotFoundError
@@ -120,8 +119,6 @@ def __init__(self, path, **kwargs):
         """
         super().__init__(path, **kwargs)
 
-        self.logger = config.getConfig('logger')
-
         # We want to make a list of paths of files in this item, if multiple,
         # or adjacent items in the folder if the item is a single file.  We
         # filter files with names that have a preferred extension.

sources/tiff/large_image_source_tiff/__init__.py

@@ -28,7 +28,6 @@
 import PIL.Image
 import tifftools
 
-from large_image import config
 from large_image.cache_util import LruCacheMetaclass, methodcache
 from large_image.constants import TILE_FORMAT_NUMPY, TILE_FORMAT_PIL, SourcePriority
 from large_image.exceptions import TileSourceError, TileSourceFileNotFoundError
@@ -89,7 +88,7 @@ def __init__(self, path, **kwargs):  # noqa
             self._initWithTiffTools()
             return
         except Exception as exc:
-            config.getConfig('logger').debug('Cannot read with tifftools route; %r', exc)
+            self.logger.debug('Cannot read with tifftools route; %r', exc)
 
         alldir = []
         try:
@@ -109,7 +108,7 @@ def __init__(self, path, **kwargs):  # noqa
                 raise TileSourceFileNotFoundError(self._largeImagePath) from None
             msg = "File %s didn't meet requirements for tile source: %s" % (
                 self._largeImagePath, lastException)
-            config.getConfig('logger').debug(msg)
+            self.logger.debug(msg)
             raise TileSourceError(msg)
         # Sort the known directories by image area (width * height).  Given
         # equal area, sort by the level.
@@ -393,7 +392,7 @@ def _checkForInefficientDirectories(self, warn=True):
         self._skippedLevels = maxMissing
         if maxMissing >= self._maxSkippedLevels:
             if warn:
-                config.getConfig('logger').warning(
+                self.logger.warning(
                     'Tiff image is missing many lower resolution levels (%d).  '
                     'It will be inefficient to read lower resolution tiles.', maxMissing)
             self._inefficientWarning = True
@@ -501,7 +500,7 @@ def _addAssociatedImage(self, directoryNum, mustBeTiled=False, topImage=None):
         except Exception:
             # If we fail for other reasons, don't raise an exception, but log
             # what happened.
-            config.getConfig('logger').exception(
+            self.logger.exception(
                 'Could not use non-tiled TIFF image as an associated image.')
 
     def _parseImageXml(self, xml, topImage):

sources/tiff/large_image_source_tiff/tiff_reader.py

@@ -43,7 +43,7 @@
     # ImportError.  We convert this to an ImportError, so that we will print a
     # more lucid error message and just fail to load this one tile source
     # instead of failing to load the whole plugin.
-    config.getConfig('logger').warning(
+    config.getLogger().warning(
         'Failed to import libtiff; try upgrading the python module (%s)' % exc)
     raise ImportError(str(exc))
 
@@ -97,7 +97,7 @@ def __init__(self, filePath, directoryNum, mustBeTiled=True, subDirectoryNum=0,
         :raises: InvalidOperationTiffError or IOTiffError or
             ValidationTiffError
         """
-        self.logger = config.getConfig('logger')
+        self.logger = config.getLogger()
         # create local cache to store Jpeg tables and getTileByteCountsType
         self.cache = cachetools.LRUCache(10)
         self._mustBeTiled = mustBeTiled

utilities/converter/large_image_converter/__main__.py

@@ -230,7 +230,7 @@ def main(args=sys.argv[1:]):
     try:
         import large_image
 
-        li_logger = large_image.config.getConfig('logger')
+        li_logger = large_image.config.getLogger()
         li_logger.setLevel(max(1, logging.CRITICAL - (opts.verbose - opts.silent) * 10))
     except ImportError:
         pass