PR IntelRealSense#12132 from Eran: DDS test retries

unit-tests/dds/test-device-init.py

@@ -2,6 +2,7 @@
 # Copyright(c) 2022 Intel Corporation. All Rights Reserved.
 
 #test:donotrun:!dds
+#test:retries 2
 
 import pyrealdds as dds
 from rspy import log, test

unit-tests/dds/test-librs-device-properties.py

@@ -2,6 +2,7 @@
 # Copyright(c) 2022 Intel Corporation. All Rights Reserved.
 
 #test:donotrun:!dds
+#test:retries 2
 
 from rspy import log, test
 log.nested = 'C  '

unit-tests/dds/test-metadata-syncer.py

@@ -2,6 +2,7 @@
 # Copyright(c) 2023 Intel Corporation. All Rights Reserved.
 
 #test:donotrun:!dds
+#test:retries 2
 
 import pyrealdds as dds
 from rspy import log, test

unit-tests/dds/test-watcher.py

@@ -2,6 +2,7 @@
 # Copyright(c) 2022 Intel Corporation. All Rights Reserved.
 
 #test:donotrun:!dds
+#test:retries 2
 
 from rspy import log, test
 

unit-tests/py/rspy/devices.py

@@ -70,6 +70,8 @@ def __init__( self, sn, dev ):
         self._name = None
         if dev.supports( rs.camera_info.name ):
             self._name = dev.get_info( rs.camera_info.name )
+            if self._name.startswith( 'Intel RealSense ' ):
+                self._name = self._name[16:]
         self._product_line = None
         if dev.supports( rs.camera_info.product_line ):
             self._product_line = dev.get_info( rs.camera_info.product_line )

unit-tests/py/rspy/libci.py

@@ -89,6 +89,7 @@ def __init__( self, context ):
         self._tags = set()
         self._flags = set()
         self._timeout = 200
+        self._retries = 0
         self._context = context
         self._donotrun = False
 
@@ -99,6 +100,8 @@ def debug_dump( self ):
             log.d( 'priority:', self._priority )
         if self._timeout != 200:
             log.d( 'timeout:', self._timeout )
+        if self._retries != 0:
+            log.d( 'retries:', self._retries )
         if len( self._tags ) > 1:
             log.d( 'tags:', self._tags )
         if self._flags:
@@ -119,6 +122,10 @@ def priority( self ):
     def timeout( self ):
         return self._timeout
 
+    @property
+    def retries( self ):
+        return self._retries
+
     @property
     def tags( self ):
         return self._tags
@@ -224,6 +231,12 @@ def derive_config_from_text( self, source, line_prefix ):
                 else:
                     log.e( source + '+' + str( line['index'] ) + ': timeout directive with invalid parameters:',
                            params )
+            elif directive == 'retries':
+                if len( params ) == 1 and params[0].isdigit():
+                    self._retries = int( params[0] )
+                else:
+                    log.e( source + '+' + str( line['index'] ) + ': timeout directive with invalid parameters:',
+                           params )
             elif directive == 'tag':
                 self._tags.update( map( str.lower, params ))  # tags are case-insensitive
             elif directive == 'flag':

unit-tests/run-unit-tests.py

@@ -233,19 +233,26 @@ def find_build_dir( dir ):
     os.environ["PYTHONPATH"] += os.pathsep + dir
 
 
-def configuration_str( configuration, repetition = 1, prefix = '', suffix = '' ):
+def configuration_str( configuration, repetition=1, retry=0, sns=None, prefix='', suffix='' ):
     """ Return a string repr (with a prefix and/or suffix) of the configuration or '' if it's None """
     s = ''
     if configuration is not None:
-        s += '[' + ' '.join( configuration ) + ']'
+        s += '[' + ' '.join( configuration )
+        if sns is not None:
+            s += ' -> ' + ' '.join( [f'{devices.get(sn).name}_{sn}' for sn in sns] )
+        s += ']'
+    elif sns is not None:
+        s += '[' + ' '.join( [f'{devices.get(sn).name}_{sn}' for sn in sns] ) + ']'
     if repetition:
         s += '[' + str(repetition+1) + ']'
+    if retry:
+        s += f'[retry {retry}]'
     if s:
         s = prefix + s + suffix
     return s
 
 
-def check_log_for_fails( path_to_log, testname, configuration = None, repetition = 1 ):
+def check_log_for_fails( path_to_log, testname, configuration=None, repetition=1, sns=None ):
     # Normal logs are expected to have in last line:
     #     "All tests passed (11 assertions in 1 test case)"
     # Tests that have failures, however, will show:
@@ -279,13 +286,13 @@ def check_log_for_fails( path_to_log, testname, configuration = None, repetition
             desc = str( total - passed ) + ' of ' + str( total ) + ' failed'
 
         if log.is_verbose_on():
-            log.e( log.red + testname + log.reset + ': ' + configuration_str( configuration, repetition, suffix=' ' ) + desc )
+            log.e( log.red + testname + log.reset + ': ' + configuration_str( configuration, repetition, suffix=' ', sns=sns ) + desc )
             log.i( 'Log: >>>' )
             log.out()
             file.cat( path_to_log )
             log.out( '<<<' )
         else:
-            log.e( log.red + testname + log.reset + ': ' + configuration_str( configuration, repetition,
+            log.e( log.red + testname + log.reset + ': ' + configuration_str( configuration, repetition, sns=sns,
                                                                               suffix=' ' ) + desc + '; see ' + path_to_log )
         return True
     return False
@@ -381,13 +388,12 @@ def devices_by_test_config( test, exceptions ):
             continue
 
 
-def test_wrapper( test, configuration = None, repetition = 1 ):
-    global n_tests, rslog
-    n_tests += 1
+def test_wrapper_( test, configuration=None, repetition=1, retry=0, sns=None ):
+    global rslog
     #
-    conf_str = configuration_str( configuration, repetition, suffix=' ' ) + test.name
     if not log.is_debug_on():
-        log.i( f'Running {conf_str}' )
+        conf_str = configuration_str( configuration, repetition, retry=retry, prefix='  ', sns=sns )
+        log.i( f'Running {test.name}{conf_str}' )
     #
     log_path = test.get_log()
     #
@@ -396,20 +402,32 @@ def test_wrapper( test, configuration = None, repetition = 1 ):
         opts.add( '--rslog' )
     try:
         test.run_test( configuration = configuration, log_path = log_path, opts = opts )
-        return True
     except FileNotFoundError as e:
         log.e( log.red + test.name + log.reset + ':', str( e ) + configuration_str( configuration, repetition, prefix=' ' ) )
     except subprocess.TimeoutExpired:
         log.e( log.red + test.name + log.reset + ':', configuration_str( configuration, repetition, suffix=' ' ) + 'timed out' )
     except subprocess.CalledProcessError as cpe:
-        if not check_log_for_fails( log_path, test.name, configuration, repetition ):
+        if not check_log_for_fails( log_path, test.name, configuration, repetition, sns=sns ):
             # An unexpected error occurred
             log.e( log.red + test.name + log.reset + ':',
                    configuration_str( configuration, repetition, suffix=' ' ) + 'exited with non-zero value (' + str(
                        cpe.returncode ) + ')' )
+    else:
+        return True
     return False
 
 
+def test_wrapper( test, configuration=None, repetition=1, sns=None ):
+    global n_tests
+    n_tests += 1
+    for retry in range( test.config.retries + 1 ):
+        if test_wrapper_( test, configuration, repetition, retry, sns ):
+            return True
+        log._n_errors -= 1
+        time.sleep( 1 )  # small pause between tries
+    log._n_errors += 1
+    return False
+
 # Run all tests
 try:
     list_only = list_tags or list_tests
@@ -520,7 +538,7 @@ def test_wrapper( test, configuration = None, repetition = 1 ):
                     except RuntimeError as e:
                         log.w( log.red + test.name + log.reset + ': ' + str( e ) )
                     else:
-                        test_ok = test_wrapper( test, configuration, repetition ) and test_ok
+                        test_ok = test_wrapper( test, configuration, repetition, sns=serial_numbers ) and test_ok
                     finally:
                         log.debug_unindent()
             if not test_ok:

wrappers/python/pyrs_device.cpp

@@ -40,7 +40,10 @@ void init_device(py::module &m) {
         .def(BIND_DOWNCAST(device, firmware_logger))
         .def("__repr__", [](const rs2::device &self) {
             std::ostringstream ss;
-            ss << "<" SNAME ".device: " << self.get_info(RS2_CAMERA_INFO_NAME);
+            auto name = self.get_info( RS2_CAMERA_INFO_NAME );
+            if( 0 == strncmp( name, "Intel RealSense ", 16 ) )
+                name += 16;
+            ss << "<" SNAME ".device: " << name;
             if (self.supports(RS2_CAMERA_INFO_SERIAL_NUMBER))
                 ss << " (S/N: " << self.get_info(RS2_CAMERA_INFO_SERIAL_NUMBER);
             else
@@ -52,6 +55,8 @@ void init_device(py::module &m) {
                 ss << "  UNLOCKED";
             if( self.supports( RS2_CAMERA_INFO_USB_TYPE_DESCRIPTOR ) )
                 ss << "  on USB" << self.get_info( RS2_CAMERA_INFO_USB_TYPE_DESCRIPTOR );
+            else if( self.supports( RS2_CAMERA_INFO_PHYSICAL_PORT ) )
+                ss << "  @ " << self.get_info( RS2_CAMERA_INFO_PHYSICAL_PORT );
             ss << ")>";
             return ss.str();
         })