fix: I/O failures causing a crash + unify logging and assertions

ue4cli/CachedDataManager.py

@@ -1,6 +1,7 @@
 from .ConfigurationManager import ConfigurationManager
 from .JsonDataManager import JsonDataManager
-import os, shutil
+from .Utility import Utility
+import os
 
 class CachedDataManager(object):
 	"""
@@ -13,7 +14,7 @@ def clearCache():
 		Clears any cached data we have stored about specific engine versions
 		"""
 		if os.path.exists(CachedDataManager._cacheDir()) == True:
-			shutil.rmtree(CachedDataManager._cacheDir())
+			Utility.removeDir(CachedDataManager._cacheDir())
 
 	@staticmethod
 	def getCachedDataKey(engineVersionHash, key):

ue4cli/JsonDataManager.py

@@ -1,6 +1,7 @@
 from .UnrealManagerException import UnrealManagerException
 from .Utility import Utility
-import json, os, platform
+from .UtilityException import UtilityException
+import json, os
 
 class JsonDataManager(object):
 	"""
@@ -12,6 +13,19 @@ def __init__(self, jsonFile):
 		Creates a new JsonDataManager instance for the specified JSON file
 		"""
 		self.jsonFile = jsonFile
+
+	def loads(self):
+		"""
+		Reads and loads owned jsonFile
+		"""
+		try:
+			path = self.jsonFile
+			file = Utility.readFile(path)
+			return json.loads(file)
+		except json.JSONDecodeError as e:
+			# FIXME: This is the only place outside of Utility class where we use UtilityException.
+			# Not worth to create new Exception class for only one single case, at least not now.
+			raise UtilityException(f'failed to load "{str(path)}" due to: ({type(e).__name__}) {str(e)}')
 
 	def getKey(self, key):
 		"""
@@ -28,10 +42,7 @@ def getDictionary(self):
 		Retrieves the entire data dictionary
 		"""
 		if os.path.exists(self.jsonFile):
-			try:
-				return json.loads(Utility.readFile(self.jsonFile))
-			except json.JSONDecodeError as err:
-				raise UnrealManagerException('malformed JSON configuration file "{}" ({})'.format(self.jsonFile, err))
+			return self.loads()
 		else:
 			return {}
 
@@ -51,7 +62,7 @@ def setDictionary(self, data):
 		# Create the directory containing the JSON file if it doesn't already exist
 		jsonDir = os.path.dirname(self.jsonFile)
 		if os.path.exists(jsonDir) == False:
-			os.makedirs(jsonDir)
+			Utility.makeDirs(jsonDir)
 
 		# Store the dictionary
 		Utility.writeFile(self.jsonFile, json.dumps(data))

ue4cli/UE4BuildInterrogator.py

@@ -2,11 +2,15 @@
 from .UnrealManagerException import UnrealManagerException
 from .CachedDataManager import CachedDataManager
 from .Utility import Utility
-import json, os, platform, shutil, tempfile
+from .UtilityException import UtilityException
+from .JsonDataManager import JsonDataManager
+import os, tempfile
 
 class UE4BuildInterrogator(object):
 
 	def __init__(self, engineRoot, engineVersion, engineVersionHash, runUBTFunc):
+		# WARN: os.path.realpath can potentially fail with OSError,
+		# but if it ever happens, this is most likely bug in our code
 		self.engineRoot = os.path.realpath(engineRoot)
 		self.engineSourceDir = 'Engine/Source/'
 		self.engineVersion = engineVersion
@@ -160,7 +164,7 @@ def _getThirdPartyLibs(self, platformIdentifier, configuration):
 		sentinelBackup = sentinelFile + '.bak'
 		renameSentinel = os.path.exists(sentinelFile) and os.environ.get('UE4CLI_SENTINEL_RENAME', '0') == '1'
 		if renameSentinel == True:
-			shutil.move(sentinelFile, sentinelBackup)
+			Utility.moveFile(sentinelFile, sentinelBackup)
 
 		# Invoke UnrealBuildTool in JSON export mode (make sure we specify gathering mode, since this is a prerequisite of JSON export)
 		# (Ensure we always perform sentinel file cleanup even when errors occur)
@@ -172,10 +176,10 @@ def _getThirdPartyLibs(self, platformIdentifier, configuration):
 				self.runUBTFunc('UE4Editor', platformIdentifier, configuration, args)
 		finally:
 			if renameSentinel == True:
-				shutil.move(sentinelBackup, sentinelFile)
+				Utility.moveFile(sentinelBackup, sentinelFile)
 
 		# Parse the JSON output
-		result = json.loads(Utility.readFile(jsonFile))
+		result = JsonDataManager(jsonFile).loads()
 
 		# Extract the list of third-party library modules
 		# (Note that since UE4.21, modules no longer have a "Type" field, so we must
@@ -188,7 +192,7 @@ def _getThirdPartyLibs(self, platformIdentifier, configuration):
 
 		# Remove the temp directory
 		try:
-			shutil.rmtree(tempDir)
+			Utility.removeDir(tempDir)
 		except:
 			pass
 

ue4cli/UnrealManagerBase.py

@@ -5,7 +5,9 @@
 from .CachedDataManager import CachedDataManager
 from .CMakeCustomFlags import CMakeCustomFlags
 from .Utility import Utility
-import glob, hashlib, json, os, re, shutil, sys
+from .JsonDataManager import JsonDataManager
+from .UtilityException import UtilityException
+import glob, hashlib, json, os, re
 
 class UnrealManagerBase(object):
 	"""
@@ -44,13 +46,13 @@ def setEngineRootOverride(self, rootDir):
 		# Set the new root directory
 		rootDir = os.path.abspath(rootDir)
 		ConfigurationManager.setConfigKey('rootDirOverride', rootDir)
-		print('Set engine root path override: {}'.format(rootDir))
+		Utility.printStderr('Setting engine root path override:', str(rootDir))
 
 		# Check that the specified directory is valid and warn the user if it is not
 		try:
 			self.getEngineVersion()
 		except:
-			print('Warning: the specified directory does not appear to contain a valid version of the Unreal Engine.')
+			raise UnrealManagerException('the specified directory does not appear to contain a valid version of the Unreal Engine.')
 
 	def clearEngineRootOverride(self):
 		"""
@@ -94,7 +96,7 @@ def getEngineVersion(self, outputFormat = 'full'):
 
 		# Verify that the requested output format is valid
 		if outputFormat not in formats:
-			raise Exception('unreconised version output format "{}"'.format(outputFormat))
+			raise UnrealManagerException('unreconised version output format "{}"'.format(outputFormat))
 
 		return formats[outputFormat]
 
@@ -316,8 +318,7 @@ def generateProjectFiles(self, dir=os.getcwd(), args=[]):
 
 		# If the project is a pure Blueprint project, then we cannot generate project files
 		if os.path.exists(os.path.join(dir, 'Source')) == False:
-			Utility.printStderr('Pure Blueprint project, nothing to generate project files for.')
-			return
+			raise UnrealManagerException('Pure Blueprint project, nothing to generate project files for.')
 
 		# Generate the project files
 		genScript = self.getGenerateScript()
@@ -334,8 +335,8 @@ def cleanDescriptor(self, dir=os.getcwd()):
 
 		# Because performing a clean will also delete the engine build itself when using
 		# a source build, we simply delete the `Binaries` and `Intermediate` directories
-		shutil.rmtree(os.path.join(dir, 'Binaries'), ignore_errors=True)
-		shutil.rmtree(os.path.join(dir, 'Intermediate'), ignore_errors=True)
+		Utility.removeDir(os.path.join(dir, 'Binaries'), ignore_errors=True)
+		Utility.removeDir(os.path.join(dir, 'Intermediate'), ignore_errors=True)
 
 		# If we are cleaning a project, also clean any plugins
 		if self.isProject(descriptor):
@@ -354,8 +355,7 @@ def buildDescriptor(self, dir=os.getcwd(), configuration='Development', target='
 
 		# If the project or plugin is Blueprint-only, there is no C++ code to build
 		if os.path.exists(os.path.join(dir, 'Source')) == False:
-			Utility.printStderr('Pure Blueprint {}, nothing to build.'.format(descriptorType))
-			return
+			raise UnrealManagerException('Pure Blueprint {}, nothing to build.'.format(descriptorType))
 
 		# Verify that the specified build configuration is valid
 		if configuration not in self.validBuildConfigurations():
@@ -538,7 +538,7 @@ def listAutomationTests(self, projectFile):
 		# Detect if the Editor terminated abnormally (i.e. not triggered by `automation quit`)
 		# In Unreal Engine 4.27.0, the exit method changed from RequestExit to RequestExitWithStatus
 		if 'PlatformMisc::RequestExit(' not in logOutput.stdout and 'PlatformMisc::RequestExitWithStatus(' not in logOutput.stdout:
-			raise RuntimeError(
+			raise UnrealManagerException(
 				'failed to retrieve the list of automation tests!' +
 				' stdout was: "{}", stderr was: "{}"'.format(logOutput.stdout, logOutput.stderr)
 			)
@@ -556,7 +556,7 @@ def automationTests(self, dir=os.getcwd(), args=[]):
 
 		# Verify that at least one argument was supplied
 		if len(args) == 0:
-			raise RuntimeError('at least one test name must be specified')
+			raise UnrealManagerException('at least one test name must be specified')
 
 		# Gather any additional arguments to pass directly to the Editor
 		extraArgs = []
@@ -567,7 +567,12 @@ def automationTests(self, dir=os.getcwd(), args=[]):
 
 		# Build the project if it isn't already built
 		Utility.printStderr('Ensuring project is built...')
-		self.buildDescriptor(dir, suppressOutput=True)
+		try:
+			self.buildDescriptor(dir, suppressOutput=True)
+		except UnrealManagerException:
+			# FIXME: Ideally, we should NOT catch every UnrealManagerException here
+			# This is currently a limitation of our API that uses only one Exception class for multiple different cases
+			pass
 
 		# Determine which arguments we are passing to the automation test commandlet
 		projectFile = self.getProjectDescriptor(dir)
@@ -605,7 +610,7 @@ def automationTests(self, dir=os.getcwd(), args=[]):
 			# Detect abnormal exit conditions (those not triggered by `automation quit`)
 			# In Unreal Engine 4.27.0, the exit method changed from RequestExit to RequestExitWithStatus
 			if 'PlatformMisc::RequestExit(' not in logOutput.stdout and 'PlatformMisc::RequestExitWithStatus(' not in logOutput.stdout:
-				sys.exit(1)
+				raise UnrealManagerException('abnormal exit condition detected')
 
 			# Since UE4 doesn't consistently produce accurate exit codes across all platforms, we need to rely on
 			# text-based heuristics to detect failed automation tests or errors related to not finding any tests to run
@@ -618,12 +623,12 @@ def automationTests(self, dir=os.getcwd(), args=[]):
 			]
 			for errorStr in errorStrings:
 				if errorStr in logOutput.stdout:
-					sys.exit(1)
+					raise UnrealManagerException('abnormal exit condition detected')
 
 			# If an explicit exit code was specified in the output text then identify it and propagate it
-			match = re.search('TEST COMPLETE\\. EXIT CODE: ([0-9]+)', logOutput.stdout + logOutput.stderr)
+			match = re.search('TEST COMPLETE\\. EXIT CODE: ([1-9]+)', logOutput.stdout + logOutput.stderr)
 			if match is not None:
-				sys.exit(int(match.group(1)))
+				raise UnrealManagerException('abnormal exit condition detected')
 
 	# "Protected" methods
 
@@ -638,7 +643,7 @@ def _getEngineVersionDetails(self):
 		Parses the JSON version details for the latest installed version of UE4
 		"""
 		versionFile = os.path.join(self.getEngineRoot(), 'Engine', 'Build', 'Build.version')
-		return json.loads(Utility.readFile(versionFile))
+		return JsonDataManager(versionFile).loads()
 
 	def _getEngineVersionHash(self):
 		"""

ue4cli/UnrealManagerWindows.py

@@ -42,7 +42,7 @@ def getGenerateScript(self):
 		except:
 			pass
 
-		raise UnrealManagerException('could not detect the location of GenerateProjectFiles.bat or UnrealVersionSelector.exe.\nThis typically indicates that .uproject files are not correctly associated with UE4.')
+		raise UnrealManagerException('could not detect the location of GenerateProjectFiles.bat or UnrealVersionSelector.exe. This typically indicates that .uproject files are not correctly associated with UE4.')
 
 	def getRunUATScript(self):
 		return self.getEngineRoot() + '\\Engine\\Build\\BatchFiles\\RunUAT.bat'
@@ -52,7 +52,7 @@ def generateProjectFiles(self, dir=os.getcwd(), args=[]):
 		# If we are using our custom batch file, use the appropriate arguments
 		genScript = self.getGenerateScript()
 		projectFile = self.getProjectDescriptor(dir)
-		print(projectFile)
+		Utility.printStderr('Using project file:', projectFile)
 		if '.ue4\\GenerateProjectFiles.bat' in genScript:
 			Utility.run([genScript, projectFile], raiseOnError=True)
 		else:
@@ -91,7 +91,7 @@ def _customBatchScriptDir(self):
 		# If the script directory doesn't already exist, attempt to create it
 		scriptDir = os.path.join(os.environ['HOMEDRIVE'] + os.environ['HOMEPATH'], '.ue4')
 		try:
-			os.makedirs(scriptDir)
+			Utility.makeDirs(scriptDir)
 		except:
 			pass
 

ue4cli/Utility.py

@@ -1,4 +1,5 @@
-import os, platform, shlex, subprocess, sys
+from .UtilityException import UtilityException
+import os, platform, shlex, subprocess, sys, shutil
 
 class CommandOutput(object):
 	"""
@@ -21,23 +22,39 @@ def printStderr(*args, **kwargs):
 		Prints to stderr instead of stdout
 		"""
 		if os.environ.get('UE4CLI_QUIET', '0') != '1':
-			print(*args, file=sys.stderr, **kwargs)
+			print('(ue4cli)', *args, file=sys.stderr, **kwargs)
 
 	@staticmethod
 	def readFile(filename):
 		"""
 		Reads data from a file
 		"""
-		with open(filename, 'rb') as f:
-			return f.read().decode('utf-8')
+		try:
+			with open(filename, 'rb') as f:
+				return f.read().decode('utf-8')
+		except OSError as e:
+			raise UtilityException(f'failed to read file "{str(filename)}" due to: ({type(e).__name__}) {str(e)}')
 
 	@staticmethod
 	def writeFile(filename, data):
 		"""
 		Writes data to a file
 		"""
-		with open(filename, 'wb') as f:
-			f.write(data.encode('utf-8'))
+		try:
+			with open(filename, 'wb') as f:
+				f.write(data.encode('utf-8'))
+		except OSError as e:
+			raise UtilityException(f'failed to write file "{str(filename)}" due to: ({type(e).__name__}) {str(e)}')
+
+	@staticmethod
+	def moveFile(src, dst):
+		"""
+		Moves file from 'src' to 'dst'
+		"""
+		try:
+			shutil.move(src, dst)
+		except OSError as e:
+			raise UtilityException(f'failed to move file from "{str(src)}" to "{str(dst)}" due to: ({type(e).__name__}) {str(e)}')
 
 	@staticmethod
 	def patchFile(filename, replacements):
@@ -52,6 +69,26 @@ def patchFile(filename, replacements):
 
 		Utility.writeFile(filename, patched)
 
+	@staticmethod
+	def removeDir(path, ignore_errors=False):
+		"""
+		Recursively remove directory tree
+		"""
+		try:
+			shutil.rmtree(path, ignore_errors)
+		except OSError as e:
+			raise UtilityException(f'failed to remove directory "{str(path)}" due to: ({type(e).__name__}) {str(e)}')
+
+	@staticmethod
+	def makeDirs(name, mode=0o777, exist_ok=False):
+		"""
+		Makes directory
+		"""
+		try:
+			os.makedirs(name, mode, exist_ok)
+		except OSError as e:
+			raise UtilityException(f'failed to create directory "{str(name)}" due to: ({type(e).__name__}) {str(e)}')
+
 	@staticmethod
 	def forwardSlashes(paths):
 		"""
@@ -123,12 +160,12 @@ def capture(command, input=None, cwd=None, shell=False, raiseOnError=False):
 
 		# If the child process failed and we were asked to raise an exception, do so
 		if raiseOnError == True and proc.returncode != 0:
-			raise Exception(
-				'child process ' + str(command) +
-				' failed with exit code ' + str(proc.returncode) +
-				'\nstdout: "' + stdout + '"' +
-				'\nstderr: "' + stderr + '"'
-			)
+			Utility.printStderr("Warning: child process failure encountered!")
+			Utility.printStderr("printing stdout..")
+			print(stdout)
+			Utility.printStderr("printing stderr..")
+			print(stderr)
+			raise UtilityException(f'child process {str(command)} failed with exit code {str(proc.returncode)}')
 
 		return CommandOutput(proc.returncode, stdout, stderr)
 
@@ -143,7 +180,7 @@ def run(command, cwd=None, shell=False, raiseOnError=False):
 
 		returncode = subprocess.call(command, cwd=cwd, shell=shell)
 		if raiseOnError == True and returncode != 0:
-			raise Exception('child process ' + str(command) + ' failed with exit code ' + str(returncode))
+			raise UtilityException(f'child process {str(command)} failed with exit code {str(returncode)}')
 		return returncode
 
 	@staticmethod
@@ -152,4 +189,4 @@ def _printCommand(command):
 		Prints a command if verbose output is enabled
 		"""
 		if os.environ.get('UE4CLI_VERBOSE', '0') == '1':
-			Utility.printStderr('[UE4CLI] EXECUTE COMMAND:', command)
+			Utility.printStderr('EXECUTE COMMAND:', command)

ue4cli/UtilityException.py

@@ -0,0 +1,2 @@
+class UtilityException(Exception):
+	pass