From f176262147ee08cfdc7701d9980b5e00b687ad12 Mon Sep 17 00:00:00 2001 From: Vincent Chen Date: Wed, 31 Jan 2024 14:46:58 -0800 Subject: [PATCH 1/4] remove Composer imports --- llmfoundry/utils/huggingface_hub_utils.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/llmfoundry/utils/huggingface_hub_utils.py b/llmfoundry/utils/huggingface_hub_utils.py index 07a9c3900e..0877d813bd 100644 --- a/llmfoundry/utils/huggingface_hub_utils.py +++ b/llmfoundry/utils/huggingface_hub_utils.py @@ -83,6 +83,13 @@ def process_file( node.module.startswith('composer') or node.module.startswith('omegaconf')): nodes_to_remove.append(node) + # Remove names starting with 'Composer' from ImportFrom nodes + elif isinstance(node, ast.ImportFrom): + # Filter out names that start with 'Composer' + node.names = [name for name in node.names if not name.name.startswith('Composer')] + # If all names are removed, mark the whole import statement for removal + if not node.names: + nodes_to_remove.append(node) # Remove the Composer* class elif (isinstance(node, ast.ClassDef) and node.name.startswith('Composer')): @@ -90,9 +97,19 @@ def process_file( # Remove the __all__ declaration in any __init__.py files, whose # enclosing module will be converted to a single file of the same name elif (isinstance(node, ast.Assign) and len(node.targets) == 1 and - isinstance(node.targets[0], ast.Name) and - node.targets[0].id == '__all__'): + isinstance(node.targets[0], ast.Name) and + node.targets[0].id == '__all__'): nodes_to_remove.append(node) + # Update __all__ by removing entries that start with 'Composer' + __all__names = [elt.s for elt in node.value.elts if not elt.s.startswith('Composer')] + # Create a new __all__ assignment node if there are any names left + if __all__names: + new_all_node = ast.Assign( + targets=[ast.Name(id='__all__', ctx=ast.Store())], + value=ast.List(elts=[ast.Str(s=name) for name in __all__names], ctx=ast.Load()) + ) + ast.fix_missing_locations(new_all_node) + tree.body.insert(0, new_all_node) transformer = DeleteSpecificNodes(nodes_to_remove) new_tree = transformer.visit(tree) From d058ead3a8e4d21f886a96ccca3a5f7c2f82979d Mon Sep 17 00:00:00 2001 From: Vincent Chen Date: Wed, 31 Jan 2024 15:00:39 -0800 Subject: [PATCH 2/4] lint --- llmfoundry/utils/huggingface_hub_utils.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/llmfoundry/utils/huggingface_hub_utils.py b/llmfoundry/utils/huggingface_hub_utils.py index 0877d813bd..00f939637b 100644 --- a/llmfoundry/utils/huggingface_hub_utils.py +++ b/llmfoundry/utils/huggingface_hub_utils.py @@ -86,7 +86,10 @@ def process_file( # Remove names starting with 'Composer' from ImportFrom nodes elif isinstance(node, ast.ImportFrom): # Filter out names that start with 'Composer' - node.names = [name for name in node.names if not name.name.startswith('Composer')] + node.names = [ + name for name in node.names + if not name.name.startswith('Composer') + ] # If all names are removed, mark the whole import statement for removal if not node.names: nodes_to_remove.append(node) @@ -97,17 +100,22 @@ def process_file( # Remove the __all__ declaration in any __init__.py files, whose # enclosing module will be converted to a single file of the same name elif (isinstance(node, ast.Assign) and len(node.targets) == 1 and - isinstance(node.targets[0], ast.Name) and - node.targets[0].id == '__all__'): + isinstance(node.targets[0], ast.Name) and + node.targets[0].id == '__all__'): nodes_to_remove.append(node) # Update __all__ by removing entries that start with 'Composer' - __all__names = [elt.s for elt in node.value.elts if not elt.s.startswith('Composer')] + __all__names = [ + elt.s + for elt in node.value.elts + if not elt.s.startswith('Composer') + ] # Create a new __all__ assignment node if there are any names left if __all__names: new_all_node = ast.Assign( targets=[ast.Name(id='__all__', ctx=ast.Store())], - value=ast.List(elts=[ast.Str(s=name) for name in __all__names], ctx=ast.Load()) - ) + value=ast.List( + elts=[ast.Str(s=name) for name in __all__names], + ctx=ast.Load())) ast.fix_missing_locations(new_all_node) tree.body.insert(0, new_all_node) From 11e25605383dd96a3104e829c1da166e3272180f Mon Sep 17 00:00:00 2001 From: Vincent Chen Date: Wed, 31 Jan 2024 16:16:39 -0800 Subject: [PATCH 3/4] precommit lint --- llmfoundry/utils/huggingface_hub_utils.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/llmfoundry/utils/huggingface_hub_utils.py b/llmfoundry/utils/huggingface_hub_utils.py index 00f939637b..6f7de8fb3a 100644 --- a/llmfoundry/utils/huggingface_hub_utils.py +++ b/llmfoundry/utils/huggingface_hub_utils.py @@ -86,10 +86,7 @@ def process_file( # Remove names starting with 'Composer' from ImportFrom nodes elif isinstance(node, ast.ImportFrom): # Filter out names that start with 'Composer' - node.names = [ - name for name in node.names - if not name.name.startswith('Composer') - ] + node.names = [name for name in node.names if not name.name.startswith('Composer')] # If all names are removed, mark the whole import statement for removal if not node.names: nodes_to_remove.append(node) @@ -103,19 +100,13 @@ def process_file( isinstance(node.targets[0], ast.Name) and node.targets[0].id == '__all__'): nodes_to_remove.append(node) - # Update __all__ by removing entries that start with 'Composer' - __all__names = [ - elt.s - for elt in node.value.elts - if not elt.s.startswith('Composer') - ] + __all__names = [elt.s for elt in node.value.elts if not elt.s.startswith('Composer')] # Create a new __all__ assignment node if there are any names left if __all__names: new_all_node = ast.Assign( targets=[ast.Name(id='__all__', ctx=ast.Store())], - value=ast.List( - elts=[ast.Str(s=name) for name in __all__names], - ctx=ast.Load())) + value=ast.List(elts=[ast.Str(s=name) for name in __all__names], ctx=ast.Load()) + ) ast.fix_missing_locations(new_all_node) tree.body.insert(0, new_all_node) From 2d962194d12e3689d471a34bde055f77536ac43c Mon Sep 17 00:00:00 2001 From: Vincent Chen Date: Thu, 1 Feb 2024 10:28:30 -0800 Subject: [PATCH 4/4] lint --- llmfoundry/utils/huggingface_hub_utils.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/llmfoundry/utils/huggingface_hub_utils.py b/llmfoundry/utils/huggingface_hub_utils.py index 6f7de8fb3a..78cb7d8591 100644 --- a/llmfoundry/utils/huggingface_hub_utils.py +++ b/llmfoundry/utils/huggingface_hub_utils.py @@ -86,7 +86,10 @@ def process_file( # Remove names starting with 'Composer' from ImportFrom nodes elif isinstance(node, ast.ImportFrom): # Filter out names that start with 'Composer' - node.names = [name for name in node.names if not name.name.startswith('Composer')] + node.names = [ + name for name in node.names + if not name.name.startswith('Composer') + ] # If all names are removed, mark the whole import statement for removal if not node.names: nodes_to_remove.append(node) @@ -100,13 +103,17 @@ def process_file( isinstance(node.targets[0], ast.Name) and node.targets[0].id == '__all__'): nodes_to_remove.append(node) - __all__names = [elt.s for elt in node.value.elts if not elt.s.startswith('Composer')] + __all__names = [] + for elt in node.value.elts: + if not elt.s.startswith('Composer'): + __all__names.append(elt.s) # Create a new __all__ assignment node if there are any names left if __all__names: new_all_node = ast.Assign( targets=[ast.Name(id='__all__', ctx=ast.Store())], - value=ast.List(elts=[ast.Str(s=name) for name in __all__names], ctx=ast.Load()) - ) + value=ast.List( + elts=[ast.Str(s=name) for name in __all__names], + ctx=ast.Load())) ast.fix_missing_locations(new_all_node) tree.body.insert(0, new_all_node)