Skip to content

Commit

Permalink
s4cmd: avoid AttributeError on newer botocore
Browse files Browse the repository at this point in the history
Newer botocore has removed `vendored` entirely; try to import carefully.
Makes s4cmd work again on botocore-1.29.84.

```
Traceback (most recent call last):
  File "/usr/lib/python-exec/python3.10/s4cmd", line 33, in <module>
    sys.exit(load_entry_point('s4cmd==2.1.0', 'console_scripts', 's4cmd')())
  File "/usr/lib/python-exec/python3.10/s4cmd", line 25, in importlib_load_entry_point
    return next(matches).load()
  File "/usr/lib/python3.10/importlib/metadata/__init__.py", line 171, in load
    module = import_module(match.group('module'))
  File "/usr/lib/python3.10/importlib/__init__.py", line 126, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
  File "/usr/lib64/python-exec/python3.10/s4cmd.py", line 255, in <module>
    class BotoClient(object):
  File "/usr/lib64/python-exec/python3.10/s4cmd.py", line 274, in BotoClient
    botocore.vendored.requests.packages.urllib3.exceptions.ReadTimeoutError,
AttributeError: module 'botocore' has no attribute 'vendored'
```

Signed-off-by: Robin H. Johnson <[email protected]>
  • Loading branch information
robbat2 committed Mar 19, 2023
1 parent 91cf847 commit 4a1323f
Showing 1 changed file with 10 additions and 6 deletions.
16 changes: 10 additions & 6 deletions s4cmd.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/python3.10

#
# Copyright 2012-2018 BloomReach, Inc.
Expand Down Expand Up @@ -60,8 +60,8 @@ def cmp(a, b):
TEMP_FILES = set()

# Environment variable names for S3 credentials.
S3_ACCESS_KEY_NAME = "AWS_ACCESS_KEY_ID"
S3_SECRET_KEY_NAME = "AWS_SECRET_ACCESS_KEY"
S3_ACCESS_KEY_NAME = "S3_ACCESS_KEY"
S3_SECRET_KEY_NAME = "S3_SECRET_KEY"
S4CMD_ENV_KEY = "S4CMD_OPTS"


Expand Down Expand Up @@ -271,9 +271,13 @@ class BotoClient(object):
S3RetryableErrors = (
socket.timeout,
socket.error if IS_PYTHON2 else ConnectionError,
botocore.vendored.requests.packages.urllib3.exceptions.ReadTimeoutError,
botocore.exceptions.IncompleteReadError
)
try:
# Newer boto removed botocore.vendored entirely.
S3RetryableErrors += (botocore.vendored.requests.packages.urllib3.exceptions.ReadTimeoutError,)
except AttributeError:
pass

# List of API functions we use in s4cmd.
ALLOWED_CLIENT_METHODS = [
Expand Down Expand Up @@ -655,7 +659,7 @@ def s3_keys_from_s3cfg(opt):
config = ConfigParser.ConfigParser()
config.read(s3cfg_path)
keys = config.get("default", "access_key"), config.get("default", "secret_key")
debug("read S3 keys from %s file", s3cfg_path)
debug("read S3 keys from $HOME/.s3cfg file")
return keys
except Exception as e:
info("could not read S3 keys from %s file; skipping (%s)", s3cfg_path, e)
Expand Down Expand Up @@ -1158,7 +1162,7 @@ def partial_match(self, path, filter_path):
fi = filter_path.split(PATH_SEP)

# Here, if we are in recursive mode, we allow the pi to be longer than fi.
# Otherwise, length of pi should be equal or less than the length of fi.
# Otherwise, length of pi should be equal or less than the lenght of fi.
min_len = min(len(pi), len(fi))
matched = fnmatch.fnmatch(PATH_SEP.join(pi[0:min_len]), PATH_SEP.join(fi[0:min_len]))
return matched and (self.opt.recursive or len(pi) <= len(fi))
Expand Down

0 comments on commit 4a1323f

Please sign in to comment.