Skip to content

Commit

Permalink
Fix my mistake: map and reduce exe stdout will be buried in output file
Browse files Browse the repository at this point in the history
  • Loading branch information
awdeorio committed Dec 12, 2024
1 parent b2bef06 commit cd4584d
Show file tree
Hide file tree
Showing 3 changed files with 1 addition and 7 deletions.
6 changes: 1 addition & 5 deletions madoop/mapreduce.py
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,6 @@ def map_single_chunk(exe, input_path, output_path, chunk):
f"Command returned non-zero: "
f"{exe} < {input_path} > {output_path}\n"
f"{err}\n"
f"{err.stdout.decode()}" if err.stdout else ""
f"{err.stderr.decode()}" if err.stderr else ""
) from err
except OSError as err:
Expand Down Expand Up @@ -342,10 +341,8 @@ def partition_keys_custom(
return_code = process.wait()
if return_code:
stderr_output = process.stderr.read()
if len(stderr_output) != 0:
stderr_output = '\n' + stderr_output
raise MadoopError(
f"Partition executable returned non-zero: {str(partitioner)}"
f"Partition executable returned non-zero: {str(partitioner)}\n"
f"{stderr_output}"
)

Expand Down Expand Up @@ -445,7 +442,6 @@ def reduce_single_file(exe, input_path, output_path):
f"Command returned non-zero: "
f"{exe} < {input_path} > {output_path}\n"
f"{err}\n"
f"{err.stdout.decode()}" if err.stdout else ""
f"{err.stderr.decode()}" if err.stderr else ""
) from err
except OSError as err:
Expand Down
1 change: 0 additions & 1 deletion tests/testdata/word_count/map_error_msg.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,5 @@
# Avoid error on executable check which has an empty string input
input_lines_n = sum(1 for _ in sys.stdin)
if input_lines_n > 1:
sys.stdout.write("Map error message to stdout\n")
sys.stderr.write("Map error message to stderr\n")
sys.exit(1)
1 change: 0 additions & 1 deletion tests/testdata/word_count/reduce_error_msg.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,5 @@
# Avoid error on executable check which has an empty string input
input_lines_n = sum(1 for _ in sys.stdin)
if input_lines_n > 1:
sys.stdout.write("Reduce error message to stdout\n")
sys.stderr.write("Reduce error message to stderr\n")
sys.exit(1)

0 comments on commit cd4584d

Please sign in to comment.