diff --git a/tests/baseline_images/test_regressor/test_prediction_error/test_peplot_no_lines.png b/tests/baseline_images/test_regressor/test_prediction_error/test_peplot_no_lines.png
index 0ed3013ea..a0f423af5 100644
Binary files a/tests/baseline_images/test_regressor/test_prediction_error/test_peplot_no_lines.png and b/tests/baseline_images/test_regressor/test_prediction_error/test_peplot_no_lines.png differ
diff --git a/tests/baseline_images/test_regressor/test_prediction_error/test_peplot_no_shared_limits.png b/tests/baseline_images/test_regressor/test_prediction_error/test_peplot_no_shared_limits.png
index af1ab2702..70b6c3b2a 100644
Binary files a/tests/baseline_images/test_regressor/test_prediction_error/test_peplot_no_shared_limits.png and b/tests/baseline_images/test_regressor/test_prediction_error/test_peplot_no_shared_limits.png differ
diff --git a/tests/baseline_images/test_regressor/test_prediction_error/test_prediction_error.png b/tests/baseline_images/test_regressor/test_prediction_error/test_prediction_error.png
index 747d6a5b9..48b0457d0 100644
Binary files a/tests/baseline_images/test_regressor/test_prediction_error/test_prediction_error.png and b/tests/baseline_images/test_regressor/test_prediction_error/test_prediction_error.png differ
diff --git a/tests/baseline_images/test_regressor/test_prediction_error/test_prediction_error_numpy.png b/tests/baseline_images/test_regressor/test_prediction_error/test_prediction_error_numpy.png
index 82cf833cd..82e25ca8b 100644
Binary files a/tests/baseline_images/test_regressor/test_prediction_error/test_prediction_error_numpy.png and b/tests/baseline_images/test_regressor/test_prediction_error/test_prediction_error_numpy.png differ
diff --git a/tests/baseline_images/test_regressor/test_prediction_error/test_prediction_error_pandas.png b/tests/baseline_images/test_regressor/test_prediction_error/test_prediction_error_pandas.png
index f6be119e0..989736c48 100644
Binary files a/tests/baseline_images/test_regressor/test_prediction_error/test_prediction_error_pandas.png and b/tests/baseline_images/test_regressor/test_prediction_error/test_prediction_error_pandas.png differ
diff --git a/tests/baseline_images/test_regressor/test_prediction_error/test_prediction_error_quick_method.png b/tests/baseline_images/test_regressor/test_prediction_error/test_prediction_error_quick_method.png
index 85619cdfa..acd4c6dcb 100644
Binary files a/tests/baseline_images/test_regressor/test_prediction_error/test_prediction_error_quick_method.png and b/tests/baseline_images/test_regressor/test_prediction_error/test_prediction_error_quick_method.png differ
diff --git a/yellowbrick/regressor/prediction_error.py b/yellowbrick/regressor/prediction_error.py
index 79b08ba76..2922d0b70 100644
--- a/yellowbrick/regressor/prediction_error.py
+++ b/yellowbrick/regressor/prediction_error.py
@@ -216,12 +216,13 @@ def draw(self, y, y_pred):
                 label="best fit",
             )
 
-        # Set the axes limits based on the range of X and Y data
+        # Set the axes limits based on the overall max/min values of
+        # concatenated X and Y data
         # NOTE: shared_limits will be accounted for in finalize()
-        # TODO: do better than add one for really small residuals
-        self.ax.set_xlim(y.min() - 1, y.max() + 1)
-        self.ax.set_ylim(y_pred.min() - 1, y_pred.max() + 1)
-
+        if self.shared_limits is True:
+            self.ax.set_xlim(min(min(y), min(y_pred)), max(max(y), max(y_pred)))
+            self.ax.set_ylim(self.ax.get_xlim())
+            
         return self.ax
 
     def finalize(self, **kwargs):