From a10bd4a020b8edc650b9b0f7462de04856a5ca1c Mon Sep 17 00:00:00 2001
From: jpg <europe.jpg@gmail.com>
Date: Tue, 6 Aug 2019 19:03:16 +0200
Subject: [PATCH] Improved the handling of errors while parsing recursively.
 Updated tests. + various formatting fixes

---
 .travis.yml                                   |  2 +-
 src/SitemapParser.php                         | 40 +++++++++++--------
 .../Exceptions/SitemapParserException.php     |  1 +
 .../Exceptions/TransferException.php          | 13 ++++++
 src/SitemapParser/UrlParser.php               |  1 +
 tests/DownloadTest.php                        |  3 +-
 tests/ExceptionEncodingTest.php               |  1 +
 tests/InvalidURLTest.php                      |  1 +
 tests/RecursiveTest.php                       | 12 +++++-
 tests/RobotsTxtTest.php                       |  4 +-
 tests/SitemapIndexTest.php                    |  4 +-
 tests/StrictTest.php                          |  4 +-
 tests/StringTest.php                          |  4 +-
 tests/URLSetTest.php                          |  4 +-
 14 files changed, 63 insertions(+), 31 deletions(-)
 create mode 100644 src/SitemapParser/Exceptions/TransferException.php

diff --git a/.travis.yml b/.travis.yml
index 28e281f..23eff7e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,11 +1,11 @@
 sudo: false
 language: php
 php:
+  - 7.3
   - 7.2
   - 7.1
   - 7.0
   - 5.6
-  - hhvm
 install:
   - composer install
 after_script:
diff --git a/src/SitemapParser.php b/src/SitemapParser.php
index 0c242d6..4bea782 100644
--- a/src/SitemapParser.php
+++ b/src/SitemapParser.php
@@ -1,9 +1,10 @@
 <?php
+
 namespace vipnytt;
 
 use GuzzleHttp;
 use SimpleXMLElement;
-use vipnytt\SitemapParser\Exceptions\SitemapParserException;
+use vipnytt\SitemapParser\Exceptions;
 use vipnytt\SitemapParser\UrlParser;
 
 /**
@@ -101,13 +102,13 @@ class SitemapParser
      *
      * @param string $userAgent User-Agent to send with every HTTP(S) request
      * @param array $config Configuration options
-     * @throws SitemapParserException
+     * @throws Exceptions\SitemapParserException
      */
     public function __construct($userAgent = self::DEFAULT_USER_AGENT, array $config = [])
     {
         mb_language("uni");
         if (!mb_internal_encoding(self::ENCODING)) {
-            throw new SitemapParserException('Unable to set internal character encoding to `' . self::ENCODING . '`');
+            throw new Exceptions\SitemapParserException('Unable to set internal character encoding to `' . self::ENCODING . '`');
         }
         $this->userAgent = $userAgent;
         $this->config = $config;
@@ -118,7 +119,7 @@ public function __construct($userAgent = self::DEFAULT_USER_AGENT, array $config
      *
      * @param string $url
      * @return void
-     * @throws SitemapParserException
+     * @throws Exceptions\SitemapParserException
      */
     public function parseRecursive($url)
     {
@@ -126,7 +127,12 @@ public function parseRecursive($url)
         while (count($todo = $this->getQueue()) > 0) {
             $sitemaps = $this->sitemaps;
             $urls = $this->urls;
-            $this->parse($todo[0]);
+            try {
+                $this->parse($todo[0]);
+            } catch (Exceptions\TransferException $e) {
+                // Keep crawling
+                continue;
+            }
             $this->sitemaps = array_merge_recursive($sitemaps, $this->sitemaps);
             $this->urls = array_merge_recursive($urls, $this->urls);
         }
@@ -161,14 +167,15 @@ public function getQueue()
      * @param string $url URL to parse
      * @param string|null $urlContent URL body content (provide to skip download)
      * @return void
-     * @throws SitemapParserException
+     * @throws Exceptions\TransferException
+     * @throws Exceptions\SitemapParserException
      */
     public function parse($url, $urlContent = null)
     {
         $this->clean();
         $this->currentURL = $url;
-        $response = (is_string($urlContent)) ? $urlContent : $this->getContent();
         $this->history[] = $this->currentURL;
+        $response = is_string($urlContent) ? $urlContent : $this->getContent();
         if ($this->urlValidate($this->currentURL) && parse_url($this->currentURL, PHP_URL_PATH) === self::ROBOTSTXT_PATH) {
             $this->parseRobotstxt($response);
             return;
@@ -201,13 +208,14 @@ protected function clean()
      * Request the body content of an URL
      *
      * @return string Raw body content
-     * @throws SitemapParserException
+     * @throws Exceptions\TransferException
+     * @throws Exceptions\SitemapParserException
      */
     protected function getContent()
     {
         $this->currentURL = $this->urlEncode($this->currentURL);
         if (!$this->urlValidate($this->currentURL)) {
-            throw new SitemapParserException('Invalid URL');
+            throw new Exceptions\SitemapParserException('Invalid URL');
         }
         try {
             if (!isset($this->config['guzzle']['headers']['User-Agent'])) {
@@ -217,9 +225,9 @@ protected function getContent()
             $res = $client->request('GET', $this->currentURL, $this->config['guzzle']);
             return $res->getBody();
         } catch (GuzzleHttp\Exception\TransferException $e) {
-            if (stripos($e->getMessage(), 'cURL error 6:') === false && $e->getCode() != 404) {
-                throw new SitemapParserException($e->getMessage());
-            }
+            throw new Exceptions\TransferException('Unable to fetch URL contents', 0, $e);
+        } catch (GuzzleHttp\Exception\GuzzleException $e) {
+            throw new Exceptions\SitemapParserException('GuzzleHttp exception', 0, $e);
         }
     }
 
@@ -309,7 +317,7 @@ protected function generateXMLObject($xml)
         // strip XML comments from files
         // if they occur at the beginning of the file it will invalidate the XML
         // this occurs with certain versions of Yoast
-        $xml = preg_replace('/\s*\<\!\-\-((?!\-\-\>)[\s\S])*\-\-\>\s*/', '', (string) $xml);
+        $xml = preg_replace('/\s*\<\!\-\-((?!\-\-\>)[\s\S])*\-\-\>\s*/', '', (string)$xml);
         try {
             libxml_use_internal_errors(true);
             return new SimpleXMLElement($xml, LIBXML_NOCDATA);
@@ -351,9 +359,9 @@ protected function isSitemapURL($url)
     {
         $path = parse_url($this->urlEncode($url), PHP_URL_PATH);
         return $this->urlValidate($url) && (
-            mb_substr($path, -mb_strlen(self::XML_EXTENSION) - 1) == '.' . self::XML_EXTENSION ||
-            mb_substr($path, -mb_strlen(self::XML_EXTENSION_COMPRESSED) - 1) == '.' . self::XML_EXTENSION_COMPRESSED
-        );
+                mb_substr($path, -mb_strlen(self::XML_EXTENSION) - 1) == '.' . self::XML_EXTENSION ||
+                mb_substr($path, -mb_strlen(self::XML_EXTENSION_COMPRESSED) - 1) == '.' . self::XML_EXTENSION_COMPRESSED
+            );
     }
 
     /**
diff --git a/src/SitemapParser/Exceptions/SitemapParserException.php b/src/SitemapParser/Exceptions/SitemapParserException.php
index 8eec58b..8a1acea 100644
--- a/src/SitemapParser/Exceptions/SitemapParserException.php
+++ b/src/SitemapParser/Exceptions/SitemapParserException.php
@@ -1,4 +1,5 @@
 <?php
+
 namespace vipnytt\SitemapParser\Exceptions;
 
 use Exception;
diff --git a/src/SitemapParser/Exceptions/TransferException.php b/src/SitemapParser/Exceptions/TransferException.php
new file mode 100644
index 0000000..ee1b1d6
--- /dev/null
+++ b/src/SitemapParser/Exceptions/TransferException.php
@@ -0,0 +1,13 @@
+<?php
+
+namespace vipnytt\SitemapParser\Exceptions;
+
+/**
+ * TransferException class
+ *
+ * @license https://opensource.org/licenses/MIT MIT license
+ * @link https://github.com/VIPnytt/SitemapParser
+ */
+class TransferException extends SitemapParserException
+{
+}
diff --git a/src/SitemapParser/UrlParser.php b/src/SitemapParser/UrlParser.php
index a6a56e8..25199da 100644
--- a/src/SitemapParser/UrlParser.php
+++ b/src/SitemapParser/UrlParser.php
@@ -1,4 +1,5 @@
 <?php
+
 namespace vipnytt\SitemapParser;
 
 /**
diff --git a/tests/DownloadTest.php b/tests/DownloadTest.php
index 7154443..1478077 100644
--- a/tests/DownloadTest.php
+++ b/tests/DownloadTest.php
@@ -36,8 +36,7 @@ public function testDownload($url)
      * Generate test data
      * @return array
      */
-    public
-    function generateDataForTest()
+    public function generateDataForTest()
     {
         return [
             [
diff --git a/tests/ExceptionEncodingTest.php b/tests/ExceptionEncodingTest.php
index d1c80d4..86d1df8 100644
--- a/tests/ExceptionEncodingTest.php
+++ b/tests/ExceptionEncodingTest.php
@@ -1,4 +1,5 @@
 <?php
+
 namespace vipnytt\SitemapParser\Tests;
 
 use PHPUnit\Framework\TestCase;
diff --git a/tests/InvalidURLTest.php b/tests/InvalidURLTest.php
index 9bef8e5..c0095e3 100644
--- a/tests/InvalidURLTest.php
+++ b/tests/InvalidURLTest.php
@@ -1,4 +1,5 @@
 <?php
+
 namespace vipnytt\SitemapParser\Tests;
 
 use PHPUnit\Framework\TestCase;
diff --git a/tests/RecursiveTest.php b/tests/RecursiveTest.php
index c94b9c2..a75eddf 100644
--- a/tests/RecursiveTest.php
+++ b/tests/RecursiveTest.php
@@ -1,4 +1,5 @@
 <?php
+
 namespace vipnytt\SitemapParser\Tests;
 
 use PHPUnit\Framework\TestCase;
@@ -36,11 +37,18 @@ public function testRecursive($url)
      * Generate test data
      * @return array
      */
-    public
-    function generateDataForTest()
+    public function generateDataForTest()
     {
         return [
             [
+                'https://edenapartmentsqueenanne.com/sitemap_index.xml',
+                'https://livingnongmo.org/sitemap.xml',
+                'https://loganwestom.com/sitemap_index.xml',
+                'https://sawyerflats.com/sitemap.xml',
+                'https://www.bellinghambaymarathon.org/sitemap_index.xml',
+                'https://www.coachforteens.com/sitemap_index.xml',
+                'https://www.hallerpostapts.com/sitemap_index.xml',
+                'https://www.nongmoproject.org/sitemap.xml',
                 'https://www.xml-sitemaps.com/robots.txt',
             ]
         ];
diff --git a/tests/RobotsTxtTest.php b/tests/RobotsTxtTest.php
index fc8e9ee..77617c1 100644
--- a/tests/RobotsTxtTest.php
+++ b/tests/RobotsTxtTest.php
@@ -1,4 +1,5 @@
 <?php
+
 namespace vipnytt\SitemapParser\Tests;
 
 use PHPUnit\Framework\TestCase;
@@ -25,8 +26,7 @@ public function testRobotsTxt($url, $body, $result)
      * Generate test data
      * @return array
      */
-    public
-    function generateDataForTest()
+    public function generateDataForTest()
     {
         return [
             [
diff --git a/tests/SitemapIndexTest.php b/tests/SitemapIndexTest.php
index 44f24f4..91e3c98 100644
--- a/tests/SitemapIndexTest.php
+++ b/tests/SitemapIndexTest.php
@@ -1,4 +1,5 @@
 <?php
+
 namespace vipnytt\SitemapParser\Tests;
 
 use PHPUnit\Framework\TestCase;
@@ -25,8 +26,7 @@ public function testSitemapIndex($url, $body, $result)
      * Generate test data
      * @return array
      */
-    public
-    function generateDataForTest()
+    public function generateDataForTest()
     {
         return [
             [
diff --git a/tests/StrictTest.php b/tests/StrictTest.php
index 63e7fd8..abb70ae 100644
--- a/tests/StrictTest.php
+++ b/tests/StrictTest.php
@@ -1,4 +1,5 @@
 <?php
+
 namespace vipnytt\SitemapParser\Tests;
 
 use PHPUnit\Framework\TestCase;
@@ -24,8 +25,7 @@ public function testStrict($url, $body)
      * Generate test data
      * @return array
      */
-    public
-    function generateDataForTest()
+    public function generateDataForTest()
     {
         return [
             [
diff --git a/tests/StringTest.php b/tests/StringTest.php
index 9b6bbf9..0b526db 100644
--- a/tests/StringTest.php
+++ b/tests/StringTest.php
@@ -1,4 +1,5 @@
 <?php
+
 namespace vipnytt\SitemapParser\Tests;
 
 use PHPUnit\Framework\TestCase;
@@ -37,8 +38,7 @@ public function testString($url)
      * Generate test data
      * @return array
      */
-    public
-    function generateDataForTest()
+    public function generateDataForTest()
     {
         return [
             [
diff --git a/tests/URLSetTest.php b/tests/URLSetTest.php
index 7d75bed..18a4638 100644
--- a/tests/URLSetTest.php
+++ b/tests/URLSetTest.php
@@ -1,4 +1,5 @@
 <?php
+
 namespace vipnytt\SitemapParser\Tests;
 
 use PHPUnit\Framework\TestCase;
@@ -25,8 +26,7 @@ public function testURLSet($url, $body, $result)
      * Generate test data
      * @return array
      */
-    public
-    function generateDataForTest()
+    public function generateDataForTest()
     {
         return [
             [