Merge branch 'develop'

Norconex · Aug 29, 2016 · a29a022 · a29a022
2 parents e83bd9f + fe513d9
commit a29a022
Show file tree

Hide file tree

Showing 6 changed files with 92 additions and 7 deletions.
diff --git a/norconex-commons-lang/pom.xml b/norconex-commons-lang/pom.xml
@@ -19,15 +19,15 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>com.norconex.commons</groupId>
   <artifactId>norconex-commons-lang</artifactId>
-  <version>1.11.0</version>
+  <version>1.12.0</version>
   <packaging>jar</packaging>
   <name>Norconex Commons Lang</name>
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     <site.baseurl/>
-    <currentStableVersion>1.11.0</currentStableVersion>
+    <currentStableVersion>1.12.0</currentStableVersion>
   </properties>
   <inceptionYear>2008</inceptionYear>
 

diff --git a/norconex-commons-lang/src/changes/changes.xml b/norconex-commons-lang/src/changes/changes.xml
@@ -7,6 +7,19 @@
   </properties>
   <body>
 
+    <release version="1.12.0" date="2016-08-29" description="Feature release">
+      <action dev="essiembre" type="add">
+        New URLNormalizer#addDomainTrailingSlash() method. 
+      </action>
+      <action dev="essiembre" type="update">
+        HttpURL now preserves uppercase characters in protocol. 
+      </action>
+      <action dev="essiembre" type="fix">
+        Fixed HttpURL constructing URLs with null elements when protocol
+        was not lowercase (e.g. HTTP). 
+      </action>
+    </release>
+
     <release version="1.11.0" date="2016-08-25" description="Feature release">
       <action dev="essiembre" type="add">
         New URLNormalizer#removeTrailingSlash() method. 

diff --git a/norconex-commons-lang/src/main/java/com/norconex/commons/lang/url/HttpURL.java b/norconex-commons-lang/src/main/java/com/norconex/commons/lang/url/HttpURL.java
@@ -104,18 +104,18 @@ public HttpURL(String url, String encoding) {
         } else {
             this.encoding = encoding;
         }
-        if (url.startsWith("http")) {
+        if (StringUtils.startsWithIgnoreCase(url, PROTOCOL_HTTP)) {
             URL urlwrap;
             try {
                 urlwrap = new URL(url);
             } catch (MalformedURLException e) {
                 throw new URLException("Could not interpret URL: " + url, e);
             }
-            protocol = urlwrap.getProtocol();
+            protocol = StringUtils.substringBefore(url, ":");
             host = urlwrap.getHost();
             port = urlwrap.getPort();
             if (port < 0) {
-                if (url.startsWith(PROTOCOL_HTTPS)) {
+                if (StringUtils.startsWithIgnoreCase(url, PROTOCOL_HTTPS)) {
                     port = DEFAULT_HTTPS_PORT;
                 } else {
                     port = DEFAULT_HTTP_PORT;
@@ -384,8 +384,9 @@ public String toString() {
      * @since 1.8.0
      */
     public boolean isPortDefault() {
-        return PROTOCOL_HTTPS.equalsIgnoreCase(protocol) && port == DEFAULT_HTTPS_PORT
-                || "http".equalsIgnoreCase(protocol)
+        return PROTOCOL_HTTPS.equalsIgnoreCase(protocol) 
+                && port == DEFAULT_HTTPS_PORT
+                || PROTOCOL_HTTP.equalsIgnoreCase(protocol)
                         && port == DEFAULT_HTTP_PORT;
     }
 

diff --git a/norconex-commons-lang/src/main/java/com/norconex/commons/lang/url/URLNormalizer.java b/norconex-commons-lang/src/main/java/com/norconex/commons/lang/url/URLNormalizer.java
@@ -340,6 +340,29 @@ public URLNormalizer addDirectoryTrailingSlash() {
         }
         return this;
     }
+
+    /**
+     * <p>Adds a trailing slash (/) right after the domain for URLs with no 
+     * path, before any fragment (#) or query string (?).</p>
+     *   
+     * <p><b>Please Note:</b> Adding a trailing slash to URLs could 
+     * potentially break its semantic equivalence.</p>
+     * <code>http://www.example.com &rarr; 
+     *       http://www.example.com/</code>
+     * @return this instance
+     * @since 1.12.0
+     */
+    public URLNormalizer addDomainTrailingSlash() {
+        String urlRoot = HttpURL.getRoot(url);
+        String path = toURL().getPath();
+        if (StringUtils.isNotBlank(path)) {
+            // there is a path so do nothing
+            return this;
+        }
+        String urlRootAndPath = urlRoot + "/";
+        url = StringUtils.replaceOnce(url, urlRoot, urlRootAndPath);
+        return this;
+    }
 
     /**
      * <p>Adds a trailing slash (/) to a URL ending with a directory.  A URL is 

diff --git a/norconex-commons-lang/src/test/java/com/norconex/commons/lang/url/HttpURLTest.java b/norconex-commons-lang/src/test/java/com/norconex/commons/lang/url/HttpURLTest.java
@@ -48,6 +48,13 @@ public void tearDown() throws Exception {
         t = null;
     }
 
+    @Test
+    public void testKeepProtocolUpperCase() {
+        s = "HTTP://www.example.com";
+        t = "HTTP://www.example.com";
+        assertEquals(t, new HttpURL(s).toString());
+    }
+
     @Test
     public void testToAbsoluteRelativeToProtocol() {
         s = "//www.relative.com/e/f.html";

diff --git a/norconex-commons-lang/src/test/java/com/norconex/commons/lang/url/URLNormalizerTest.java b/norconex-commons-lang/src/test/java/com/norconex/commons/lang/url/URLNormalizerTest.java
@@ -86,6 +86,41 @@ public void testAllAtOnce() {
           assertEquals(t,  n.toURI().toString());
     }
 
+    @Test
+    public void testAddDomainTrailingSlash() {
+        s = "http://www.example.com";
+        t = "http://www.example.com/";
+        assertEquals(t, n(s).addDomainTrailingSlash().toString());
+
+        s = "http://www.example.com/";
+        t = "http://www.example.com/";
+        assertEquals(t, n(s).addDomainTrailingSlash().toString());
+
+        s = "http://www.example.com/blah";
+        t = "http://www.example.com/blah";
+        assertEquals(t, n(s).addDomainTrailingSlash().toString());
+
+        s = "http://www.example.com/blah/path";
+        t = "http://www.example.com/blah/path";
+        assertEquals(t, n(s).addDomainTrailingSlash().toString());
+
+        s = "http://www.example.com?param1=value1&param2=value2";
+        t = "http://www.example.com/?param1=value1&param2=value2";
+        assertEquals(t, n(s).addDomainTrailingSlash().toString());
+
+        s = "http://www.example.com/?param1=value1&param2=value2";
+        t = "http://www.example.com/?param1=value1&param2=value2";
+        assertEquals(t, n(s).addDomainTrailingSlash().toString());
+
+        s = "http://www.example.com#hash";
+        t = "http://www.example.com/#hash";
+        assertEquals(t, n(s).addDomainTrailingSlash().toString());
+
+        s = "http://www.example.com/#hash";
+        t = "http://www.example.com/#hash";
+        assertEquals(t, n(s).addDomainTrailingSlash().toString());
+    }
+
 
     @Test
     public void testEncodeUTF8Characters() {
@@ -99,6 +134,12 @@ public void testEncodeNonURICharacters() {
         s = "http://www.example.com/^a [b]/c?d e=";
         t = "http://www.example.com/%5Ea%20%5Bb%5D/c?d+e=";
         assertEquals(t, n(s).encodeNonURICharacters().toString());
+
+        //Test for https://github.com/Norconex/collector-http/issues/294
+        //Was failing when HTTP was uppercase
+        s = "HTTP://www.Example.com/";
+        t = "HTTP://www.Example.com/";
+        assertEquals(t, n(s).encodeNonURICharacters().toString());
     }
 
     @Test