Merge pull request #3 from gfx/fix_utf8_handling

fix utf8 handling in C code

ext/greenmat/autolink.c

@@ -16,6 +16,7 @@
 
 #include "buffer.h"
 #include "autolink.h"
+#include "houdini.h"
 
 #include <string.h>
 #include <stdlib.h>
@@ -169,7 +170,7 @@ sd_autolink__www(
 {
 	size_t link_end;
 
-	if (max_rewind > 0 && !ispunct(data[-1]) && !isspace(data[-1]))
+	if (max_rewind > 0 && !ispunct(data[-1]) && is_non_space(data[-1]))
 		return 0;
 
 	if (size < 4 || memcmp(data, "www.", strlen("www.")) != 0)
@@ -180,7 +181,7 @@ sd_autolink__www(
 	if (link_end == 0)
 		return 0;
 
-	while (link_end < size && !isspace(data[link_end]))
+	while (link_end < size && is_non_space(data[link_end]))
 		link_end++;
 
 	link_end = autolink_delim(data, link_end, max_rewind, size);
@@ -222,7 +223,7 @@ sd_autolink__email(
 		return 0;
 
 	for (link_end = 0; link_end < size; ++link_end) {
-		uint8_t c = data[link_end];
+		char c = data[link_end];
 
 		if (isalnum(c))
 			continue;
@@ -280,7 +281,7 @@ sd_autolink__url(
 		return 0;
 
 	link_end += domain_len;
-	while (link_end < size && !isspace(data[link_end]))
+	while (link_end < size && is_non_space(data[link_end]))
 		link_end++;
 
 	link_end = autolink_delim(data, link_end, max_rewind, size);

ext/greenmat/houdini.h

@@ -1,6 +1,7 @@
 #ifndef HOUDINI_H__
 #define HOUDINI_H__
 
+#include <stdlib.h>
 #include "buffer.h"
 
 #ifdef __cplusplus
@@ -22,6 +23,8 @@ extern void houdini_escape_html(struct buf *ob, const uint8_t *src, size_t size)
 extern void houdini_escape_html0(struct buf *ob, const uint8_t *src, size_t size, int secure);
 extern void houdini_escape_href(struct buf *ob, const uint8_t *src, size_t size);
 
+#define is_non_space(c) (!isspace(c) || (c) >= 0x7f)
+
 #ifdef __cplusplus
 }
 #endif

ext/greenmat/html.c

@@ -137,7 +137,7 @@ rndr_blockcode(struct buf *ob, const struct buf *text, const struct buf *lang, v
 
 			if (i < lang->size) {
 				size_t org = i;
-				while (i < lang->size && !isspace(lang->data[i]))
+				while (i < lang->size && is_non_space(lang->data[i]))
 					i++;
 
 				if (lang->data[org] == '.')

greenmat.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_development_dependency "nokogiri", "~> 1.6.0"
+  s.add_development_dependency "rake", "~> 10.0"
   s.add_development_dependency "rake-compiler", "~> 0.8.3"
   s.add_development_dependency "rspec", "~> 3.2"
   s.add_development_dependency "test-unit", "~> 2.5.4"

test/markdown_test.rb

@@ -131,6 +131,24 @@ def test_whitespace_after_urls
     html_equal exp, rd
   end
 
+  def test_auto_linked_www_utf8_issue
+		rd = render_with({ autolink: true }, "www.example.com/码")
+		exp = %{<p><a href="http://www.example.com/%E7%A0%81">www.example.com/码</a></p>\n}
+    assert_equal exp, rd
+  end
+
+  def test_auto_linked_url_utf8_issue
+		rd = render_with({ autolink: true }, "http://example.com/码")
+    exp = %{<p><a href="http://example.com/%E7%A0%81">http://example.com/码</a></p>\n}
+    assert_equal exp, rd
+  end
+
+  def test_auto_linked_email_utf8_issue
+    rd = render_with({ autolink: true }, "[email protected]あ")
+    exp = %{<p><a href="mailto:[email protected]">[email protected]</a>あ</p>\n}
+    assert_equal exp, rd
+  end
+
   def test_memory_leak_when_parsing_char_links
     @markdown.render(<<-leaks)
 2. Identify the wild-type cluster and determine all clusters
@@ -243,6 +261,12 @@ def test_that_fenced_flag_works_without_space
     assert !out.include?("<pre><code>")
   end
 
+  def test_that_fenced_flag_works_with_utf8
+    text = "```ム\ncode\n```"
+    out = Greenmat::Markdown.new(Greenmat::Render::HTML, :fenced_code_blocks => true).render(text)
+    assert out.include?(%{<pre><code class="ム">})
+  end
+
   def test_that_indented_flag_works
     text = <<indented
 This is a simple text