read hex byte into control for unicode (#29)

- possibly fixes #26

rtftxt/rtftxt.go

@@ -126,7 +126,16 @@ func tokenizeControl(r peekingReader.Reader) (string, int, error) {
 		case b == '\'' && buf.Len() == 0:
 			isHex = true
 			buf.WriteByte(b)
-			r.ReadByte()
+			r.ReadByte() // consume valid character
+			// read 2 bytes for hex
+			for i := 0; i < 2; i++ {
+				b, err = r.ReadByte() // consume valid digit
+				if err != nil {
+					return "", -1, err
+				}
+				buf.WriteByte(b)
+			}
+			return buf.String(), -1, nil
 		case b >= '0' && b <= '9' || b == '-':
 			if numStart == -1 {
 				numStart = buf.Len()

rtftxt/rtftxt_test.go

@@ -8,6 +8,7 @@ import (
 
 const txt = `Of course, we frequently hear about larger brands pushing out a ton of amazing content, and they're often used as examples of how to do content right. `
 const red = `Restore The Selling Balance. Ad Technology doesn't have to be faceless. Our platform is designed to connect media companies directly to advertisers.`
+const unicode = `Beställer! Bulkpackning  `
 
 func TestToStr(t *testing.T) {
 	s, err := rtftxt.ToStr(`../test/np.new.rtf`)
@@ -29,4 +30,12 @@ func TestToStr(t *testing.T) {
 	if err == nil {
 		t.Error("Nonexisting file does not throw error")
 	}
+
+	s, err = rtftxt.ToStr(`../test/infotext.rtf`)
+	if err != nil {
+		t.Error(err)
+	}
+	if s != unicode {
+		t.Error("doesn't match", s)
+	}
 }

test/infotext.rtf

@@ -0,0 +1,5 @@
+{\rtf1\ansi\ansicpg1252\deff0\deflang1053{\fonttbl{\f0\fnil\fcharset0 Arial;}}
+\viewkind4\uc1\pard\fs48 Best\'e4ller!
+\par Bulkpackning
+\par 
+\par }