Fix handling of file bodies with multi-byte characters (#450)

Always calculate the length of file contents from their bytes
representation to not cut off characters at the end when the string
contains multi-byte characters.

Fix #443

plugin/clang/cindex32.py

@@ -1968,7 +1968,7 @@ def from_source(cls, filename, args=None, unsaved_files=None, options=0,
 
                 unsaved_array[i].name = name.encode('utf8')
                 unsaved_array[i].contents = contents.encode('utf8')
-                unsaved_array[i].length = len(contents)
+                unsaved_array[i].length = len(unsaved_array[i].contents)
 
         ptr = conf.lib.clang_parseTranslationUnit(index, filename.encode('utf8'), args_array,
                                     len(args), unsaved_array,
@@ -2150,7 +2150,8 @@ def reparse(self, unsaved_files=None, options=0):
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode('utf-8')
                 unsaved_files_array[i].contents = value.encode('utf-8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_reparseTranslationUnit(self, len(unsaved_files),
                 unsaved_files_array, options)
 
@@ -2214,7 +2215,8 @@ def codeComplete(self, path, line, column, unsaved_files=None,
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode('utf8')
                 unsaved_files_array[i].contents = value.encode('utf8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_codeCompleteAt(self, path.encode('utf8'), line, column,
                 unsaved_files_array, len(unsaved_files), options)
         if ptr:

plugin/clang/cindex33.py

@@ -2048,7 +2048,7 @@ def from_source(cls, filename, args=None, unsaved_files=None, options=0,
 
                 unsaved_array[i].name = name.encode('utf8')
                 unsaved_array[i].contents = contents.encode('utf8')
-                unsaved_array[i].length = len(contents)
+                unsaved_array[i].length = len(unsaved_array[i].contents)
 
         ptr = conf.lib.clang_parseTranslationUnit(index, filename.encode('utf8'), args_array,
                                     len(args), unsaved_array,
@@ -2230,7 +2230,8 @@ def reparse(self, unsaved_files=None, options=0):
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode('utf-8')
                 unsaved_files_array[i].contents = value.encode('utf-8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_reparseTranslationUnit(self, len(unsaved_files),
                 unsaved_files_array, options)
 
@@ -2294,7 +2295,8 @@ def codeComplete(self, path, line, column, unsaved_files=None,
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode('utf8')
                 unsaved_files_array[i].contents = value.encode('utf8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_codeCompleteAt(self, path.encode('utf8'), line, column,
                 unsaved_files_array, len(unsaved_files), options)
         if ptr:

plugin/clang/cindex34.py

@@ -2181,7 +2181,7 @@ def from_source(cls, filename, args=None, unsaved_files=None, options=0,
 
                 unsaved_array[i].name = name.encode('utf8')
                 unsaved_array[i].contents = contents.encode('utf8')
-                unsaved_array[i].length = len(contents)
+                unsaved_array[i].length = len(unsaved_array[i].contents)
 
         ptr = conf.lib.clang_parseTranslationUnit(index, filename.encode('utf8'), args_array,
                                     len(args), unsaved_array,
@@ -2363,7 +2363,8 @@ def reparse(self, unsaved_files=None, options=0):
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode('utf-8')
                 unsaved_files_array[i].contents = value.encode('utf-8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_reparseTranslationUnit(self, len(unsaved_files),
                 unsaved_files_array, options)
 
@@ -2427,7 +2428,8 @@ def codeComplete(self, path, line, column, unsaved_files=None,
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode('utf8')
                 unsaved_files_array[i].contents = value.encode('utf8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_codeCompleteAt(self, path.encode('utf8'), line, column,
                 unsaved_files_array, len(unsaved_files), options)
         if ptr:

plugin/clang/cindex35.py

@@ -2253,7 +2253,7 @@ def from_source(cls, filename, args=None, unsaved_files=None, options=0,
 
                 unsaved_array[i].name = name.encode('utf8')
                 unsaved_array[i].contents = contents.encode('utf8')
-                unsaved_array[i].length = len(contents)
+                unsaved_array[i].length = len(unsaved_array[i].contents)
 
         ptr = conf.lib.clang_parseTranslationUnit(index, filename.encode('utf8'), args_array,
                                     len(args), unsaved_array,
@@ -2435,7 +2435,8 @@ def reparse(self, unsaved_files=None, options=0):
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode('utf-8')
                 unsaved_files_array[i].contents = value.encode('utf-8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_reparseTranslationUnit(self, len(unsaved_files),
                 unsaved_files_array, options)
 
@@ -2499,7 +2500,8 @@ def codeComplete(self, path, line, column, unsaved_files=None,
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode('utf8')
                 unsaved_files_array[i].contents = value.encode('utf8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_codeCompleteAt(self, path.encode('utf8'), line, column,
                 unsaved_files_array, len(unsaved_files), options)
         if ptr:

plugin/clang/cindex36.py

@@ -2294,7 +2294,7 @@ def from_source(cls, filename, args=None, unsaved_files=None, options=0,
 
                 unsaved_array[i].name = name.encode('utf8')
                 unsaved_array[i].contents = contents.encode('utf8')
-                unsaved_array[i].length = len(contents)
+                unsaved_array[i].length = len(unsaved_array[i].contents)
 
         ptr = conf.lib.clang_parseTranslationUnit(index, filename.encode('utf8'), args_array,
                                     len(args), unsaved_array,
@@ -2476,7 +2476,8 @@ def reparse(self, unsaved_files=None, options=0):
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode('utf-8')
                 unsaved_files_array[i].contents = value.encode('utf-8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_reparseTranslationUnit(self, len(unsaved_files),
                 unsaved_files_array, options)
 
@@ -2540,7 +2541,8 @@ def codeComplete(self, path, line, column, unsaved_files=None,
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode('utf8')
                 unsaved_files_array[i].contents = value.encode('utf8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_codeCompleteAt(self, path.encode('utf8'), line, column,
                 unsaved_files_array, len(unsaved_files), options)
         if ptr:

plugin/clang/cindex37.py

@@ -2321,7 +2321,7 @@ def from_source(cls, filename, args=None, unsaved_files=None, options=0,
 
                 unsaved_array[i].name = name.encode('utf8')
                 unsaved_array[i].contents = contents.encode('utf8')
-                unsaved_array[i].length = len(contents)
+                unsaved_array[i].length = len(unsaved_array[i].contents)
 
         ptr = conf.lib.clang_parseTranslationUnit(index, filename.encode('utf8'), args_array,
                                     len(args), unsaved_array,
@@ -2503,7 +2503,8 @@ def reparse(self, unsaved_files=None, options=0):
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode('utf-8')
                 unsaved_files_array[i].contents = value.encode('utf-8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_reparseTranslationUnit(self, len(unsaved_files),
                 unsaved_files_array, options)
 
@@ -2567,7 +2568,8 @@ def codeComplete(self, path, line, column, unsaved_files=None,
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode('utf8')
                 unsaved_files_array[i].contents = value.encode('utf8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_codeCompleteAt(self, path.encode('utf8'), line, column,
                 unsaved_files_array, len(unsaved_files), options)
         if ptr:

plugin/clang/cindex38.py

@@ -2355,7 +2355,7 @@ def from_source(cls, filename, args=None, unsaved_files=None, options=0,
 
                 unsaved_array[i].name = name.encode('utf-8')
                 unsaved_array[i].contents = contents.encode('utf-8')
-                unsaved_array[i].length = len(contents)
+                unsaved_array[i].length = len(unsaved_array[i].contents)
 
         if filename is not None:
             filename = filename.encode('utf-8')
@@ -2541,7 +2541,8 @@ def reparse(self, unsaved_files=None, options=0):
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode('utf-8')
                 unsaved_files_array[i].contents = value.encode('utf-8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_reparseTranslationUnit(self, len(unsaved_files),
                 unsaved_files_array, options)
 
@@ -2604,7 +2605,8 @@ def codeComplete(self, path, line, column, unsaved_files=None,
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode('utf-8')
                 unsaved_files_array[i].contents = value.encode('utf-8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_codeCompleteAt(self, path.encode('utf-8'), line, column,
                 unsaved_files_array, len(unsaved_files), options)
         if ptr:

plugin/clang/cindex39.py

@@ -2651,7 +2651,7 @@ def from_source(cls, filename, args=None, unsaved_files=None, options=0,
 
                 unsaved_array[i].name = name.encode ('utf-8')
                 unsaved_array[i].contents = contents.encode ('utf-8')
-                unsaved_array[i].length = len(contents)
+                unsaved_array[i].length = len(unsaved_array[i].contents)
 
         if filename is not None:
             filename = filename.encode ('utf-8')
@@ -2837,7 +2837,8 @@ def reparse(self, unsaved_files=None, options=0):
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode ('utf-8')
                 unsaved_files_array[i].contents = value.encode ('utf-8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_reparseTranslationUnit(self, len(unsaved_files),
                 unsaved_files_array, options)
 
@@ -2901,7 +2902,8 @@ def codeComplete(self, path, line, column, unsaved_files=None,
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode ('utf-8')
                 unsaved_files_array[i].contents = value.encode('utf-8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_codeCompleteAt(self, path.encode ('utf-8'), line, column,
                 unsaved_files_array, len(unsaved_files), options)
         if ptr:

plugin/clang/cindex40.py

@@ -2565,7 +2565,7 @@ def from_source(cls, filename, args=None, unsaved_files=None, options=0,
 
                 unsaved_array[i].name = name.encode('utf-8')
                 unsaved_array[i].contents = contents.encode('utf-8')
-                unsaved_array[i].length = len(contents)
+                unsaved_array[i].length = len(unsaved_array[i].contents)
 
         if filename is not None:
             filename = filename.encode('utf-8')
@@ -2751,7 +2751,8 @@ def reparse(self, unsaved_files=None, options=0):
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode('utf-8')
                 unsaved_files_array[i].contents = value.encode('utf-8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_reparseTranslationUnit(self, len(unsaved_files),
                 unsaved_files_array, options)
 
@@ -2815,7 +2816,8 @@ def codeComplete(self, path, line, column, unsaved_files=None,
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = name.encode('utf-8')
                 unsaved_files_array[i].contents = value.encode('utf-8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_codeCompleteAt(self, path.encode('utf-8'), line, column,
                 unsaved_files_array, len(unsaved_files), options)
         if ptr:

plugin/clang/cindex50.py

@@ -2704,7 +2704,7 @@ def from_source(cls, filename, args=None, unsaved_files=None, options=0,
 
                 unsaved_array[i].name = b(name)
                 unsaved_array[i].contents = b(contents)
-                unsaved_array[i].length = len(contents)
+                unsaved_array[i].length = len(unsaved_array[i].contents)
 
         ptr = conf.lib.clang_parseTranslationUnit(index, filename, args_array,
                                     len(args), unsaved_array,
@@ -2884,9 +2884,10 @@ def reparse(self, unsaved_files=None, options=0):
                     print(value)
                 if not isinstance(value, str):
                     raise TypeError('Unexpected unsaved file contents.')
-                unsaved_files_array[i].name = name.encode('utf-8')
-                unsaved_files_array[i].contents = value.encode('utf-8')
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].name = b(name)
+                unsaved_files_array[i].contents = b(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_reparseTranslationUnit(self, len(unsaved_files),
                 unsaved_files_array, options)
 
@@ -2950,7 +2951,8 @@ def codeComplete(self, path, line, column, unsaved_files=None,
                     raise TypeError('Unexpected unsaved file contents.')
                 unsaved_files_array[i].name = b(name)
                 unsaved_files_array[i].contents = b(value)
-                unsaved_files_array[i].length = len(value)
+                unsaved_files_array[i].length = \
+                    len(unsaved_files_array[i].contents)
         ptr = conf.lib.clang_codeCompleteAt(self, path, line, column,
                 unsaved_files_array, len(unsaved_files), options)
         if ptr: