-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathtesttokenize.py
123 lines (109 loc) · 4.31 KB
/
testtokenize.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
import vdfutils
from vdfutils import _Field, _OpenBrace, _CloseBrace, VDFConsistencyFailure
_tokenize_vdf = vdfutils._tokenize_vdf
# General
assert list(_tokenize_vdf('')) == []
assert list(_tokenize_vdf(' ')) == []
assert list(_tokenize_vdf('abc')) == [_Field('abc')]
assert list(_tokenize_vdf('abc def')) == [_Field('abc'), _Field('def')]
assert list(_tokenize_vdf('abc def ghi')) == [
_Field('abc'), _Field('def'), _Field('ghi')
]
assert list(_tokenize_vdf('abc def {} ghi')) == [
_Field('abc'), _Field('def'), _OpenBrace(), _CloseBrace(), _Field('ghi')
]
assert list(_tokenize_vdf('abc def {} {ghi}')) == [
_Field('abc'), _Field('def'), _OpenBrace(), _CloseBrace(), _OpenBrace(),
_Field('ghi'), _CloseBrace()
]
assert list(_tokenize_vdf('abc def {} {"ghi"}')) == [
_Field('abc'), _Field('def'), _OpenBrace(), _CloseBrace(), _OpenBrace(),
_Field('ghi'), _CloseBrace()
]
assert list(_tokenize_vdf('"abc def {} {ghi}"')) == [
_Field('abc def {} {ghi}')
]
assert list(_tokenize_vdf('"abc def {} ""{ghi}"')) == [
_Field('abc def {} '), _Field('{ghi}')
]
assert list(_tokenize_vdf('"abc"def {} ""{ghi}')) == [
_Field('abc'), _Field('def'), _OpenBrace(), _CloseBrace(), _Field(''),
_OpenBrace(), _Field('ghi'), _CloseBrace()
]
assert list(_tokenize_vdf('"abc"def"ghi"jkl"mno"')) == [
_Field('abc'), _Field('def'), _Field('ghi'), _Field('jkl'), _Field('mno')
]
assert list(_tokenize_vdf('"\\abc\\""def"ghi"jkl"m\\no"')) == [
_Field('\\abc"'), _Field('def'), _Field('ghi'),
_Field('jkl'), _Field('m\no')
]
assert list(_tokenize_vdf('""""""')) == [
_Field(''), _Field(''), _Field('')
]
# Escape chars
assert list(_tokenize_vdf('\\')) == [_Field('\\')]
assert list(_tokenize_vdf('\\n')) == [_Field('\n')]
assert list(_tokenize_vdf('\\t')) == [_Field('\t')]
assert list(_tokenize_vdf('\\"')) == [_Field('"')]
assert list(_tokenize_vdf('\\\\\\"')) == [_Field('\\"')]
assert list(_tokenize_vdf('\\\\')) == [_Field('\\')]
assert list(_tokenize_vdf('\\\\\\')) == [_Field('\\\\')]
assert list(_tokenize_vdf('\\\\\\\\')) == [_Field('\\\\')]
assert list(_tokenize_vdf('\\\\\\\\t')) == [_Field('\\\\t')]
assert list(_tokenize_vdf('\\\\\\\\\\t')) == [_Field('\\\\\t')]
assert list(_tokenize_vdf('"\\t\\n\\"\\\\\\\\\\"\\n\\t"')) == [
_Field('\t\n\"\\\\\"\n\t')
]
assert list(_tokenize_vdf('"\\\\"\n"abc"')) == [_Field('\\'), _Field('abc')]
# Escape chars (disabled)
assert list(_tokenize_vdf('\\', escape=False)) == [_Field('\\')]
assert list(_tokenize_vdf('\\n', escape=False)) == [_Field('\\n')]
assert list(_tokenize_vdf('\\t', escape=False)) == [_Field('\\t')]
try:
list(_tokenize_vdf('\\"', escape=False))
assert False
except VDFConsistencyFailure as e:
assert e.message == 'Mismatched quotes!'
try:
list(_tokenize_vdf('\\\\\\"', escape=False))
assert False
except VDFConsistencyFailure as e:
assert e.message == 'Mismatched quotes!'
assert list(_tokenize_vdf('\\\\', escape=False)) == [_Field('\\\\')]
assert list(_tokenize_vdf('\\\\\\', escape=False)) == [_Field('\\\\\\')]
assert list(_tokenize_vdf('\\\\\\\\', escape=False)) == [_Field('\\\\\\\\')]
assert list(_tokenize_vdf('\\\\\\\\t', escape=False)) == [_Field('\\\\\\\\t')]
assert list(_tokenize_vdf('\\\\\\\\\\t', escape=False)) == [
_Field('\\\\\\\\\\t')
]
assert list(_tokenize_vdf('"\\t\\n\\"\\\\\\\\\\"\\n\\t"', escape=False)) == [
_Field('\\t\\n\\'), _Field('\\\\\\\\\\'), _Field('\\n\\t')
]
assert list(_tokenize_vdf('"\\\\"\n"abc"', escape=False)) == [
_Field('\\\\'), _Field('abc')
]
# Bad VDF
try:
list(_tokenize_vdf('"'))
assert False
except VDFConsistencyFailure as e:
assert e.message == 'Mismatched quotes!'
try:
list(_tokenize_vdf('"abc def {} ""{ghi}""'))
assert False
except VDFConsistencyFailure as e:
assert e.message == 'Mismatched quotes!'
try:
list(_tokenize_vdf('"""""'))
assert False
except VDFConsistencyFailure as e:
assert e.message == 'Mismatched quotes!'
# Comments
assert list(_tokenize_vdf('//')) == []
assert list(_tokenize_vdf('// abc')) == []
assert list(_tokenize_vdf('// abc def')) == []
assert list(_tokenize_vdf('// abc def\nghi')) == [_Field('ghi')]
assert list(_tokenize_vdf('/ abc def\nghi')) == [_Field('ghi')]
assert list(_tokenize_vdf('"/"')) == [_Field('/')]
assert list(_tokenize_vdf('/"')) == []
print "All tests passed!"