Merge pull request #2 from Patrick-Lapid/tokenization

Allows multiple txt files

preprocessing/test.py

@@ -2,14 +2,22 @@
 # nltk.download('punkt')
 from nltk.tokenize import sent_tokenize, word_tokenize
 from nltk.util import filestring
+usrInput = input("Enter file name: ")
+fileNames = [usrInput]
+
+
+while usrInput != "":
+    fileNames.append(usrInput)
+    usrInput = input("Enter file name (ENTER to terminate): ")
+
 
-fileName = input("Enter File Name: ")
 try: 
-    with open(fileName, encoding='utf-8') as file:
-        text = file.read()
-        words = word_tokenize(text)
-        sentences = sent_tokenize(text)
-        print('Words: ', words, '\nSentences: ', sentences)
+    for fileName in fileNames:
+        with open(fileName, encoding='utf-8') as file:
+            text = file.read()
+            words = word_tokenize(text)
+            sentences = sent_tokenize(text)
+            print('Words: ', words, '\nSentences: ', sentences)
 
 except IOError as e:
     print("I/O error({0}): {1}".format(e.errno, e.strerror))

preprocessing/test2.txt

@@ -0,0 +1,3 @@
+apple
+orange
+banana