added test file

ufosc · Sep 15, 2021 · aaf164d · aaf164d
1 parent 8f94d2c
commit aaf164d
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 0 deletions.
diff --git a/preprocessing/test.py b/preprocessing/test.py
@@ -0,0 +1,18 @@
+import nltk
+# nltk.download('punkt')
+from nltk.tokenize import sent_tokenize, word_tokenize
+from nltk.util import filestring
+
+fileName = input("Enter File Name: ")
+try: 
+    with open(fileName, encoding='utf-8') as file:
+        text = file.read()
+        words = word_tokenize(text)
+        sentences = sent_tokenize(text)
+        print('Words: ', words, '\nSentences: ', sentences)
+
+except IOError as e:
+    print("I/O error({0}): {1}".format(e.errno, e.strerror))
+except OSError:
+    print(f"OS error trying to open {fileName}")
+
diff --git a/preprocessing/test.txt b/preprocessing/test.txt
@@ -0,0 +1,6 @@
+This is a test text file.
+
+This is a new line.
+
+
+Goodbye