feat: refactoring raw data

README.md

@@ -2,6 +2,7 @@
 This is a learning repository about DVC Data Version Control and Luigi Pipelines
 
 - luigi, dvc, pre-commit
-- setup https://pre-commit.com/
+- https://luigi.readthedocs.io/
+- setup https://pre-commit.com/, https://pre-commit.com/hooks.html
 - setup https://github.com/Kaggle/kaggle-api
 - `kaggle competitions download -c sentiment-analysis-on-movie-reviews -p data`

requirements.txt

@@ -2,3 +2,4 @@ pre-commit==3.5.0
 kaggle==1.5.16
 dvc==3.28.0
 luigi==3.4.0
+scikit-learn==1.3.2

source/data.py

@@ -0,0 +1,32 @@
+import luigi
+import zipfile
+
+
+class SentimentAnalysisZipFile(luigi.ExternalTask):
+    """
+    Raw data zip file
+    """
+
+    def output(self):
+        return luigi.LocalTarget("../data/sentiment-analysis-on-movie-reviews.zip")
+
+
+class ExtractRawData(luigi.Task):
+    """
+    Extract raw data from zip file
+    """
+
+    def requires(self):
+        return SentimentAnalysisZipFile()
+
+    def output(self):
+        return {
+            "test": luigi.LocalTarget("../data/output/test.tsv.zip"),
+            "train": luigi.LocalTarget("../data/output/train.tsv.zip"),
+            "sampleSubission": luigi.LocalTarget("../data/output/sampleSubmission.csv"),
+        }
+
+    def run(self):
+        # Unzip data file
+        with zipfile.ZipFile(self.input().path, "r") as zip_ref:
+            zip_ref.extractall("../data/output/")

source/preprocessing.py

@@ -0,0 +1 @@
+import luigi