Fix path issue for deploy (#92)

* Update deploy.sh * Update deploy.sh * Update .gcloudignore * Update .gcloudignore * Update .gcloudignore * Update .gcloudignore * Update .gcloudignore * directory support * Update configurationManagerTest.js * Update index.js * Update configurationManager.js * Update configurationManager.js
GoogleCloudPlatform · Dec 18, 2019 · d583194 · d583194
1 parent 03b7b6f
commit d583194
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 22 deletions.
diff --git a/ingestion/batch/.gcloudignore b/ingestion/batch/.gcloudignore
@@ -18,6 +18,12 @@ node_modules
 .DS_Store
 *.png
 *.md
+*.bak
 .eslintrc.json
+.eslintignore
 .nyc_output
-/shared/test
+/bin
+/test
+/shared/test
+/shared/config
+/shared/ciphertexts
diff --git a/ingestion/batch/bin/deploy.sh b/ingestion/batch/bin/deploy.sh
@@ -103,31 +103,31 @@ if [ -z "$FUNCTION_REGION" ]; then
     exit 7
 else
     echo "Function region: ${FUNCTION_REGION}"
-    FUNCTION_SHARED="../function/shared"
+    FUNCTION_SHARED="../../batch/shared"
     if [ -d "${FUNCTION_SHARED}" ]; then
         rm -R "${FUNCTION_SHARED}"
     fi
     # Symlinks do not work, have to physical copy the directory
     echo "Copying shared module into function directory"
-    cp -R ../../shared "${FUNCTION_SHARED}/"
+    cp -R ../../../shared "${FUNCTION_SHARED}/"
 
     echo "Creating backup of package.json"
-    cp ../function/package.json ../function/package.json.bak
+    cp ../../batch/package.json ../../batch/package.json.bak
     UNAME=$(uname | awk '{print tolower($0)}')
     if [ "$UNAME" == "darwin" ]; then
         # macOS
         echo 'Running on macOS, performing package.json replacement for bqds-shared module'
-        sed -i '' -E 's/(file:)(\.\.\/\.\.\/)(shared)/\1\3/g' ../function/package.json
+        sed -i '' -E 's/(file:)(\.\.\/\.\.\/)(shared)/\1\3/g' ../../batch/package.json
     else
         # linux
         echo 'Running on linux, performing package.json replacement for bqds-shared module'
-        sed -i -E 's/(file:)(\.\.\/\.\.\/)(shared)/\1\3/g' ../function/package.json
+        sed -i -E 's/(file:)(\.\.\/\.\.\/)(shared)/\1\3/g' ../../batch/package.json
     fi
 
-    gcloud functions deploy ${FUNCTION_NAME:-processUpload} --region=${FUNCTION_REGION} --memory=256MB --source=../function --runtime=nodejs8 --entry-point=processEvent --timeout=540s --trigger-bucket="${BUCKET_NAME}" --quiet --set-env-vars=VERBOSE_MODE=true,ARCHIVE_FILES=false
+    gcloud functions deploy ${FUNCTION_NAME:-processUpload} --region=${FUNCTION_REGION} --memory=256MB --source=../../batch --runtime=nodejs8 --entry-point=processEvent --timeout=540s --trigger-bucket="${BUCKET_NAME}" --quiet --set-env-vars=VERBOSE_MODE=true,ARCHIVE_FILES=false
 
     echo "Restoring original package.json"
-    mv -f ../function/package.json.bak ../function/package.json
+    mv -f ../../batch/package.json.bak ../../batch/package.json
 
     echo "Removing shared directory from function directory"
     rm -R "${FUNCTION_SHARED}"

diff --git a/ingestion/batch/configurationManager.js b/ingestion/batch/configurationManager.js
@@ -116,7 +116,7 @@ async function validateOptions(options, validateStorage) {
                     }
                 }
             }
-            
+
             if (attributes.datasetId) {
                 if (attributes.datasetId.length > 1024) {
                     errors.push(`DatasetId '${attributes.datasetId}' exceeds maximum allowable length of 1024: ${attributes.datasetId.length}}`);
@@ -137,17 +137,21 @@ async function validateOptions(options, validateStorage) {
         }
     }
 
-    if (attributes && attributes.isDataFile === false) {
+    if (attributes && attributes.isArchiveFile === true) {
         console.log(`Ignoring archived file: '${options.fileName} in bucket: ${options.bucketName}'`);
-        return { isValid: true, isDataFile: false };
+        return { isValid: false, hasException: false, isArchiveFile: true };
+    }
+    else if (attributes && attributes.isDirectoryPath === true) {
+        console.log(`Ignoring directory path: '${options.fileName} in bucket: ${options.bucketName}'`);
+        return { isValid: false, isDirectoryPath: true, hasException: false };
     }
-    else if (errors.length === 0) {
+    else if (attributes && attributes.isDataFile === true && errors.length === 0) {
         console.log(`Options validation succeeded: ${info.join(", ")}`);
-        return { isValid: true, isDataFile: true, info: info, warn: warn };
+        return { isValid: true, isDataFile: true, info: info, warn: warn, hasException: false };
     }
     else {
         console.log(`Options validation failed: ${errors.join(", ")}`);
-        return { isValid: false, isDataFile: true, errors: errors, info: info, warn: warn };
+        return { isValid: false, errors: errors, info: info, warn: warn, hasException: true };
     }
 }
 
@@ -163,15 +167,23 @@ function parseDerivedFileAttributes(options) {
     const schemaFileBucketPath = path.join(bucketPath, "..", "config", `schema.json`);
     const transformFileBucketPath = path.join(bucketPath, "..", "config", `transform.sql`);
     const archivePath = path.join(bucketPath, "archive", `${basename}`);
-    const isDataFile = (pathParts.length === 4 && underscore.first(pathParts).toLowerCase() === "bqds" && pathParts.pop().toLowerCase() === "data");
+    const isDataFile = (pathParts.length === 4 && pathParts[0].toLowerCase() === "bqds" && pathParts[3].toLowerCase() === "data");
+    const isArchived = (pathParts.length === 5 && pathParts[0].toLowerCase() === "bqds" && pathParts[3].toLowerCase() === "data" && pathParts[4].toLowerCase() === "archive");
+
+    let isDirectoryPath = false;
+    if (options.fileName.endsWith("/")) {
+        isDirectoryPath = true;
+    }
 
     return {
         datasetId: datasetId,
         destinationTableId: destinationTableId,
         schemaPath: schemaFileBucketPath,
         transformPath: transformFileBucketPath,
         archivePath: archivePath,
-        isDataFile: isDataFile
+        isDataFile: isDataFile,
+        isArchiveFile: isArchived,
+        isDirectoryPath: isDirectoryPath
     };
 }
 

diff --git a/ingestion/batch/index.js b/ingestion/batch/index.js
@@ -57,10 +57,12 @@ async function processTriggerEvent(event, context) {
         fileName: event.name
     };
     const result = await configManager.validateOptions(options, true);
-    if (!result.isValid) {
+    if (result.hasException) {
         throw new Error(`Validation error for fileName: ${options.fileName}: ${JSON.stringify(result)}`);
     }
-    await processFile(options, true);
+    else if (result.isValid) {
+        await processFile(options, true);
+    }
 }
 
 /**
@@ -71,7 +73,7 @@ async function processTriggerEvent(event, context) {
 async function processHttpEvent(request, response) {
     const options = request.body || {};
     const result = await configManager.validateOptions(options, true);
-    if (!result.isValid) {
+    if (!result.isValid || result.hasException) {
         response.status(400).send({ errors: result.errors });
         return;
     }

diff --git a/ingestion/batch/test/configurationManagerTest.js b/ingestion/batch/test/configurationManagerTest.js
@@ -103,11 +103,10 @@ it("options are valid with archived file", async () => {
     const options = {
         eventId: 1,
         bucketName: bucketName,
-        fileName: "/bqds/dataset/table/data/archive/myFile.txt"
+        fileName: "/bqds/dataset/table/data/archive/myFile.csv"
     };
     const v = await configManager.validateOptions(options, false);
-    console.log(JSON.stringify(v));
-    expect(v.isDataFile).is.false;
+    expect(v.isArchiveFile).is.true;
 });
 
 it("options are invalid with bad path", async () => {
@@ -129,6 +128,16 @@ it("options are invalid", async () => {
     expect(result.errors.length).is.equal(3);
 });
 
+it("options file is a path directory", async () => {
+    const options = {
+        eventId: 1,
+        bucketName: bucketName,
+        fileName: "/bqds/bqds/table/data/"
+    };
+    const result = await configManager.validateOptions(options, false);
+    expect(result.isDirectoryPath).is.true;
+});
+
 if (argv.runCloudTests) {
     let configFileName = `bqds/${uuid}/table/config/schema.json`;