From 078786dd76b213efcd8e1dd91fe3c3b402af3a98 Mon Sep 17 00:00:00 2001 From: avatar-lavventura Date: Fri, 16 Mar 2018 19:17:42 +0300 Subject: [PATCH] some update --- .gitignore | 4 +++ Driver.py | 12 +++---- README.md | 5 ++- driverFunc.py | 78 +++++++++++++++++++++++----------------------- endCode.py | 9 +++--- initialize.sh | 1 - ipfsGet.sh | 4 ++- ownCloudExample.py | 3 -- 8 files changed, 59 insertions(+), 57 deletions(-) diff --git a/.gitignore b/.gitignore index 89fd2f2d..71e479d0 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,7 @@ contracts/populus\.ini contractCalls/\.node-xmlhttprequest-sync-13830 \.#main\.js + +contract/\.cache/v/populus/project/compiled_contracts + +contract/\.cache/v/populus/project/compiled_contracts_mtime diff --git a/Driver.py b/Driver.py index e7273f9f..435bfd04 100755 --- a/Driver.py +++ b/Driver.py @@ -173,7 +173,7 @@ def isSlurmOn(): # Waits here until new job submitted into the cluster returnVal = contractCall('echo "$header; console.log( \'\' + eBlocBroker.LogJob($blockReadFrom, \'$jobsReadFromPath\') )"'); - if os.path.isfile(jobsReadFromPath): #{ Waits until generated file on log is completed + if os.path.isfile(jobsReadFromPath): #{Waits until generated file on log is completed fR = open(jobsReadFromPath, 'r' ) blockReadFrom = fR.read().rstrip('\n'); fR.close(); @@ -185,7 +185,6 @@ def isSlurmOn(): for i in range(0, (len(submittedJobs) - 1)): #{ logTest("------------------------------------------------------------------") submittedJob = submittedJobs[i].split(' '); - print(submittedJob[5]) if (clusterID == submittedJob[1]): # Only obtain jobs that are submitted to the cluster logTest("BlockNum: " + submittedJob[0] + " " + submittedJob[1] + " " + submittedJob[2] + " " + submittedJob[3] + " " + submittedJob[4]); @@ -203,16 +202,17 @@ def isSlurmOn(): if (jobInfo[0] == str(constants.job_state_code['PENDING'])): if (submittedJob[4] == '0'): logTest("New job has been recieved. IPFS call |" + time.ctime()) - driverFunc.driverIpfsCall(submittedJob[2], submittedJob[3], submittedJob[4], submittedJob[5]); #TODO: could be called as a thread but its already fast + driverFunc.driverIpfsCall(submittedJob[2], submittedJob[3], submittedJob[4], submittedJob[5]); elif (submittedJob[4] == '1'): logTest("New job has been recieved. EUDAT call |" + time.ctime()); driverFunc.driverEudatCall(submittedJob[2], submittedJob[3]); - #thread.start_new_thread(driverFunc.driverEudatCall, (submittedJob[2], submittedJob[3], clusterID) ) #works + #thread.start_new_thread(driverFunc.driverEudatCall, (submittedJob[2], submittedJob[3])) elif (submittedJob[4] == '2'): logTest("New job has been recieved. IPFS with miniLock call |" + time.ctime()); - driverFunc.driverIpfsCall(submittedJob[2], submittedJob[3], submittedJob[4], submittedJob[5]); + driverFunc.driverIpfsCall(submittedJob[2], submittedJob[3], submittedJob[4], submittedJob[5]); + #thread.start_new_thread(driverFunc.driverIpfsCall, (submittedJob[2], submittedJob[3], submittedJob[4], submittedJob[5])) else: - logTest("Job is already captured and in process"); + logTest("Job is already captured and in process or completed"); #} if( submittedJob != 0 and (int(maxVal) != 0) ): #{ diff --git a/README.md b/README.md index 80156cfc..732dd74f 100755 --- a/README.md +++ b/README.md @@ -283,9 +283,8 @@ Your miniLock ID: LRFbCrhCeN2uVCdDXd2bagoCM1fVcGvUzwhfVdqfyVuhi ###### How to encripty your folder using miniLock ```bash -myMiniLockId="LRFbCrhCeN2uVCdDXd2bagoCM1fVcGvUzwhfVdqfyVuhi" -clusterMiniLockId="9VZyJy1gRFJfdDtAjRitqmjSxPjSAjBR6BxH59UeNgKzQ"; -encrypyFolderPath="./ipfsCode" +clusterMiniLockId = "9VZyJy1gRFJfdDtAjRitqmjSxPjSAjBR6BxH59UeNgKzQ"; +encrypyFolderPath = "./ipfsCode" tar -cvzf $encrypyFolderPath.tar.gz $encrypyFolderPath mlck encrypt -f $encrypyFolderPath.tar.gz $clusterMiniLockId --passphrase="$(cat mlck_password.txt)" diff --git a/driverFunc.py b/driverFunc.py index 849a1e79..f6bc8a7a 100755 --- a/driverFunc.py +++ b/driverFunc.py @@ -12,13 +12,13 @@ indexGlobal = ""; def logTest(strIn): - print(strIn) - txFile = open( constants.LOG_PATH + '/transactions/' + jobKeyGlobal + '_' + indexGlobal + '_driverOutput' +'.txt', 'a'); + #print(strIn) + txFile = open(constants.LOG_PATH + '/transactions/' + jobKeyGlobal + '_' + indexGlobal + '_driverOutput' +'.txt', 'a'); txFile.write(strIn + "\n"); txFile.close(); # If non-thread tests are running - txFile = open( constants.LOG_PATH + '/transactions/clusterOut.txt', 'a'); + txFile = open(constants.LOG_PATH + '/transactions/clusterOut.txt', 'a'); txFile.write(strIn + "\n"); txFile.close(); @@ -27,15 +27,15 @@ def enum(**named_values): def contractCall(val): printFlag=1; - ret = os.popen( val + "| node").read().rstrip('\n').replace(" ", ""); + ret = os.popen(val + "| node").read().rstrip('\n').replace(" ", ""); while(True): - if( not(ret == "notconnected" or ret == "") ): + if (not(ret == "notconnected" or ret == "")): break; else: - if(printFlag == 1): + if (printFlag == 1): logTest("Error: Please run Parity or Geth on the background.**************************************************************") printFlag = 0; - ret = os.popen( val + "| node").read().rstrip('\n').replace(" ", ""); + ret = os.popen(val + "| node").read().rstrip('\n').replace(" ", ""); time.sleep(1); return ret; @@ -43,8 +43,8 @@ def contractCall(val): def isIpfsDaemonOn(): check = os.popen("ps aux | grep \'ipfs daemon\' | grep -v \'grep\' ").read().rstrip('\n'); - if (len(check) == 0): - logTest( "Error: IPFS does not work on the background. Please do: ipfs daemon & " ) + if (len(check)== 0): + logTest("Error: IPFS does not work on the background. Please do: ipfs daemon & " ) return False return True; @@ -62,7 +62,7 @@ def driverEudatCall(jobKey, index): os.environ['folderIndex'] = "1"; os.environ['miniLockId'] = "-1"; os.environ['whoami'] = constants.WHOAMI - whoami = os.system("whoami") # To learn running as root or userName + whoami = os.system("whoami")# To learn running as root or userName jobKeyTemp = jobKey.split('='); owner = jobKeyTemp[0] @@ -70,7 +70,7 @@ def driverEudatCall(jobKey, index): header = "var eBlocBroker = require('" + eblocPath + "/eBlocBrokerHeader.js')"; os.environ['header'] = header; - f = open(eblocPath + '/eudatPassword.txt', 'r') # Password is read from the file. password.txt is have only user access + f = open(eblocPath + '/eudatPassword.txt', 'r') # Password is read from the file. password.txt is have only user access password = f.read().rstrip('\n').replace(" ", ""); f.close() logTest("Login into owncloud") @@ -82,15 +82,15 @@ def driverEudatCall(jobKey, index): logTest("finding_acceptId") acceptFlag = 0; eudatFolderName = "" - for i in range( len(shareList)-1, -1, -1 ): # Starts iterating from last item to first one + for i in range(len(shareList)-1, -1, -1 ): # Starts iterating from last item to first one inputFolderName = shareList[i]['name'] inputFolderName = inputFolderName[1:] # Removes '/' on the beginning inputId = shareList[i]['id'] inputOwner = shareList[i]['owner'] shareToken = shareList[i]['share_token'] - if( (inputFolderName == folderName) and (inputOwner == owner) ): - logTest("Here:_" + inputId + "_ShareToken:_" + shareToken) + if ((inputFolderName == folderName)and (inputOwner == owner)): + logTest("Here:_" + inputId + "_ShareToken:_" + shareToken) os.environ['shareToken'] = str(shareToken); os.environ['eudatFolderName'] = str(inputFolderName); eudatFolderName = inputFolderName; @@ -107,20 +107,20 @@ def driverEudatCall(jobKey, index): if not os.path.isdir(localOwnCloudPathFolder): # If folder does not exist os.makedirs(localOwnCloudPathFolder) - os.popen( "wget https://b2drop.eudat.eu/s/$shareToken/download --output-document=$localOwnCloudPathFolder/output.zip" ).read() # Downloads shared file as zip + os.popen("wget -q https://b2drop.eudat.eu/s/$shareToken/download --output-document=$localOwnCloudPathFolder/output.zip" ).read()# Downloads shared file as zip #run.tar.gz check yap. - #checkRunExist = os.popen( "unzip -l $localOwnCloudPathFolder/output.zip | grep $eudatFolderName/run.sh" ).read() # Checks does zip contains run.sh file - #if( not eudatFolderName + "/run.sh" in checkRunExist ): + #checkRunExist = os.popen("unzip -l $localOwnCloudPathFolder/output.zip | grep $eudatFolderName/run.sh" ).read()# Checks does zip contains run.sh file + #if (not eudatFolderName + "/run.sh" in checkRunExist ): #logTest("Error: Folder does not contain run.sh file or client does not run ipfs daemon on the background.") #return; #detects error on the SLURM side. os.popen("unzip $localOwnCloudPathFolder/output.zip -d $localOwnCloudPathFolder/.").read() - os.popen("mv $localOwnCloudPathFolder/$eudatFolderName/* $localOwnCloudPathFolder/ ").read() + os.popen("mv $localOwnCloudPathFolder/$eudatFolderName/* $localOwnCloudPathFolder/ ").read() os.popen("rm $localOwnCloudPathFolder/output.zip" ) os.popen("rmdir $localOwnCloudPathFolder/$eudatFolderName" ) myDate = os.popen('LANG=en_us_88591 && date +"%b %d %k:%M:%S:%N %Y"' ).read().rstrip('\n'); #logTest(myDate); - txFile = open( localOwnCloudPathFolder + '/modifiedDate.txt', 'w'); txFile.write(myDate + '\n'); txFile.close(); + txFile = open(localOwnCloudPathFolder + '/modifiedDate.txt', 'w'); txFile.write(myDate + '\n'); txFile.close(); time.sleep(0.2) #ipfs.tar.gz var mi diye bak!!!!! #os.popen("tar -xf $localOwnCloudPathFolder/ipfs.tar.gz -C $localOwnCloudPathFolder/" ).read() @@ -145,13 +145,13 @@ def driverEudatCall(jobKey, index): os.environ['jobCoreNum'] = jobCoreNum; logTest("Job's Core Number: " + jobCoreNum) - os.chdir(localOwnCloudPathFolder) # 'cd' into the working path and call sbatch from there - if(whoami == "root"): + os.chdir(localOwnCloudPathFolder)# 'cd' into the working path and call sbatch from there + if (whoami == "root"): jobId = os.popen('sbatch -U root -N$jobCoreNum $localOwnCloudPathFolder/${jobKey}_${index}_${folderIndex}_${shareToken}_$miniLockId.sh --mail-type=ALL | cut -d " " -f4-').read().rstrip('\n'); else: jobId = os.popen('sbatch -N$jobCoreNum $localOwnCloudPathFolder/${jobKey}_${index}_${folderIndex}_${shareToken}_$miniLockId.sh --mail-type=ALL | cut -d " " -f4-').read().rstrip('\n'); os.environ['jobId'] = jobId; - logTest( "jobId: "+ str(jobId) ); + logTest("jobId: "+ str(jobId)); if not jobId.isdigit(): # Detected an error on the SLURM side @@ -172,7 +172,7 @@ def driverIpfsCall(ipfsHash, index, ipfsType, miniLockId): os.environ['eblocPath'] = eblocPath os.environ['shareToken'] = "-1" os.environ['whoami'] = constants.WHOAMI - whoami = os.system( "whoami" ) + whoami = os.system("whoami" ) if (ipfsType == '0'): os.environ['miniLockId'] = "-1" @@ -180,7 +180,7 @@ def driverIpfsCall(ipfsHash, index, ipfsType, miniLockId): os.environ['miniLockId'] = miniLockId header = "var eBlocBroker = require('" + eblocPath + "/eBlocBrokerHeader.js')"; os.environ['header'] = header; - logTest( "ipfsHash: " + ipfsHash); + logTest("ipfsHash: " + ipfsHash); jobSavePath = ipfsHashes + '/' + ipfsHash + "_" + index; os.environ['jobSavePath'] = jobSavePath @@ -192,36 +192,36 @@ def driverIpfsCall(ipfsHash, index, ipfsType, miniLockId): else: os.system(" mkdir $mkdirPath"); - os.chdir( jobSavePath ) + os.chdir(jobSavePath ) if os.path.isfile(ipfsHash): - os.system( 'rm $ipfsHash' ); + os.system('rm $ipfsHash'); ipfsCallCounter=0; isIPFSHashExist="" if (whoami == "root"): - isIPFSHashExist = os.popen( "sudo -u $whoami bash $eblocPath/ipfsStat.sh $ipfsHash" ).read(); + isIPFSHashExist = os.popen("sudo -u $whoami bash $eblocPath/ipfsStat.sh $ipfsHash" ).read(); else: - isIPFSHashExist = os.popen( " bash $eblocPath/ipfsStat.sh $ipfsHash" ).read(); + isIPFSHashExist = os.popen(" bash $eblocPath/ipfsStat.sh $ipfsHash" ).read(); logTest(isIPFSHashExist); if (constants.IPFS_USE == 1): while(True): - if(isIpfsDaemonOn): + if (isIpfsDaemonOn): break; if ("CumulativeSize" in isIPFSHashExist): if (whoami == "root"): - os.system( 'sudo -u $whoami bash $eblocPath/ipfsGet.sh $ipfsHash $jobSavePath'); + os.system('sudo -u $whoami bash $eblocPath/ipfsGet.sh $ipfsHash $jobSavePath'); else: - os.system( ' bash $eblocPath/ipfsGet.sh $ipfsHash $jobSavePath'); + os.system(' bash $eblocPath/ipfsGet.sh $ipfsHash $jobSavePath'); if (ipfsType == '2'): # case for the ipfsMiniLock - res = os.popen( 'mlck decrypt -f $jobSavePath/$ipfsHash --passphrase="exfoliation econometrics revivifying obsessions transverse salving dishes" --output-file=$jobSavePath/output.tar.gz' ).read(); - os.system( 'rm $jobSavePath/$ipfsHash' ); - os.system( 'tar -xvf $jobSavePath/output.tar.gz && rm $jobSavePath/output.tar.gz' ); - print( res ) + res = os.popen('mlck decrypt -f $jobSavePath/$ipfsHash --passphrase="exfoliation econometrics revivifying obsessions transverse salving dishes" --output-file=$jobSavePath/output.tar.gz' ).read(); + os.system('rm $jobSavePath/$ipfsHash' ); + os.system('tar -xf $jobSavePath/output.tar.gz && rm $jobSavePath/output.tar.gz' ); + print(res) if not os.path.isfile('run.sh'): return @@ -231,17 +231,17 @@ def driverIpfsCall(ipfsHash, index, ipfsType, miniLockId): return myDate = os.popen('LANG=en_us_88591 && date +"%b %d %k:%M:%S:%N %Y"' ).read().rstrip('\n'); logTest(myDate); - txFile = open('modifiedDate.txt', 'w'); txFile.write( myDate + '\n' ); txFile.close(); + txFile = open('modifiedDate.txt', 'w'); txFile.write(myDate + '\n' ); txFile.close(); time.sleep(0.2) os.system("cp run.sh ${ipfsHash}_${index}_${folderIndex}_${shareToken}_$miniLockId.sh"); - jobInfo = contractCall('echo "$header; console.log( \'\' + eBlocBroker.getJobInfo( \'$clusterID\', \'$ipfsHash\', \'$index\' ) )"'); + jobInfo = contractCall('echo "$header; console.log(\'\' + eBlocBroker.getJobInfo(\'$clusterID\', \'$ipfsHash\', \'$index\' ))"'); jobInfo = jobInfo.split(','); jobCoreNum = jobInfo[1] os.environ['jobCoreNum'] = jobCoreNum; - logTest( "RequestedCoreNum: " + str(jobCoreNum) ) + logTest("RequestedCoreNum: " + str(jobCoreNum)) # SLURM submit job if (whoami == "root"): @@ -255,7 +255,7 @@ def driverIpfsCall(ipfsHash, index, ipfsType, miniLockId): sys.exit(); # Detects na error on the SLURM side if (whoami == "root"): - os.popen( "sudo chown $whoami: $jobSavePath") + os.popen("sudo chown $whoami: $jobSavePath") # To test driverFunc.py executed as script. if __name__ == '__main__': #{ diff --git a/endCode.py b/endCode.py index fd596d3a..80284994 100755 --- a/endCode.py +++ b/endCode.py @@ -79,10 +79,12 @@ def endCall(jobKey, index, storageType, shareToken, miniLockId, folderName): logTest("TimeLimit: " + clientTimeLimit); if jobInfo[0] == str(constants.job_state_code['RUNNING']): + logTest("Job started running"); break; # Wait until does values updated on the blockchain if time.time() > timeout: - break + logTest("Timeout "); + sys.exit() if jobInfo[0] == constants.job_state_code['COMPLETED']: logTest( "Error: Already completed job..."); @@ -96,7 +98,6 @@ def endCall(jobKey, index, storageType, shareToken, miniLockId, folderName): logTest("Error: Please run Parity or Geth on the background.****************************") jobInfo = os.popen('python $contractCallPath/getJobInfo.py $clusterID $jobKey $index').read().rstrip('\n').replace(" ","")[1:-1]; time.sleep(1) - jobInfo = jobInfo.split(','); time.sleep(10) # Short sleep here so this loop is not keeping CPU busy #} @@ -181,12 +182,12 @@ def endCall(jobKey, index, storageType, shareToken, miniLockId, folderName): elapsedRawTime = int(elapsedDay)* 1440 + int(elapsedHour) * 60 + int(elapsedMinute) + 1; logTest("ElapsedRawTime: " + str(elapsedRawTime)) - if(elapsedRawTime > int(clientTimeLimit)): + if (elapsedRawTime > int(clientTimeLimit)): elapsedRawTime = clientTimeLimit os.environ['elapsedRawTime'] = str(elapsedRawTime); logTest("ElapsedRawTime: " + str(elapsedRawTime)) - logTest("jobInfo: " + str(jobInfo)) + logTest("jobInfo: " + jobInfo) if storageType == '0' or storageType == '2': transactionHash = os.popen('node $eblocPath/eBlocBrokerNodeCall.js receiptCheck $jobKey $index $elapsedRawTime $newHash $storageType $endTimeStamp').read().rstrip('\n').replace(" ", ""); diff --git a/initialize.sh b/initialize.sh index b2070c23..bad95664 100644 --- a/initialize.sh +++ b/initialize.sh @@ -28,7 +28,6 @@ sudo chmod +x $currentDir/slurmScript.sh # User Name Setup:------------------------------------ lineOld="whoami"; -lineNew="whoami"; lineNew=$(logname); sed -i.bak 's/'$lineOld'/'$lineNew'/' $currentDir/nodePaths.js && rm $currentDir/nodePaths.js.bak diff --git a/ipfsGet.sh b/ipfsGet.sh index 6bb3c850..7d8e5de7 100755 --- a/ipfsGet.sh +++ b/ipfsGet.sh @@ -3,4 +3,6 @@ IPFS_PATH=$HOME"/.ipfs" export IPFS_PATH ipfsHash=$1 -ipfs get $ipfsHash --output=$2 + +#ipfs get $ipfsHash --output=$2 &> /dev/null #TEST +ipfs get $ipfsHash --output=$2 #Development diff --git a/ownCloudExample.py b/ownCloudExample.py index 1f5688fe..443d3f6a 100755 --- a/ownCloudExample.py +++ b/ownCloudExample.py @@ -14,11 +14,8 @@ oc = owncloud.Client('https://b2drop.eudat.eu/') oc.login('aalimog1@binghamton.edu', password ); -#oc.mkdir("helloo") oc.share_file_with_user( 'helloWorld', '3d8e2dc2-b855-1036-807f-9dbd8c6b1579@b2drop.eudat.eu', remote_user=True ) -#oc.share_file_with_user( "hello", "3d8e2dc2-b855-1036-807f-9dbd8c6b1579@b2drop.eudat.eu") - ''' z = oc.list_open_remote_share();