Skip to content

Commit

Permalink
some update
Browse files Browse the repository at this point in the history
  • Loading branch information
@avatar-lavventura
avatar-lavventura committed Mar 16, 2018
1 parent 0c6ecfa commit 078786d
Show file tree
Hide file tree
Showing 8 changed files with 59 additions and 57 deletions.
4 changes: 4 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,7 @@ contracts/populus\.ini
contractCalls/\.node-xmlhttprequest-sync-13830

\.#main\.js

contract/\.cache/v/populus/project/compiled_contracts

contract/\.cache/v/populus/project/compiled_contracts_mtime
12 changes: 6 additions & 6 deletions Driver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ def isSlurmOn():
# Waits here until new job submitted into the cluster
returnVal = contractCall('echo "$header; console.log( \'\' + eBlocBroker.LogJob($blockReadFrom, \'$jobsReadFromPath\') )"');

if os.path.isfile(jobsReadFromPath): #{ Waits until generated file on log is completed
if os.path.isfile(jobsReadFromPath): #{Waits until generated file on log is completed
fR = open(jobsReadFromPath, 'r' )
blockReadFrom = fR.read().rstrip('\n');
fR.close();
Expand All @@ -185,7 +185,6 @@ def isSlurmOn():
for i in range(0, (len(submittedJobs) - 1)): #{
logTest("------------------------------------------------------------------")
submittedJob = submittedJobs[i].split(' ');
print(submittedJob[5])

if (clusterID == submittedJob[1]): # Only obtain jobs that are submitted to the cluster
logTest("BlockNum: " + submittedJob[0] + " " + submittedJob[1] + " " + submittedJob[2] + " " + submittedJob[3] + " " + submittedJob[4]);
Expand All @@ -203,16 +202,17 @@ def isSlurmOn():
if (jobInfo[0] == str(constants.job_state_code['PENDING'])):
if (submittedJob[4] == '0'):
logTest("New job has been recieved. IPFS call |" + time.ctime())
driverFunc.driverIpfsCall(submittedJob[2], submittedJob[3], submittedJob[4], submittedJob[5]); #TODO: could be called as a thread but its already fast
driverFunc.driverIpfsCall(submittedJob[2], submittedJob[3], submittedJob[4], submittedJob[5]);
elif (submittedJob[4] == '1'):
logTest("New job has been recieved. EUDAT call |" + time.ctime());
driverFunc.driverEudatCall(submittedJob[2], submittedJob[3]);
#thread.start_new_thread(driverFunc.driverEudatCall, (submittedJob[2], submittedJob[3], clusterID) ) #works
#thread.start_new_thread(driverFunc.driverEudatCall, (submittedJob[2], submittedJob[3]))
elif (submittedJob[4] == '2'):
logTest("New job has been recieved. IPFS with miniLock call |" + time.ctime());
driverFunc.driverIpfsCall(submittedJob[2], submittedJob[3], submittedJob[4], submittedJob[5]);
driverFunc.driverIpfsCall(submittedJob[2], submittedJob[3], submittedJob[4], submittedJob[5]);
#thread.start_new_thread(driverFunc.driverIpfsCall, (submittedJob[2], submittedJob[3], submittedJob[4], submittedJob[5]))
else:
logTest("Job is already captured and in process");
logTest("Job is already captured and in process or completed");
#}

if( submittedJob != 0 and (int(maxVal) != 0) ): #{
Expand Down
5 changes: 2 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,9 +283,8 @@ Your miniLock ID: LRFbCrhCeN2uVCdDXd2bagoCM1fVcGvUzwhfVdqfyVuhi
###### How to encripty your folder using miniLock
```bash
myMiniLockId="LRFbCrhCeN2uVCdDXd2bagoCM1fVcGvUzwhfVdqfyVuhi"
clusterMiniLockId="9VZyJy1gRFJfdDtAjRitqmjSxPjSAjBR6BxH59UeNgKzQ";
encrypyFolderPath="./ipfsCode"
clusterMiniLockId = "9VZyJy1gRFJfdDtAjRitqmjSxPjSAjBR6BxH59UeNgKzQ";
encrypyFolderPath = "./ipfsCode"
tar -cvzf $encrypyFolderPath.tar.gz $encrypyFolderPath
mlck encrypt -f $encrypyFolderPath.tar.gz $clusterMiniLockId --passphrase="$(cat mlck_password.txt)"
Expand Down
78 changes: 39 additions & 39 deletions driverFunc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,13 @@
indexGlobal = "";

def logTest(strIn):
print(strIn)
txFile = open( constants.LOG_PATH + '/transactions/' + jobKeyGlobal + '_' + indexGlobal + '_driverOutput' +'.txt', 'a');
#print(strIn)
txFile = open(constants.LOG_PATH + '/transactions/' + jobKeyGlobal + '_' + indexGlobal + '_driverOutput' +'.txt', 'a');
txFile.write(strIn + "\n");
txFile.close();

# If non-thread tests are running
txFile = open( constants.LOG_PATH + '/transactions/clusterOut.txt', 'a');
txFile = open(constants.LOG_PATH + '/transactions/clusterOut.txt', 'a');
txFile.write(strIn + "\n");
txFile.close();

Expand All @@ -27,24 +27,24 @@ def enum(**named_values):

def contractCall(val):
printFlag=1;
ret = os.popen( val + "| node").read().rstrip('\n').replace(" ", "");
ret = os.popen(val + "| node").read().rstrip('\n').replace(" ", "");
while(True):
if( not(ret == "notconnected" or ret == "") ):
if (not(ret == "notconnected" or ret == "")):
break;
else:
if(printFlag == 1):
if (printFlag == 1):
logTest("Error: Please run Parity or Geth on the background.**************************************************************")
printFlag = 0;
ret = os.popen( val + "| node").read().rstrip('\n').replace(" ", "");
ret = os.popen(val + "| node").read().rstrip('\n').replace(" ", "");
time.sleep(1);
return ret;

# Checks does IPFS work on the background or not
def isIpfsDaemonOn():
check = os.popen("ps aux | grep \'ipfs daemon\' | grep -v \'grep\' ").read().rstrip('\n');

if (len(check) == 0):
logTest( "Error: IPFS does not work on the background. Please do: ipfs daemon & " )
if (len(check)== 0):
logTest("Error: IPFS does not work on the background. Please do: ipfs daemon & " )
return False
return True;

Expand All @@ -62,15 +62,15 @@ def driverEudatCall(jobKey, index):
os.environ['folderIndex'] = "1";
os.environ['miniLockId'] = "-1";
os.environ['whoami'] = constants.WHOAMI
whoami = os.system("whoami") # To learn running as root or userName
whoami = os.system("whoami")# To learn running as root or userName

jobKeyTemp = jobKey.split('=');
owner = jobKeyTemp[0]
folderName = jobKeyTemp[1]

header = "var eBlocBroker = require('" + eblocPath + "/eBlocBrokerHeader.js')"; os.environ['header'] = header;

f = open(eblocPath + '/eudatPassword.txt', 'r') # Password is read from the file. password.txt is have only user access
f = open(eblocPath + '/eudatPassword.txt', 'r') # Password is read from the file. password.txt is have only user access
password = f.read().rstrip('\n').replace(" ", ""); f.close()

logTest("Login into owncloud")
Expand All @@ -82,15 +82,15 @@ def driverEudatCall(jobKey, index):
logTest("finding_acceptId")
acceptFlag = 0;
eudatFolderName = ""
for i in range( len(shareList)-1, -1, -1 ): # Starts iterating from last item to first one
for i in range(len(shareList)-1, -1, -1 ): # Starts iterating from last item to first one
inputFolderName = shareList[i]['name']
inputFolderName = inputFolderName[1:] # Removes '/' on the beginning
inputId = shareList[i]['id']
inputOwner = shareList[i]['owner']
shareToken = shareList[i]['share_token']

if( (inputFolderName == folderName) and (inputOwner == owner) ):
logTest("Here:_" + inputId + "_ShareToken:_" + shareToken)
if ((inputFolderName == folderName)and (inputOwner == owner)):
logTest("Here:_" + inputId + "_ShareToken:_" + shareToken)
os.environ['shareToken'] = str(shareToken);
os.environ['eudatFolderName'] = str(inputFolderName);
eudatFolderName = inputFolderName;
Expand All @@ -107,20 +107,20 @@ def driverEudatCall(jobKey, index):
if not os.path.isdir(localOwnCloudPathFolder): # If folder does not exist
os.makedirs(localOwnCloudPathFolder)

os.popen( "wget https://b2drop.eudat.eu/s/$shareToken/download --output-document=$localOwnCloudPathFolder/output.zip" ).read() # Downloads shared file as zip
os.popen("wget -q https://b2drop.eudat.eu/s/$shareToken/download --output-document=$localOwnCloudPathFolder/output.zip" ).read()# Downloads shared file as zip

#run.tar.gz check yap.
#checkRunExist = os.popen( "unzip -l $localOwnCloudPathFolder/output.zip | grep $eudatFolderName/run.sh" ).read() # Checks does zip contains run.sh file
#if( not eudatFolderName + "/run.sh" in checkRunExist ):
#checkRunExist = os.popen("unzip -l $localOwnCloudPathFolder/output.zip | grep $eudatFolderName/run.sh" ).read()# Checks does zip contains run.sh file
#if (not eudatFolderName + "/run.sh" in checkRunExist ):
#logTest("Error: Folder does not contain run.sh file or client does not run ipfs daemon on the background.")
#return; #detects error on the SLURM side.

os.popen("unzip $localOwnCloudPathFolder/output.zip -d $localOwnCloudPathFolder/.").read()
os.popen("mv $localOwnCloudPathFolder/$eudatFolderName/* $localOwnCloudPathFolder/ ").read()
os.popen("mv $localOwnCloudPathFolder/$eudatFolderName/* $localOwnCloudPathFolder/ ").read()
os.popen("rm $localOwnCloudPathFolder/output.zip" )
os.popen("rmdir $localOwnCloudPathFolder/$eudatFolderName" )
myDate = os.popen('LANG=en_us_88591 && date +"%b %d %k:%M:%S:%N %Y"' ).read().rstrip('\n'); #logTest(myDate);
txFile = open( localOwnCloudPathFolder + '/modifiedDate.txt', 'w'); txFile.write(myDate + '\n'); txFile.close();
txFile = open(localOwnCloudPathFolder + '/modifiedDate.txt', 'w'); txFile.write(myDate + '\n'); txFile.close();
time.sleep(0.2)
#ipfs.tar.gz var mi diye bak!!!!!
#os.popen("tar -xf $localOwnCloudPathFolder/ipfs.tar.gz -C $localOwnCloudPathFolder/" ).read()
Expand All @@ -145,13 +145,13 @@ def driverEudatCall(jobKey, index):
os.environ['jobCoreNum'] = jobCoreNum;
logTest("Job's Core Number: " + jobCoreNum)

os.chdir(localOwnCloudPathFolder) # 'cd' into the working path and call sbatch from there
if(whoami == "root"):
os.chdir(localOwnCloudPathFolder)# 'cd' into the working path and call sbatch from there
if (whoami == "root"):
jobId = os.popen('sbatch -U root -N$jobCoreNum $localOwnCloudPathFolder/${jobKey}_${index}_${folderIndex}_${shareToken}_$miniLockId.sh --mail-type=ALL | cut -d " " -f4-').read().rstrip('\n');
else:
jobId = os.popen('sbatch -N$jobCoreNum $localOwnCloudPathFolder/${jobKey}_${index}_${folderIndex}_${shareToken}_$miniLockId.sh --mail-type=ALL | cut -d " " -f4-').read().rstrip('\n');
os.environ['jobId'] = jobId;
logTest( "jobId: "+ str(jobId) );
logTest("jobId: "+ str(jobId));

if not jobId.isdigit():
# Detected an error on the SLURM side
Expand All @@ -172,15 +172,15 @@ def driverIpfsCall(ipfsHash, index, ipfsType, miniLockId):
os.environ['eblocPath'] = eblocPath
os.environ['shareToken'] = "-1"
os.environ['whoami'] = constants.WHOAMI
whoami = os.system( "whoami" )
whoami = os.system("whoami" )

if (ipfsType == '0'):
os.environ['miniLockId'] = "-1"
else:
os.environ['miniLockId'] = miniLockId

header = "var eBlocBroker = require('" + eblocPath + "/eBlocBrokerHeader.js')"; os.environ['header'] = header;
logTest( "ipfsHash: " + ipfsHash);
logTest("ipfsHash: " + ipfsHash);

jobSavePath = ipfsHashes + '/' + ipfsHash + "_" + index;
os.environ['jobSavePath'] = jobSavePath
Expand All @@ -192,36 +192,36 @@ def driverIpfsCall(ipfsHash, index, ipfsType, miniLockId):
else:
os.system(" mkdir $mkdirPath");

os.chdir( jobSavePath )
os.chdir(jobSavePath )
if os.path.isfile(ipfsHash):
os.system( 'rm $ipfsHash' );
os.system('rm $ipfsHash');

ipfsCallCounter=0;

isIPFSHashExist=""
if (whoami == "root"):
isIPFSHashExist = os.popen( "sudo -u $whoami bash $eblocPath/ipfsStat.sh $ipfsHash" ).read();
isIPFSHashExist = os.popen("sudo -u $whoami bash $eblocPath/ipfsStat.sh $ipfsHash" ).read();
else:
isIPFSHashExist = os.popen( " bash $eblocPath/ipfsStat.sh $ipfsHash" ).read();
isIPFSHashExist = os.popen(" bash $eblocPath/ipfsStat.sh $ipfsHash" ).read();

logTest(isIPFSHashExist);

if (constants.IPFS_USE == 1):
while(True):
if(isIpfsDaemonOn):
if (isIpfsDaemonOn):
break;

if ("CumulativeSize" in isIPFSHashExist):
if (whoami == "root"):
os.system( 'sudo -u $whoami bash $eblocPath/ipfsGet.sh $ipfsHash $jobSavePath');
os.system('sudo -u $whoami bash $eblocPath/ipfsGet.sh $ipfsHash $jobSavePath');
else:
os.system( ' bash $eblocPath/ipfsGet.sh $ipfsHash $jobSavePath');
os.system(' bash $eblocPath/ipfsGet.sh $ipfsHash $jobSavePath');

if (ipfsType == '2'): # case for the ipfsMiniLock
res = os.popen( 'mlck decrypt -f $jobSavePath/$ipfsHash --passphrase="exfoliation econometrics revivifying obsessions transverse salving dishes" --output-file=$jobSavePath/output.tar.gz' ).read();
os.system( 'rm $jobSavePath/$ipfsHash' );
os.system( 'tar -xvf $jobSavePath/output.tar.gz && rm $jobSavePath/output.tar.gz' );
print( res )
res = os.popen('mlck decrypt -f $jobSavePath/$ipfsHash --passphrase="exfoliation econometrics revivifying obsessions transverse salving dishes" --output-file=$jobSavePath/output.tar.gz' ).read();
os.system('rm $jobSavePath/$ipfsHash' );
os.system('tar -xf $jobSavePath/output.tar.gz && rm $jobSavePath/output.tar.gz' );
print(res)

if not os.path.isfile('run.sh'):
return
Expand All @@ -231,17 +231,17 @@ def driverIpfsCall(ipfsHash, index, ipfsType, miniLockId):
return

myDate = os.popen('LANG=en_us_88591 && date +"%b %d %k:%M:%S:%N %Y"' ).read().rstrip('\n'); logTest(myDate);
txFile = open('modifiedDate.txt', 'w'); txFile.write( myDate + '\n' ); txFile.close();
txFile = open('modifiedDate.txt', 'w'); txFile.write(myDate + '\n' ); txFile.close();
time.sleep(0.2)

os.system("cp run.sh ${ipfsHash}_${index}_${folderIndex}_${shareToken}_$miniLockId.sh");

jobInfo = contractCall('echo "$header; console.log( \'\' + eBlocBroker.getJobInfo( \'$clusterID\', \'$ipfsHash\', \'$index\' ) )"');
jobInfo = contractCall('echo "$header; console.log(\'\' + eBlocBroker.getJobInfo(\'$clusterID\', \'$ipfsHash\', \'$index\' ))"');
jobInfo = jobInfo.split(',');
jobCoreNum = jobInfo[1]

os.environ['jobCoreNum'] = jobCoreNum;
logTest( "RequestedCoreNum: " + str(jobCoreNum) )
logTest("RequestedCoreNum: " + str(jobCoreNum))

# SLURM submit job
if (whoami == "root"):
Expand All @@ -255,7 +255,7 @@ def driverIpfsCall(ipfsHash, index, ipfsType, miniLockId):
sys.exit(); # Detects na error on the SLURM side

if (whoami == "root"):
os.popen( "sudo chown $whoami: $jobSavePath")
os.popen("sudo chown $whoami: $jobSavePath")

# To test driverFunc.py executed as script.
if __name__ == '__main__': #{
Expand Down
9 changes: 5 additions & 4 deletions endCode.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,10 +79,12 @@ def endCall(jobKey, index, storageType, shareToken, miniLockId, folderName):
logTest("TimeLimit: " + clientTimeLimit);

if jobInfo[0] == str(constants.job_state_code['RUNNING']):
logTest("Job started running");
break; # Wait until does values updated on the blockchain

if time.time() > timeout:
break
logTest("Timeout ");
sys.exit()

if jobInfo[0] == constants.job_state_code['COMPLETED']:
logTest( "Error: Already completed job...");
Expand All @@ -96,7 +98,6 @@ def endCall(jobKey, index, storageType, shareToken, miniLockId, folderName):
logTest("Error: Please run Parity or Geth on the background.****************************")
jobInfo = os.popen('python $contractCallPath/getJobInfo.py $clusterID $jobKey $index').read().rstrip('\n').replace(" ","")[1:-1];
time.sleep(1)

jobInfo = jobInfo.split(',');
time.sleep(10) # Short sleep here so this loop is not keeping CPU busy
#}
Expand Down Expand Up @@ -181,12 +182,12 @@ def endCall(jobKey, index, storageType, shareToken, miniLockId, folderName):
elapsedRawTime = int(elapsedDay)* 1440 + int(elapsedHour) * 60 + int(elapsedMinute) + 1;
logTest("ElapsedRawTime: " + str(elapsedRawTime))

if(elapsedRawTime > int(clientTimeLimit)):
if (elapsedRawTime > int(clientTimeLimit)):
elapsedRawTime = clientTimeLimit

os.environ['elapsedRawTime'] = str(elapsedRawTime);
logTest("ElapsedRawTime: " + str(elapsedRawTime))
logTest("jobInfo: " + str(jobInfo))
logTest("jobInfo: " + jobInfo)

if storageType == '0' or storageType == '2':
transactionHash = os.popen('node $eblocPath/eBlocBrokerNodeCall.js receiptCheck $jobKey $index $elapsedRawTime $newHash $storageType $endTimeStamp').read().rstrip('\n').replace(" ", "");
Expand Down
1 change: 0 additions & 1 deletion initialize.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ sudo chmod +x $currentDir/slurmScript.sh

# User Name Setup:------------------------------------
lineOld="whoami";
lineNew="whoami";
lineNew=$(logname);

sed -i.bak 's/'$lineOld'/'$lineNew'/' $currentDir/nodePaths.js && rm $currentDir/nodePaths.js.bak
Expand Down
4 changes: 3 additions & 1 deletion ipfsGet.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,6 @@
IPFS_PATH=$HOME"/.ipfs"
export IPFS_PATH
ipfsHash=$1
ipfs get $ipfsHash --output=$2

#ipfs get $ipfsHash --output=$2 &> /dev/null #TEST
ipfs get $ipfsHash --output=$2 #Development
3 changes: 0 additions & 3 deletions ownCloudExample.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,8 @@
oc = owncloud.Client('https://b2drop.eudat.eu/')
oc.login('[email protected]', password );

#oc.mkdir("helloo")
oc.share_file_with_user( 'helloWorld', '[email protected]', remote_user=True )

#oc.share_file_with_user( "hello", "[email protected]")

'''
z = oc.list_open_remote_share();
Expand Down

0 comments on commit 078786d

Please sign in to comment.