Improve Dockerfile.

- Fix watch scripts
ebloc · Jun 16, 2022 · 8dacdc5 · 8dacdc5
1 parent aa99b80
commit 8dacdc5
Show file tree

Hide file tree

Showing 53 changed files with 360 additions and 339 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,16 +1,16 @@
 # syntax=docker/dockerfile:1
 FROM golang:latest
 RUN apt-get install -y ca-certificates
-RUN wget --no-check-certificate -q "https://dist.ipfs.io/go-ipfs/v0.11.0/go-ipfs_v0.11.0_linux-amd64.tar.gz" \
- && tar -xvf "go-ipfs_v0.11.0_linux-amd64.tar.gz" \
- && rm -f go-ipfs_v0.11.0_linux-amd64.tar.gz
-WORKDIR go-ipfs
-RUN make install \
+RUN wget --no-check-certificate -q "https://dist.ipfs.io/go-ipfs/v0.13.0/go-ipfs_v0.13.0_linux-amd64.tar.gz" \
+ && tar -xf "go-ipfs_v0.13.0_linux-amd64.tar.gz" \
+ && rm -f go-ipfs_v0.13.0_linux-amd64.tar.gz \
+ && cd go-ipfs \
+ && make install \
  && ./install.sh
 
-RUN git clone https://github.com/prasmussen/gdrive.git /workspace
-WORKDIR /workspace/gdrive
-RUN go env -w GO111MODULE=auto \
+RUN git clone https://github.com/prasmussen/gdrive.git /workspace/gdrive \
+ && cd /workspace/gdrive \
+ && go env -w GO111MODULE=auto \
  && go get github.com/prasmussen/gdrive
 
 FROM python:3.7
@@ -151,7 +151,7 @@ RUN brownie init \
   && cd /workspace//ebloc-broker/contract \
   && brownie compile
 
-# orginize Slurm files
+# orginize slurm files
 RUN chown root:munge -R /etc/munge /etc/munge/munge.key /var/lib/munge  # works but root is alright?
 WORKDIR /var/log/slurm
 WORKDIR /var/run/supervisor
@@ -165,15 +165,16 @@ COPY --chown=slurm docker/slurm/files/slurm/slurmdbd.conf /etc/slurm/slurmdbd.co
 RUN chmod 0600 /etc/slurm/slurmdbd.conf
 
 ## finally  # sysctl -w net.core.rmem_max=2500000  ?
-RUN ipfs version \
- && ipfs init --profile=server,badgerds \
+RUN gdrive version \
+ && ipfs version \
+ && ipfs init \
  && ipfs config Reprovider.Strategy roots \
  && ipfs config Routing.Type none \
  && ganache --version \
  && /workspace/ebloc-broker/broker/bash_scripts/ubuntu_clean.sh >/dev/null 2>&1 \
- && du -sh / 2>&1 | grep -v "cannot" \
  && echo "alias ls='ls -h --color=always -v --author --time-style=long-iso'" >> ~/.bashrc \
- && echo "export SQUEUE_FORMAT=\"%8i %9u %5P %2t %12M %12l %5D %3C %30j\"v" >> ~/.bashrc
+ && echo "export SQUEUE_FORMAT=\"%8i %9u %5P %2t %12M %12l %5D %3C %30j\"v" >> ~/.bashrc \
+ && du -sh / 2>&1 | grep -v "cannot"
 
 WORKDIR /workspace/ebloc-broker/broker
 CMD ["/bin/bash"]

diff --git a/broker/Driver.py b/broker/Driver.py
@@ -114,12 +114,12 @@ def _tools(block_continue):  # noqa
 
             if not output:
                 log(
-                    f"E: provider's registered gmail=[magenta]{gmail}[/magenta] does not match\n"
-                    f"   with the set gdrive's gmail=[magenta]{gdrive_gmail}[/magenta]"
+                    f"E: provider's registered gmail=[m]{gmail}[/m] does not match\n"
+                    f"   with the set gdrive's gmail=[m]{gdrive_gmail}[/m]"
                 )
                 raise QuietExit
 
-            log(f"==> provider_gmail=[magenta]{gmail}")
+            log(f"==> provider_gmail=[m]{gmail}")
 
         if env.IS_IPFS_USE:
             if not os.path.isfile(env.GPG_PASS_FILE):
@@ -211,8 +211,8 @@ def process_logged_job(self, idx):
         index = self.logged_job.args["index"]
         self.job_block_number = self.logged_job["blockNumber"]
         self.cloud_storage_id = self.logged_job.args["cloudStorageID"]
-        log(f"## job_key=[magenta]{job_key}[/magenta] | index={index}", "b")
-        log(f"   received_block_number={self.job_block_number}", "b")
+        log(f"## job_key=[m]{job_key}[/m] | index={index}", "b")
+        log(f"   received_bn={self.job_block_number}", "b")
         log(f"   tx_hash={self.logged_job['transactionHash'].hex()} | log_index={self.logged_job['logIndex']}", "b")
         log(f"   provider={self.logged_job.args['provider']}", "b")
         log(f"   received={self.logged_job.args['received']}", "b")
@@ -302,7 +302,7 @@ def process_logged_jobs(self):
             except Exception as e:
                 print_tb(e)
                 log(str(e))
-                breakpoint()  # DEBUG
+                # breakpoint()  # DEBUG
 
 
 def run_driver(given_bn):

diff --git a/broker/_utils/_log.py b/broker/_utils/_log.py
@@ -24,12 +24,11 @@
 thread_log_files: Dict[str, str] = {}
 custom_theme = Theme(
     {
-        "info": "dim cyan",
-        "warning": "magenta",
+        "info": "bold magenta",
+        # "info": "bold dim magenta",
         "danger": "bold red",
         "b": "bold",
         "m": "magenta",
-        # "magenta": "#ff79c6",
     }
 )
 console = Console(theme=custom_theme)
@@ -64,29 +63,23 @@ def __init__(self):
         self.LOG_FILENAME: Union[str, pathlib.Path] = ""
         self.console: Dict[str, Console] = {}
 
-    def print_color(self, text: str, color=None, is_bold=True, end=None) -> None:
+    def print_color(self, text: str, color=None, is_bold=True, end="\n") -> None:
         """Print string in color format."""
         if text[0:3] in ["==>", "#> ", "## "]:
             if color and text == "==> ":
-                print(f"[bold {color}]{text[0:3]}[/bold {color}]", end="", flush=True)
+                console.print(f"[bold][{color}]{text[0:3]}[{color}][/bold]", end="")
             else:
-                print(f"[bold blue]{text[0:3]}[/bold blue]", end="", flush=True)
+                console.print(f"[bold blue]{text[0:3]}[/bold blue]", end="")
 
             text = text[3:]
         elif text[0:2] == "E:":
-            print("[bold red]E:[/bold red]", end="", flush=True)
+            console.print("[bold red]E:[/bold red]", end="")
             text = text[2:]
 
-        if end is None:
-            if is_bold:
-                print(f"[bold {color}]{text}[/bold {color}]")
-            else:
-                print(f"[{color}]{text}[/{color}]")
+        if is_bold:
+            console.print(f"[bold][{color}]{text}[{color}][/bold]", end=end)
         else:
-            if is_bold:
-                print(f"[bold {color}]{text}[/bold {color}]", end="", flush=True)
-            else:
-                print(f"[{color}]{text}[/{color}]", end="")
+            console.print(f"[{color}]{text}[/{color}]", end=end)
 
     def pre_color_check(self, text, color, is_bold):
         """Check color for substring."""
@@ -165,14 +158,14 @@ def console_ruler(msg="", character="=", color="cyan", fn=""):
         ll.console[fn] = Console(file=open(fn, "a"), force_terminal=True, theme=custom_theme)
 
     if msg:
-        console.rule(f"[bold {color}]{msg}", characters=character)
-        ll.console[fn].rule(f"[bold {color}]{msg}", characters=character)
+        console.rule(f"[bold][{color}]{msg}", characters=character)
+        ll.console[fn].rule(f"[bold][{color}]{msg}", characters=character)
     else:
         console.rule(characters=character)
         ll.console[fn].rule(characters=character)
 
 
-def _log(text, color, is_bold, flush, fn, end, is_write=True, is_output=True):
+def _log(text, color, is_bold, fn, end="\n", is_write=True, is_output=True):
     if not is_output:
         is_print = is_output
     else:
@@ -192,11 +185,8 @@ def _log(text, color, is_bold, flush, fn, end, is_write=True, is_output=True):
         if is_print:
             if not IS_THREADING_MODE_PRINT or threading.current_thread().name == "MainThread":
                 if is_bullet:
-                    print(
-                        f"[bold {_color}]{is_r}{text[:_len]}[/bold {_color}][{color}]{text[_len:]}[/{color}]",
-                        end=end,
-                        flush=flush,
-                    )
+                    _msg = f"[bold][{_color}]{is_r}{text[:_len]}[/{_color}][/bold][{color}]{text[_len:]}[/{color}]"
+                    console.print(_msg, end=end)
                 else:
                     ll.print_color(str(text), color, is_bold=is_bold, end=end)
 
@@ -209,48 +199,37 @@ def _log(text, color, is_bold, flush, fn, end, is_write=True, is_output=True):
         if is_write and IS_WRITE:
             if is_bullet:
                 ll.console[fn].print(
-                    f"[bold {_color}]{is_r}{text[:_len]}[/bold {_color}][{color}]{_text}[/{color}]",
+                    f"[bold][{_color}]{is_r}{text[:_len]}[/{_color}][/bold][{color}]{_text}[/{color}]",
                     end=end,
                     soft_wrap=True,
                 )
             else:
                 if color:
-                    ll.console[fn].print(f"[bold {color}]{_text}[/bold {color}]", end="", soft_wrap=True)
+                    ll.console[fn].print(f"[bold][{color}]{_text}[/{color}][/bold]", end=end, soft_wrap=True)
                 else:
-                    ll.console[fn].print(_text, end="", soft_wrap=True)
+                    ll.console[fn].print(_text, end=end, soft_wrap=True)
     else:
         text_to_write = ""
         if is_bullet:
-            text_to_write = f"[bold {_color}]{is_r}{_text[:_len]}[/bold {_color}][bold]{_text[_len:]}[/bold]"
+            text_to_write = f"[bold][{_color}]{is_r}{_text[:_len]}[/{_color}][/bold][bold]{_text[_len:]}[/bold]"
         else:
             if _color:
                 text_to_write = f"[{_color}]{_text}[/{_color}]"
             else:
                 text_to_write = _text
 
         if is_print:
-            if end == "":
-                print(text_to_write, end="")
-            else:
-                print(text_to_write, flush=flush)
+            console.print(text_to_write, end=end)
 
         if is_write and IS_WRITE:
             ll.console[fn].print(text_to_write, end=end, soft_wrap=True)
 
-    if end is None:
-        if is_write and IS_WRITE:
-            ll.console[fn].print("")
-
-        if color and is_bullet:
-            print()
-
 
 def log(
     text="",
     color=None,
     fn=None,
-    end=None,
-    flush=False,
+    end="\n",
     is_write=True,
     where_back=0,
     is_code=False,
@@ -264,9 +243,10 @@ def log(
     * colors:
     __ https://rich.readthedocs.io/en/latest/appendix/colors.html#appendix-colors
 
-    :param text: string to print
-    :param color: color of the complete string
-    :param fn: filename to write
+    :param end: (str, optional) Character to write at end of output. Defaults to "\\n".
+    :param text: String to print
+    :param color: Color of the complete string
+    :param fn: Filename to write
     """
     is_bold: bool = False
     if color in ["bold", "b"]:
@@ -331,7 +311,7 @@ def log(
         if is_write and IS_WRITE:
             ll.console[fn].print(text)
     else:
-        _log(text, color, is_bold, flush, fn, end, is_write, is_output)
+        _log(text, color, is_bold, fn, end, is_write, is_output)
 
 
 def WHERE(back=0):

diff --git a/broker/_utils/tools.py b/broker/_utils/tools.py
@@ -138,7 +138,7 @@ def print_tb(message=None, is_print_exc=True) -> None:
     sep_terminate = "raise Terminate"
     tb_text = "".join(traceback.format_exc())
     if sep_terminate in tb_text:
-        tb_text = tb_text.split(sep_terminate, 1)[0] + "raise [magenta]Terminate[/magenta]()"
+        tb_text = tb_text.split(sep_terminate, 1)[0] + "raise [m]Terminate[/m]()"
 
     if is_print_exc and tb_text != "NoneType: None\n":
         log(tb_text.rstrip(), "bold", where_back=1)
@@ -167,12 +167,12 @@ def _remove(path: str, is_verbose=False) -> None:
             shutil.rmtree(path)
         else:
             if is_verbose:
-                log(f"warning: {WHERE(1)} Nothing removed, following path does not exist:\n[magenta]{path}")
+                log(f"warning: {WHERE(1)} Nothing removed, following path does not exist:\n[m]{path}")
 
             return
 
         if is_verbose:
-            log(f"#> {WHERE(1)} following path:\n[magenta]{path}[/magenta] is removed")
+            log(f"#> {WHERE(1)} following path:\n[m]{path}[/m] is removed")
     except OSError as e:
         # Suppress the exception if it is a file not found error.
         # Otherwise, re-raise the exception.

diff --git a/broker/bash_scripts/clean_for_new_test.sh b/broker/bash_scripts/clean_for_new_test.sh
@@ -79,7 +79,7 @@ echo "#> Running: ~/ebloc-broker/broker/python_scripts/clean_gdrive.py"
 ~/ebloc-broker/broker/python_scripts/clean_gdrive.py
 echo "[  OK  ]"
 
-for i in `gpg --list-keys --with-colons --fingerprint  | sed -n 's/^fpr:::::::::\([[:alnum:]]\+\):/\1/p'`; do
+for i in `gpg --list-keys --with-colons --fingerprint | sed -n 's/^fpr:::::::::\([[:alnum:]]\+\):/\1/p'`; do
     gpg --batch --delete-key "$i" 2>/dev/null
 done
 

diff --git a/broker/bash_scripts/decrypt_gpg.sh b/broker/bash_scripts/decrypt_gpg.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 
-for dir in */*; do
-    gpg --verbose --batch --yes --output=$(echo $dir | rev | cut -c5- | rev) --pinentry-mode loopback --passphrase-file=/home/alper/.ebloc-broker/.gpg_pass.txt --decrypt "$dir"
+for fn in */*; do
+    echo "$fn"
+    gpg --verbose --batch --yes --output=$(echo $fn | rev | cut -c5- | rev) --pinentry-mode loopback --passphrase-file=/home/alper/.ebloc-broker/.gpg_pass.txt --decrypt "$fn"
 done
 rm */*.diff.gz.gpg
diff --git a/broker/bash_scripts/fetch_ipfs_hashes.sh b/broker/bash_scripts/fetch_ipfs_hashes.sh
@@ -1,5 +1,6 @@
 #!/bin/bash
 
+# run at requester-node
 while read p; do
   ipfs get "$p"
 done <result_ipfs_hashes.txt
@@ -14,3 +15,12 @@ cd ../ipfs_gpg
 rm */*.diff.gz
 rmdir * >/dev/null 2>&1
 cd ..
+
+cd ipfs_gpg
+for fn in */*; do
+    echo "$fn"
+    gpg --verbose --batch --yes --output=$(echo $fn | rev | cut -c5- | rev) --pinentry-mode loopback \
+        --passphrase-file=/home/alper/.ebloc-broker/.gpg_pass.txt --decrypt "$fn"
+done
+rm */*.diff.gz.gpg
+cd ..
diff --git a/broker/bash_scripts/killall.sh b/broker/bash_scripts/killall.sh
@@ -19,4 +19,4 @@ killall python 2> /dev/null
 killall python3 2> /dev/null
 echo "## killall all jobs in squeue"
 squeue | tail -n+2 | awk '{print $1}' | xargs scancel 2> /dev/null
-printf "killall for ebloc-broker test  [  ${GREEN}OK${NC} ]\n"
+printf "killall for ebloc-broker test  [  ${GREEN}OK${NC}  ]\n"
diff --git a/broker/bash_scripts/slurm_mail_prog.sh b/broker/bash_scripts/slurm_mail_prog.sh
@@ -42,7 +42,7 @@ if [[ $event == *"COMPLETED"* ]] || [[ $event == *"FAILED"* ]]; then
     fi
     arg0=$(echo $name | cut -d "$SEP" -f 1)  # job_key
     arg1=$(echo $name | cut -d "$SEP" -f 2)  # index
-    arg2=$(echo $name | cut -d "$SEP" -f 3)  # received_block_number
+    arg2=$(echo $name | cut -d "$SEP" -f 3)  # received_bn
     msg="$state fn=$name\n"
     msg="${msg}./end_code.py $arg0 $arg1 $arg2 \"$name\" $slurm_job_id"
     echo $msg | mail -s "Message Subject" $EMAIL
@@ -57,7 +57,7 @@ if [[ $event == *"TIMEOUT"* ]]; then
     name=$(echo "$c"   | grep -o -P '(?<=Name=).*(?=.sh Failed)')
     arg0=$(echo $name | cut -d "$SEP" -f 1)  # job_key
     arg1=$(echo $name | cut -d "$SEP" -f 2)  # index
-    arg2=$(echo $name | cut -d "$SEP" -f 3)  # received_block_number
+    arg2=$(echo $name | cut -d "$SEP" -f 3)  # received_bn
     msg="TIMEOUT fn=$name\n"
     msg="${msg}./end_code.py $arg0 $arg1 $arg2 \"$name\" $slurm_job_id"
     echo $msg | mail -s "Message Subject" $EMAIL
@@ -72,7 +72,7 @@ if [[ $event == *"CANCELLED"* ]]; then
     name=$(echo "$c"  | grep -o -P '(?<=Name=).*(?=.sh Ended)')
     arg0=$(echo $name | cut -d "$SEP" -f 1)  # job_key
     arg1=$(echo $name | cut -d "$SEP" -f 2)  # index
-    arg2=$(echo $name | cut -d "$SEP" -f 3)  # received_block_number
+    arg2=$(echo $name | cut -d "$SEP" -f 3)  # received_bn
     msg="CANCELLED fn=$name\n"
     msg="${msg}./end_code.py $arg0 $arg1 $arg2 \"$name\" $slurm_job_id"
     echo $msg | mail -s "Message Subject" $EMAIL

diff --git a/broker/drivers/driver_gc.py b/broker/drivers/driver_gc.py
@@ -19,12 +19,10 @@ def main():
     for document in cursor:
         # print(document)
         # TODO: requester paramer as get_storage_duration
-        received_block_number, storage_duration = Ebb.get_job_storage_duration(
-            env.PROVIDER_ID, document["sourceCodeHash"]
-        )
-        end_block_time = received_block_number + storage_duration * cfg.ONE_HOUR_BLOCK_DURATION
+        received_bn, storage_duration = Ebb.get_job_storage_duration(env.PROVIDER_ID, document["sourceCodeHash"])
+        end_block_time = received_bn + storage_duration * cfg.ONE_HOUR_BLOCK_DURATION
         storageID = document["storageID"]
-        if end_block_time < block_number and received_block_number != 0:
+        if end_block_time < block_number and received_bn != 0:
             if storageID in (StorageID.IPFS, StorageID.IPFS_GPG):
                 ipfsHash = document["jobKey"]
                 print(run(["ipfs", "pin", "rm", ipfsHash]))
@@ -39,7 +37,7 @@ def main():
                 print(cached_file_name)
                 _remove(cached_file_name)
 
-            print(received_block_number)
+            print(received_bn)
             coll.delete_one({"jobKey": ipfsHash})
 
 

diff --git a/broker/drivers/gdrive.py b/broker/drivers/gdrive.py
@@ -192,7 +192,7 @@ def get_data_init(self, key, _id, is_job_key=False):
 
         mime_type = gdrive.get_file_info(gdrive_output, _type="Mime")
         folder_name = gdrive.get_file_info(gdrive_output, _type="Name")
-        log(f"==> mime_type=[magenta]{mime_type}")
+        log(f"==> mime_type=[m]{mime_type}")
         if is_job_key:
             # key for the sourceCode tar.gz file is obtained
             try:
@@ -241,7 +241,7 @@ def get_data(self, key, _id, is_job_key=False):
         # folder is already stored by its code_hash
         code_hash = name.replace(".tar.gz", "")
         log(f"==> name={name}")
-        log(f"==> mime_type=[magenta]{mime_type}")
+        log(f"==> mime_type=[m]{mime_type}")
         if _id == 0:
             # source code folder, ignore downloading result-*
             name = f"{name}.tar.gz"