From aca7d02f110406384b46dba367d26d9ef5d94d6b Mon Sep 17 00:00:00 2001 From: rootTHC Date: Thu, 5 Nov 2020 21:02:48 +0000 Subject: [PATCH 01/21] new --- packaging/Makefile | 2 +- packaging/debian/control | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 packaging/debian/control diff --git a/packaging/Makefile b/packaging/Makefile index 5179446d..92a4cfe6 100755 --- a/packaging/Makefile +++ b/packaging/Makefile @@ -1,5 +1,5 @@ -VERSION=1.4.15 +VERSION=1.4.23 BIN_NAME=gsocket PKG_NAME=${BIN_NAME}-${VERSION} diff --git a/packaging/debian/control b/packaging/debian/control new file mode 100644 index 00000000..eb1ae458 --- /dev/null +++ b/packaging/debian/control @@ -0,0 +1 @@ +... From 4ca336470b6f58930591d4c69ceb5b52a30074bf Mon Sep 17 00:00:00 2001 From: SkyperTHC <5938498+SkyperTHC@users.noreply.github.com> Date: Thu, 5 Nov 2020 21:04:32 +0000 Subject: [PATCH 02/21] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 907580b1..04ca9905 100755 --- a/README.md +++ b/README.md @@ -223,7 +223,7 @@ GSOCKET_ARGS="-k MySecret3 -lqD -d 127.1 -p22" /bin/bash -c "exec -a rsyslogp / 8. I did not invent SRP. It's part of OpenSSL :> --- - If netcat is a swiss army knife then gs-netcat is a germanian battle axe... + If netcat is a swiss army knife then gs-netcat is a germanic battle axe... --acpizer/UnitedCrackingForce Join us From ce02f3c47714cb8446de4b50f48deb131022e6b8 Mon Sep 17 00:00:00 2001 From: rootTHC Date: Fri, 6 Nov 2020 04:44:31 +0000 Subject: [PATCH 03/21] test --- configure.ac | 1 + 1 file changed, 1 insertion(+) diff --git a/configure.ac b/configure.ac index 6b37d9e7..ecb84d88 100755 --- a/configure.ac +++ b/configure.ac @@ -141,6 +141,7 @@ fi AC_OUTPUT([Makefile lib/Makefile tools/Makefile man/Makefile]) + echo " ${PACKAGE_NAME}-${PACKAGE_VERSION} has been configured: From f66c44e67899ba4f7204add4abf48807c808ef99 Mon Sep 17 00:00:00 2001 From: rootTHC Date: Tue, 10 Nov 2020 12:42:26 +0000 Subject: [PATCH 04/21] docker --- packaging/docker/gsocket-tor/Dockerfile | 11 +++++++++++ packaging/docker/gsocket-tor/bashrc | 9 +++++++++ packaging/docker/gsocket-tor/make.sh | 4 ++++ packaging/docker/gsocket/Dockerfile | 17 +++++++++++++++++ packaging/docker/gsocket/bashrc | 4 ++++ packaging/docker/gsocket/gs-motd | 17 +++++++++++++++++ packaging/docker/gsocket/make.sh | 4 ++++ 7 files changed, 66 insertions(+) create mode 100644 packaging/docker/gsocket-tor/Dockerfile create mode 100644 packaging/docker/gsocket-tor/bashrc create mode 100755 packaging/docker/gsocket-tor/make.sh create mode 100644 packaging/docker/gsocket/Dockerfile create mode 100644 packaging/docker/gsocket/bashrc create mode 100644 packaging/docker/gsocket/gs-motd create mode 100755 packaging/docker/gsocket/make.sh diff --git a/packaging/docker/gsocket-tor/Dockerfile b/packaging/docker/gsocket-tor/Dockerfile new file mode 100644 index 00000000..6b3a70bf --- /dev/null +++ b/packaging/docker/gsocket-tor/Dockerfile @@ -0,0 +1,11 @@ +FROM hackerschoice/gsocket + +COPY bashrc /root/.bashrc + +WORKDIR /root/ +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends \ + tor \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/ + diff --git a/packaging/docker/gsocket-tor/bashrc b/packaging/docker/gsocket-tor/bashrc new file mode 100644 index 00000000..627bd83b --- /dev/null +++ b/packaging/docker/gsocket-tor/bashrc @@ -0,0 +1,9 @@ + +[[ -f /etc/gs-motd ]] && printf "$(cat /etc/gs-motd)" +PS1='${debian_chroot:+($debian_chroot)}\u@\h-\e[0;32mTOR\e[0m:\e[0;33m\w\e[0m\$ ' +export GSOCKET_SOCKS_IP=127.0.0.1 +export GSOCKET_SOCKS_PORT=9050 +pidof tor >/dev/null || tor --quiet & +echo -e "TOR : \033[1;32menabled\033[0m - to disable execute 'unset GSOCKET_SOCKS_IP'" + + diff --git a/packaging/docker/gsocket-tor/make.sh b/packaging/docker/gsocket-tor/make.sh new file mode 100755 index 00000000..54cbfdf3 --- /dev/null +++ b/packaging/docker/gsocket-tor/make.sh @@ -0,0 +1,4 @@ +#! /bin/bash + +docker build -t hackerschoice/gsocket-tor . + diff --git a/packaging/docker/gsocket/Dockerfile b/packaging/docker/gsocket/Dockerfile new file mode 100644 index 00000000..ebcdc22b --- /dev/null +++ b/packaging/docker/gsocket/Dockerfile @@ -0,0 +1,17 @@ +FROM debian + +# Must be debian compiled binaries: +COPY gs-netcat gs-sftp gs-mount blitz gs_funcs gs_uchroot_so /usr/local/bin/ +COPY gs-motd /etc/ +COPY bashrc /root/.bashrc + +WORKDIR /root/ +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends \ + binutils \ + openssl \ + rsync \ + openssh-server \ + sshfs \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/ diff --git a/packaging/docker/gsocket/bashrc b/packaging/docker/gsocket/bashrc new file mode 100644 index 00000000..f3147818 --- /dev/null +++ b/packaging/docker/gsocket/bashrc @@ -0,0 +1,4 @@ + +[[ -f /etc/gs-motd ]] && printf "$(cat /etc/gs-motd)" +echo -e "TOR : \033[1;31mDISABLED\033[0m - use hackerschoice/gsocket-tor for TOR support." + diff --git a/packaging/docker/gsocket/gs-motd b/packaging/docker/gsocket/gs-motd new file mode 100644 index 00000000..57ff3be5 --- /dev/null +++ b/packaging/docker/gsocket/gs-motd @@ -0,0 +1,17 @@ +# Use this command (from your host OS) to share your hosts's ~/hax directory +# with this image (optional) + [\033[0:33mhost\033[0m ] $ \033[1:34mdocker run --rm -it -v ~/hax:/hax hackerschoice/gsocket\033[0m + +Test your setup: + [\033[0:33mdocker\033[0m] $ \033[1:34mgs-sftp -s thctestserver\033[0m + +Transfer files to a friend who has 'blitz -s foobar -l' running: + [\033[0:33mdocker\033[0m] $ \033[1:34mblitz -s foobar /hax/./mp3/*\033[0m + +Login to a friend's computer who has 'gs-netcat -s foobar -il' running: + [\033[0:33mdocker\033[0m] $ \033[1:34mgs-netcat -s foobar -i\033[0m + +Help : gs-netcat -m | more +Commands: gs-netcat, gs-sftp, gs-mount, blitz +Latest : \033[1;35mhttps://github.com/hackerschoice/gsocket\033[0m +Shoutz : Yogee for ideas & testing\n diff --git a/packaging/docker/gsocket/make.sh b/packaging/docker/gsocket/make.sh new file mode 100755 index 00000000..9d7e4874 --- /dev/null +++ b/packaging/docker/gsocket/make.sh @@ -0,0 +1,4 @@ +#! /bin/bash + +docker build -t hackerschoice/gsocket . + From bf6af9a0b67ba6a02ff45282c3e21281dbf36673 Mon Sep 17 00:00:00 2001 From: rootTHC Date: Tue, 10 Nov 2020 15:28:32 +0000 Subject: [PATCH 05/21] docker --- packaging/docker/gsocket-tor/Dockerfile | 3 +-- packaging/docker/gsocket-tor/bashrc | 9 --------- packaging/docker/gsocket/bashrc | 14 +++++++++++++- packaging/docker/gsocket/gs-motd | 9 +++++++-- 4 files changed, 21 insertions(+), 14 deletions(-) delete mode 100644 packaging/docker/gsocket-tor/bashrc diff --git a/packaging/docker/gsocket-tor/Dockerfile b/packaging/docker/gsocket-tor/Dockerfile index 6b3a70bf..ef907924 100644 --- a/packaging/docker/gsocket-tor/Dockerfile +++ b/packaging/docker/gsocket-tor/Dockerfile @@ -1,11 +1,10 @@ FROM hackerschoice/gsocket -COPY bashrc /root/.bashrc - WORKDIR /root/ RUN apt-get update -y \ && apt-get install -y --no-install-recommends \ tor \ + && touch /root/.gs_with_tor \ && apt-get clean \ && rm -rf /var/lib/apt/lists/ diff --git a/packaging/docker/gsocket-tor/bashrc b/packaging/docker/gsocket-tor/bashrc deleted file mode 100644 index 627bd83b..00000000 --- a/packaging/docker/gsocket-tor/bashrc +++ /dev/null @@ -1,9 +0,0 @@ - -[[ -f /etc/gs-motd ]] && printf "$(cat /etc/gs-motd)" -PS1='${debian_chroot:+($debian_chroot)}\u@\h-\e[0;32mTOR\e[0m:\e[0;33m\w\e[0m\$ ' -export GSOCKET_SOCKS_IP=127.0.0.1 -export GSOCKET_SOCKS_PORT=9050 -pidof tor >/dev/null || tor --quiet & -echo -e "TOR : \033[1;32menabled\033[0m - to disable execute 'unset GSOCKET_SOCKS_IP'" - - diff --git a/packaging/docker/gsocket/bashrc b/packaging/docker/gsocket/bashrc index f3147818..b0dcc7a4 100644 --- a/packaging/docker/gsocket/bashrc +++ b/packaging/docker/gsocket/bashrc @@ -1,4 +1,16 @@ [[ -f /etc/gs-motd ]] && printf "$(cat /etc/gs-motd)" -echo -e "TOR : \033[1;31mDISABLED\033[0m - use hackerschoice/gsocket-tor for TOR support." + +export SHELL=/bin/bash +export TERM=xterm-256color + +if [[ -f ~/.gs_with_tor ]]; then + PS1='${debian_chroot:+($debian_chroot)}\u@\h-\e[0;32mTOR\e[0m:\e[0;33m\w\e[0m\$ ' + export GSOCKET_SOCKS_IP=127.0.0.1 + export GSOCKET_SOCKS_PORT=9050 + pidof tor >/dev/null || { tor --quiet & } + echo -e "TOR : \033[1;32menabled\033[0m - to disable execute 'unset GSOCKET_SOCKS_IP'" +else + echo -e "TOR : \033[1;31mDISABLED\033[0m - use hackerschoice/gsocket-tor for TOR support." +fi diff --git a/packaging/docker/gsocket/gs-motd b/packaging/docker/gsocket/gs-motd index 57ff3be5..c8b533b8 100644 --- a/packaging/docker/gsocket/gs-motd +++ b/packaging/docker/gsocket/gs-motd @@ -1,6 +1,8 @@ -# Use this command (from your host OS) to share your hosts's ~/hax directory +# Use this command (on your host OS) to share your hosts's ~/hax directory # with this image (optional) - [\033[0:33mhost\033[0m ] $ \033[1:34mdocker run --rm -it -v ~/hax:/hax hackerschoice/gsocket\033[0m + [\033[0:33mhost\033[0m ] $ \033[1:34mdocker run --rm -it --name gs -v ~/hax:/hax hackerschoice/gsocket\033[0m +# And this command to have a second shell: + [\033[0:33mhost\033[0m ] $ \033[1:34mdocker exec -it gs bash\033[0m Test your setup: [\033[0:33mdocker\033[0m] $ \033[1:34mgs-sftp -s thctestserver\033[0m @@ -8,6 +10,9 @@ Test your setup: Transfer files to a friend who has 'blitz -s foobar -l' running: [\033[0:33mdocker\033[0m] $ \033[1:34mblitz -s foobar /hax/./mp3/*\033[0m +FTP to a friend (securely) who has 'gs-sftp -s foobar -l' running: + [\033[0:33mdocker\033[0m] $ \033[1:34mgs-sftp -s foobar\033[0m + Login to a friend's computer who has 'gs-netcat -s foobar -il' running: [\033[0:33mdocker\033[0m] $ \033[1:34mgs-netcat -s foobar -i\033[0m From 620e8a3e6a11a19c0cb15c3b2a8467c8ba5a88be Mon Sep 17 00:00:00 2001 From: rootTHC Date: Wed, 9 Dec 2020 14:15:38 +0000 Subject: [PATCH 06/21] filetransfer init --- tools/filetransfer-test.c | 228 ++++++++++++++++ tools/filetransfer.c | 543 ++++++++++++++++++++++++++++++++++++++ tools/filetransfer.h | 99 +++++++ 3 files changed, 870 insertions(+) create mode 100644 tools/filetransfer-test.c create mode 100644 tools/filetransfer.c create mode 100644 tools/filetransfer.h diff --git a/tools/filetransfer-test.c b/tools/filetransfer-test.c new file mode 100644 index 00000000..88033ecc --- /dev/null +++ b/tools/filetransfer-test.c @@ -0,0 +1,228 @@ +/* + * Test program to test gs filetransfer sub-system used by gs-netcat. + * + * Client: socat SYSTEM:'./filetransfer-test c test1.dat' TCP:127.1:1337 + * Server: socat TCP-LISTEN:1337 SYSTEM:'./filetransfer-test s' + * ...or in one line: + +mkdir -p test +rm -rf test/test*.dat +socat SYSTEM:'./filetransfer-test c test4k.dat test8k.dat 2>client.log' SYSTEM:'(cd test; ../filetransfer-test s 2>server.log)' + +rm -rf test/test*.dat +dd bs=1k count=5 if=test8k.dat of=test/test8k.dat +socat SYSTEM:'./filetransfer-test c test4k.dat test8k.dat 2>client.log' SYSTEM:'(cd test; ../filetransfer-test s 2>server.log)' +md5sum -q test8k.dat test/test8k.dat + */ + +#include "common.h" +#include "filetransfer.h" +#include "utils.h" + +#define BUF_LEN (1024) + +static GS_PKT pkt; +static GS_FT ft; +static int is_server; +static uint8_t wbuf[GS_PKT_MAX_SIZE]; +static size_t wlen; + +// static void file_error(uint32_t id, const char *str); + +/* SERVER receiving PUT from client */ +static void +pkt_cb_put(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) +{ + struct _gs_ft_put *p = (struct _gs_ft_put *)data; + struct _gs_ft_put hdr; + + if (len < sizeof hdr + 1) + return; // protocol error + + // HEXDUMP(data, len); + if (data[len - 1] != '\0') + return; // protocol error + + memcpy(&hdr, data, sizeof hdr); + // FIXME: sanitize file name + DEBUGF("CB-PUT - umask 0x%x, id %u, '%s'\n", ntohl(hdr.umask), ntohl(hdr.id), p->name); + GS_FT_add_file(&ft, ntohl(hdr.id), (char *)p->name, ntohl(hdr.umask)); +} + +static void +pkt_cb_switch(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) +{ + struct _gs_ft_switch *p = (struct _gs_ft_switch *)data; + struct _gs_ft_switch hdr; + + if (len < sizeof hdr) + return; // protocol error + + memcpy(&hdr, data, sizeof hdr); + GS_FT_switch(&ft, ntohl(hdr.id), ntohll(hdr.fsize)); +} + +/* SERVER receiving DATA from client */ +static void +pkt_cb_data(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) +{ + // DEBUGF("Data chn %d, len %zu\n", chn, len); + GS_FT_data(&ft, data, len); +} + +/* CLIENT receiving ACCEPT from server */ +static void +pkt_cb_accept(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) +{ + struct _gs_ft_accept hdr; + + if (len < sizeof hdr) + return; // protocol error + + memcpy(&hdr, data, sizeof hdr); + DEBUGF("acc offset = %lld\n", ntohll(hdr.offset)); + + GS_FT_accept(&ft, ntohl(hdr.id), ntohll(hdr.offset)); +} + +// Client & Server +static void +pkt_cb_error(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) +{ + struct _gs_ft_error *p = (struct _gs_ft_error *)data; + struct _gs_ft_error hdr; + + + if (len < sizeof hdr + 1) + return; // protocol error + if (data[len - 1] != '\0') + return; // protocol error + + memcpy(&hdr, data, sizeof hdr); + DEBUGF_R("CB-ERROR: id %u code %u (%s)\n", ntohl(hdr.id), hdr.code, p->str); + + GS_FT_del_file(&ft, ntohl(hdr.id)); +} + +int +mk_packet(void) +{ + struct gs_pkt_chn_hdr *hdr = (struct gs_pkt_chn_hdr *)wbuf; + int pkt_type; + size_t sz; + + sz = GS_FT_packet(&ft, wbuf + sizeof *hdr, sizeof wbuf - sizeof *hdr, &pkt_type); + XASSERT(sz <= sizeof wbuf - sizeof *hdr, "Oops, GS_FT_packet() to long. sz=%zu.\n", sz); + + // DEBUGF("sz %zu, type %d\n", sz, pkt_type); + switch (pkt_type) + { + case GS_FT_TYPE_NONE: + return 0; + case GS_FT_TYPE_PUT: + hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_PUT); + break; + case GS_FT_TYPE_ERROR: + hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_ERROR); + break; + case GS_FT_TYPE_SWITCH: + hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_SWITCH); + break; + case GS_FT_TYPE_ACCEPT: + hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_ACCEPT); + break; + case GS_FT_TYPE_DATA: + hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_DATA); + break; + case GS_FT_TYPE_DONE: + DEBUGF_G("all done??\n"); + return -1; + default: + ERREXIT("unknown type %d\n", pkt_type); + } + + hdr->esc = GS_PKT_ESC; + hdr->len = htons(sz); + + wlen = sizeof *hdr + sz; + + return 0; +} + +int +main(int arc, char *argv[]) +{ + int is_extra_puts = 0; + uint8_t src[BUF_LEN]; + uint8_t dst[2 * sizeof src]; + ssize_t sz; + size_t dsz; + int ret; + + GS_library_init(stderr, stderr); + gopt.err_fp = stderr; + gopt.log_fp = stderr; + + GS_PKT_init(&pkt); + GS_FT_init(&ft); + + GS_PKT_assign_chn(&pkt, GS_FT_CHN_ERROR, pkt_cb_error, NULL); + if (*argv[1] == 'c') + { + GS_PKT_assign_chn(&pkt, GS_FT_CHN_ACCEPT, pkt_cb_accept, NULL); + // Add files to queue... + char **ptr = &argv[2]; + while (*ptr != NULL) + { + if (GS_FT_put(&ft, *ptr) != 0) + DEBUGF_Y("Not found: %s\n", *ptr); + ptr++; + } + // GS_FT_put(&ft, "test1k.dat"); + // GS_FT_put(&ft, "test4k.dat"); + } else { + GS_PKT_assign_chn(&pkt, GS_FT_CHN_PUT, pkt_cb_put, NULL); + GS_PKT_assign_chn(&pkt, GS_FT_CHN_DATA, pkt_cb_data, NULL); + GS_PKT_assign_chn(&pkt, GS_FT_CHN_SWITCH, pkt_cb_switch, NULL); + is_server = 1; + } + + while (1) + { + // If there is data to write then write data first. + if (wlen > 0) + { + sz = write(1, wbuf, wlen); + // DEBUGF("write %zu\n", sz); + if (sz <= 0) + ERREXIT("write()\n"); + wlen = 0; + } + if (mk_packet() != 0) + { + if (is_server) + break; + + if (is_extra_puts >= 1) + break; + is_extra_puts++; + if (GS_FT_put(&ft, "test1k-extra1.dat") != 0) + DEBUGF_Y("Not found: test1k-extra1.dat\n"); + if (GS_FT_put(&ft, "test1k-extra2.dat") != 0) + DEBUGF_Y("Not found: test1k-extra2.dat\n"); + continue; + } + if (wlen > 0) + continue; + + sz = read(0, src, sizeof src); + // DEBUGF_B("read %zu\n", sz); + if (sz <= 0) + ERREXIT("read()\n"); + ret = GS_PKT_decode(&pkt, src, sz, dst, &dsz); + if (ret != 0) + ERREXIT("GS_PKT_decode()\n"); + } + + return 0; +} diff --git a/tools/filetransfer.c b/tools/filetransfer.c new file mode 100644 index 00000000..5b88fcea --- /dev/null +++ b/tools/filetransfer.c @@ -0,0 +1,543 @@ + +#include "common.h" +#include "utils.h" +#include "filetransfer.h" + +static void ft_del(GS_LIST_ITEM *li); + +// FIXME: must make sure that server cant request transfer from client! + +void +GS_FT_init(GS_FT *ft) +{ + memset(ft, 0, sizeof *ft); + GS_LIST_init(&ft->fqueue, 0); + GS_LIST_init(&ft->fputs, 0); + GS_LIST_init(&ft->faccepted, 0); + + GS_LIST_init(&ft->fadded, 0); + GS_LIST_init(&ft->freceiving, 0); +} + +#if 0 +uint16_t +GS_FT_mk_put(GS_FT *ft, void *dst, size_t len, const char *name) +{ + size_t n; + struct _gs_ft_put *hdr = (struct _gs_ft_put *)dst; + + if (len < sizeof *hdr + 1) + return 0; // protcol error + + hdr->umask = 3133; + hdr->id = htonl(ft->g_id); + ft->g_id += 1; + + n = MIN(len - sizeof *hdr, strlen(name) + 1); + // DEBUGF("name len %zu + hdr %zu\n", n, sizeof *hdr); + memcpy(hdr->name, name, n - 1); + hdr->name[n] = '\0'; + + return sizeof *hdr + n; +} + +uint16_t +GS_FT_mk_error(GS_FT *ft, void *dst, size_t len, uint32_t id, uint8_t code, const char *str) +{ + size_t n; + struct _gs_ft_error *hdr = (struct _gs_ft_error *)dst; + + if (len < sizeof *hdr + 1) + return 0; // protcol error + + hdr->id = htonl(id); + hdr->code = code; + + n = MIN(len - sizeof *hdr, strlen(str) + 1); + // DEBUGF("name len %zu + hdr %zu\n", n, sizeof *hdr); + memcpy(hdr->str, str, n - 1); + hdr->str[n] = '\0'; + + return sizeof *hdr + n; +} +#endif + + +/* + * SERVER + * Return < 0 on error. + * Return bytes already here. + */ +int64_t +GS_FT_add_file(GS_FT *ft, uint32_t id, const char *fname, uint32_t umask) +{ + // FIXME: get absolute path and save absolute path + + int ret; + int64_t fz = 0; + struct stat res; + ret = stat(fname, &res); + if (ret != 0) + { + if (errno != ENOENT) + return -GS_FT_ERR_PERM; // Exists but stat() failed + } else { + // FILE exists + if (!S_ISREG(res.st_mode)) + return -GS_FT_ERR_BADF; // Not a regular file + // File is a regular file. + fz = res.st_size; + } + + struct _gs_ft_file *f; + f = calloc(1, sizeof *f); + f->name = strdup(fname); + f->umask = umask; + char buf[PATH_MAX]; + snprintf(buf, sizeof buf, "%s/%s", getcwd(buf, sizeof buf), fname); + f->realname = strdup(buf); + f->offset = fz; + + f->li = GS_LIST_add(&ft->fadded, NULL, f, id); + + return fz; +} + +/* + * CLIENT: Add this file to queue. + */ +int +GS_FT_put(GS_FT *ft, const char *fname) +{ + int ret; + struct stat res; + + // Get absolute and real path as CWD may change before + // upload starts. + char *realfname; + realfname = realpath(fname, NULL); + if (realfname == NULL) + return -3; + + ret = stat(fname, &res); + if (ret != 0) + return -1; + + if (!S_ISREG(res.st_mode)) + return -2; + + struct _gs_ft_file *f; + f = calloc(1, sizeof *f); + f->name = strdup(fname); + f->realname = realfname; + f->fsize = res.st_size; + f->umask = 31337; // FIXME + + f->li = GS_LIST_add(&ft->fqueue, NULL, f, ft->g_id); + ft->g_id += 1; + ft->is_put_done = 0; + + return 0; +} + +// SERVER +void +GS_FT_data(GS_FT *ft, const void *data, size_t len) +{ + struct _gs_ft_file *f = ft->active_receiving; + size_t sz; + + if (f == NULL) + { + DEBUGF_R("Oops. Receivng data but no active receiving file\n"); + return; + } + + + XASSERT(f->fp != NULL, "fp is NULL\n"); + if (f->offset + len > f->fsize) + { + DEBUGF_R("Oops. More data than we want!\n"); + len = f->fsize - f->offset; + } + + sz = fwrite(data, 1, len, f->fp); + f->offset += sz; + + if (sz != len) + { + // FIXME: return error to peer that write has failed... + goto completed; + } + + if (f->offset < f->fsize) + return; // still data missing... + +completed: + ft->active_receiving = NULL; + ft_del(f->li); +} + +// CLIENT +void +GS_FT_accept(GS_FT *ft, uint32_t id, int64_t offset) +{ + GS_LIST_ITEM *li; + + li = GS_LIST_by_id(&ft->fputs, id); + if (li == NULL) + { + DEBUGF_R("Unknown file id %u\n", id); + return; // actually a protocol error.... + } + + GS_LIST_move(&ft->faccepted, li); + + struct _gs_ft_file *f = (struct _gs_ft_file *)li->data; + f->offset = offset; +} + +// SERVER +void +GS_FT_switch(GS_FT *ft, uint32_t id, int64_t fsize) +{ + GS_LIST_ITEM *li; + + li = GS_LIST_by_id(&ft->freceiving, id); + if (li == NULL) + { + DEBUGF_R("Unknown file id %u\n", id); + return; // actually a protocol error.... + } + + // Switch from active receiving file to new file + struct _gs_ft_file *new = (struct _gs_ft_file *)li->data; + + DEBUGF_W("Switching to id %u '%s' (got %"PRId64", want %"PRId64")\n", id, new->name, new->offset, fsize); + + if ((ft->active_receiving != NULL) && (ft->active_receiving != new)) + { + fclose(ft->active_receiving->fp); + } + + ft->active_receiving = NULL; + + // Existing file is larger to fsize send by peer. + if (new->offset > fsize) + goto err; + + if (fsize == new->offset) + { + DEBUGF_G("file already fully received\n"); + ft_del(li); + return; + } + + new->fsize = fsize; + new->fp = fopen(new->realname, "a"); + // new->fp = fopen(new->realname, "r+"); + if (new->fp == NULL) + { + DEBUGF("fopen(%s) failed: %s\n", new->realname, strerror(errno)); + goto err; + } + + ft->active_receiving = new; + return; + +err: + // FIXME: Transmit error back to peer. + // Use a List-queue with error messages (or just 1 error message?) + ft_del(li); +} +#if 0 +/* + * Remote reported error on this file. Remove from active list. + */ +void +GS_FT_error(GS_FT *ft, uint32_t id) +{ + GS_LIST_ITEM *li; + + li = GS_LIST_by_id(&ft->flist, id); + if (li == NULL) + return; + f = (struct _gs_ft_file *)li->data; + + ft_del(li); + + ft->n_denied += 1; +} +#endif + +static void +file_free(struct _gs_ft_file *f) +{ + XFREE(f->name); + XFREE(f->realname); + XFREE(f); +} +/* + * Remove file from queue. + */ +static void +ft_del(GS_LIST_ITEM *li) +{ + struct _gs_ft_file *f = (struct _gs_ft_file *)li->data; + + if (f->fp != NULL) + { + fclose(f->fp); + f->fp = NULL; + } + + file_free(f); + GS_LIST_del(li); +} + +/* + * Error received from Server or Client. + * Remove and free item. + */ +void +GS_FT_del_file(GS_FT *ft, uint32_t id) +{ + GS_LIST_ITEM *li; + struct _gs_ft_file *f; + + li = GS_LIST_by_id(&ft->fqueue, id); + if (li == NULL) + { + li = GS_LIST_by_id(&ft->fputs, id); + if (li == NULL) + { + li = GS_LIST_by_id(&ft->fadded, id); + if (li == NULL) + { + li = GS_LIST_by_id(&ft->freceiving, id); + if (li == NULL) + { + DEBUGF_R("id %u not found\n", id); + return; // not found + } + } + } + } + + f = (struct _gs_ft_file *)li->data; + if (f == ft->active_put_file) + ft->active_put_file = NULL; + + file_free(f); + GS_LIST_del(li); +} + + +/* + * Create an error package. Return 0 on error or the length of patcket + * on success.. + */ +static int +mk_error(void *dst, size_t len, uint32_t id, uint8_t code) +{ + struct _gs_ft_error err; + + if (len < sizeof (err) + 1) + { + DEBUGF_R("Oops, not enough space?\n"); + return 0; + } + + memset(&err, 0, sizeof err); + err.id = htonl(id); + err.code = code; + memcpy(dst, &err, sizeof err); + ((uint8_t *)dst)[sizeof err] = '\0'; + + return sizeof err + 1; // 0-terminate +} + + +static int +ft_error(GS_FT *ft, void *dst, size_t len, GS_LIST_ITEM *li, uint8_t code, int *pkt_type) +{ + int ret; + + XASSERT(li != NULL, "Oops. li is NULL\n"); + ret = mk_error(dst, len, li->id, code); + *pkt_type = GS_FT_TYPE_ERROR; + + ft_del(li); + if ((ft->fputs.n_items == 0) && (ft->faccepted.n_items == 0)) + ft->is_put_done = 1; + + return ret; +} +/* + * Create a data packet (from job/queue that need attendtion). + * Return the length. + * Return 0 if no data needs to be written or an error occured. + * check pkt_type to determine the error code. + * + * Set TYPE to SWITCH or DATA depending on the type of packet created. + * Set to DONE when done and NONE when nothing is to be done. + */ +size_t +GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) +{ + struct _gs_ft_file *f; + size_t sz; + + // DEBUGF("puts %d, accepted %d, done %d\n", ft->fputs.n_items, ft->faccepted.n_items, ft->is_put_done); + if (ft->is_put_done) + { + *pkt_type = GS_FT_TYPE_DONE; + return 0; + } + + *pkt_type = GS_FT_TYPE_NONE; + + // Check if any files in the queue that need to be 'put' + // on offer to the remote side (and then awaiting 'accept'). + if (ft->fqueue.n_items > 0) + { + GS_LIST_ITEM *li = NULL; + + DEBUGF("%d items in queue (waiting for put to be send)\n", ft->fqueue.n_items); + li = GS_LIST_next(&ft->fqueue, NULL); + f = (struct _gs_ft_file *)li->data; + + struct _gs_ft_put put; + struct _gs_ft_put *p = (struct _gs_ft_put *)dst; + memset(&put, 0, sizeof put); + put.umask = htonl(f->umask); + put.id = htonl(li->id); + memcpy(dst, &put, sizeof put); + + sz = MIN(len - sizeof put, strlen(f->name) + 1); + // DEBUGF("name len %zu + hdr %zu\n", n, sizeof *hdr); + memcpy(p->name, f->name, sz - 1); + p->name[sz - 1] = '\0'; + + GS_LIST_move(&ft->fputs, li); + *pkt_type = GS_FT_TYPE_PUT; + return sizeof put + sz; + } + + if (ft->fadded.n_items > 0) + { + GS_LIST_ITEM *li = NULL; + li = GS_LIST_next(&ft->fadded, NULL); + f = (struct _gs_ft_file *)li->data; + + struct _gs_ft_accept acc; + memset(&acc, 0, sizeof acc); + acc.id = htonl(li->id); + acc.offset = htonll(f->offset); + memcpy(dst, &acc, sizeof acc); + // HERE: Server. Inform client that we accepted. + + GS_LIST_move(&ft->freceiving, li); + *pkt_type = GS_FT_TYPE_ACCEPT; + return sizeof acc; + } + + if (ft->faccepted.n_items > 0) + { + // Here: No files waiting to offer ('put') to remote. + if (ft->active_put_file == NULL) + { + GS_LIST_ITEM *li = NULL; + int ret; + + // FIXME: find file with least amount of outstanding data + li = GS_LIST_next(&ft->faccepted, NULL); + f = (struct _gs_ft_file *)li->data; + + // Open file and seek to location + f->fp = fopen(f->name, "r"); + if (f->fp == NULL) + return ft_error(ft, dst, len, f->li, GS_FT_ERR_PERM, pkt_type); + + ret = fseek(f->fp, 0, SEEK_END); + if (ret != 0) + return ft_error(ft, dst, len, f->li, GS_FT_ERR_BADF, pkt_type); + f->fsize = ftell(f->fp); + // Peer already has all data (or more). + if (f->fsize <= f->offset) + return ft_error(ft, dst, len, f->li, GS_FT_ERR_NODATA, pkt_type); + + ret = fseek(f->fp, f->offset, SEEK_SET); + if (ret != 0) + return ft_error(ft, dst, len, f->li, GS_FT_ERR_BADF, pkt_type); + + struct _gs_ft_switch sw; + memset(&sw, 0, sizeof sw); + sw.id = htonl(li->id); + sw.fsize = htonll(f->fsize); + memcpy(dst, &sw, sizeof sw); + + ft->active_put_file = f; + *pkt_type = GS_FT_TYPE_SWITCH; + + return (sizeof sw); + } + + // HERE: active file + *pkt_type = GS_FT_TYPE_DATA; + f = ft->active_put_file; + + DEBUGF_C("at %ld\n", ftell(f->fp)); + sz = fread(dst, 1, len, f->fp); + if (sz <= 0) + { + // HERE: Read error or file may have shrunk. + ft->active_put_file = NULL; + return ft_error(ft, dst, len, f->li, GS_FT_ERR_BADF, pkt_type); + } + f->offset += sz; + + // File completed. No more data. + if (f->offset >= f->fsize) + { + ft_del(f->li); + if ((ft->fputs.n_items == 0) && (ft->faccepted.n_items == 0)) + ft->is_put_done = 1; + ft->active_put_file = NULL; + } + return sz; + } + + return 0; +} + +static void +free_gsl(GS_LIST *gsl) +{ + GS_LIST_ITEM *li = GS_LIST_next(gsl, NULL); + + for (; li != NULL; li = GS_LIST_next(gsl, li)) + { + struct _gs_ft_file *f = (struct _gs_ft_file *)li->data; + + file_free(f); + GS_LIST_del(li); + } +} + +void +GS_FT_free(GS_FT *ft) +{ + free_gsl(&ft->fqueue); + free_gsl(&ft->fputs); + free_gsl(&ft->faccepted); + + free_gsl(&ft->fadded); + free_gsl(&ft->freceiving); + + ft->active_put_file = NULL; +} + + + + diff --git a/tools/filetransfer.h b/tools/filetransfer.h new file mode 100644 index 00000000..7c041092 --- /dev/null +++ b/tools/filetransfer.h @@ -0,0 +1,99 @@ +#ifndef __GS_FILETRANSFER_H__ +#define __GS_FILETRANSFER_H__ 1 + +#define GS_FT_CHN_PUT (0) +#define GS_FT_CHN_ACCEPT (1) +#define GS_FT_CHN_DATA (3) +#define GS_FT_CHN_ERROR (4) +#define GS_FT_CHN_SWITCH (5) + +struct _gs_ft_file +{ + GS_LIST_ITEM *li; + char *name; + char *realname; // realpath() resolved + uint32_t umask; + FILE *fp; + off_t offset; // Offset to start transmitting + off_t fsize; // Total file size (from client) +}; + +typedef struct +{ + GS_LIST fqueue; // Client List of files to be transfered + GS_LIST fputs; // Client list of files we requested transfer (put sent) + GS_LIST faccepted; // Client List of accepted files + + GS_LIST fadded; // Server Side list of ready files + GS_LIST freceiving; // Server Side list of receiving files + int g_id; + struct _gs_ft_file *active_put_file; // Current active file + struct _gs_ft_file *active_receiving; // + int is_put_done; // No more files to transmit +} GS_FT; + + +// CLIENT -> Server: put a file to server. +struct _gs_ft_put +{ + uint32_t umask; + uint32_t id; + uint8_t reserved[32 - 4 - 4]; + uint8_t name[0]; // 0-terminated file name +} __attribute__((__packed__)); + +// SERVER -> Client: Accept file. +struct _gs_ft_accept +{ + uint32_t id; + uint8_t res[4]; + int64_t offset; + uint8_t crcNOTUSED[4]; + uint8_t res2[4]; +} __attribute__((__packed__)); + +// CLIENT -> Server: Following data is for this file id. +struct _gs_ft_switch +{ + uint32_t id; + uint8_t res[4]; + int64_t fsize; // total file size +} __attribute__((__packed__)); + +struct _gs_ft_error +{ + uint8_t code; + uint8_t res[3]; // reerved + uint32_t id; + uint8_t str[0]; // 0-terminated error string (not used) +} __attribute__((__packed__)); + +#define GS_FT_ERR_UNKNOWN (0) +#define GS_FT_ERR_PERM (1) +#define GS_FT_ERR_NOENT (2) +#define GS_FT_ERR_BADF (9) +#define GS_FT_ERR_NODATA (10) + +void GS_FT_init(GS_FT *ft); +void GS_FT_free(GS_FT *ft); +int64_t GS_FT_add_file(GS_FT *ft, uint32_t id, const char *fname, uint32_t umask); +int GS_FT_put(GS_FT *ft, const char *fname); +void GS_FT_switch(GS_FT *ft, uint32_t id, int64_t fsize); +void GS_FT_accept(GS_FT *ft, uint32_t id, int64_t offset); +void GS_FT_data(GS_FT *ft, const void *data, size_t len); +void GS_FT_del_file(GS_FT *ft, uint32_t id); +size_t GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type); + +#define GS_FT_TYPE_NONE (0) +#define GS_FT_TYPE_SWITCH (1) +#define GS_FT_TYPE_DATA (2) +#define GS_FT_TYPE_ERROR (3) +#define GS_FT_TYPE_PUT (4) +#define GS_FT_TYPE_DONE (5) +#define GS_FT_TYPE_ACCEPT (6) + +// Packet Bulding functions +//uint16_t GS_FT_mk_put(GS_FT *ft, void *dst, size_t len, const char *fname); +//uint16_t GS_FT_mk_error(GS_FT *ft, void *dst, size_t len, uint32_t id, uint8_t code, const char *str); + +#endif /* !__GS_FILETRANSFER_H__*/ From c8671084e5f7c5fe7f1428f243d2a461691fc14a Mon Sep 17 00:00:00 2001 From: rootTHC Date: Mon, 21 Dec 2020 16:14:23 +0000 Subject: [PATCH 07/21] almost there! --- configure.ac | 2 +- include/gsocket/gsocket.h | 2 + include/gsocket/list.h | 2 + include/gsocket/packet.h | 19 ++ lib/gsocket-util.c | 32 ++ lib/list.c | 27 +- lib/packet.c | 6 + tools/4_gs-netcat.c | 1 + tools/Makefile.am | 9 +- tools/common.h | 13 +- tools/console.c | 239 +++++++++++---- tools/filetransfer-test.c | 87 +++--- tools/filetransfer.c | 620 ++++++++++++++++++++++++++++---------- tools/filetransfer.h | 98 +++++- tools/run_ft_tests.sh | 166 ++++++++++ tools/utils.c | 22 -- 16 files changed, 1040 insertions(+), 305 deletions(-) create mode 100755 tools/run_ft_tests.sh diff --git a/configure.ac b/configure.ac index 0e3af241..bdfce4e6 100755 --- a/configure.ac +++ b/configure.ac @@ -143,7 +143,7 @@ if test x"$STATIC" = x"yes"; then fi AS_IF([test x$debug = xtrue], AC_SUBST(PROGRAMS_TEST_LIB, "list-test event-test")) -AS_IF([test x$debug = xtrue], AC_SUBST(PROGRAMS_TEST_TOOLS, "packet-test readline-test console_display-test")) +AS_IF([test x$debug = xtrue], AC_SUBST(PROGRAMS_TEST_TOOLS, "packet-test readline-test console_display-test filetransfer-test")) AC_OUTPUT([Makefile lib/Makefile tools/Makefile man/Makefile]) diff --git a/include/gsocket/gsocket.h b/include/gsocket/gsocket.h index 86a41e33..d5c78ee9 100644 --- a/include/gsocket/gsocket.h +++ b/include/gsocket/gsocket.h @@ -371,6 +371,8 @@ uint32_t GS_hton(const char *hostname); void GS_SELECT_FD_SET_W(GS *gs); void GS_daemonize(FILE *logfp); +uint64_t GS_usec(void); +void GS_format_bps(char *dst, size_t size, int64_t bytes); const char *GS_gen_secret(void); const char *GS_user_secret(GS_CTX *ctx, const char *file, const char *sec_str); diff --git a/include/gsocket/list.h b/include/gsocket/list.h index c32330d6..7ee4bec4 100644 --- a/include/gsocket/list.h +++ b/include/gsocket/list.h @@ -26,10 +26,12 @@ typedef struct int GS_LIST_init(GS_LIST *gsl, int opt); GS_LIST_ITEM *GS_LIST_add(GS_LIST *gsl, GS_LIST_ITEM *src_li, void *data, uint64_t id); +void GS_LIST_move(GS_LIST *gsl, GS_LIST_ITEM *li); int GS_LIST_del(GS_LIST_ITEM *li); int GS_LIST_del_all(GS_LIST *gsl, int deep); GS_LIST_ITEM *GS_LIST_next(GS_LIST *gsl, GS_LIST_ITEM *li); GS_LIST_ITEM *GS_LIST_by_pos(GS_LIST *gsl, int pos); +GS_LIST_ITEM *GS_LIST_by_id(GS_LIST *gsl, uint64_t id); void GS_LIST_relink(GS_LIST_ITEM *li, uint64_t id); diff --git a/include/gsocket/packet.h b/include/gsocket/packet.h index 70aaeaa7..94e03218 100644 --- a/include/gsocket/packet.h +++ b/include/gsocket/packet.h @@ -8,6 +8,10 @@ #ifndef GS_PKT_ESC # define GS_PKT_ESC 0xFB // escape character #endif + +#define GS_PKT_MSG_HDR_LEN (2) +#define GS_PKT_CHN_HDR_LEN (4) + typedef void (*gspkt_cb_t)(uint8_t type, const uint8_t *data, size_t len, void *arg); /* @@ -25,7 +29,22 @@ typedef struct void *args[256]; } GS_PKT; +struct gs_pkt_msg_hdr +{ + uint8_t esc; + uint8_t type; +} __attribute__((__packed__)); + +struct gs_pkt_chn_hdr +{ + uint8_t esc; + uint8_t type; + uint16_t len; +} __attribute__((__packed__)); + + #define GS_PKT_IS_CHANNEL(a) (((a) >> 7) & 0x01) +#define GS_PKT_CHN2TYPE(a) (GS_PKT_MAX_MSG + a) int GS_PKT_init(GS_PKT *pkt); int GS_PKT_close(GS_PKT *pkt); diff --git a/lib/gsocket-util.c b/lib/gsocket-util.c index 68dc0a54..e4a07d20 100755 --- a/lib/gsocket-util.c +++ b/lib/gsocket-util.c @@ -172,6 +172,38 @@ GS_user_secret(GS_CTX *ctx, const char *sec_file, const char *sec_str) return ptr; } +uint64_t +GS_usec(void) +{ + struct timeval tv; + + gettimeofday(&tv, NULL); + return GS_TV_TO_USEC(&tv); +} + +// 7 readable characters + 0 +static const char unit[] = "BKMGT"; +void +GS_format_bps(char *dst, size_t size, int64_t bytes) +{ + int i; + + if (bytes < 1000) + { + snprintf(dst, size, "%3d.0 B", (int)bytes); + return; + } + bytes *= 100; + + for (i = 0; bytes >= 100*1000 && unit[i] != 'T'; i++) + bytes = (bytes + 512) / 1024; + snprintf(dst, size, "%3lld.%1lld%c%s", + (long long) (bytes + 5) / 100, + (long long) (bytes + 5) / 10 % 10, + unit[i], + i ? "B" : " "); +} + /* * Duplicate the process. Child returns. Parent monitors child * and re-spwans child if it dies. diff --git a/lib/list.c b/lib/list.c index b40982f5..4516b5e4 100644 --- a/lib/list.c +++ b/lib/list.c @@ -106,7 +106,18 @@ GS_LIST_relink(GS_LIST_ITEM *li, uint64_t id) } /* - * Add an item to the list. Sorted by id. Lowest at top. + * Move ITEM to a new another list. + */ +void +GS_LIST_move(GS_LIST *gsl, GS_LIST_ITEM *li) +{ + if (li->gsl == gsl) + return; + gs_list_unlink(li); + GS_LIST_add(gsl, li, li->data, li->id); +} +/* + * Add an item to the list. Sorted by id. Smallest at top. */ GS_LIST_ITEM * GS_LIST_add(GS_LIST *gsl, GS_LIST_ITEM *src_li, void *data, uint64_t id) @@ -155,6 +166,20 @@ GS_LIST_by_pos(GS_LIST *gsl, int pos) return li; } +GS_LIST_ITEM * +GS_LIST_by_id(GS_LIST *gsl, uint64_t id) +{ + GS_LIST_ITEM *li = GS_LIST_next(gsl, NULL); + + for (; li != NULL; li = GS_LIST_next(gsl, li)) + { + if (id == li->id) + return li; + } + + return NULL; +} + int GS_LIST_del(GS_LIST_ITEM *del_li) { diff --git a/lib/packet.c b/lib/packet.c index a42e5f9b..6fb4e542 100644 --- a/lib/packet.c +++ b/lib/packet.c @@ -178,7 +178,10 @@ GS_PKT_decode_single(GS_PKT *pkt, const uint8_t *src, size_t slen, uint8_t *dst, /* First character after ESC is TYPE || CHN */ pkt->type = *src; if (pkt->type == 0x0) + { + DEBUGF_R("ERROR TYPE IS 0x00\n"); return -1; // Protocol Error (FATAL) + } if (GS_PKT_IS_CHANNEL(pkt->type)) { @@ -221,6 +224,9 @@ GS_PKT_decode_single(GS_PKT *pkt, const uint8_t *src, size_t slen, uint8_t *dst, return src - src_orig; } +/* + * Return 0 on success. + */ int GS_PKT_decode(GS_PKT *pkt, const uint8_t *src, size_t slen, uint8_t *dst, size_t *dlen) { diff --git a/tools/4_gs-netcat.c b/tools/4_gs-netcat.c index a343412b..c038554d 100755 --- a/tools/4_gs-netcat.c +++ b/tools/4_gs-netcat.c @@ -169,6 +169,7 @@ cb_atexit(void) { CONSOLE_reset(); stty_reset(); + printf("\n[Bye]\n"); } /* *********************** FD READ / WRITE ******************************/ diff --git a/tools/Makefile.am b/tools/Makefile.am index d891461d..33f0658c 100755 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -1,5 +1,5 @@ noinst_PROGRAMS = gs-helloworld gs-pipe gs-full-pipe @PROGRAMS_TEST_TOOLS@ -EXTRA_PROGRAMS = packet-test readline-test console_display-test +EXTRA_PROGRAMS = packet-test readline-test console_display-test filetransfer-test bin_PROGRAMS = gs-netcat gs_uchroot_so gs_helloworld_SOURCES = 1_gs-helloworld.c utils.c @@ -11,7 +11,7 @@ gs_pipe_LDADD = ../lib/libgsocket.a gs_full_pipe_SOURCES = 3_gs-full-pipe.c utils.c gs_full_pipe_LDADD = ../lib/libgsocket.a -gs_netcat_SOURCES = 4_gs-netcat.c utils.c socks.c console.c ids.c event_mgr.c pkt_mgr.c console_display.c +gs_netcat_SOURCES = 4_gs-netcat.c utils.c socks.c console.c ids.c event_mgr.c pkt_mgr.c console_display.c filetransfer.c gs_netcat_LDADD = ../lib/libgsocket.a dist_bin_SCRIPTS = blitz gs-sftp gs-mount gs_funcs @@ -29,6 +29,9 @@ readline_test_LDADD = ../lib/libgsocket.a console_display_test_SOURCES = console_display.c console_display-test.c utils.c console_display_test_LDADD = ../lib/libgsocket.a +filetransfer_test_SOURCES = filetransfer.c filetransfer-test.c utils.c +filetransfer_test_LDADD = ../lib/libgsocket.a + EXTRA_DIST = run_all_tests.sh -noinst_HEADERS = common.h utils.h socks.h console.h ids.h event_mgr.h pkt_mgr.h gs-netcat.h console_display.h man_gs-netcat.h +noinst_HEADERS = common.h utils.h socks.h console.h ids.h event_mgr.h pkt_mgr.h gs-netcat.h console_display.h filetransfer.h man_gs-netcat.h AM_CFLAGS = -I../include diff --git a/tools/common.h b/tools/common.h index 3e20b1f5..cee772c8 100755 --- a/tools/common.h +++ b/tools/common.h @@ -262,16 +262,23 @@ extern struct _gopt gopt; fd = -1; \ } while (0) +#define XFCLOSE(fp) do { \ + if (fp == NULL) { DEBUGF_R("*** WARNING *** Closing BAD fp\n"); break; } \ + fclose(fp); \ + fp = NULL; \ +} while (0) + + #define XFD_SET(fd, set) do { \ if (fd < 0) { DEBUGF_R("WARNING: FD_SET(%d, )\n", fd); break; } \ FD_SET(fd, set); \ } while (0) #ifdef DEBUG -# define HEXDUMP(a, len) do { \ - int n = 0; \ +# define HEXDUMP(a, _len) do { \ + size_t _n = 0; \ xfprintf(gopt.err_fp, "%s:%d HEX ", __FILE__, __LINE__); \ - while (n < len) xfprintf(gopt.err_fp, "%2.2x", ((unsigned char *)a)[n++]); \ + while (_n < (_len)) xfprintf(gopt.err_fp, "%2.2x", ((unsigned char *)a)[_n++]); \ xfprintf(gopt.err_fp, "\n"); \ } while (0) # define HEXDUMPF(a, len, m...) do{xfprintf(gopt.err_fp, m); HEXDUMP(a, len);}while(0) diff --git a/tools/console.c b/tools/console.c index d48ec84b..5d86804e 100644 --- a/tools/console.c +++ b/tools/console.c @@ -31,7 +31,7 @@ static void console_start(void); static void console_stop(void); static int console_command(struct _peer *p, const char *cmd); -static void get_cursor_pos(int *row, int *col); +// static void get_cursor_pos(int *row, int *col); static uint8_t chr_last; static int tty_fd = -1; @@ -122,6 +122,7 @@ console_init(int fd) return; } +#if 0 static int readstring(int fd, char *buf, size_t sz, const char *str) { @@ -174,6 +175,7 @@ readstring(int fd, char *buf, size_t sz, const char *str) // } return rv; } +#endif static ssize_t tty_write(void *src, size_t len) @@ -192,7 +194,7 @@ static int is_cursor_in_console; static void console_cursor_off(void) { - tty_write("\x1B[u", 3); // Move cursor to upper tier + tty_write("\x1B""8", 2); // Move cursor to upper tier is_cursor_in_console = 0; } @@ -247,7 +249,6 @@ mk_statusbar(void) else VSADDF(ptr, end, vc, "[%3dms]", (int)ms); - DEBUGF("Total User: %u\n", ci.n_users); if (ci.load >= 1000) VSADDF(ptr, end, vc, "[Load %02.02f][User(%u) ", (float)ci.load / 100, ci.n_users); else @@ -269,8 +270,9 @@ mk_statusbar(void) VSADDF(ptr, end, vc, "*idle*"); VSADDF(ptr, end, vc, "]"); + // BYTES/sec char buf[8]; - format_bps(buf, sizeof buf, (int64_t)ci.bps); + GS_format_bps(buf, sizeof buf, (int64_t)ci.bps); VSADDF(ptr, end, vc, "[%s/s]", buf); // Fill until end @@ -337,6 +339,7 @@ CONSOLE_resize(struct _peer *p) char *end = buf + sizeof buf; int delta; + DEBUGF_R("RESIZE to %d;%d\n", gopt.winsize.ws_col, gopt.winsize.ws_row); if (gopt.is_console) { delta = gopt.winsize.ws_row - gopt.winsize_prev.ws_row; @@ -348,7 +351,7 @@ CONSOLE_resize(struct _peer *p) // Assign scrolling area. Will reset cursor to 1;1 SXPRINTF(ptr, end - ptr, "\x1b[1;%dr", gopt.winsize.ws_row - GS_CONSOLE_ROWS); // Restore cursor to upper tier - SXPRINTF(ptr, end - ptr, "\x1B[u"); + SXPRINTF(ptr, end - ptr, "\x1B""8"); // Clear screen SXPRINTF(ptr, end - ptr, "\x1B[J"); } @@ -356,13 +359,15 @@ CONSOLE_resize(struct _peer *p) if (delta < 0) { // Shorter: + DEBUGF_R("Shorter. ScrollingArea to %d\n", gopt.winsize.ws_row - GS_CONSOLE_ROWS); if (is_cursor_in_console) { - SXPRINTF(ptr, end - ptr, "\x1B[u\x1B[J"); + DEBUGF_R("cursor is IN console\n"); + SXPRINTF(ptr, end - ptr, "\x1B""8""\x1B[J"); SXPRINTF(ptr, end - ptr, "\x1B[%dA", 0-delta); - SXPRINTF(ptr, end - ptr, "\x1B[s"); + SXPRINTF(ptr, end - ptr, "\x1B""7"); SXPRINTF(ptr, end - ptr, "\x1b[1;%dr", gopt.winsize.ws_row - GS_CONSOLE_ROWS); - SXPRINTF(ptr, end - ptr, "\x1B[u"); + SXPRINTF(ptr, end - ptr, "\x1B""8"); // WORKING // SXPRINTF(ptr, end - ptr, "\x1B[u\x1B[J"); @@ -372,9 +377,10 @@ CONSOLE_resize(struct _peer *p) // SXPRINTF(ptr, end - ptr, "\x1B[u"); } else { // WORKING + DEBUGF_R("cursor is UPPER TIER\n"); SXPRINTF(ptr, end - ptr, "\x1B[%dS", 0-delta); SXPRINTF(ptr, end - ptr, "\x1b[1;%dr", gopt.winsize.ws_row - GS_CONSOLE_ROWS); - SXPRINTF(ptr, end - ptr, "\x1B[u\x1B[%dA", 0-delta); + SXPRINTF(ptr, end - ptr, "\x1B""8""\x1B[%dA", 0-delta); } } tty_write(buf, ptr - buf); @@ -417,7 +423,7 @@ GS_sb_draw(int force) static void -GS_prompt_draw(int row, int force) +GS_prompt_draw(int force) { char buf[512]; char *ptr = buf;; @@ -428,7 +434,7 @@ GS_prompt_draw(int row, int force) ci.is_prompt_redraw_needed = 0; ptr = buf; - SXPRINTF(ptr, end - ptr, "\x1B[%d;1f" GS_CONSOLE_PROMPT "%s", row + GS_CONSOLE_ROWS, rl.vline); + SXPRINTF(ptr, end - ptr, "\x1B[%d;1f" GS_CONSOLE_PROMPT "%s", gopt.winsize.ws_row /*last*/, rl.vline); tty_write(buf, ptr - buf); } @@ -436,7 +442,7 @@ GS_prompt_draw(int row, int force) * Position active cursor to user input in console */ static void -GS_prompt_cursor(int row) +GS_prompt_cursor(void) { char buf[64]; char *ptr = buf; @@ -449,20 +455,17 @@ GS_prompt_cursor(int row) static void console_draw(int fd, int force) { - int row = gopt.winsize.ws_row - (GS_CONSOLE_ROWS - 1); - if (gopt.is_console == 0) return; - DEBUGF_R("CONSOLE DRAW (force=%d)\n", force); - // GS_condis_pos(row + 1, gopt.winsize.ws_col); + // DEBUGF_R("CONSOLE DRAW (force=%d)\n", force); int cursor_to_prompt = 0; cursor_to_prompt += ci.is_sb_redraw_needed; cursor_to_prompt += gs_condis.is_redraw_needed; if (is_cursor_in_console == 0) - tty_write("\x1B[s", 3); + tty_write("\x1B""7", 2); // Status Bar (Normally black on blue) GS_sb_draw(force); @@ -471,15 +474,15 @@ console_draw(int fd, int force) GS_condis_draw(&gs_condis, force); // Prompt - GS_prompt_draw(row, force); + GS_prompt_draw(force); // Restore cursor position if (is_cursor_in_console == 0) { - tty_write("\x1B[u", 3); + tty_write("\x1B""8", 2); } else { if (cursor_to_prompt) - GS_prompt_cursor(row); + GS_prompt_cursor(); } } @@ -542,7 +545,7 @@ CONSOLE_check_esc(uint8_t c, uint8_t *submit) { int esc; - DEBUGF_Y("key = 0x%02x\n", c); + // DEBUGF_Y("key = 0x%02x\n", c); if (chr_last == GS_CONSOLE_ESC) { if (check_arrow(&esc, c) == 0) @@ -606,11 +609,13 @@ CONSOLE_reset(void) if (tty_fd < 0) return; + DEBUGF_R("Resetting scolling area (rows %d)\n", gopt.winsize.ws_row); if (gopt.is_console) { - /* Reset scrolling area */ ptr = buf; - SXPRINTF(ptr, end - ptr, "\x1B[r"); + // Reset scrolling area. Will set cursor to 1;1. + // SXPRINTF(ptr, end - ptr, "\x1B[r"); + SXPRINTF(ptr, end - ptr, "\x1b[1;%dr", gopt.winsize.ws_row); /* Move cursor to last line */ SXPRINTF(ptr, end - ptr, "\x1B[9999;9999H"); /* Restore cursor */ @@ -634,9 +639,119 @@ static struct _pat cls_pattern[] = { {"\x1B[J", 3, 1}, // Clear screen from cursor down {"\x1B[2J", 4, 1}, // Clear entire screen {"\x1B[?1049h", 8, 2}, // Switch Alternate Screen Buffer (clears screen) - {"\x1B[?1049l", 8, 3} // Switch Normal Screen Buffer (clears screen) + {"\x1B[?1049l", 8, 3}, // Switch Normal Screen Buffer (clears screen) + {"\x1B""c", 2, 4} // Reset terminal to initial state }; +#ifdef DEBUG +/* + * For debugging only. + * Find the next ansi sequence. + * Return the length of the sequence. Set 'is_ansi' if it's an ansi sequence. + * The sequence can be a non-ansi sequence (e.g. normal data) or an ansi sequnce. + */ +static size_t +ansi_next(void *data, size_t len, int *is_ansi) +{ + static int in_esc; + static int in_esc_pos; + uint8_t *src = (uint8_t *)data; + uint8_t *src_orig = src; + uint8_t *src_end = src + len; + + in_esc_pos = 0; + in_esc = 0; + *is_ansi = 0; + if (*src == '\x1B') + { + src++; + *is_ansi = 1; + in_esc = 1; + } + + for (; src < src_end; src++) + { + if (in_esc == 0) + { + if (*src != '\x1B') + continue; + + *is_ansi = 0; + return src - src_orig; + } + + // HERE: in escape sequence + // Check when escape finishes + if (*src == '\x1B') + break; + in_esc_pos++; + + if (in_esc_pos == 1) + { + // Check if multi character esc sequence + if (*src == '[') + continue; + if (*src == '(') + continue; + if (*src == ')') + continue; + if (*src == '#') + continue; // Esc-#2 + if (*src == '6') + continue; // Esc-6n + if (*src == '5') + continue; + if (*src == '0') + continue; + if (*src == '3') + continue; + + break; + } + + if (in_esc_pos >= 2) + { + if ((*src >= '0') && (*src <= '9')) + continue; + if (*src == ';') + continue; + if (*src == '?') + continue; + + break; + } + + break; + } + return src - src_orig + 1; +} + +// For debugging +static void +ansi_output(void *data, size_t len) +{ + uint8_t *src = (uint8_t *)data; + uint8_t *src_end = src + len; + int is_ansi; + size_t n; + char buf[64]; + + while (src < src_end) + { + n = ansi_next(src, src_end - src, &is_ansi); + XASSERT(n > 0, "n is 0\n"); + if (is_ansi) + { + snprintf(buf, sizeof buf, "%.*s", (int)(n - 1), src + 1); + DEBUGF_B("ansi: %s\n", buf); + } + else + HEXDUMP(src, n); + src += n; + } +} +#endif + static uint8_t cls_buf[8]; static size_t cls_pos; /* @@ -656,12 +771,12 @@ static size_t cls_pos; * amount => Amount of data save to process (remaining is part of an * unfinished ansi sequence). */ -static int in_esc; -static int in_esc_pos; static void ansi_parse(void *data, size_t len, size_t *amount, int *cls_code) { + static int in_esc; + static int in_esc_pos; uint8_t *src = (uint8_t *)data; uint8_t *src_orig = src; uint8_t *src_end = src + len; @@ -732,8 +847,8 @@ ansi_parse(void *data, size_t len, size_t *amount, int *cls_code) cls_buf[cls_pos] = *src; cls_pos++; - // Any sequence we are interested in is at least 3 chars long - if (cls_pos < 3) + // Any sequence we are interested in is at least 2 chars long + if (cls_pos < 2) goto skip; //Check if any ESC sequence matches @@ -794,6 +909,9 @@ ansi_write(int fd, void *data, size_t len, int *cls_code) if (write(fd, data, amount) != amount) return -1; +#ifdef DEBUG + ansi_output(data, amount); +#endif if (amount < len) { @@ -844,7 +962,7 @@ CONSOLE_write(int fd, void *data, size_t len) /* Move cursor to upper tier if cursor inside console */ if (is_cursor_in_console) - tty_write("\x1B[u", 3); // Restore cursor + tty_write("\x1B""8", 2); // Restore cursor ssize_t sz; sz = ansi_write(fd, data, len, &is_detected_clearscreen); @@ -858,15 +976,17 @@ CONSOLE_write(int fd, void *data, size_t len) // HEXDUMP(data, MIN(16, len)); if (is_cursor_in_console) - tty_write("\x1B[s", 3); // Save cursor position + tty_write("\x1B""7", 2); // Save cursor position + + // Now check if console needs to be re-drawn if (is_detected_clearscreen == 2) { // Switch to Alternate Screen Buffer detected is_console_before_sb = gopt.is_console; } - // DEBUGF_G("cls = %d before = %d, now %d\n", is_detected_clearscreen, is_console_before_sb, gopt.is_console); + // DEBUGF_G("cls = %d iscon-before = %d, iscon-now %d\n", is_detected_clearscreen, is_console_before_sb, gopt.is_console); if (is_detected_clearscreen == 3) { // Switched to Normal Screen Buffer detected @@ -880,6 +1000,13 @@ CONSOLE_write(int fd, void *data, size_t len) } } + if (is_detected_clearscreen == 4) + { + DEBUGF_R("RESET of terminal detected.\n"); + if (gopt.is_console) + console_start(); + } + if (gopt.is_console == 0) return sz; @@ -950,6 +1077,7 @@ CONSOLE_readline(struct _peer *p, void *data, size_t len) return 1; } +#if 0 static void get_cursor_pos(int *row, int *col) { @@ -966,6 +1094,7 @@ get_cursor_pos(int *row, int *col) DEBUGF_G("Current Cursor row=%d col=%d\n", *row, *col); } +#endif /* * Set up terminal to display console (e.g. scroll upper tier up @@ -980,43 +1109,17 @@ console_start(void) int row; row = gopt.winsize.ws_row - GS_CONSOLE_ROWS; -#if 0 - // Get cursor's current location. - // int rv; - int current_row; - int current_col; - get_cursor_pos(¤t_row, ¤t_col); - - // Current cursor is inside console's space. Scroll up... - if (current_row > row) - SXPRINTF(ptr, end - ptr, "\x1b[%dS", current_row - row); - - // Reduce scrolling area (will reset cursor to 0;0) - SXPRINTF(ptr, end - ptr, "\x1b[1;%dr", row); - - // We scrolled up. Cursor was in console area. Set to last row. - int new_row = current_row; // Default: Leave cursor where it was - if (current_row > row) - new_row = row; // Set cursor to last row of new scrolling area - - // Adjust cursor to new location after [r moved it to 0;0 - SXPRINTF(ptr, end - ptr, "\x1b[%d;%dH", new_row, current_col); - - // Save the cursor location from upper tier - SXPRINTF(ptr, end - ptr, "\x1B[s"); -#else - int i; // Scroll up i lines for (i = 0; i < GS_CONSOLE_ROWS; i++) SXPRINTF(ptr, end - ptr, "\x1B""D"); - // Move cursor up as well. Save cursor thereafter. - SXPRINTF(ptr, end - ptr, "\x1B[%dA\x1B[s", GS_CONSOLE_ROWS); - // Reset scrolling area. Will set cursor to 1;1. + // Move cursor up. Then save cursor pos. + SXPRINTF(ptr, end - ptr, "\x1B[%dA\x1B""7", GS_CONSOLE_ROWS); + // Set scrolling area. Will set cursor to 1;1. + DEBUGF("Setting Scrolling area to %d\n", row); SXPRINTF(ptr, end - ptr, "\x1b[1;%dr", row); // Restore cursor to saved location - SXPRINTF(ptr, end - ptr, "\x1B[u"); -#endif + SXPRINTF(ptr, end - ptr, "\x1B""8"); tty_write(buf, ptr - buf); } @@ -1037,7 +1140,7 @@ console_stop(void) // Reset scroll size SXPRINTF(ptr, end - ptr, "\x1B[r"); // Restore cursor to upper tier (shell) - SXPRINTF(ptr, end - ptr, "\x1B[u"); + SXPRINTF(ptr, end - ptr, "\x1B""8"); tty_write(buf, ptr - buf); is_cursor_in_console = 0; } @@ -1050,7 +1153,6 @@ hard_quit(void) { CONSOLE_reset(); stty_reset(); - printf("\n[Bye]\n"); exit(0); // hard exit. } @@ -1067,6 +1169,15 @@ CONSOLE_action(struct _peer *p, uint8_t key) if (key == 'q') hard_quit(); +#ifdef DEBUG + if (key == 'l') + { + DEBUGF_B("redraw\n"); + mk_statusbar(); + console_draw(p->fd_out, 1); + } +#endif + if (key == 'c') { /* Trigger: Send new window size to peer */ diff --git a/tools/filetransfer-test.c b/tools/filetransfer-test.c index 88033ecc..246e8ffc 100644 --- a/tools/filetransfer-test.c +++ b/tools/filetransfer-test.c @@ -1,18 +1,11 @@ /* * Test program to test gs filetransfer sub-system used by gs-netcat. * - * Client: socat SYSTEM:'./filetransfer-test c test1.dat' TCP:127.1:1337 - * Server: socat TCP-LISTEN:1337 SYSTEM:'./filetransfer-test s' - * ...or in one line: mkdir -p test rm -rf test/test*.dat -socat SYSTEM:'./filetransfer-test c test4k.dat test8k.dat 2>client.log' SYSTEM:'(cd test; ../filetransfer-test s 2>server.log)' +socat SYSTEM:'./filetransfer-test c test4k.dat test8k.dat 2>client.log' SYSTEM:'(cd test; ../filetransfer-test s 2>../server.log)' -rm -rf test/test*.dat -dd bs=1k count=5 if=test8k.dat of=test/test8k.dat -socat SYSTEM:'./filetransfer-test c test4k.dat test8k.dat 2>client.log' SYSTEM:'(cd test; ../filetransfer-test s 2>server.log)' -md5sum -q test8k.dat test/test8k.dat */ #include "common.h" @@ -27,8 +20,6 @@ static int is_server; static uint8_t wbuf[GS_PKT_MAX_SIZE]; static size_t wlen; -// static void file_error(uint32_t id, const char *str); - /* SERVER receiving PUT from client */ static void pkt_cb_put(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) @@ -39,27 +30,20 @@ pkt_cb_put(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) if (len < sizeof hdr + 1) return; // protocol error - // HEXDUMP(data, len); - if (data[len - 1] != '\0') - return; // protocol error - memcpy(&hdr, data, sizeof hdr); - // FIXME: sanitize file name - DEBUGF("CB-PUT - umask 0x%x, id %u, '%s'\n", ntohl(hdr.umask), ntohl(hdr.id), p->name); - GS_FT_add_file(&ft, ntohl(hdr.id), (char *)p->name, ntohl(hdr.umask)); + GS_FT_add_file(&ft, ntohl(hdr.id), (char *)p->name, len - sizeof hdr - 1, ntohll(hdr.fsize), ntohl(hdr.mtime), ntohl(hdr.fperm)); } static void pkt_cb_switch(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) { - struct _gs_ft_switch *p = (struct _gs_ft_switch *)data; struct _gs_ft_switch hdr; if (len < sizeof hdr) return; // protocol error memcpy(&hdr, data, sizeof hdr); - GS_FT_switch(&ft, ntohl(hdr.id), ntohll(hdr.fsize)); + GS_FT_switch(&ft, ntohl(hdr.id), /*ntohll(hdr.fsize), */ntohll(hdr.offset)); } /* SERVER receiving DATA from client */ @@ -80,9 +64,7 @@ pkt_cb_accept(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) return; // protocol error memcpy(&hdr, data, sizeof hdr); - DEBUGF("acc offset = %lld\n", ntohll(hdr.offset)); - - GS_FT_accept(&ft, ntohl(hdr.id), ntohll(hdr.offset)); + GS_FT_accept(&ft, ntohl(hdr.id), ntohll(hdr.offset_dst)); } // Client & Server @@ -92,16 +74,40 @@ pkt_cb_error(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) struct _gs_ft_error *p = (struct _gs_ft_error *)data; struct _gs_ft_error hdr; - if (len < sizeof hdr + 1) return; // protocol error - if (data[len - 1] != '\0') - return; // protocol error memcpy(&hdr, data, sizeof hdr); - DEBUGF_R("CB-ERROR: id %u code %u (%s)\n", ntohl(hdr.id), hdr.code, p->str); + GS_FT_status(&ft, ntohl(hdr.id), hdr.code, (char *)p->str, len - sizeof hdr - 1); +} + +// Output total stats +static void +stats_total(GS_FT_stats_total *st) +{ + DEBUGF_Y("Speed : %s\n", st->speed_str); + DEBUGF_Y("Amount : %"PRIu64"/%"PRIu64"usec\n", st->xfer_amount, st->xfer_duration); + DEBUGF_Y("Success: %d\n", st->n_files_success); + DEBUGF_Y("Errors : %d\n", st->n_files_error); +} - GS_FT_del_file(&ft, ntohl(hdr.id)); +// On file transfer completion (for each file) +static void +cb_stats(struct _gs_ft_stats *s) +{ + DEBUGF_C("%u stats: %s\n", s->id, s->f->name); + DEBUGF_C("Speed: %s\n", s->speed_str); + + stats_total(&ft.stats_total); +} + +// Status and Error messages +static void +cb_status(void *ft_ptr, struct _gs_ft_status *s) +{ + DEBUGF_M("Status: %u\n", s->code); + DEBUGF_M("error : '%s'\n", s->err_str); + DEBUGF_M("File : %s\n", s->file->name); } int @@ -118,9 +124,12 @@ mk_packet(void) switch (pkt_type) { case GS_FT_TYPE_NONE: + // Nothing to write...waiting for peer's reply. + DEBUGF_G("TYPE NONE\n"); return 0; case GS_FT_TYPE_PUT: hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_PUT); + // HEXDUMP(wbuf, sizeof *hdr + sz); break; case GS_FT_TYPE_ERROR: hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_ERROR); @@ -135,7 +144,8 @@ mk_packet(void) hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_DATA); break; case GS_FT_TYPE_DONE: - DEBUGF_G("all done??\n"); + // CLIENT only. Server always keeps listening. + DEBUGF_G("All done.\n"); return -1; default: ERREXIT("unknown type %d\n", pkt_type); @@ -149,6 +159,7 @@ mk_packet(void) return 0; } + int main(int arc, char *argv[]) { @@ -164,11 +175,11 @@ main(int arc, char *argv[]) gopt.log_fp = stderr; GS_PKT_init(&pkt); - GS_FT_init(&ft); GS_PKT_assign_chn(&pkt, GS_FT_CHN_ERROR, pkt_cb_error, NULL); if (*argv[1] == 'c') { + GS_FT_init(&ft, cb_stats, cb_status); GS_PKT_assign_chn(&pkt, GS_FT_CHN_ACCEPT, pkt_cb_accept, NULL); // Add files to queue... char **ptr = &argv[2]; @@ -178,9 +189,8 @@ main(int arc, char *argv[]) DEBUGF_Y("Not found: %s\n", *ptr); ptr++; } - // GS_FT_put(&ft, "test1k.dat"); - // GS_FT_put(&ft, "test4k.dat"); } else { + GS_FT_init(&ft, NULL, cb_status); GS_PKT_assign_chn(&pkt, GS_FT_CHN_PUT, pkt_cb_put, NULL); GS_PKT_assign_chn(&pkt, GS_FT_CHN_DATA, pkt_cb_data, NULL); GS_PKT_assign_chn(&pkt, GS_FT_CHN_SWITCH, pkt_cb_switch, NULL); @@ -198,11 +208,16 @@ main(int arc, char *argv[]) ERREXIT("write()\n"); wlen = 0; } - if (mk_packet() != 0) + + ret = mk_packet(); + + if ((is_server == 0) && (ret != 0)) { - if (is_server) - break; + // No more files to transfer + // (All data send. Not waiting for any reply). + break; + // HERE: test adding files after transfer completed... if (is_extra_puts >= 1) break; is_extra_puts++; @@ -216,7 +231,6 @@ main(int arc, char *argv[]) continue; sz = read(0, src, sizeof src); - // DEBUGF_B("read %zu\n", sz); if (sz <= 0) ERREXIT("read()\n"); ret = GS_PKT_decode(&pkt, src, sz, dst, &dsz); @@ -224,5 +238,8 @@ main(int arc, char *argv[]) ERREXIT("GS_PKT_decode()\n"); } + // stats_total(&ft.stats_total); + GS_FT_free(&ft); + return 0; } diff --git a/tools/filetransfer.c b/tools/filetransfer.c index 5b88fcea..0ba80339 100644 --- a/tools/filetransfer.c +++ b/tools/filetransfer.c @@ -4,87 +4,90 @@ #include "filetransfer.h" static void ft_del(GS_LIST_ITEM *li); +static void qerr_add(GS_FT *ft, uint32_t id, uint8_t code, const char *str); +static void mk_stats_total(GS_FT *ft); +static mode_t GS_fperm2mode(uint32_t u); +static uint32_t GS_mode2fperm(mode_t m); -// FIXME: must make sure that server cant request transfer from client! + +#if 0 +0 FIXME: must make sure that server cant request transfer from client! +- test when file becomes unavaialble after after it was added. +- Sym Link +- max buffer size 64 macro somewhere needed +- queue up all 'put' requests and send as 1 large message or loop around write() while we can. +- What do we do if a pathname/filename does not fit into wbuf? Filename can be rather long (like 4k?) + and larger than channel buffer + (add files until it does not fit. Return to caller the ID that failed and let caller decide + if he likes to remove that ID from our list or just let caller try again until it fits...) + is there a limt? + + +- implement GS_FT_get() + +TEST CASES: +1. pathname + filename 4096 long +#2. dest file not writeable +#3. re-start transmission +4. same file in command line +5. src file can not be opened or read error. +6. write to symlink +7. zero file size +8. retain timestamp +#endif void -GS_FT_init(GS_FT *ft) +GS_FT_init(GS_FT *ft, gsft_cb_stats_t func_stats, gsft_cb_status_t func_status) { memset(ft, 0, sizeof *ft); GS_LIST_init(&ft->fqueue, 0); GS_LIST_init(&ft->fputs, 0); GS_LIST_init(&ft->faccepted, 0); + GS_LIST_init(&ft->fcompleted, 0); GS_LIST_init(&ft->fadded, 0); GS_LIST_init(&ft->freceiving, 0); -} - -#if 0 -uint16_t -GS_FT_mk_put(GS_FT *ft, void *dst, size_t len, const char *name) -{ - size_t n; - struct _gs_ft_put *hdr = (struct _gs_ft_put *)dst; - if (len < sizeof *hdr + 1) - return 0; // protcol error + GS_LIST_init(&ft->qerrs, 0); - hdr->umask = 3133; - hdr->id = htonl(ft->g_id); - ft->g_id += 1; - - n = MIN(len - sizeof *hdr, strlen(name) + 1); - // DEBUGF("name len %zu + hdr %zu\n", n, sizeof *hdr); - memcpy(hdr->name, name, n - 1); - hdr->name[n] = '\0'; - - return sizeof *hdr + n; + ft->func_stats = func_stats; + ft->func_status = func_status; } -uint16_t -GS_FT_mk_error(GS_FT *ft, void *dst, size_t len, uint32_t id, uint8_t code, const char *str) -{ - size_t n; - struct _gs_ft_error *hdr = (struct _gs_ft_error *)dst; - - if (len < sizeof *hdr + 1) - return 0; // protcol error - - hdr->id = htonl(id); - hdr->code = code; - - n = MIN(len - sizeof *hdr, strlen(str) + 1); - // DEBUGF("name len %zu + hdr %zu\n", n, sizeof *hdr); - memcpy(hdr->str, str, n - 1); - hdr->str[n] = '\0'; - - return sizeof *hdr + n; -} -#endif - - /* * SERVER * Return < 0 on error. - * Return bytes already here. + * Return 0 otherwise. */ -int64_t -GS_FT_add_file(GS_FT *ft, uint32_t id, const char *fname, uint32_t umask) +int +GS_FT_add_file(GS_FT *ft, uint32_t id, const char *fname, size_t len, int64_t fsize, uint32_t mtime, uint32_t fperm) { - // FIXME: get absolute path and save absolute path - int ret; int64_t fz = 0; + + if (fname[len] != '\0') + return -1; // protocol error. Not 0 terminated. + + // FIXME: sanitize file name + DEBUGF_Y("#%u ADD-FILE - fperm 0%o, '%s'\n", id, fperm, fname); + struct stat res; ret = stat(fname, &res); + if (ret != 0) { if (errno != ENOENT) + { + qerr_add(ft, id, GS_FT_ERR_PERM, NULL); return -GS_FT_ERR_PERM; // Exists but stat() failed + } } else { // FILE exists if (!S_ISREG(res.st_mode)) + { + qerr_add(ft, id, GS_FT_ERR_BADF, NULL); return -GS_FT_ERR_BADF; // Not a regular file + } // File is a regular file. fz = res.st_size; } @@ -92,7 +95,9 @@ GS_FT_add_file(GS_FT *ft, uint32_t id, const char *fname, uint32_t umask) struct _gs_ft_file *f; f = calloc(1, sizeof *f); f->name = strdup(fname); - f->umask = umask; + f->mode = GS_fperm2mode(fperm); + f->mtime = mtime; + f->fsize = fsize; char buf[PATH_MAX]; snprintf(buf, sizeof buf, "%s/%s", getcwd(buf, sizeof buf), fname); f->realname = strdup(buf); @@ -100,11 +105,71 @@ GS_FT_add_file(GS_FT *ft, uint32_t id, const char *fname, uint32_t umask) f->li = GS_LIST_add(&ft->fadded, NULL, f, id); - return fz; + return 0; +} + + +struct _mperm +{ + mode_t mode; + uint32_t perm; +}; + +struct _mperm x_mperm[] = { + {S_ISUID, 04000}, + {S_ISGID, 02000}, + {S_ISVTX, 01000}, + + {S_IRUSR, 00400}, + {S_IWUSR, 00200}, + {S_IXUSR, 00100}, + + {S_IRGRP, 00040}, + {S_IWGRP, 00020}, + {S_IXGRP, 00010}, + + {S_IROTH, 00004}, + {S_IWOTH, 00002}, + {S_IXOTH, 00001} +}; + +// Host to network byte order for st_mode file permission +static uint32_t +GS_mode2fperm(mode_t m) +{ + uint32_t u = 0; + int n; + + m &= ~S_IFMT; + for (n = 0; n < sizeof x_mperm / sizeof *x_mperm; n++) + { + if (m & x_mperm[n].mode) + u |= x_mperm[n].perm; + } + + DEBUGF_B("mode2fperm 0%o\n", u); + return u; } +// Network to host byte order for st_mode file permission +static mode_t +GS_fperm2mode(uint32_t u) +{ + mode_t m = 0; + int n; + + DEBUGF_B("fperm2mode 0%o\n", u); + + for (n = 0; n < sizeof x_mperm / sizeof *x_mperm; n++) + { + if (u & x_mperm[n].perm) + m |= x_mperm[n].mode; + } + + return m; +} /* - * CLIENT: Add this file to queue. + * CLIENT: Add this file (not directory) to queue. */ int GS_FT_put(GS_FT *ft, const char *fname) @@ -126,20 +191,81 @@ GS_FT_put(GS_FT *ft, const char *fname) if (!S_ISREG(res.st_mode)) return -2; +STOP HERE: +- if we want to retain umask/mtime of directories then we need to create them on remote as well +and need a flag +- empty directory (ignore for now?! Create on server side on the fly when needed) +- /./etc/hosts (create directoyr on the fly when needed - for now. later implement on client side to + add directories up to filename to create them and globbing can also add directories...) struct _gs_ft_file *f; f = calloc(1, sizeof *f); - f->name = strdup(fname); + char *str = strdup(fname); + f->name = strdup(basename(str)); + free(str); f->realname = realfname; f->fsize = res.st_size; - f->umask = 31337; // FIXME + f->mode = res.st_mode; + f->mtime = res.st_mtime; + + DEBUGF_Y("mode = %o\n", res.st_mode & ~S_IFMT); + DEBUGF_Y("#%u name = %s\n", ft->g_id, f->name); f->li = GS_LIST_add(&ft->fqueue, NULL, f, ft->g_id); ft->g_id += 1; - ft->is_put_done = 0; + + ft->n_files_waiting += 1; return 0; } + +static void +qerr_add(GS_FT *ft, uint32_t id, uint8_t code, const char *str) +{ + struct _gs_ft_qerr *qerr; + + qerr = calloc(1, sizeof *qerr); + qerr->id = id; + qerr->code = code; + qerr->str = NULL; + if (str != NULL) + qerr->str = strdup(str); + + // Must add in sequence of occurance (add_count) + GS_LIST_add(&ft->qerrs, NULL, qerr, ft->qerrs.add_count); +} + +static void +do_error(GS_FT *ft, GS_LIST_ITEM *li, uint32_t code, const char *str) +{ + qerr_add(ft, li->id, code, str); + ft_del(li); +} + +static void +do_complete(GS_FT *ft, struct _gs_ft_file *f) +{ + if (f->fp != NULL) + { + fflush(f->fp); + fchmod(fileno(f->fp), f->mode & ~S_IFMT); + if (f->mtime != 0) + { + DEBUGF_B("Setting time to %ld\n", f->mtime); + struct timeval t[] = {{f->mtime, 0}, {f->mtime, 0}}; + futimes(fileno(f->fp), t); + } + } + do_error(ft, f->li, GS_FT_ERR_COMPLETED, NULL); +} + +static void +qerr_free(struct _gs_ft_qerr *qerr) +{ + XFREE(qerr->str); + XFREE(qerr); +} + // SERVER void GS_FT_data(GS_FT *ft, const void *data, size_t len) @@ -149,15 +275,14 @@ GS_FT_data(GS_FT *ft, const void *data, size_t len) if (f == NULL) { - DEBUGF_R("Oops. Receivng data but no active receiving file\n"); + DEBUGF_R("Receiving data but no active receiving file\n"); return; } - XASSERT(f->fp != NULL, "fp is NULL\n"); if (f->offset + len > f->fsize) { - DEBUGF_R("Oops. More data than we want!\n"); + DEBUGF_R("More data than we want!\n"); len = f->fsize - f->offset; } @@ -166,24 +291,27 @@ GS_FT_data(GS_FT *ft, const void *data, size_t len) if (sz != len) { - // FIXME: return error to peer that write has failed... - goto completed; + do_error(ft, f->li, GS_FT_ERR_BADF, NULL); + ft->active_receiving = NULL; + return; } if (f->offset < f->fsize) return; // still data missing... -completed: + DEBUGF_B("Server: All data received (%"PRIu64" of %"PRIu64")\n", f->offset, f->fsize); + do_complete(ft, f); ft->active_receiving = NULL; - ft_del(f->li); } // CLIENT void -GS_FT_accept(GS_FT *ft, uint32_t id, int64_t offset) +GS_FT_accept(GS_FT *ft, uint32_t id, int64_t offset_dst) { GS_LIST_ITEM *li; + DEBUGF("#%u acc offset_dst = %"PRId64"\n", id, offset_dst); + li = GS_LIST_by_id(&ft->fputs, id); if (li == NULL) { @@ -194,12 +322,12 @@ GS_FT_accept(GS_FT *ft, uint32_t id, int64_t offset) GS_LIST_move(&ft->faccepted, li); struct _gs_ft_file *f = (struct _gs_ft_file *)li->data; - f->offset = offset; + f->offset = offset_dst; } // SERVER void -GS_FT_switch(GS_FT *ft, uint32_t id, int64_t fsize) +GS_FT_switch(GS_FT *ft, uint32_t id, int64_t offset) { GS_LIST_ITEM *li; @@ -213,7 +341,7 @@ GS_FT_switch(GS_FT *ft, uint32_t id, int64_t fsize) // Switch from active receiving file to new file struct _gs_ft_file *new = (struct _gs_ft_file *)li->data; - DEBUGF_W("Switching to id %u '%s' (got %"PRId64", want %"PRId64")\n", id, new->name, new->offset, fsize); + DEBUGF_W("Switching to id %u '%s' (got %"PRId64", want %"PRId64")\n", id, new->name, new->offset, new->fsize); if ((ft->active_receiving != NULL) && (ft->active_receiving != new)) { @@ -222,53 +350,62 @@ GS_FT_switch(GS_FT *ft, uint32_t id, int64_t fsize) ft->active_receiving = NULL; - // Existing file is larger to fsize send by peer. - if (new->offset > fsize) - goto err; + // Server: Existing file is larger. Overwrite. + if (new->offset > new->fsize) + { + DEBUGF_W("File larger. Overwritting...\n"); + XASSERT(offset == 0, "OFFSET not 0 but server's file is larger\n"); + new->offset = 0; + offset = 0; + } - if (fsize == new->offset) + if ((new->offset == new->fsize) && (new->fsize != 0)) { - DEBUGF_G("file already fully received\n"); - ft_del(li); + // FIXME: currently _not_ updating mtime/fperm if file is already on peer side. + // Do we want this? (if so then move this code block after fopen().) + do_complete(ft, new); return; } - new->fsize = fsize; - new->fp = fopen(new->realname, "a"); - // new->fp = fopen(new->realname, "r+"); + // new->fsize = fsize; + if (offset == 0) + { + DEBUGF_G("New file\n"); + new->fp = fopen(new->realname, "w"); + } else { + // Check fsize of local file. + DEBUGF_G("Appending file\n"); + struct stat res; + if (stat(new->realname, &res) != 0) + goto err; + if (res.st_size != offset) + { + // Size changed + do_error(ft, new->li, GS_FT_ERR_BADFSIZE, NULL); + return; + } + new->fp = fopen(new->realname, "a"); + } + if (new->fp == NULL) { DEBUGF("fopen(%s) failed: %s\n", new->realname, strerror(errno)); goto err; } + if (new->fsize == 0) + { + // Zero sized file. Completed. + do_complete(ft, new); + return; + } + ft->active_receiving = new; return; err: - // FIXME: Transmit error back to peer. - // Use a List-queue with error messages (or just 1 error message?) - ft_del(li); -} -#if 0 -/* - * Remote reported error on this file. Remove from active list. - */ -void -GS_FT_error(GS_FT *ft, uint32_t id) -{ - GS_LIST_ITEM *li; - - li = GS_LIST_by_id(&ft->flist, id); - if (li == NULL) - return; - f = (struct _gs_ft_file *)li->data; - - ft_del(li); - - ft->n_denied += 1; + do_error(ft, new->li, GS_FT_ERR_PERM, NULL); } -#endif static void file_free(struct _gs_ft_file *f) @@ -277,6 +414,17 @@ file_free(struct _gs_ft_file *f) XFREE(f->realname); XFREE(f); } + +// Reduce counter of outstanding files/errors. +static void +ft_done(GS_FT *ft) +{ + if (ft->n_files_waiting > 0) + ft->n_files_waiting -= 1; + else + DEBUGF_R("Oops, n_files_waiting == %d\n", ft->n_files_waiting); +} + /* * Remove file from queue. */ @@ -285,99 +433,181 @@ ft_del(GS_LIST_ITEM *li) { struct _gs_ft_file *f = (struct _gs_ft_file *)li->data; - if (f->fp != NULL) - { - fclose(f->fp); - f->fp = NULL; - } + XFCLOSE(f->fp); file_free(f); GS_LIST_del(li); } +// Human readable bps string +static void +mk_bps(char *str, size_t sz, uint64_t duration, uint64_t amount, int err) +{ + if (err != 0) + { + snprintf(str, sz, "ERROR"); + return; + } + if (duration > 0) + GS_format_bps(str, sz, (amount * 1000000 / duration)); + else + snprintf(str, sz, "SKIPPED"); +} + +// Generate stats per file and call call-back. +static void +mk_stats(GS_FT *ft, uint32_t id, struct _gs_ft_file *f, int err) +{ + struct _gs_ft_stats s; + + memset(&s, 0, sizeof s); + s.id = id; + s.f = f; + s.xfer_amount = f->xfer_amount; + if (f->usec_start > f->usec_end) + f->usec_end = GS_usec(); + + if (f->usec_suspend_start != 0) + { + DEBUGF_R("Oops, Reporting stats on a suspended file\n"); + f->usec_suspend_duration += (GS_usec() - f->usec_suspend_start); + } + + s.xfer_duration = (f->usec_end - f->usec_start) - f->usec_suspend_duration; + mk_bps(s.speed_str, sizeof s.speed_str, s.xfer_duration, f->xfer_amount, err); + + // Global stats for all files + ft->stats_total.xfer_duration += s.xfer_duration; + ft->stats_total.xfer_amount += f->xfer_amount; + if (err == 0) + ft->stats_total.n_files_success += 1; + else + ft->stats_total.n_files_error += 1; + mk_stats_total(ft); + + // Call call-back + if (ft->func_stats != NULL) + (*ft->func_stats)(&s); +} + +// Generate total stats +static void +mk_stats_total(GS_FT *ft) +{ + GS_FT_stats_total *st = &ft->stats_total; + + mk_bps(st->speed_str, sizeof st->speed_str, st->xfer_duration, st->xfer_amount, st->n_files_success==0?1:0); +} + /* * Error received from Server or Client. * Remove and free item. */ void -GS_FT_del_file(GS_FT *ft, uint32_t id) +GS_FT_status(GS_FT *ft, uint32_t id, uint8_t code, const char *err_str, size_t len) { GS_LIST_ITEM *li; - struct _gs_ft_file *f; + int err = 1; - li = GS_LIST_by_id(&ft->fqueue, id); + if (err_str[len] != '\0') + return; // protocol error. Not 0 terminated. + DEBUGF_R("#%u STATUS: %u (%s)\n", id, code, err_str); + + li = GS_LIST_by_id(&ft->fcompleted, id); if (li == NULL) { - li = GS_LIST_by_id(&ft->fputs, id); + li = GS_LIST_by_id(&ft->fqueue, id); if (li == NULL) { - li = GS_LIST_by_id(&ft->fadded, id); + li = GS_LIST_by_id(&ft->fputs, id); if (li == NULL) { - li = GS_LIST_by_id(&ft->freceiving, id); + li = GS_LIST_by_id(&ft->fadded, id); if (li == NULL) { - DEBUGF_R("id %u not found\n", id); - return; // not found + li = GS_LIST_by_id(&ft->freceiving, id); + if (li == NULL) + { + DEBUGF_R("id %u not found\n", id); + return; // not found + } } } } + } else { + // Was waiting for 'complete' signal. No error if 'complete' is a success. + if (code == GS_FT_ERR_COMPLETED) + err = 0; } - f = (struct _gs_ft_file *)li->data; - if (f == ft->active_put_file) + // Make status + struct _gs_ft_status s; + memset(&s, 0, sizeof s); + s.code = code; + s.file = li->data; + // FIXME: sanitize error string + snprintf(s.err_str, sizeof s.err_str, "%s", err_str); + if (ft->func_stats != NULL) + (*ft->func_status)(ft, &s); + + // Report stats to caller + mk_stats(ft, id, li->data, err); + + if (li->data == ft->active_put_file) ft->active_put_file = NULL; - file_free(f); - GS_LIST_del(li); + ft_done(ft); + ft_del(li); } - /* - * Create an error package. Return 0 on error or the length of patcket - * on success.. + * Make an error packet. Return length. */ -static int -mk_error(void *dst, size_t len, uint32_t id, uint8_t code) +static size_t +mk_error(void *dst, size_t len, uint32_t id, uint8_t code, const char *str) { + size_t sz; struct _gs_ft_error err; - if (len < sizeof (err) + 1) - { - DEBUGF_R("Oops, not enough space?\n"); - return 0; - } - memset(&err, 0, sizeof err); err.id = htonl(id); err.code = code; memcpy(dst, &err, sizeof err); - ((uint8_t *)dst)[sizeof err] = '\0'; - return sizeof err + 1; // 0-terminate -} + sz = 1; + struct _gs_ft_error *p = (struct _gs_ft_error *)dst; + if ((str != NULL) && (strlen(str) > 0)) + { + sz = MIN(len - sizeof err, strlen(str) + 1); + memcpy(p->str, str, sz - 1); + } + p->str[sz - 1] = '\0'; + return sizeof err + sz; +} -static int -ft_error(GS_FT *ft, void *dst, size_t len, GS_LIST_ITEM *li, uint8_t code, int *pkt_type) +/* + * Make an error packet. Remove errornous file from queue + */ +static size_t +ft_mk_error(GS_FT *ft, void *dst, size_t len, int *pkt_type, GS_LIST_ITEM *li, uint8_t code, const char *str) { - int ret; + size_t sz; - XASSERT(li != NULL, "Oops. li is NULL\n"); - ret = mk_error(dst, len, li->id, code); *pkt_type = GS_FT_TYPE_ERROR; + sz = mk_error(dst, len, li->id, code, str); + ft_done(ft); ft_del(li); - if ((ft->fputs.n_items == 0) && (ft->faccepted.n_items == 0)) - ft->is_put_done = 1; - return ret; + return sz; } + /* - * Create a data packet (from job/queue that need attendtion). + * Create a data packet (from job/queue that need attention). * Return the length. * Return 0 if no data needs to be written or an error occured. - * check pkt_type to determine the error code. + * check pkt_type to for packet code that needs to be written. * * Set TYPE to SWITCH or DATA depending on the type of packet created. * Set to DONE when done and NONE when nothing is to be done. @@ -388,30 +618,48 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) struct _gs_ft_file *f; size_t sz; - // DEBUGF("puts %d, accepted %d, done %d\n", ft->fputs.n_items, ft->faccepted.n_items, ft->is_put_done); - if (ft->is_put_done) + // DEBUGF("puts %d, accepted %d, len %zu\n", ft->fputs.n_items, ft->faccepted.n_items, len); + + *pkt_type = GS_FT_TYPE_NONE; + if (len < GS_FT_MIN_BUF_SIZE) { - *pkt_type = GS_FT_TYPE_DONE; return 0; } + // XASSERT(len >= GS_FT_MIN_BUF_SIZE, "len to small\n"); - *pkt_type = GS_FT_TYPE_NONE; + // Check if any queue'd errors needs sending. + // Server & Client + if (ft->qerrs.n_items > 0) + { + GS_LIST_ITEM *li = NULL; + li = GS_LIST_next(&ft->qerrs, NULL); + struct _gs_ft_qerr *qerr = (struct _gs_ft_qerr *)li->data; + + sz = mk_error(dst, len, qerr->id, qerr->code, qerr->str); + + *pkt_type = GS_FT_TYPE_ERROR; + GS_LIST_del(li); + return sz; + // return sizeof err + sz; + } // Check if any files in the queue that need to be 'put' // on offer to the remote side (and then awaiting 'accept'). + // Client if (ft->fqueue.n_items > 0) { - GS_LIST_ITEM *li = NULL; - DEBUGF("%d items in queue (waiting for put to be send)\n", ft->fqueue.n_items); + GS_LIST_ITEM *li = NULL; li = GS_LIST_next(&ft->fqueue, NULL); f = (struct _gs_ft_file *)li->data; struct _gs_ft_put put; struct _gs_ft_put *p = (struct _gs_ft_put *)dst; memset(&put, 0, sizeof put); - put.umask = htonl(f->umask); + put.fperm = htonl(GS_mode2fperm(f->mode)); put.id = htonl(li->id); + put.fsize = htonll(f->fsize); + put.mtime = htonl(f->mtime); memcpy(dst, &put, sizeof put); sz = MIN(len - sizeof put, strlen(f->name) + 1); @@ -424,6 +672,7 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) return sizeof put + sz; } + // Server if (ft->fadded.n_items > 0) { GS_LIST_ITEM *li = NULL; @@ -433,7 +682,7 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) struct _gs_ft_accept acc; memset(&acc, 0, sizeof acc); acc.id = htonl(li->id); - acc.offset = htonll(f->offset); + acc.offset_dst = htonll(f->offset); memcpy(dst, &acc, sizeof acc); // HERE: Server. Inform client that we accepted. @@ -442,11 +691,12 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) return sizeof acc; } + // Client if (ft->faccepted.n_items > 0) { - // Here: No files waiting to offer ('put') to remote. if (ft->active_put_file == NULL) { + // HERE: Currently not transmitting any file data => Select new file. GS_LIST_ITEM *li = NULL; int ret; @@ -455,29 +705,50 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) f = (struct _gs_ft_file *)li->data; // Open file and seek to location - f->fp = fopen(f->name, "r"); + f->fp = fopen(f->realname, "r"); if (f->fp == NULL) - return ft_error(ft, dst, len, f->li, GS_FT_ERR_PERM, pkt_type); + return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_PERM, NULL); ret = fseek(f->fp, 0, SEEK_END); if (ret != 0) - return ft_error(ft, dst, len, f->li, GS_FT_ERR_BADF, pkt_type); + return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_BADF, NULL); f->fsize = ftell(f->fp); - // Peer already has all data (or more). - if (f->fsize <= f->offset) - return ft_error(ft, dst, len, f->li, GS_FT_ERR_NODATA, pkt_type); + // Peer already has this file. + // Overwrite if remote size is smaller _or_ larger. + // f->fsize == local size, f->offset == remote size + if ((f->fsize == f->offset) && (f->fsize != 0)) + { + DEBUGF("#%u Skipping %s (already on peer)\n", (unsigned int)f->li->id, f->name); + mk_stats(ft, li->id, f, 0 /*success*/); + return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_NODATA, NULL); + } + + // Remote size is larger. Overwrite from beginning. + if (f->fsize < f->offset) + f->offset = 0; + + // Remote size is smaller. Restart transmission. ret = fseek(f->fp, f->offset, SEEK_SET); if (ret != 0) - return ft_error(ft, dst, len, f->li, GS_FT_ERR_BADF, pkt_type); + return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_BADF, NULL); struct _gs_ft_switch sw; memset(&sw, 0, sizeof sw); sw.id = htonl(li->id); - sw.fsize = htonll(f->fsize); + sw.offset = htonll(f->offset); memcpy(dst, &sw, sizeof sw); - ft->active_put_file = f; + // Handle zero size files + if (f->fsize == 0) + { + GS_LIST_move(&ft->fcompleted, f->li); + ft->active_put_file = NULL; + } else { + // HERE: fsize is not zero. + ft->active_put_file = f; + } + *pkt_type = GS_FT_TYPE_SWITCH; return (sizeof sw); @@ -487,27 +758,45 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) *pkt_type = GS_FT_TYPE_DATA; f = ft->active_put_file; - DEBUGF_C("at %ld\n", ftell(f->fp)); sz = fread(dst, 1, len, f->fp); + if (sz <= 0) { // HERE: Read error or file may have shrunk. ft->active_put_file = NULL; - return ft_error(ft, dst, len, f->li, GS_FT_ERR_BADF, pkt_type); + return ft_mk_error(ft, dst, len, pkt_type, f->li, GS_FT_ERR_BADF, NULL); } f->offset += sz; + // -----BEGIN Log statistics----- + if (f->usec_start == 0) + { + f->usec_start = GS_usec(); + } + + if (f->usec_suspend_start != 0) + { + f->usec_suspend_duration += (GS_usec() - f->usec_suspend_start); + f->usec_suspend_start = 0; + } + f->xfer_amount += sz; + // -----END Log statistics----- + // File completed. No more data. if (f->offset >= f->fsize) { - ft_del(f->li); - if ((ft->fputs.n_items == 0) && (ft->faccepted.n_items == 0)) - ft->is_put_done = 1; + GS_LIST_move(&ft->fcompleted, f->li); ft->active_put_file = NULL; } return sz; } + if (ft->n_files_waiting == 0) + { + *pkt_type = GS_FT_TYPE_DONE; + return 0; + } + return 0; } @@ -531,10 +820,19 @@ GS_FT_free(GS_FT *ft) free_gsl(&ft->fqueue); free_gsl(&ft->fputs); free_gsl(&ft->faccepted); + free_gsl(&ft->fcompleted); free_gsl(&ft->fadded); free_gsl(&ft->freceiving); + GS_LIST_ITEM *li; + for (li = GS_LIST_next(&ft->qerrs, NULL); li != NULL; li = GS_LIST_next(&ft->qerrs, li)) + { + qerr_free((struct _gs_ft_qerr *)li->data); + + GS_LIST_del(li); + } + ft->active_put_file = NULL; } diff --git a/tools/filetransfer.h b/tools/filetransfer.h index 7c041092..e1847fbd 100644 --- a/tools/filetransfer.h +++ b/tools/filetransfer.h @@ -7,47 +7,109 @@ #define GS_FT_CHN_ERROR (4) #define GS_FT_CHN_SWITCH (5) +// Number of bytes needed for largest message (could be data) +#define GS_FT_MIN_BUF_SIZE (64) + struct _gs_ft_file { GS_LIST_ITEM *li; char *name; char *realname; // realpath() resolved - uint32_t umask; + mode_t mode; + time_t mtime; FILE *fp; - off_t offset; // Offset to start transmitting + off_t offset; // Offset to read(=client)/write(=server) off_t fsize; // Total file size (from client) + + // statistics + // Note: xfer might be suspended by 'switch' to another file. + // Log suspended time in usec_suspend. + uint64_t usec_start; + uint64_t usec_end; + uint64_t usec_suspend_start; + uint64_t usec_suspend_duration; + int64_t xfer_amount; // Actual data on the wire +}; + +struct _gs_ft_stats +{ + uint32_t id; + struct _gs_ft_file *f; + uint64_t xfer_duration; // Actual transfer time (without suspension) + uint64_t xfer_amount; // Actual data transfered + char speed_str[8]; // Speed (bps). Human readable string. }; +typedef void (*gsft_cb_stats_t)(struct _gs_ft_stats *s); + +// Updated after each file completion +typedef struct +{ + uint64_t xfer_duration; + uint64_t xfer_amount; + char speed_str[8]; + int n_files_success; // transferred or skipped so far + int n_files_error; +} GS_FT_stats_total; + +/* + * Queue'ed error's that need to be send to peer. + */ +struct _gs_ft_qerr +{ + uint32_t id; + uint8_t code; + char *str; +}; + +struct _gs_ft_status +{ + uint8_t code; + struct _gs_ft_file *file; + char err_str[128]; // 0-terminated error string +}; +typedef void (*gsft_cb_status_t)(void *ft_ptr, struct _gs_ft_status *s); + typedef struct { GS_LIST fqueue; // Client List of files to be transfered GS_LIST fputs; // Client list of files we requested transfer (put sent) GS_LIST faccepted; // Client List of accepted files + GS_LIST fcompleted; // Completed. Waiting for 'ERR_COMPLETED' GS_LIST fadded; // Server Side list of ready files GS_LIST freceiving; // Server Side list of receiving files int g_id; struct _gs_ft_file *active_put_file; // Current active file struct _gs_ft_file *active_receiving; // - int is_put_done; // No more files to transmit -} GS_FT; + gsft_cb_stats_t func_stats; + gsft_cb_status_t func_status; + GS_LIST qerrs; // queue'd errors + + int n_files_waiting; // Files waiting for completion or error + + // Statistics total (all files) + GS_FT_stats_total stats_total; +} GS_FT; // CLIENT -> Server: put a file to server. struct _gs_ft_put { - uint32_t umask; uint32_t id; - uint8_t reserved[32 - 4 - 4]; + uint32_t fperm; + int64_t fsize; + uint32_t mtime; + uint8_t reserved[32 - 4 - 4 - 4 - 8]; uint8_t name[0]; // 0-terminated file name } __attribute__((__packed__)); -// SERVER -> Client: Accept file. +// SERVER -> Client: Accept file. (reply to PUT) struct _gs_ft_accept { uint32_t id; uint8_t res[4]; - int64_t offset; + int64_t offset_dst; // Server side fsize uint8_t crcNOTUSED[4]; uint8_t res2[4]; } __attribute__((__packed__)); @@ -57,39 +119,45 @@ struct _gs_ft_switch { uint32_t id; uint8_t res[4]; - int64_t fsize; // total file size + // int64_t fsize; // total file size + int64_t offset; // offset to start } __attribute__((__packed__)); struct _gs_ft_error { + uint32_t id; uint8_t code; uint8_t res[3]; // reerved - uint32_t id; uint8_t str[0]; // 0-terminated error string (not used) } __attribute__((__packed__)); #define GS_FT_ERR_UNKNOWN (0) #define GS_FT_ERR_PERM (1) #define GS_FT_ERR_NOENT (2) +#define GS_FT_ERR_BADFSIZE (3) // Size on server is larger #define GS_FT_ERR_BADF (9) #define GS_FT_ERR_NODATA (10) +#define GS_FT_ERR_COMPLETED (128) // All data written successfully -void GS_FT_init(GS_FT *ft); +void GS_FT_init(GS_FT *ft, gsft_cb_stats_t func, gsft_cb_status_t); void GS_FT_free(GS_FT *ft); -int64_t GS_FT_add_file(GS_FT *ft, uint32_t id, const char *fname, uint32_t umask); +int GS_FT_add_file(GS_FT *ft, uint32_t id, const char *fname, size_t len, int64_t fsize, uint32_t mtime, uint32_t fperm); int GS_FT_put(GS_FT *ft, const char *fname); -void GS_FT_switch(GS_FT *ft, uint32_t id, int64_t fsize); +void GS_FT_switch(GS_FT *ft, uint32_t id, /*int64_t fsize, */int64_t offset); void GS_FT_accept(GS_FT *ft, uint32_t id, int64_t offset); void GS_FT_data(GS_FT *ft, const void *data, size_t len); -void GS_FT_del_file(GS_FT *ft, uint32_t id); +void GS_FT_status(GS_FT *ft, uint32_t id, uint8_t code, const char *err_str, size_t len); +// void GS_FT_del_file(GS_FT *ft, uint32_t id); size_t GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type); +int GS_FT_WANT_WRITE(GS_FT *ft); +// Packet types #define GS_FT_TYPE_NONE (0) #define GS_FT_TYPE_SWITCH (1) #define GS_FT_TYPE_DATA (2) #define GS_FT_TYPE_ERROR (3) #define GS_FT_TYPE_PUT (4) -#define GS_FT_TYPE_DONE (5) +#define GS_FT_TYPE_DONE (5) // Inform caller that stack is done #define GS_FT_TYPE_ACCEPT (6) // Packet Bulding functions diff --git a/tools/run_ft_tests.sh b/tools/run_ft_tests.sh new file mode 100755 index 00000000..92548cbb --- /dev/null +++ b/tools/run_ft_tests.sh @@ -0,0 +1,166 @@ +#! /bin/bash + +command -v md5 >/dev/null 2>&1 && MD5(){ md5 -q "${1}";} +command -v md5sum >/dev/null 2>&1 && MD5() { md5sum "${1}" | cut -f1 -d' ';} + +IODIR=ft_test_io +OK="....[\033[1;32mOK\033[0m]" +FAIL="[\033[1;31mFAILED\033[0m]" +ECHO="echo -e" + +mk_dummy() +{ + [ -f "$1" ] || dd bs=1024 count=$2 if=/dev/urandom of="$1" 2>/dev/null +} + +mk_dummy test1k.dat 1 +mk_dummy test4k.dat 4 +mk_dummy test8k.dat 8 + +test_start() +{ + mkdir -p "${IODIR}" &>/dev/null + rm -rf "${IODIR}/"*.dat &>/dev/null + [[ x"$1" != x ]] && $ECHO $* +} + +fail() +{ + $ECHO "${FAIL}"-$* + exit 255 +} + +# code file1 file2 +md5fail() +{ + [[ "$(MD5 ${2})" != "$(MD5 ${3})" ]] && fail $1; +} + +run_put() +{ + socat SYSTEM:"./filetransfer-test c $* 2>client.log" SYSTEM:"(cd ${IODIR}; ../filetransfer-test s 2>../server.log)" +} + + +# tests="1.0 " +# tests+="1.1 " +# tests+="1.2 " +tests+="1.3 " +# tests+="2.1 2.2 " +# tests+="2.3 " +# tests+="3.1 3.2 3.3 " +# tests+="4.1 " +# tests+="4.2 " +# tests+="4.3 " + +if [[ "$tests" =~ '1.0 ' ]]; then +test_start -n "Running #1.0 (put 1 file).................................." +run_put test1k.dat +md5fail 1 test1k.dat "${IODIR}/test1k.dat" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '1.1 ' ]]; then +test_start -n "Running #1.1 (put 2 files)................................." +run_put test4k.dat test8k.dat +md5fail 1 test4k.dat "${IODIR}/test4k.dat" +md5fail 2 test8k.dat "${IODIR}/test8k.dat" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '1.2 ' ]]; then +test_start -n "Running #1.2 (non-exist)..................................." +run_put not-exists.dat +[[ -f "${IODIR}/not-exists.dat" ]] && fail 1 +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '1.3 ' ]]; then +test_start -n "Running #1.3 (absolute file)..............................." +# run_put /etc/hosts +cp test1k.dat ./foo/bar/test1k.dat +run_put ./foo/bar/test1k.dat +# run_put /./etc/hosts ../tools/test1k.dat +ls -al "${IODIR}/foo/bar" +exit +[[ -f "${IODIR}/etc/hosts" ]] && fail 1 +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '2.1 ' ]]; then +test_start -n "Running #2.1 (src is larger, restart)......................" +dd bs=1k count=5 if=test8k.dat of="${IODIR}/test8k.dat" &>/dev/null +run_put test4k.dat test8k.dat +md5fail 1 test8k.dat "${IODIR}/test8k.dat" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '2.2 ' ]]; then +test_start -n "Running #2.2 (dst is larger, overwrite)...................." +cp test8k.dat "${IODIR}/test4k.dat" +run_put test4k.dat +md5fail 1 test4k.dat "${IODIR}/test4k.dat" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '2.3 ' ]]; then +test_start -n "Running #2.3 (zero src size)..............................." +touch zero.dat +run_put zero.dat +md5fail 1 zero.dat "${IODIR}/zero.dat" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '3.1 ' ]]; then +test_start -n "Running #3.2 (write-error 0-sized dst)....................." +touch "${IODIR}/test4k.dat" +chmod 400 "${IODIR}/test4k.dat" +run_put test4k.dat +[[ x`stat -f%z "${IODIR}/test4k.dat"` = x0 ]] || fail 1 +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '3.2 ' ]]; then +test_start -n "Running #3.2 (write-error partial)........................." +cp test4k.dat "${IODIR}/test8k.dat" +chmod 400 "${IODIR}/test8k.dat" +run_put test8k.dat +md5fail 1 test4k.dat "${IODIR}/test8k.dat" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '3.3 ' ]]; then +test_start -n "Running #3.3 (dir not writeable)..........................." +chmod a-w "${IODIR}" +run_put test4k.dat +chmod u+w "${IODIR}" +[[ -f "${IODIR}/test4k.dat" ]] && fail 1 +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '4.1 ' ]]; then +test_start -n "Running #4.1 (permission).................................." +chmod 462 test4k.dat +# chmod u+s test4k.dat # On MacOS our own app can not set +s... +run_put test4k.dat +[[ x`stat -f%A "test4k.dat"` = x`stat -f%A "${IODIR}/test4k.dat"` ]] || fail 1 +chmod 644 "${IODIR}/test4k.dat" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '4.2 ' ]]; then +test_start -n "Running #4.2 (mtime)......................................." +touch -r /etc/hosts test4k.dat +run_put test4k.dat +[[ x`stat -f%a "test4k.dat"` = x`stat -f%a "${IODIR}/test4k.dat"` ]] || fail 1 +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '4.3 ' ]]; then +test_start -n "Running #4.2 (zero-size, mtime)............................" +touch -r /etc/hosts zero.dat +run_put zero.dat +[[ x`stat -f%a "zero.dat"` = x`stat -f%a "${IODIR}/zero.dat"` ]] || fail 1 +$ECHO "${OK}" +fi + diff --git a/tools/utils.c b/tools/utils.c index 367cd36e..223c7e52 100755 --- a/tools/utils.c +++ b/tools/utils.c @@ -1015,28 +1015,6 @@ sanitize_fname_to_str(uint8_t *str, size_t len) } -static const char unit[] = "BKMGT"; -void -format_bps(char *buf, size_t size, int64_t bytes) -{ - int i; - - if (bytes < 1000) - { - snprintf(buf, size, "%3d.0 B", (int)bytes); - return; - } - bytes *= 100; - - for (i = 0; bytes >= 100*1000 && unit[i] != 'T'; i++) - bytes = (bytes + 512) / 1024; - snprintf(buf, size, "%3lld.%1lld%c%s", - (long long) (bytes + 5) / 100, - (long long) (bytes + 5) / 10 % 10, - unit[i], - i ? "B" : " "); -} - From 6cfdd3c3b4b03b6b8d2889581b902ff377a8f750 Mon Sep 17 00:00:00 2001 From: rootTHC Date: Tue, 22 Dec 2020 09:24:39 +0000 Subject: [PATCH 08/21] test scripts --- tools/filetransfer-test.c | 20 +++++++++ tools/filetransfer.c | 88 ++++++++++++++++++++++++++++++++------- tools/run_ft_tests.sh | 47 ++++++++++++++++----- 3 files changed, 128 insertions(+), 27 deletions(-) diff --git a/tools/filetransfer-test.c b/tools/filetransfer-test.c index 246e8ffc..9bca99d2 100644 --- a/tools/filetransfer-test.c +++ b/tools/filetransfer-test.c @@ -159,6 +159,24 @@ mk_packet(void) return 0; } +static void +do_test(void) +{ + // char *fname = "/tmp/foo/./bar/hosts"; + char fname[80] = "This is/./www.tutorialspoint.com/./website"; + char *str; + char s[4] = "/./"; + + DEBUGF("mkar\n"); + str = strtok(fname, s); + DEBUGF("mkar\n"); + while (str != NULL) + { + DEBUGF("'%s'\n", str); + str = strtok(NULL, s); + } + exit(0); +} int main(int arc, char *argv[]) @@ -174,6 +192,8 @@ main(int arc, char *argv[]) gopt.err_fp = stderr; gopt.log_fp = stderr; + // do_test(); + GS_PKT_init(&pkt); GS_PKT_assign_chn(&pkt, GS_FT_CHN_ERROR, pkt_cb_error, NULL); diff --git a/tools/filetransfer.c b/tools/filetransfer.c index 0ba80339..7d241866 100644 --- a/tools/filetransfer.c +++ b/tools/filetransfer.c @@ -12,7 +12,7 @@ static uint32_t GS_mode2fperm(mode_t m); #if 0 0 FIXME: must make sure that server cant request transfer from client! -- test when file becomes unavaialble after after it was added. +- test when file becomes unavaialble after it was added. - Sym Link - max buffer size 64 macro somewhere needed - queue up all 'put' requests and send as 1 large message or loop around write() while we can. @@ -21,6 +21,9 @@ static uint32_t GS_mode2fperm(mode_t m); (add files until it does not fit. Return to caller the ID that failed and let caller decide if he likes to remove that ID from our list or just let caller try again until it fits...) is there a limt? +- retain fperm/mtime on directories +- empty directories + - implement GS_FT_get() @@ -32,8 +35,8 @@ TEST CASES: 4. same file in command line 5. src file can not be opened or read error. 6. write to symlink -7. zero file size -8. retain timestamp +#7. zero file size +#8. retain timestamp #endif void @@ -191,24 +194,47 @@ GS_FT_put(GS_FT *ft, const char *fname) if (!S_ISREG(res.st_mode)) return -2; -STOP HERE: -- if we want to retain umask/mtime of directories then we need to create them on remote as well -and need a flag -- empty directory (ignore for now?! Create on server side on the fly when needed) -- /./etc/hosts (create directoyr on the fly when needed - for now. later implement on client side to - add directories up to filename to create them and globbing can also add directories...) struct _gs_ft_file *f; f = calloc(1, sizeof *f); - char *str = strdup(fname); - f->name = strdup(basename(str)); - free(str); + + /* + * Consider these possibilities (examples) + * /tmp/foo/bar/hosts + * /./tmp/foo/bar/hosts + * /tmp/foo/./bar/./hosts + * /tmp/./foo/bar/hosts + * hosts + * foo/bar/host + */ + // Find token after last occurance of '/./' + const char *str = fname; + char *token; + int found = 0; + while (1) + { + token = strstr(str, "/./"); + if (token == NULL) + break; + found = 1; + str = token + 3; // skip '/./' + } + // str contains everything after '/./' or fname if '/./' not found. + if (found == 0) + { + // HERE: No '/./'. Use basename (file only, no directory part) + char *s = strdup(fname); // basename() might modify str :/ + f->name = strdup(basename(s)); + free(s); + } else { + f->name = strdup(str); + } + f->realname = realfname; f->fsize = res.st_size; f->mode = res.st_mode; f->mtime = res.st_mtime; - DEBUGF_Y("mode = %o\n", res.st_mode & ~S_IFMT); - + // DEBUGF_Y("mode = %o\n", res.st_mode & ~S_IFMT); DEBUGF_Y("#%u name = %s\n", ft->g_id, f->name); f->li = GS_LIST_add(&ft->fqueue, NULL, f, ft->g_id); ft->g_id += 1; @@ -325,6 +351,35 @@ GS_FT_accept(GS_FT *ft, uint32_t id, int64_t offset_dst) f->offset = offset_dst; } +/* + * Create all directories up to file + * /tmp/foo/bar/test.dat would create /tmp/foo/bar/ + * /tmp/foo/bar/test.dat/ would create /tmp/foo/bar/test.dat/ + */ +static void +mkdirp(const char *file) +{ + char *f = strdup(file); + + char *ptr = f; + while (1) + { + ptr = index(ptr, '/'); + if (ptr == NULL) + break; + *ptr = '\0'; + if (*f != 0) + { + DEBUGF_W("mkdir(%s)\n", f); + mkdir(f, 0755); + } + *ptr = '/'; + ptr += 1; + } + + free(f); +} + // SERVER void GS_FT_switch(GS_FT *ft, uint32_t id, int64_t offset) @@ -370,7 +425,8 @@ GS_FT_switch(GS_FT *ft, uint32_t id, int64_t offset) // new->fsize = fsize; if (offset == 0) { - DEBUGF_G("New file\n"); + DEBUGF_G("New file (%s)\n", new->name); + mkdirp(new->name); new->fp = fopen(new->realname, "w"); } else { // Check fsize of local file. @@ -618,7 +674,7 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) struct _gs_ft_file *f; size_t sz; - // DEBUGF("puts %d, accepted %d, len %zu\n", ft->fputs.n_items, ft->faccepted.n_items, len); + DEBUGF("GS_FT_packet() %d, accepted %d, len %zu\n", ft->fputs.n_items, ft->faccepted.n_items, len); *pkt_type = GS_FT_TYPE_NONE; if (len < GS_FT_MIN_BUF_SIZE) diff --git a/tools/run_ft_tests.sh b/tools/run_ft_tests.sh index 92548cbb..feaa42e5 100755 --- a/tools/run_ft_tests.sh +++ b/tools/run_ft_tests.sh @@ -4,6 +4,7 @@ command -v md5 >/dev/null 2>&1 && MD5(){ md5 -q "${1}";} command -v md5sum >/dev/null 2>&1 && MD5() { md5sum "${1}" | cut -f1 -d' ';} IODIR=ft_test_io +IODIRSRC=ft_test_src OK="....[\033[1;32mOK\033[0m]" FAIL="[\033[1;31mFAILED\033[0m]" ECHO="echo -e" @@ -19,8 +20,8 @@ mk_dummy test8k.dat 8 test_start() { + rm -rf "${IODIR}/" &>/dev/null mkdir -p "${IODIR}" &>/dev/null - rm -rf "${IODIR}/"*.dat &>/dev/null [[ x"$1" != x ]] && $ECHO $* } @@ -45,10 +46,11 @@ run_put() # tests="1.0 " # tests+="1.1 " # tests+="1.2 " -tests+="1.3 " +# tests+="1.3 " # tests+="2.1 2.2 " # tests+="2.3 " # tests+="3.1 3.2 3.3 " +tests+="3.4 " # tests+="4.1 " # tests+="4.2 " # tests+="4.3 " @@ -75,14 +77,29 @@ run_put not-exists.dat $ECHO "${OK}" fi +run_put_fail() +{ + rm -rf "${IODIR}/" &>/dev/null + mkdir -p "${IODIR}" &>/dev/null + run_put "$2" + [[ -f "$3" ]] || fail "$1" + rm -f "$3" +} + if [[ "$tests" =~ '1.3 ' ]]; then test_start -n "Running #1.3 (absolute file)..............................." -# run_put /etc/hosts -cp test1k.dat ./foo/bar/test1k.dat -run_put ./foo/bar/test1k.dat -# run_put /./etc/hosts ../tools/test1k.dat -ls -al "${IODIR}/foo/bar" -exit +mkdir -p "${IODIRSRC}/foo/bar" +cp test1k.dat "${IODIRSRC}/foo/bar/test1k.dat" + +run_put_fail 1 "${IODIRSRC}/foo/bar/test1k.dat" "${IODIR}/test1k.dat" +run_put_fail 2 "${IODIRSRC}/foo/bar/./test1k.dat" "${IODIR}/test1k.dat" +run_put_fail 3 "./${IODIRSRC}/foo/bar/test1k.dat" "${IODIR}/test1k.dat" +run_put_fail 4 "${IODIRSRC}/foo/./bar/test1k.dat" "${IODIR}/bar/test1k.dat" +run_put_fail 5 "././${IODIRSRC}/foo/bar/test1k.dat" "${IODIR}/${IODIRSRC}/foo/bar/test1k.dat" +run_put_fail 6 "${IODIRSRC}/foo/../foo/bar/test1k.dat" "${IODIR}/test1k.dat" +run_put_fail 7 "${IODIRSRC}/foo/../foo/./bar/test1k.dat" "${IODIR}/bar/test1k.dat" +run_put_fail 8 "${PWD}/${IODIRSRC}/foo/bar/test1k.dat" "${IODIR}/test1k.dat" +# run_put foo/./../foo/../foo/bar/test1k.dat # escape. wanted behavior (?). [[ -f "${IODIR}/etc/hosts" ]] && fail 1 $ECHO "${OK}" fi @@ -133,18 +150,26 @@ if [[ "$tests" =~ '3.3 ' ]]; then test_start -n "Running #3.3 (dir not writeable)..........................." chmod a-w "${IODIR}" run_put test4k.dat -chmod u+w "${IODIR}" [[ -f "${IODIR}/test4k.dat" ]] && fail 1 $ECHO "${OK}" fi +if [[ "$tests" =~ '3.4 ' ]]; then +test_start -n "Running #3.4 (src not readable)............................" +chmod a-r test4k.dat +run_put test4k.dat +[[ -f "${IODIR}/test4k.dat" ]] && fail 1 +chmod a+r test4k.dat +$ECHO "${OK}" +fi + if [[ "$tests" =~ '4.1 ' ]]; then test_start -n "Running #4.1 (permission).................................." chmod 462 test4k.dat # chmod u+s test4k.dat # On MacOS our own app can not set +s... run_put test4k.dat [[ x`stat -f%A "test4k.dat"` = x`stat -f%A "${IODIR}/test4k.dat"` ]] || fail 1 -chmod 644 "${IODIR}/test4k.dat" +chmod 644 test4k.dat $ECHO "${OK}" fi @@ -157,7 +182,7 @@ $ECHO "${OK}" fi if [[ "$tests" =~ '4.3 ' ]]; then -test_start -n "Running #4.2 (zero-size, mtime)............................" +test_start -n "Running #4.3 (zero-size, mtime)............................" touch -r /etc/hosts zero.dat run_put zero.dat [[ x`stat -f%a "zero.dat"` = x`stat -f%a "${IODIR}/zero.dat"` ]] || fail 1 From 5d37431cc1b9fa0997f4ba7b7ab1d3024ac34aed Mon Sep 17 00:00:00 2001 From: rootTHC Date: Sat, 9 Jan 2021 11:41:38 +0000 Subject: [PATCH 09/21] filetransfer slowly comming --- include/gsocket/buf.h | 26 +++++ include/gsocket/gsocket.h | 1 + include/gsocket/packet.h | 3 +- lib/Makefile.am | 2 +- lib/buf.c | 82 +++++++++++++++ lib/packet.c | 7 +- tools/Makefile.am | 4 +- tools/filetransfer-test.c | 207 ++++++++++++++++++++++++++++---------- tools/filetransfer.c | 65 ++++++++---- tools/filetransfer.h | 17 ++-- tools/run_ft_tests.sh | 96 +++++++++++++++--- 11 files changed, 411 insertions(+), 99 deletions(-) create mode 100644 include/gsocket/buf.h create mode 100644 lib/buf.c diff --git a/include/gsocket/buf.h b/include/gsocket/buf.h new file mode 100644 index 00000000..382bdcb3 --- /dev/null +++ b/include/gsocket/buf.h @@ -0,0 +1,26 @@ +#ifndef __GS_BUF_H__ +#define __GS_BUF_H__ 1 + + +typedef struct +{ + void *data; + size_t sz_total; + size_t sz_used; + + size_t sz_max_add; +} GS_BUF; + +void GS_BUF_init(GS_BUF *gsb, size_t sz_min_free); +void GS_BUF_free(GS_BUF *gsb); +int GS_BUF_resize(GS_BUF *gsb, size_t sz_new); +int GS_BUF_add(GS_BUF *gsb, size_t len); +int GS_BUF_add_data(GS_BUF *gsb, void *data, size_t len); +int GS_BUF_del(GS_BUF *gsb, size_t len); + +#define GS_BUF_UNUSED(gsb) ((gsb)->sz_total - (gsb)->sz_used) +#define GS_BUF_RSRC(gsb) (gsb)->data +#define GS_BUF_WDST(gsb) ((uint8_t *)(gsb)->data + (gsb)->sz_used) +#define GS_BUF_USED(gsb) (gsb)->sz_used + +#endif /* !__GS_BUF_H__ */ diff --git a/include/gsocket/gsocket.h b/include/gsocket/gsocket.h index d5c78ee9..67e73135 100644 --- a/include/gsocket/gsocket.h +++ b/include/gsocket/gsocket.h @@ -56,6 +56,7 @@ #include #include #include +#include /* ########################### * ### PROTOCOL DEFINITION ### diff --git a/include/gsocket/packet.h b/include/gsocket/packet.h index 94e03218..7bdb3ca7 100644 --- a/include/gsocket/packet.h +++ b/include/gsocket/packet.h @@ -1,7 +1,8 @@ #ifndef __GS_PACKET_H__ #define __GS_PACKET_H__ 1 -#define GS_PKT_MAX_SIZE (2048) +#define GS_PKT_MAX_SIZE (2048) // content length without pkt-header (2 or 4 bytes) +#define GS_PKT_HDR_MAX_SIZE (4) #define GS_PKT_MAX_MSG 128 // type = 0..127 #define GS_PKT_MAX_CHN 128 // type = 128..255 // #define GS_PKT_ESC 'e' // TESTING ONLY diff --git a/lib/Makefile.am b/lib/Makefile.am index 788e29d0..a1bea783 100755 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -2,7 +2,7 @@ noinst_LIBRARIES = libgsocket.a noinst_PROGRAMS = @PROGRAMS_TEST_LIB@ EXTRA_PROGRAMS = list-test event-test -libgsocket_a_SOURCES = gsocket-util.c gsocket-select.c gsocket-sha256.c gsocket-ssl.c gsocket-engine.c packet.c gs-readline.c list.c event.c +libgsocket_a_SOURCES = gsocket-util.c gsocket-select.c gsocket-sha256.c gsocket-ssl.c gsocket-engine.c packet.c gs-readline.c list.c event.c buf.c list_test_SOURCES = list-test.c list_test_LDADD = libgsocket.a diff --git a/lib/buf.c b/lib/buf.c new file mode 100644 index 00000000..da8457cf --- /dev/null +++ b/lib/buf.c @@ -0,0 +1,82 @@ +/* + * A FIFO like buffer to. Used by file transfer as a write-buffer to queue + * control messages (such as chn_accept, chn_error, ...). + */ +#include "gs-common.h" +#include +#include "gs-externs.h" + +// #define GS_BUF_CHUNK_SIZE (4096) + +void +GS_BUF_init(GS_BUF *gsb, size_t sz_max_add) +{ + memset(gsb, 0, sizeof *gsb); + gsb->sz_max_add = sz_max_add; + + gsb->sz_total = 16*1024*1024; // FIXME + gsb->data = malloc(gsb->sz_total); // FIXME + + GS_BUF_resize(gsb, 0); +} + +void +GS_BUF_free(GS_BUF *gsb) +{ + XFREE(gsb->data); + memset(gsb, 0, sizeof *gsb); +} + +// Adjust size to have at least sz_min_free available. +int +GS_BUF_resize(GS_BUF *gsb, size_t sz_new) +{ + if (GS_BUF_UNUSED(gsb) >= sz_new + gsb->sz_max_add) + return 0; + + gsb->sz_total = gsb->sz_used + sz_new + gsb->sz_max_add; + DEBUGF_R("realloc to %zu, used %zu\n", gsb->sz_total, gsb->sz_used); + gsb->data = realloc(gsb->data, gsb->sz_total); + + return 0; +} + +int +GS_BUF_add(GS_BUF *gsb, size_t len) +{ + // Bail. There is sz_max_add space available but looks like caller wrote + // more ata... + XASSERT(len <= GS_BUF_UNUSED(gsb), "Not enough space in buffer\n"); + + gsb->sz_used += len; + + // Resize + GS_BUF_resize(gsb, 0); + + return 0; +} + +int +GS_BUF_add_data(GS_BUF *gsb, void *data, size_t len) +{ + GS_BUF_resize(gsb, len); + memcpy((uint8_t *)gsb->data + gsb->sz_used, data, len); + + gsb->sz_used += len; + + return 0; +} + +/* + * Consume data from beginning. + */ +int +GS_BUF_del(GS_BUF *gsb, size_t len) +{ + XASSERT(gsb->sz_used >= len, "Cant. used=%zu, len=%zu\n", gsb->sz_used, len); + gsb->sz_used -= len; + memmove(gsb->data, (uint8_t *)gsb->data + len, gsb->sz_used); + + return 0; +} + diff --git a/lib/packet.c b/lib/packet.c index 6fb4e542..90b1b84f 100644 --- a/lib/packet.c +++ b/lib/packet.c @@ -126,8 +126,10 @@ GS_PKT_decode_single(GS_PKT *pkt, const uint8_t *src, size_t slen, uint8_t *dst, /* Check for BO (should not never happen) */ size_t available = sizeof pkt->inband - pkt->len; - XASSERT(len <= available, "len = %zd, left = %zd\n", len, available); + XASSERT(len <= available, "len = %zu, left = %zu\n", len, available); + XASSERT(len <= pkt->esc_len_rem, "len=%zu, len_rem=%zu\n", len, pkt->esc_len_rem); + // DEBUGF("Copying %zu to inband data (total after: %zu, dsz=%zu)\n", len, pkt->len + len, dst - dst_orig); memcpy(pkt->inband + pkt->len, src, len); pkt->len += len; src += len; @@ -140,7 +142,7 @@ GS_PKT_decode_single(GS_PKT *pkt, const uint8_t *src, size_t slen, uint8_t *dst, memcpy(&nlen, &pkt->inband, 2); pkt->is_got_chn_len = 1; pkt->esc_len_rem = ntohs(nlen); - // DEBUGF("Len remaining: %zu\n", pkt->esc_len_rem); + // DEBUGF_B("Len of channel message: %zu (dsz=%zu)\n", pkt->esc_len_rem, dst - dst_orig); pkt->len = 0; if (pkt->esc_len_rem != 0) continue; @@ -156,6 +158,7 @@ GS_PKT_decode_single(GS_PKT *pkt, const uint8_t *src, size_t slen, uint8_t *dst, val = pkt->type; if (val >= GS_PKT_MAX_MSG) val -= GS_PKT_MAX_MSG; + // DEBUGF("PKT cb type %d, data left=%zu (dsz=%zu)\n", pkt->type, send - src, dst - dst_orig); (*pkt->funcs[pkt->type])(val, pkt->inband, pkt->len, pkt->args[pkt->type]); } diff --git a/tools/Makefile.am b/tools/Makefile.am index 33f0658c..37889e8c 100755 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -29,9 +29,9 @@ readline_test_LDADD = ../lib/libgsocket.a console_display_test_SOURCES = console_display.c console_display-test.c utils.c console_display_test_LDADD = ../lib/libgsocket.a -filetransfer_test_SOURCES = filetransfer.c filetransfer-test.c utils.c +filetransfer_test_SOURCES = filetransfer.c filetransfer-test.c utils.c globbing.c filetransfer_test_LDADD = ../lib/libgsocket.a EXTRA_DIST = run_all_tests.sh -noinst_HEADERS = common.h utils.h socks.h console.h ids.h event_mgr.h pkt_mgr.h gs-netcat.h console_display.h filetransfer.h man_gs-netcat.h +noinst_HEADERS = common.h utils.h socks.h console.h ids.h event_mgr.h pkt_mgr.h gs-netcat.h console_display.h filetransfer.h man_gs-netcat.h globbing.h AM_CFLAGS = -I../include diff --git a/tools/filetransfer-test.c b/tools/filetransfer-test.c index 9bca99d2..96278853 100644 --- a/tools/filetransfer-test.c +++ b/tools/filetransfer-test.c @@ -6,19 +6,28 @@ mkdir -p test rm -rf test/test*.dat socat SYSTEM:'./filetransfer-test c test4k.dat test8k.dat 2>client.log' SYSTEM:'(cd test; ../filetransfer-test s 2>../server.log)' +socat SYSTEM:'./filetransfer-test c /usr/share/man/./ 2>client.log' SYSTEM:'(cd test; ../filetransfer-test s 2>../server.log)' +STOP HERE (not working): +socat SYSTEM:'./filetransfer-test c /usr/share/man/mann 2>client.log' SYSTEM:'(cd test; ../filetransfer-test s 2>../server.log)' +is this a blocking problem? or some odd files that are not normal files? + +FIXME: +- test that buffer does not go heywire when write() blocks or when write() is incomplete (8192). + (i really should PAUSE transfer if write() is incomplete???) +- */ #include "common.h" #include "filetransfer.h" #include "utils.h" -#define BUF_LEN (1024) +#define BUF_LEN (250) +// #define BUF_LEN (GS_PKT_MAX_SIZE) static GS_PKT pkt; static GS_FT ft; -static int is_server; -static uint8_t wbuf[GS_PKT_MAX_SIZE]; -static size_t wlen; +static GS_BUF gsb; +static fd_set rfds, wfds; /* SERVER receiving PUT from client */ static void @@ -110,26 +119,35 @@ cb_status(void *ft_ptr, struct _gs_ft_status *s) DEBUGF_M("File : %s\n", s->file->name); } +/* + * Return -1 when all done (must exit) + * Return 0 on success. + * Return 1 when waiting for data + */ int mk_packet(void) { - struct gs_pkt_chn_hdr *hdr = (struct gs_pkt_chn_hdr *)wbuf; + struct gs_pkt_chn_hdr *hdr = (struct gs_pkt_chn_hdr *)GS_BUF_WDST(&gsb); int pkt_type; size_t sz; - sz = GS_FT_packet(&ft, wbuf + sizeof *hdr, sizeof wbuf - sizeof *hdr, &pkt_type); - XASSERT(sz <= sizeof wbuf - sizeof *hdr, "Oops, GS_FT_packet() to long. sz=%zu.\n", sz); + size_t max_len; + + max_len = MIN(GS_PKT_MAX_SIZE, GS_BUF_UNUSED(&gsb) - sizeof *hdr); + + // if (GS_BUF_USED(&gsb) > 0) + // DEBUGF_Y("%zu bytes already in buffer, max_len=%zu\n", GS_BUF_USED(&gsb), max_len); + memset(GS_BUF_WDST(&gsb), 0, max_len); // FIXME + sz = GS_FT_packet(&ft, GS_BUF_WDST(&gsb) + sizeof *hdr, max_len, &pkt_type); - // DEBUGF("sz %zu, type %d\n", sz, pkt_type); switch (pkt_type) { case GS_FT_TYPE_NONE: // Nothing to write...waiting for peer's reply. - DEBUGF_G("TYPE NONE\n"); - return 0; + // DEBUGF_G("TYPE NONE\n"); + return 1; case GS_FT_TYPE_PUT: hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_PUT); - // HEXDUMP(wbuf, sizeof *hdr + sz); break; case GS_FT_TYPE_ERROR: hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_ERROR); @@ -144,7 +162,7 @@ mk_packet(void) hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_DATA); break; case GS_FT_TYPE_DONE: - // CLIENT only. Server always keeps listening. + // CLIENT only: done with all files. DEBUGF_G("All done.\n"); return -1; default: @@ -152,29 +170,54 @@ mk_packet(void) } hdr->esc = GS_PKT_ESC; - hdr->len = htons(sz); - - wlen = sizeof *hdr + sz; + uint16_t len = htons(sz); + memcpy(&hdr->len, &len, sizeof len); + // DEBUGF("Packet type=%u length %zu + %zu\n", hdr->type, sizeof *hdr, sz); + + // if (hdr->type == 131) + // { + // static FILE *dfp; + // if (dfp == NULL) + // dfp = fopen("packet-out.dat", "w"); + // fwrite(GS_BUF_WDST(&gsb), 1, sizeof *hdr + sz, dfp); fflush(dfp); + // } + // STOP HERE: packet-out.dat shows correct data but output.dat (from write()) does not.. + + XASSERT(sz + sizeof *hdr <= GS_BUF_UNUSED(&gsb), "Oops, GS_FT_packet() to long. sz=%zu, unusued=%zu.\n", sz, GS_BUF_UNUSED(&gsb)); + GS_BUF_add(&gsb, sizeof *hdr + sz); + + // if (hdr->type == 131) + // { + // static FILE *dxfp; + // if (dxfp == NULL) + // dxfp = fopen("packet-out-after.dat", "w"); + // fwrite(GS_BUF_WDST(&gsb) - sizeof *hdr - sz, 1, sizeof *hdr + sz, dxfp); fflush(dxfp); + // } return 0; } +#include "globbing.h" static void -do_test(void) +glob_cb(GS_GL *res) { - // char *fname = "/tmp/foo/./bar/hosts"; - char fname[80] = "This is/./www.tutorialspoint.com/./website"; - char *str; - char s[4] = "/./"; - - DEBUGF("mkar\n"); - str = strtok(fname, s); - DEBUGF("mkar\n"); - while (str != NULL) - { - DEBUGF("'%s'\n", str); - str = strtok(NULL, s); - } + DEBUGF("Inside Globbing CB %s\n", res->name); + if (GS_FT_put(&ft, res->name) != 0) + DEBUGF_Y("Not valid: %s\n", res->name); // not found or directory +} + + +static void +glob_cb_test(GS_GL *res) +{ + DEBUGF("Inside Globbing CB %s\n", res->name); +} +static void +do_test(const char *exp) +{ + if (exp == NULL) + exp = "/tmp/fo*"; + GS_GLOBBING(glob_cb_test, exp); exit(0); } @@ -187,56 +230,49 @@ main(int arc, char *argv[]) ssize_t sz; size_t dsz; int ret; + int n; GS_library_init(stderr, stderr); gopt.err_fp = stderr; gopt.log_fp = stderr; - // do_test(); + // do_test(argv[2]); + + GS_BUF_init(&gsb, GS_PKT_MAX_SIZE + GS_PKT_HDR_MAX_SIZE); + fcntl(1, F_SETFL, O_NONBLOCK | fcntl(1, F_GETFL, 0)); GS_PKT_init(&pkt); GS_PKT_assign_chn(&pkt, GS_FT_CHN_ERROR, pkt_cb_error, NULL); if (*argv[1] == 'c') { - GS_FT_init(&ft, cb_stats, cb_status); + GS_FT_init(&ft, cb_stats, cb_status, 0); GS_PKT_assign_chn(&pkt, GS_FT_CHN_ACCEPT, pkt_cb_accept, NULL); // Add files to queue... char **ptr = &argv[2]; while (*ptr != NULL) { - if (GS_FT_put(&ft, *ptr) != 0) - DEBUGF_Y("Not found: %s\n", *ptr); + // DEBUGF_B("'%s'\n", *ptr); + GS_GLOBBING(glob_cb, *ptr); ptr++; } } else { - GS_FT_init(&ft, NULL, cb_status); + GS_FT_init(&ft, NULL, cb_status, 1); GS_PKT_assign_chn(&pkt, GS_FT_CHN_PUT, pkt_cb_put, NULL); GS_PKT_assign_chn(&pkt, GS_FT_CHN_DATA, pkt_cb_data, NULL); GS_PKT_assign_chn(&pkt, GS_FT_CHN_SWITCH, pkt_cb_switch, NULL); - is_server = 1; } while (1) { - // If there is data to write then write data first. - if (wlen > 0) - { - sz = write(1, wbuf, wlen); - // DEBUGF("write %zu\n", sz); - if (sz <= 0) - ERREXIT("write()\n"); - wlen = 0; - } - ret = mk_packet(); - - if ((is_server == 0) && (ret != 0)) + if ((ft.is_server == 0) && (ret == -1)) { // No more files to transfer // (All data send. Not waiting for any reply). break; + #if 0 // HERE: test adding files after transfer completed... if (is_extra_puts >= 1) break; @@ -246,20 +282,81 @@ main(int arc, char *argv[]) if (GS_FT_put(&ft, "test1k-extra2.dat") != 0) DEBUGF_Y("Not found: test1k-extra2.dat\n"); continue; + #endif } - if (wlen > 0) - continue; - sz = read(0, src, sizeof src); - if (sz <= 0) - ERREXIT("read()\n"); - ret = GS_PKT_decode(&pkt, src, sz, dst, &dsz); - if (ret != 0) - ERREXIT("GS_PKT_decode()\n"); + // If there is data to write then write data first. + FD_CLR(0, &rfds); + FD_SET(0, &rfds); + + FD_CLR(1, &wfds); + if (GS_BUF_USED(&gsb) > 0) + FD_SET(1, &wfds); + // DEBUGF("Write Data pending: %zu\n", GS_BUF_USED(&gsb)); + + // Go into select if write-pending or waiting for data + if ((GS_BUF_USED(&gsb) > 0) || (ret == 1)) + { + struct timeval tv; + tv.tv_usec = 0; + tv.tv_sec = 1; + n = select(2, &rfds, &wfds, NULL, &tv); + if (n < 0) + ERREXIT("select(): %s\n", strerror(errno)); + } + + if (FD_ISSET(1, &wfds)) + { + // HERE: Write what we can from io-write buffer (max 16k writes). + // Adjust buffer of data successfully written. + // FIXME: MIN(1024,, .. to trigger bug early. Remove. Always try to write all + // sz = write(1, GS_BUF_RSRC(&gsb), MIN(1024, GS_BUF_USED(&gsb))); + sz = write(1, GS_BUF_RSRC(&gsb), GS_BUF_USED(&gsb)); + // DEBUGF("write() == %zd of %zu\n", sz, GS_BUF_USED(&gsb)); + if (sz == 0) + ERREXIT("write() EOF\n"); + + if (sz < 0) + { + if (errno == EAGAIN) + { + DEBUGF_R("WOULD BLOCK..pausing data\n"); + exit(0); // FIXME + // Stop sending data packets but keep queueing control + // packets (e.g. replies to what we read()). + GS_FT_pause_data(&ft); + } + ERREXIT("write(): %s\n", strerror(errno)); + } + + // HERE: write() was a success. Consume data. + GS_BUF_del(&gsb, sz); + // DEBUGF("Write Data pending [after write]: %zu\n", GS_BUF_USED(&gsb)); + + GS_FT_unpause_data(&ft); + } + + if (FD_ISSET(0, &rfds)) + { + sz = read(0, src, sizeof src); + // DEBUGF_G("read() == %zu\n", sz); + if (sz <= 0) + ERREXIT("read()\n"); + ret = GS_PKT_decode(&pkt, src, sz, dst, &dsz); + if (ret != 0) + ERREXIT("GS_PKT_decode()\n"); + if (dsz != 0) + { + HEXDUMP(src, sz); + HEXDUMP(dst, dsz); + ERREXIT("test program should contain only inband data...dsz=%zu\n", dsz); + } + } } // stats_total(&ft.stats_total); GS_FT_free(&ft); + GS_BUF_free(&gsb); return 0; } diff --git a/tools/filetransfer.c b/tools/filetransfer.c index 7d241866..e7943392 100644 --- a/tools/filetransfer.c +++ b/tools/filetransfer.c @@ -24,8 +24,8 @@ static uint32_t GS_mode2fperm(mode_t m); - retain fperm/mtime on directories - empty directories - - +STOP HERE 2020-12-22: +- globbing / wordexp - implement GS_FT_get() TEST CASES: @@ -33,14 +33,14 @@ TEST CASES: #2. dest file not writeable #3. re-start transmission 4. same file in command line -5. src file can not be opened or read error. -6. write to symlink +#5. src file can not be opened or read error. +#6. write to symlink - as per unix (follow symlinks) #7. zero file size #8. retain timestamp #endif void -GS_FT_init(GS_FT *ft, gsft_cb_stats_t func_stats, gsft_cb_status_t func_status) +GS_FT_init(GS_FT *ft, gsft_cb_stats_t func_stats, gsft_cb_status_t func_status, int is_server) { memset(ft, 0, sizeof *ft); GS_LIST_init(&ft->fqueue, 0); @@ -55,6 +55,7 @@ GS_FT_init(GS_FT *ft, gsft_cb_stats_t func_stats, gsft_cb_status_t func_status) ft->func_stats = func_stats; ft->func_status = func_status; + ft->is_server = is_server; } /* @@ -150,7 +151,7 @@ GS_mode2fperm(mode_t m) u |= x_mperm[n].perm; } - DEBUGF_B("mode2fperm 0%o\n", u); + // DEBUGF_B("mode2fperm 0%o\n", u); return u; } @@ -161,7 +162,7 @@ GS_fperm2mode(uint32_t u) mode_t m = 0; int n; - DEBUGF_B("fperm2mode 0%o\n", u); + // DEBUGF_B("fperm2mode 0%o\n", u); for (n = 0; n < sizeof x_mperm / sizeof *x_mperm; n++) { @@ -199,12 +200,12 @@ GS_FT_put(GS_FT *ft, const char *fname) /* * Consider these possibilities (examples) - * /tmp/foo/bar/hosts + * /tmp/foo/bar/hosts * /./tmp/foo/bar/hosts * /tmp/foo/./bar/./hosts * /tmp/./foo/bar/hosts - * hosts - * foo/bar/host + * hosts + * foo/bar/host */ // Find token after last occurance of '/./' const char *str = fname; @@ -226,6 +227,11 @@ GS_FT_put(GS_FT *ft, const char *fname) f->name = strdup(basename(s)); free(s); } else { + // Scenario: put 'foo/'' will turn into 'foo/./'' which will call + // GS_FT_put(foo/.//bar/file.txt) which must end up with 'bar/file.txt' + // and not with '/bar/file.txt' + while (*str == '/') + str++; f->name = strdup(str); } @@ -301,14 +307,16 @@ GS_FT_data(GS_FT *ft, const void *data, size_t len) if (f == NULL) { - DEBUGF_R("Receiving data but no active receiving file\n"); + DEBUGF_R("Receiving data but no active receiving file (len=%zu)\n", len); + HEXDUMP(data, MIN(len, 16)); return; } XASSERT(f->fp != NULL, "fp is NULL\n"); if (f->offset + len > f->fsize) { - DEBUGF_R("More data than we want!\n"); + DEBUGF_R("More data than we want (%"PRIu64")! (offset=%"PRIu64", len == %zu, fsize = %"PRIu64"\n", f->offset + len - f->fsize, f->offset, len, f->fsize); + HEXDUMP((uint8_t *)data + (f->fsize - f->offset), f->offset + len - f->fsize); len = f->fsize - f->offset; } @@ -674,11 +682,12 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) struct _gs_ft_file *f; size_t sz; - DEBUGF("GS_FT_packet() %d, accepted %d, len %zu\n", ft->fputs.n_items, ft->faccepted.n_items, len); + // DEBUGF("GS_FT_packet() %d, accepted %d, len %zu\n", ft->fputs.n_items, ft->faccepted.n_items, len); *pkt_type = GS_FT_TYPE_NONE; if (len < GS_FT_MIN_BUF_SIZE) { + DEBUGF_R("Does this ever happen?\n"); return 0; } // XASSERT(len >= GS_FT_MIN_BUF_SIZE, "len to small\n"); @@ -696,7 +705,6 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) *pkt_type = GS_FT_TYPE_ERROR; GS_LIST_del(li); return sz; - // return sizeof err + sz; } // Check if any files in the queue that need to be 'put' @@ -806,10 +814,17 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) } *pkt_type = GS_FT_TYPE_SWITCH; + DEBUGF_W("#%"PRIu64" Switch to %s (%"PRIu64")\n", li->id, f->name, f->fsize); return (sizeof sw); } + if (ft->is_paused_data) + { + DEBUGF_W("IS-PAUSED-DATA==1. Not sending file data....\n"); + return 0; + } + // HERE: active file *pkt_type = GS_FT_TYPE_DATA; f = ft->active_put_file; @@ -847,10 +862,14 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) return sz; } - if (ft->n_files_waiting == 0) + if (ft->is_server == 0) { - *pkt_type = GS_FT_TYPE_DONE; - return 0; + // CLIENT + if (ft->n_files_waiting == 0) + { + *pkt_type = GS_FT_TYPE_DONE; + return 0; + } } return 0; @@ -892,6 +911,18 @@ GS_FT_free(GS_FT *ft) ft->active_put_file = NULL; } +void +GS_FT_pause_data(GS_FT *ft) +{ + ft->is_paused_data = 1; +} + +void +GS_FT_unpause_data(GS_FT *ft) +{ + ft->is_paused_data = 0; +} + diff --git a/tools/filetransfer.h b/tools/filetransfer.h index e1847fbd..8e1517ee 100644 --- a/tools/filetransfer.h +++ b/tools/filetransfer.h @@ -1,11 +1,12 @@ #ifndef __GS_FILETRANSFER_H__ #define __GS_FILETRANSFER_H__ 1 -#define GS_FT_CHN_PUT (0) -#define GS_FT_CHN_ACCEPT (1) -#define GS_FT_CHN_DATA (3) -#define GS_FT_CHN_ERROR (4) -#define GS_FT_CHN_SWITCH (5) +#define GS_FT_CHN_PUT (0) // 128 +#define GS_FT_CHN_ACCEPT (1) // 129 + +#define GS_FT_CHN_DATA (3) // 131 0x83 +#define GS_FT_CHN_ERROR (4) // 132 +#define GS_FT_CHN_SWITCH (5) // 133 0x85 // Number of bytes needed for largest message (could be data) #define GS_FT_MIN_BUF_SIZE (64) @@ -86,6 +87,8 @@ typedef struct gsft_cb_status_t func_status; GS_LIST qerrs; // queue'd errors + int is_server; + int is_paused_data; // write() blocked. Queue control data. Pause sending file data int n_files_waiting; // Files waiting for completion or error @@ -139,7 +142,7 @@ struct _gs_ft_error #define GS_FT_ERR_NODATA (10) #define GS_FT_ERR_COMPLETED (128) // All data written successfully -void GS_FT_init(GS_FT *ft, gsft_cb_stats_t func, gsft_cb_status_t); +void GS_FT_init(GS_FT *ft, gsft_cb_stats_t func_stats, gsft_cb_status_t func_status, int is_server); void GS_FT_free(GS_FT *ft); int GS_FT_add_file(GS_FT *ft, uint32_t id, const char *fname, size_t len, int64_t fsize, uint32_t mtime, uint32_t fperm); int GS_FT_put(GS_FT *ft, const char *fname); @@ -149,6 +152,8 @@ void GS_FT_data(GS_FT *ft, const void *data, size_t len); void GS_FT_status(GS_FT *ft, uint32_t id, uint8_t code, const char *err_str, size_t len); // void GS_FT_del_file(GS_FT *ft, uint32_t id); size_t GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type); +void GS_FT_pause_data(GS_FT *ft); +void GS_FT_unpause_data(GS_FT *ft); int GS_FT_WANT_WRITE(GS_FT *ft); // Packet types diff --git a/tools/run_ft_tests.sh b/tools/run_ft_tests.sh index feaa42e5..e91b0ee1 100755 --- a/tools/run_ft_tests.sh +++ b/tools/run_ft_tests.sh @@ -3,11 +3,13 @@ command -v md5 >/dev/null 2>&1 && MD5(){ md5 -q "${1}";} command -v md5sum >/dev/null 2>&1 && MD5() { md5sum "${1}" | cut -f1 -d' ';} -IODIR=ft_test_io -IODIRSRC=ft_test_src +IODIR="${PWD}/ft_test_dst" +IODIRSRC="ft_test_src" +LOGDIR="${PWD}" OK="....[\033[1;32mOK\033[0m]" FAIL="[\033[1;31mFAILED\033[0m]" ECHO="echo -e" +BIN="${PWD}/filetransfer-test" mk_dummy() { @@ -18,6 +20,14 @@ mk_dummy test1k.dat 1 mk_dummy test4k.dat 4 mk_dummy test8k.dat 8 +rm -rf "${IODIRSRC}/foo" +mkdir -p "${IODIRSRC}/foo/bar" +# mkdir -p "${IODIRSRC}/foo/dir_empty" // FIXME: Empty directory not yet supported +cp test1k.dat "${IODIRSRC}/foo/bar/test1k.dat" +cp test1k.dat "${IODIRSRC}/foo/.rcfile1" +cp test1k.dat "${IODIRSRC}/foo/.rcfile1" +cp test4k.dat "${IODIRSRC}/foo/bar/.rcfile2" + test_start() { rm -rf "${IODIR}/" &>/dev/null @@ -39,21 +49,30 @@ md5fail() run_put() { - socat SYSTEM:"./filetransfer-test c $* 2>client.log" SYSTEM:"(cd ${IODIR}; ../filetransfer-test s 2>../server.log)" + # set -f disabled globbing + # socat SYSTEM:"./filetransfer-test c $* 2>client.log" SYSTEM:"(cd ${IODIR}; ../filetransfer-test s 2>../server.log)" + # socat SYSTEM:"set -f && ./filetransfer-test c $* 2>client.log" SYSTEM:"(cd ${IODIR}; ../filetransfer-test s 2>../server.log)" + socat SYSTEM:"set -f && ${BIN} c $* 2>${LOGDIR}/client.log" SYSTEM:"(cd ${IODIR}; ${BIN} s 2>${LOGDIR}/server.log)" } -# tests="1.0 " -# tests+="1.1 " -# tests+="1.2 " -# tests+="1.3 " -# tests+="2.1 2.2 " -# tests+="2.3 " -# tests+="3.1 3.2 3.3 " +tests="1.0 " +tests+="1.1 " +tests+="1.2 " +tests+="1.3 " +tests+="2.1 2.2 " +tests+="2.3 " +tests+="3.1 3.2 3.3 " tests+="3.4 " -# tests+="4.1 " -# tests+="4.2 " -# tests+="4.3 " +tests+="4.1 " +tests+="4.2 " +tests+="4.3 " +tests+="5.1 " +tests+="5.2 " +tests+="5.3 " +tests+="5.4 " +tests+="5.5 " +tests+="5.6 " if [[ "$tests" =~ '1.0 ' ]]; then test_start -n "Running #1.0 (put 1 file).................................." @@ -88,8 +107,6 @@ run_put_fail() if [[ "$tests" =~ '1.3 ' ]]; then test_start -n "Running #1.3 (absolute file)..............................." -mkdir -p "${IODIRSRC}/foo/bar" -cp test1k.dat "${IODIRSRC}/foo/bar/test1k.dat" run_put_fail 1 "${IODIRSRC}/foo/bar/test1k.dat" "${IODIR}/test1k.dat" run_put_fail 2 "${IODIRSRC}/foo/bar/./test1k.dat" "${IODIR}/test1k.dat" @@ -189,3 +206,52 @@ run_put zero.dat $ECHO "${OK}" fi +fail_file_count() +{ + # Do not quote so that globbing takes effect. + nf_src=$(find -x $2 -type f -o -type d | wc -l) + nf_dst=$(find -x $3 -type f -o -type d | wc -l) + [[ $nf_src -eq $nf_dst ]] || fail $1 +} + +if [[ "$tests" =~ '5.1 ' ]]; then +test_start -n "Running #5.1 (Globbing ./*)................................" +run_put "${IODIRSRC}/*" +fail_file_count 1 "${IODIRSRC}/*" "${IODIR}/*" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '5.2 ' ]]; then +test_start -n "Running #5.2 (Globbing ./foo/.*)..........................." +run_put "${IODIRSRC}/./foo/.*" +[[ $(find ${IODIR}/ -type f -o -type d | wc -l) -eq 3 ]] || fail 1 +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '5.3 ' ]]; then +test_start -n "Running #5.3 (Globbing .*)................................." +(cd "${IODIRSRC}/foo" && run_put ".*") +[[ $(find ${IODIR}/ -type f -o -type d | wc -l) -eq 2 ]] || fail 1 +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '5.4 ' ]]; then +test_start -n "Running #5.4 (Globbing foo)................................" +(cd "${IODIRSRC}" && run_put "foo") +fail_file_count 1 "${IODIRSRC}/foo" "${IODIR}/foo" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '5.5 ' ]]; then +test_start -n "Running #5.5 (Globbing .).................................." +(cd "${IODIRSRC}" && run_put ".") +fail_file_count 1 "${IODIRSRC}/" "${IODIR}/" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '5.6 ' ]]; then +test_start -n "Running #5.6 (Globbing foo/)..............................." +(cd "${IODIRSRC}" && run_put "foo/") +fail_file_count 1 "${IODIRSRC}/foo/" "${IODIR}/" +$ECHO "${OK}" +fi From fc9ca4f343bb13b1a251a5e1e28b3f07a7ee7b19 Mon Sep 17 00:00:00 2001 From: rootTHC Date: Tue, 26 Jan 2021 11:33:25 +0000 Subject: [PATCH 10/21] upload/download test script --- configure.ac | 2 +- lib/buf.c | 6 +- tools/Makefile.am | 2 +- tools/common.h | 3 + tools/filetransfer-test.c | 185 ++++-- tools/filetransfer.c | 1157 +++++++++++++++++++++++++++++-------- tools/filetransfer.h | 124 +++- tools/run_ft_tests.sh | 112 +++- 8 files changed, 1249 insertions(+), 342 deletions(-) diff --git a/configure.ac b/configure.ac index bdfce4e6..34152508 100755 --- a/configure.ac +++ b/configure.ac @@ -94,7 +94,7 @@ AC_CHECK_LIB([ssl], [SRP_VBASE_get1_by_user], [], [AC_MSG_ERROR([SRP not support dnl Checks for header files. AC_HEADER_STDC AC_HEADER_SYS_WAIT -AC_CHECK_HEADERS(sys/time.h unistd.h string.h utmp.h utmpx.h pty.h openssl/srp.h util.h libutil.h netinet/in_systm.h sys/loadavg.h) +AC_CHECK_HEADERS(sys/time.h unistd.h fnmatch.h string.h utmp.h utmpx.h pty.h openssl/srp.h util.h libutil.h netinet/in_systm.h sys/loadavg.h) AC_CHECK_HEADER(openssl/srp.h, [], [AC_MSG_ERROR([openssl/srp.h not found. Update OpenSSL?])]) diff --git a/lib/buf.c b/lib/buf.c index da8457cf..373623da 100644 --- a/lib/buf.c +++ b/lib/buf.c @@ -6,16 +6,14 @@ #include #include "gs-externs.h" -// #define GS_BUF_CHUNK_SIZE (4096) - void GS_BUF_init(GS_BUF *gsb, size_t sz_max_add) { memset(gsb, 0, sizeof *gsb); gsb->sz_max_add = sz_max_add; - gsb->sz_total = 16*1024*1024; // FIXME - gsb->data = malloc(gsb->sz_total); // FIXME + // gsb->sz_total = 16*1024*1024; // FIXME + // gsb->data = malloc(gsb->sz_total); // FIXME GS_BUF_resize(gsb, 0); } diff --git a/tools/Makefile.am b/tools/Makefile.am index 37889e8c..e7e306a4 100755 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -11,7 +11,7 @@ gs_pipe_LDADD = ../lib/libgsocket.a gs_full_pipe_SOURCES = 3_gs-full-pipe.c utils.c gs_full_pipe_LDADD = ../lib/libgsocket.a -gs_netcat_SOURCES = 4_gs-netcat.c utils.c socks.c console.c ids.c event_mgr.c pkt_mgr.c console_display.c filetransfer.c +gs_netcat_SOURCES = 4_gs-netcat.c utils.c socks.c console.c ids.c event_mgr.c pkt_mgr.c console_display.c filetransfer.c globbing.c gs_netcat_LDADD = ../lib/libgsocket.a dist_bin_SCRIPTS = blitz gs-sftp gs-mount gs_funcs diff --git a/tools/common.h b/tools/common.h index cee772c8..574cce4d 100755 --- a/tools/common.h +++ b/tools/common.h @@ -29,6 +29,9 @@ #ifdef HAVE_UNISTD_H # include #endif +#ifdef HAVE_FNMATCH_H +#include +#endif #include #include #include diff --git a/tools/filetransfer-test.c b/tools/filetransfer-test.c index 96278853..4d56c83c 100644 --- a/tools/filetransfer-test.c +++ b/tools/filetransfer-test.c @@ -2,34 +2,42 @@ * Test program to test gs filetransfer sub-system used by gs-netcat. * +TESTING PUT +=========== mkdir -p test rm -rf test/test*.dat socat SYSTEM:'./filetransfer-test c test4k.dat test8k.dat 2>client.log' SYSTEM:'(cd test; ../filetransfer-test s 2>../server.log)' +rm -rf test +mkdir test socat SYSTEM:'./filetransfer-test c /usr/share/man/./ 2>client.log' SYSTEM:'(cd test; ../filetransfer-test s 2>../server.log)' -STOP HERE (not working): -socat SYSTEM:'./filetransfer-test c /usr/share/man/mann 2>client.log' SYSTEM:'(cd test; ../filetransfer-test s 2>../server.log)' -is this a blocking problem? or some odd files that are not normal files? - -FIXME: -- test that buffer does not go heywire when write() blocks or when write() is incomplete (8192). - (i really should PAUSE transfer if write() is incomplete???) -- +echo Verifying... +(cd /usr/share/man/; find . -type f )| while read x; do [[ $(md5 -q test/$x) == $(md5 -q /usr/share/man/$x) ]] || { echo failed on $x; break; } done + + +TESTING GET +=========== +mkdir -p test +rm -rf test/test*.dat +socat SYSTEM:'(cd test; set -f; ../filetransfer-test C test[14]k.dat test8k.dat 2>../client.log)' SYSTEM:'./filetransfer-test s 2>server.log' +echo Verifying... +for x in test4k.dat test8k.dat; do [[ $(md5 -q "./${x}" == $(md5 -q "test/${x}" ]] || { echo failed on $x; break; } done + */ #include "common.h" #include "filetransfer.h" #include "utils.h" +#include "globbing.h" -#define BUF_LEN (250) -// #define BUF_LEN (GS_PKT_MAX_SIZE) +#define BUF_LEN (GS_PKT_MAX_SIZE) static GS_PKT pkt; static GS_FT ft; static GS_BUF gsb; static fd_set rfds, wfds; -/* SERVER receiving PUT from client */ +// SERVER receiving PUT from client static void pkt_cb_put(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) { @@ -40,7 +48,52 @@ pkt_cb_put(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) return; // protocol error memcpy(&hdr, data, sizeof hdr); - GS_FT_add_file(&ft, ntohl(hdr.id), (char *)p->name, len - sizeof hdr - 1, ntohll(hdr.fsize), ntohl(hdr.mtime), ntohl(hdr.fperm)); + GS_FT_add_file(&ft, ntohl(hdr.id), (char *)p->name, len - sizeof hdr - 1, ntohll(hdr.fsize), ntohl(hdr.mtime), ntohl(hdr.fperm), hdr.flags); +} + +// SERVER receiving DL from client +static void +pkt_cb_dl(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) +{ + struct _gs_ft_dl *d = (struct _gs_ft_dl *)data; + struct _gs_ft_dl hdr; + + if (len < sizeof hdr + 1) + return; // protocol error + + memcpy(&hdr, data, sizeof hdr); + GS_FT_dl_add_file(&ft, ntohl(hdr.id), (char *)d->name, len - sizeof hdr - 1, ntohll(hdr.offset)); +} + + +// SERVER receiving a LIST request from client +static void +pkt_cb_listrequest(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) +{ + DEBUGF_B("LIST-REQUEST received!\n"); + struct _gs_ft_list_request *lr = (struct _gs_ft_list_request *)data; + struct _gs_ft_list_request hdr; + + if (len < sizeof hdr + 1) + return; // protocol error + + memcpy(&hdr, data, sizeof hdr); + GS_FT_list_add_files(&ft, ntohl(hdr.globbing_id), (char *)lr->pattern, len - sizeof hdr - 1); +} + +// CLIENT receiving answer to LIST request +static void +pkt_cb_listreply(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) +{ + // DEBUGF_B("LIST-REPLY received\n"); + struct _gs_ft_list_reply *lr = (struct _gs_ft_list_reply *)data; + struct _gs_ft_list_reply hdr; + + if (len < sizeof hdr + 1) + return; + + memcpy(&hdr, data, sizeof hdr); + GS_FT_list_add(&ft, ntohl(hdr.globbing_id), (char *)lr->name, len - sizeof hdr - 1, ntohll(hdr.fsize), ntohl(hdr.mtime), ntohl(hdr.fperm), hdr.flags); } static void @@ -52,7 +105,7 @@ pkt_cb_switch(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) return; // protocol error memcpy(&hdr, data, sizeof hdr); - GS_FT_switch(&ft, ntohl(hdr.id), /*ntohll(hdr.fsize), */ntohll(hdr.offset)); + GS_FT_switch(&ft, ntohl(hdr.id), ntohll(hdr.offset)); } /* SERVER receiving DATA from client */ @@ -63,7 +116,7 @@ pkt_cb_data(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) GS_FT_data(&ft, data, len); } -/* CLIENT receiving ACCEPT from server */ +/* PUT, CLIENT receiving ACCEPT from server */ static void pkt_cb_accept(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) { @@ -104,7 +157,7 @@ stats_total(GS_FT_stats_total *st) static void cb_stats(struct _gs_ft_stats *s) { - DEBUGF_C("%u stats: %s\n", s->id, s->f->name); + DEBUGF_C("%u stats: %s\n", s->id, s->fname); DEBUGF_C("Speed: %s\n", s->speed_str); stats_total(&ft.stats_total); @@ -135,8 +188,8 @@ mk_packet(void) max_len = MIN(GS_PKT_MAX_SIZE, GS_BUF_UNUSED(&gsb) - sizeof *hdr); - // if (GS_BUF_USED(&gsb) > 0) - // DEBUGF_Y("%zu bytes already in buffer, max_len=%zu\n", GS_BUF_USED(&gsb), max_len); + if (GS_BUF_USED(&gsb) > 0) + DEBUGF_Y("%zu bytes already in buffer, max_len=%zu\n", GS_BUF_USED(&gsb), max_len); memset(GS_BUF_WDST(&gsb), 0, max_len); // FIXME sz = GS_FT_packet(&ft, GS_BUF_WDST(&gsb) + sizeof *hdr, max_len, &pkt_type); @@ -161,6 +214,18 @@ mk_packet(void) case GS_FT_TYPE_DATA: hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_DATA); break; + case GS_FT_TYPE_LISTREQUEST: + // GET (download), CLIENT + hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_LIST_REQUEST); + break; + case GS_FT_TYPE_LISTREPLY: + // SERVER (reply to 'get' (list request)) + hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_LIST_REPLY); + break; + case GS_FT_TYPE_DL: + // CLIENT - download by filename + hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_DL); + break; case GS_FT_TYPE_DONE: // CLIENT only: done with all files. DEBUGF_G("All done.\n"); @@ -174,34 +239,20 @@ mk_packet(void) memcpy(&hdr->len, &len, sizeof len); // DEBUGF("Packet type=%u length %zu + %zu\n", hdr->type, sizeof *hdr, sz); - // if (hdr->type == 131) - // { - // static FILE *dfp; - // if (dfp == NULL) - // dfp = fopen("packet-out.dat", "w"); - // fwrite(GS_BUF_WDST(&gsb), 1, sizeof *hdr + sz, dfp); fflush(dfp); - // } - // STOP HERE: packet-out.dat shows correct data but output.dat (from write()) does not.. - XASSERT(sz + sizeof *hdr <= GS_BUF_UNUSED(&gsb), "Oops, GS_FT_packet() to long. sz=%zu, unusued=%zu.\n", sz, GS_BUF_UNUSED(&gsb)); GS_BUF_add(&gsb, sizeof *hdr + sz); - // if (hdr->type == 131) - // { - // static FILE *dxfp; - // if (dxfp == NULL) - // dxfp = fopen("packet-out-after.dat", "w"); - // fwrite(GS_BUF_WDST(&gsb) - sizeof *hdr - sz, 1, sizeof *hdr + sz, dxfp); fflush(dxfp); - // } - return 0; } -#include "globbing.h" + +// Client, put (uploading) +// FIXME: move this internal to GS_FT (do globbing internally and make GS_FT_put() accept wildcards) static void glob_cb(GS_GL *res) { DEBUGF("Inside Globbing CB %s\n", res->name); + // PUT (upload) if (GS_FT_put(&ft, res->name) != 0) DEBUGF_Y("Not valid: %s\n", res->name); // not found or directory } @@ -213,14 +264,16 @@ glob_cb_test(GS_GL *res) DEBUGF("Inside Globbing CB %s\n", res->name); } static void -do_test(const char *exp) +do_globtest(const char *exp) { if (exp == NULL) - exp = "/tmp/fo*"; - GS_GLOBBING(glob_cb_test, exp); + ERREXIT("no arg\n"); + GS_GLOBBING(glob_cb_test, exp, 0, NULL, 0); exit(0); } +// static uint32_t globbing_id; + int main(int arc, char *argv[]) { @@ -231,12 +284,14 @@ main(int arc, char *argv[]) size_t dsz; int ret; int n; + int is_get = 0; GS_library_init(stderr, stderr); gopt.err_fp = stderr; gopt.log_fp = stderr; - // do_test(argv[2]); + if (*argv[1] == 'G') + do_globtest(argv[2]); GS_BUF_init(&gsb, GS_PKT_MAX_SIZE + GS_PKT_HDR_MAX_SIZE); @@ -244,23 +299,50 @@ main(int arc, char *argv[]) GS_PKT_init(&pkt); GS_PKT_assign_chn(&pkt, GS_FT_CHN_ERROR, pkt_cb_error, NULL); - if (*argv[1] == 'c') + if ((*argv[1] == 'c') || (*argv[1] == 'C')) { + // CLIENT GS_FT_init(&ft, cb_stats, cb_status, 0); - GS_PKT_assign_chn(&pkt, GS_FT_CHN_ACCEPT, pkt_cb_accept, NULL); - // Add files to queue... - char **ptr = &argv[2]; - while (*ptr != NULL) + + + if (*argv[1] == 'c') { - // DEBUGF_B("'%s'\n", *ptr); - GS_GLOBBING(glob_cb, *ptr); - ptr++; + // Test PUT (upload) + GS_PKT_assign_chn(&pkt, GS_FT_CHN_ACCEPT, pkt_cb_accept, NULL); + // Add files to queue... + // char **ptr = &argv[2]; + // int globbing_id = 0; + int i; + for (i = 2; argv[i] != NULL; i++) + { + GS_GLOBBING(glob_cb, argv[i], i, &ft, 0); + } + } else { + // Test GET (download) + is_get = 1; + GS_PKT_assign_chn(&pkt, GS_FT_CHN_LIST_REPLY, pkt_cb_listreply, NULL); + GS_PKT_assign_chn(&pkt, GS_FT_CHN_DATA, pkt_cb_data, NULL); + GS_PKT_assign_chn(&pkt, GS_FT_CHN_SWITCH, pkt_cb_switch, NULL); + + int i; + // GS_FT_get(&ft, "test[14]k.dat"); + // GS_FT_get(&ft, "test8k.dat"); + for (i = 2; argv[i] != NULL; i++) + { + GS_FT_get(&ft, argv[i]); + } } + } else { + // SERVER GS_FT_init(&ft, NULL, cb_status, 1); GS_PKT_assign_chn(&pkt, GS_FT_CHN_PUT, pkt_cb_put, NULL); GS_PKT_assign_chn(&pkt, GS_FT_CHN_DATA, pkt_cb_data, NULL); GS_PKT_assign_chn(&pkt, GS_FT_CHN_SWITCH, pkt_cb_switch, NULL); + + // SERVER - DOWNLOAD CAPACILITY + GS_PKT_assign_chn(&pkt, GS_FT_CHN_LIST_REQUEST, pkt_cb_listrequest, NULL); + GS_PKT_assign_chn(&pkt, GS_FT_CHN_DL, pkt_cb_dl, NULL); } while (1) @@ -292,7 +374,6 @@ main(int arc, char *argv[]) FD_CLR(1, &wfds); if (GS_BUF_USED(&gsb) > 0) FD_SET(1, &wfds); - // DEBUGF("Write Data pending: %zu\n", GS_BUF_USED(&gsb)); // Go into select if write-pending or waiting for data if ((GS_BUF_USED(&gsb) > 0) || (ret == 1)) @@ -307,10 +388,7 @@ main(int arc, char *argv[]) if (FD_ISSET(1, &wfds)) { - // HERE: Write what we can from io-write buffer (max 16k writes). // Adjust buffer of data successfully written. - // FIXME: MIN(1024,, .. to trigger bug early. Remove. Always try to write all - // sz = write(1, GS_BUF_RSRC(&gsb), MIN(1024, GS_BUF_USED(&gsb))); sz = write(1, GS_BUF_RSRC(&gsb), GS_BUF_USED(&gsb)); // DEBUGF("write() == %zd of %zu\n", sz, GS_BUF_USED(&gsb)); if (sz == 0) @@ -333,7 +411,10 @@ main(int arc, char *argv[]) GS_BUF_del(&gsb, sz); // DEBUGF("Write Data pending [after write]: %zu\n", GS_BUF_USED(&gsb)); - GS_FT_unpause_data(&ft); + if (GS_BUF_USED(&gsb) > 0) + GS_FT_pause_data(&ft); // Failed to write all data in buffer. Stop reading file data. + else + GS_FT_unpause_data(&ft); } if (FD_ISSET(0, &rfds)) diff --git a/tools/filetransfer.c b/tools/filetransfer.c index e7943392..f8be3294 100644 --- a/tools/filetransfer.c +++ b/tools/filetransfer.c @@ -2,12 +2,23 @@ #include "common.h" #include "utils.h" #include "filetransfer.h" +#include "globbing.h" static void ft_del(GS_LIST_ITEM *li); +static void free_get_li(GS_LIST_ITEM *li); +static void ft_done(GS_FT *ft); static void qerr_add(GS_FT *ft, uint32_t id, uint8_t code, const char *str); static void mk_stats_total(GS_FT *ft); +static void mk_stats(GS_FT *ft, uint32_t id, struct _gs_ft_file *f, const char *fname, int err); static mode_t GS_fperm2mode(uint32_t u); static uint32_t GS_mode2fperm(mode_t m); +static void update_stats(struct _gs_ft_file *f, size_t sz); +static const char *str_dotslash(const char *src); +static const char *str_stripslash(const char *src); +static int mkdirp(const char *dir, mode_t mode); + + + #if 0 @@ -21,12 +32,11 @@ static uint32_t GS_mode2fperm(mode_t m); (add files until it does not fit. Return to caller the ID that failed and let caller decide if he likes to remove that ID from our list or just let caller try again until it fits...) is there a limt? -- retain fperm/mtime on directories - empty directories - -STOP HERE 2020-12-22: -- globbing / wordexp -- implement GS_FT_get() +- fnmatch for receiving files (hehe. yes please). +STOP HERE: +- Test '/usr/./share/' downloads etc +- deal with server sending /etc/hosts or ../../ shit (strchr?) TEST CASES: 1. pathname + filename 4096 long @@ -43,14 +53,25 @@ void GS_FT_init(GS_FT *ft, gsft_cb_stats_t func_stats, gsft_cb_status_t func_status, int is_server) { memset(ft, 0, sizeof *ft); + + // PUT (upload) - Client side GS_LIST_init(&ft->fqueue, 0); GS_LIST_init(&ft->fputs, 0); GS_LIST_init(&ft->faccepted, 0); GS_LIST_init(&ft->fcompleted, 0); - + // PUT (upload) - Server Side GS_LIST_init(&ft->fadded, 0); GS_LIST_init(&ft->freceiving, 0); + // GET (download) - Client side + GS_LIST_init(&ft->plistreq, 0); + GS_LIST_init(&ft->plistreq_waiting, 0); + GS_LIST_init(&ft->flist, 0); + GS_LIST_init(&ft->fdl_waiting, 0); + // GET (download) - Server side + GS_LIST_init(&ft->flistreply, 0); + GS_LIST_init(&ft->fdl, 0); + GS_LIST_init(&ft->qerrs, 0); ft->func_stats = func_stats; @@ -58,57 +79,241 @@ GS_FT_init(GS_FT *ft, gsft_cb_stats_t func_stats, gsft_cb_status_t func_status, ft->is_server = is_server; } -/* - * SERVER - * Return < 0 on error. - * Return 0 otherwise. - */ -int -GS_FT_add_file(GS_FT *ft, uint32_t id, const char *fname, size_t len, int64_t fsize, uint32_t mtime, uint32_t fperm) +static struct _gs_ft_file * +file_new(const char *fname, const char *fn_local, int64_t fz_local, int64_t fz_remote, uint32_t mtime, uint32_t fperm) +{ + struct _gs_ft_file *f; + f = calloc(1, sizeof *f); + + f->mode = fperm; + f->mtime = mtime; + f->fz_local = fz_local; + f->fz_remote = fz_remote; + + if (fn_local != NULL) + { + // CLIENT + f->realname = strdup(fn_local); + } else { + // SERVER (put & get) + f->realname = strdup(fname); + } + + f->name = strdup(fname); + + return f; +} + +// SERVER. put & get +static int +gs_ft_add_file(GS_FT *ft, GS_LIST *gsl, uint32_t id, const char *fname, uint32_t mtime, uint32_t fperm, int64_t fz_remote, uint8_t flags) { + struct stat res; int ret; int64_t fz = 0; - if (fname[len] != '\0') - return -1; // protocol error. Not 0 terminated. + // First: Directory struture. + if (flags & GS_FT_FL_ISDIR) + { + // mkdirp() will remove any ordinary file that is in its way + if (mkdirp(fname, GS_fperm2mode(fperm)) != 0) + { + DEBUGF_R("mkdir(%s): %s\n", fname, strerror(errno)); + qerr_add(ft, id, GS_FT_ERR_PERM, NULL); + return -GS_FT_ERR_PERM; + } - // FIXME: sanitize file name - DEBUGF_Y("#%u ADD-FILE - fperm 0%o, '%s'\n", id, fperm, fname); + // HERE: Directory (not file). Return immediately. + return 0; + } - struct stat res; + // HERE: Remote is sending an ordinary FILE (not directory) ret = stat(fname, &res); - if (ret != 0) { if (errno != ENOENT) { + DEBUGF_R("Permission Denied: %s\n", fname); qerr_add(ft, id, GS_FT_ERR_PERM, NULL); return -GS_FT_ERR_PERM; // Exists but stat() failed } + DEBUGF_R("Not found: %s\n", fname); } else { // FILE exists if (!S_ISREG(res.st_mode)) { + DEBUGF_R("WARN: %s not a regular file\n", fname); qerr_add(ft, id, GS_FT_ERR_BADF, NULL); return -GS_FT_ERR_BADF; // Not a regular file } - // File is a regular file. + + if (S_ISDIR(res.st_mode)) + { + // Remote wants to send file but directory is in the way + DEBUGF_R("WARN: Directory is in the way: %s\n", fname); + qerr_add(ft, id, GS_FT_ERR_PERM, NULL); + return -GS_FT_ERR_PERM; + } + fz = res.st_size; } + // HERE: File (not directory) struct _gs_ft_file *f; - f = calloc(1, sizeof *f); - f->name = strdup(fname); - f->mode = GS_fperm2mode(fperm); - f->mtime = mtime; - f->fsize = fsize; - char buf[PATH_MAX]; - snprintf(buf, sizeof buf, "%s/%s", getcwd(buf, sizeof buf), fname); - f->realname = strdup(buf); - f->offset = fz; + f = file_new(fname, NULL, fz, fz_remote, mtime, GS_fperm2mode(fperm)); + f->li = GS_LIST_add(gsl, NULL, f, id); + + return 0; +} +/* + * SERVER, PUT (upload) + * Return < 0 on error. + * Return 0 otherwise. + */ +int +GS_FT_add_file(GS_FT *ft, uint32_t id, const char *fname, size_t len, int64_t fsize, uint32_t mtime, uint32_t fperm, uint8_t flags) +{ + if (fname[len] != '\0') + return -1; // protocol error. Not 0 terminated. + + // FIXME: sanitize file name + DEBUGF_Y("#%u ADD-FILE - size %"PRIu64", fperm 0%o, '%s' mtime=%d flags=0x%02x\n", id, fsize, fperm, fname, mtime, flags); + + char fn_local[4096]; + char *wdir = getwd(NULL); + snprintf(fn_local, sizeof fn_local, "%s/%s", wdir, fname); + XFREE(wdir); + + return gs_ft_add_file(ft, &ft->fadded, id, fn_local, mtime, fperm, fsize /*remote size*/, flags); +} + +/* + * SERVER, GET (download) + * Client requested this file. Add to list of files that need to be send to peer. + * Server (if idle) will send a get_switch message and start transmitting data. + */ +int +GS_FT_dl_add_file(GS_FT *ft, uint32_t id, const char *fname, size_t len, int64_t fz_remote) +{ + if (fname[len] != '\0') + return -1; // protocol error. Not 0 terminated. + + DEBUGF_Y("#%u DL-ADD-FILE - '%s' fz_remote=%"PRId64"\n", id, fname, fz_remote); + + struct stat res; + int ret; + ret = stat(fname, &res); + if (ret != 0) + { + DEBUGF_R("NOT FOUND: %s\n", fname); + char err[128]; + snprintf(err, sizeof err, "Not found: %s", fname); + qerr_add(ft, id, GS_FT_ERR_NOENT, err); + return -1; + } + + if (!S_ISREG(res.st_mode)) + { + DEBUGF_R("WARN2: %s not a regular file\n", fname); + return -2; + } + + return gs_ft_add_file(ft, &ft->fdl, id, fname, res.st_mtime, res.st_mode, fz_remote, 0 /*flags, unused*/); +} + + +/* + * CLIENT, Get (download) - receiving a file list from server + * Add a single file (that exists on server) to the list that the client may request for download. + */ +int +GS_FT_list_add(GS_FT *ft, uint32_t globbing_id, const char *fname, size_t len, int64_t fz_remote, uint32_t mtime, uint32_t fperm, uint8_t flags) +{ + struct _gs_ft_list_pattern *p = NULL; + int ret; + + if (fname[len] != '\0') + return -1; + + DEBUGF_Y("G#%u LISTREPLY - fperm 0%o, '%s'\n", globbing_id, fperm, fname); + + // Check if returned file matches what we asked for. This stops the server + // from sending files that we did not request. + GS_LIST_ITEM *li = GS_LIST_by_id(&ft->plistreq_waiting, globbing_id); + if (li == NULL) + { + DEBUGF_R("Oops. Received G#%u but no file requested?\n", globbing_id); + return -1; + } + p = (struct _gs_ft_list_pattern *)li->data; + + char fn_local[4096]; + const char *ptr; + ptr = str_dotslash(fname); + if (ptr == NULL) + { + ptr = str_stripslash(fname); + } + snprintf(fn_local, sizeof fn_local, "%s/%s", p->wdir, ptr); + + if (fnmatch(p->pattern, fn_local, 0) != 0) + { + DEBUGF_R("filename does not match request (%s != %s)\n", fname, p->pattern); + // goto done; // FIXME: C 'foo/*' would return 'bar/x.txt' but should return 'foo/bar/x.txt' + } + + // FIXME: Check for '..' and '/' and that realpath matches when send the globbing + // request to remote. Oops, this should be done when requesting file name? + if (strstr(fn_local, "..") != NULL) + { + DEBUGF_R("Bad file name (%s)...\n", fn_local); + goto done; + } + + if (flags & GS_FT_FL_ISDIR) + { + DEBUGF_C("Directory: %s\n", fn_local); + // mkdirp() will remove any ordinary file that is in its way + if (mkdirp(fn_local, GS_fperm2mode(fperm)) != 0) + { + DEBUGF_R("mkdir(%s): %s\n", fn_local, strerror(errno)); + } + + goto done; + } + + + struct stat res; + ret = stat(fn_local, &res); + int64_t fz_local = 0; + if (ret == 0) + { + if (res.st_size == fz_remote) + { + DEBUGF_R("File of equal size already exists (%s)\n", fn_local); + mk_stats(ft, -1, NULL, fn_local, 0 /*success*/); + + goto done; + } + if (res.st_size < fz_remote) + fz_local = res.st_size; + } + + struct _gs_ft_file *f; + f = file_new(fname, fn_local, fz_local, fz_remote, mtime, GS_fperm2mode(fperm)); - f->li = GS_LIST_add(&ft->fadded, NULL, f, id); + f->li = GS_LIST_add(&ft->flist, NULL, f, ft->g_id); + ft->g_id += 1; + ft->n_files_waiting += 1; + ret = 0; + +done: + if (flags & GS_FT_LISTREPLY_FL_LAST) + { + DEBUGF_G("Last file for this globbing id. Free'ing get-list\n"); + free_get_li(li); + } return 0; } @@ -172,31 +377,44 @@ GS_fperm2mode(uint32_t u) return m; } -/* - * CLIENT: Add this file (not directory) to queue. - */ -int -GS_FT_put(GS_FT *ft, const char *fname) + + +// strip all '/' from beginning of string. +static const char * +str_stripslash(const char *src) { - int ret; - struct stat res; + while (*src == '/') + src++; - // Get absolute and real path as CWD may change before - // upload starts. - char *realfname; - realfname = realpath(fname, NULL); - if (realfname == NULL) - return -3; + return src; +} - ret = stat(fname, &res); - if (ret != 0) - return -1; +// Find '/./' in string +// "/foo/./bar/test1.txt" -> "bar/test1.txt" +// "/foo/bar/./test1.txt" -> "test1.txt" +// "/foo/bar/test1.txt" -> NULL +// "test1.txt" -> NULL +static const char * +str_dotslash(const char *src) +{ - if (!S_ISREG(res.st_mode)) - return -2; + char *token; + token = strstr(src, "/./"); + if (token == NULL) + return NULL; - struct _gs_ft_file *f; - f = calloc(1, sizeof *f); + src = token + 3; // skip '/./' + src = str_stripslash(src); + + return src; +} + +// Takes a /.// construct and returns / part. +// Space is allocated to hold the new string. Caller must call free() to free it. +static char * +dotslash_filename(const char *src) +{ + char *dst = NULL; /* * Consider these possibilities (examples) @@ -208,49 +426,167 @@ GS_FT_put(GS_FT *ft, const char *fname) * foo/bar/host */ // Find token after last occurance of '/./' - const char *str = fname; - char *token; - int found = 0; - while (1) - { - token = strstr(str, "/./"); - if (token == NULL) - break; - found = 1; - str = token + 3; // skip '/./' - } + const char *str; + str = str_dotslash(src); + // str contains everything after '/./' or fname if '/./' not found. - if (found == 0) + if (str == NULL) { // HERE: No '/./'. Use basename (file only, no directory part) - char *s = strdup(fname); // basename() might modify str :/ - f->name = strdup(basename(s)); + char *s = strdup(src); // basename() might modify str + dst = strdup(basename(s)); free(s); } else { // Scenario: put 'foo/'' will turn into 'foo/./'' which will call // GS_FT_put(foo/.//bar/file.txt) which must end up with 'bar/file.txt' // and not with '/bar/file.txt' - while (*str == '/') - str++; - f->name = strdup(str); + str = str_stripslash(str); + dst = strdup(str); } + return dst; +} + +// CLIENT, put (upload) +// SERVER, get (download), after globbing. +static int +add_file_to_list(GS_FT *ft, GS_LIST *gsl, const char *fname, uint32_t globbing_id, int is_get) +{ + struct _gs_ft_file *f; + f = calloc(1, sizeof *f); + f->globbing_id = globbing_id; + + if (is_get == 1) + { + // Server, GET (download). + f->name = strdup(fname); + } else { + // Client, PUT (upload). + f->name = dotslash_filename(fname); + } + + DEBUGF_Y("#%u name = %s\n", ft->g_id, f->name); + f->li = GS_LIST_add(gsl, NULL, f, ft->g_id); + ft->g_id += 1; + + // Even if it does not exist then we still need to add it to the LISTREPLY list + // Client will request it and only then will we send an error. + // This can happen when client requests "foo[123].notexist[233].da*" and globbing + // fails. + int ret; + struct stat res; + ret = stat(fname, &res); + if (ret != 0) + { + DEBUGF_R("%s NOT FOUND\n", fname); + return -1; + } + + // Get absolute and real path as CWD may change before + // upload starts. + char *realfname; + realfname = realpath(fname, NULL); + if (realfname == NULL) + return -3; + f->realname = realfname; - f->fsize = res.st_size; + f->fz_local = res.st_size; f->mode = res.st_mode; f->mtime = res.st_mtime; - // DEBUGF_Y("mode = %o\n", res.st_mode & ~S_IFMT); - DEBUGF_Y("#%u name = %s\n", ft->g_id, f->name); - f->li = GS_LIST_add(&ft->fqueue, NULL, f, ft->g_id); - ft->g_id += 1; + return 0; +} +/* + * CLIENT: Add this file (not directory) to queue. + */ +int +GS_FT_put(GS_FT *ft, const char *fname) +{ + int ret; + + ret = add_file_to_list(ft, &ft->fqueue, fname, 0 /*unused*/, 0 /*is_get*/); + if (ret != 0) + return ret; ft->n_files_waiting += 1; return 0; } +// SERVER: Add a single file +static int +get_add_file(GS_FT *ft, const char *fname, uint32_t globbing_id) +{ + int ret; + + DEBUGF_G("Adding %s\n", fname); + + // Retrieve all info for the file. + ret = add_file_to_list(ft, &ft->flistreply, fname, globbing_id, 1 /*is_get*/); + if (ret != 0) + { + return ret; + } + + return 0; +} + +// SERVER +static void +glob_cb(GS_GL *res) +{ + GS_FT *ft = (GS_FT *)res->arg_ptr; + + DEBUGF_Y("G#%u \n", res->globbing_id); + get_add_file(ft, res->name, res->globbing_id); +} + +// SERVER: Generate file list based on globbing pattern sent by client. +int +GS_FT_list_add_files(GS_FT *ft, uint32_t globbing_id, const char *pattern, size_t len) +{ + int n_found; + + if (pattern[len] != '\0') + return -1; // protocol error. Not 0-terminated. + + DEBUGF_Y("G#%u GET-ADD-FILE: %s\n", globbing_id, pattern); + + n_found = GS_GLOBBING(glob_cb, pattern, globbing_id, ft, 0); + + if (n_found <= 0) + { + DEBUGF_R("NOT FOUND: %s\n", pattern); + char err[128]; + snprintf(err, sizeof err, "Not found: %s", pattern); + qerr_add(ft, globbing_id, GS_FT_ERR_NOENT, err); + } + + return n_found; +} + + +/* + * CLIENT: Add a get-requst to the listrequest queue. + */ +int +GS_FT_get(GS_FT *ft, const char *pattern) +{ + struct _gs_ft_list_pattern *p; + + p = calloc(1, sizeof *p); + p->pattern = strdup(pattern); + p->wdir = getwd(NULL); + p->globbing_id = ft->g_globbing_id; + ft->g_globbing_id++; + + GS_LIST_add(&ft->plistreq, NULL, p, ft->g_id); + ft->g_id += 1; + + return 0; +} + static void qerr_add(GS_FT *ft, uint32_t id, uint8_t code, const char *str) { @@ -298,13 +634,21 @@ qerr_free(struct _gs_ft_qerr *qerr) XFREE(qerr); } -// SERVER +// SERVER & CLIENT void GS_FT_data(GS_FT *ft, const void *data, size_t len) { - struct _gs_ft_file *f = ft->active_receiving; + struct _gs_ft_file *f; + struct _gs_ft_file **active; size_t sz; + if (ft->is_server) + active = &ft->active_put_file; + else + active = &ft->active_dl_file; + + f = *active; + if (f == NULL) { DEBUGF_R("Receiving data but no active receiving file (len=%zu)\n", len); @@ -313,38 +657,57 @@ GS_FT_data(GS_FT *ft, const void *data, size_t len) } XASSERT(f->fp != NULL, "fp is NULL\n"); - if (f->offset + len > f->fsize) + if (f->fz_local + len > f->fz_remote) { - DEBUGF_R("More data than we want (%"PRIu64")! (offset=%"PRIu64", len == %zu, fsize = %"PRIu64"\n", f->offset + len - f->fsize, f->offset, len, f->fsize); - HEXDUMP((uint8_t *)data + (f->fsize - f->offset), f->offset + len - f->fsize); - len = f->fsize - f->offset; + DEBUGF_R("More data than we want (%"PRIu64")! (fz_local=%"PRIu64", len == %zu, fz_remote = %"PRIu64"\n", f->fz_local + len - f->fz_remote, f->fz_local, len, f->fz_remote); + HEXDUMP((uint8_t *)data + (f->fz_remote - f->fz_local), f->fz_local + len - f->fz_remote); + len = f->fz_remote - f->fz_local; // truncating } sz = fwrite(data, 1, len, f->fp); - f->offset += sz; + f->fz_local += sz; if (sz != len) { + if (ft->is_server == 0) + mk_stats(ft, f->li->id, f->li->data, NULL, 1 /*err*/); + do_error(ft, f->li, GS_FT_ERR_BADF, NULL); - ft->active_receiving = NULL; + *active = NULL; return; } - if (f->offset < f->fsize) + if (ft->is_server == 0) + { + // CLIENT, get (download); + update_stats(f, sz); + } + + if (f->fz_local < f->fz_remote) return; // still data missing... - DEBUGF_B("Server: All data received (%"PRIu64" of %"PRIu64")\n", f->offset, f->fsize); - do_complete(ft, f); - ft->active_receiving = NULL; + DEBUGF_B("All data received (%"PRIu64" of %"PRIu64")\n", f->fz_local, f->fz_remote); + if (ft->is_server == 0) + { + // CLIENT, get (download) + mk_stats(ft, f->li->id, f->li->data, NULL, 0 /*err*/); + ft_done(ft); + ft_del(f->li); + } else { + // SERVER, put (upload) + do_complete(ft, f); + } + + *active = NULL; } // CLIENT void -GS_FT_accept(GS_FT *ft, uint32_t id, int64_t offset_dst) +GS_FT_accept(GS_FT *ft, uint32_t id, int64_t fz_remote) { GS_LIST_ITEM *li; - DEBUGF("#%u acc offset_dst = %"PRId64"\n", id, offset_dst); + DEBUGF("#%u acc fz_remote = %"PRId64"\n", id, fz_remote); li = GS_LIST_by_id(&ft->fputs, id); if (li == NULL) @@ -356,45 +719,137 @@ GS_FT_accept(GS_FT *ft, uint32_t id, int64_t offset_dst) GS_LIST_move(&ft->faccepted, li); struct _gs_ft_file *f = (struct _gs_ft_file *)li->data; - f->offset = offset_dst; + f->fz_remote = fz_remote; +} + +// unlink any file in our way and create directory. +static int +mkdir_agressive(const char *path, mode_t mode) +{ + struct stat res; + if (stat(path, &res) == 0) + { + if (S_ISDIR(res.st_mode)) + return 0; // Diretory already exists. return. + + // Remove any other file or link. + DEBUGF_R("unlink(%s)\n", path); + unlink(path); + } + + if (mkdir(path, mode) != 0) + { + DEBUGF_R("mkdir(%s) failed: %s\n", path, strerror(errno)); + return -1; + } + + return 0; } /* - * Create all directories up to file - * /tmp/foo/bar/test.dat would create /tmp/foo/bar/ + * Create all directories recursively. + * + * /tmp/foo/bar/test.dat would create /tmp/foo/bar/test.dat [directory] * /tmp/foo/bar/test.dat/ would create /tmp/foo/bar/test.dat/ + * ///// would return success ('/' always exists) */ -static void -mkdirp(const char *file) +static int +mkdirp(const char *path, mode_t mode) { - char *f = strdup(file); + int rv = 0; + char *f = strdup(path); + struct stat res; + + // Return 0 if directory already exist + if (stat(f, &res) == 0) + { + if (S_ISDIR(res.st_mode)) + { + // HERE: Directory exists. + if (mode == 0) + { + // HERE: Not allowed to overwrite permissions + // if (access(path, W_OK) != 0) + goto done; // Done have access + } else { + chmod(f, mode); + } + DEBUGF_W("%s already exists\n", f); + goto done; + } + } + + // Directory does not exist. Use default permission to create it. + if (mode == 0) + mode = 0755; + // If parent directory exist then only create last directory. + char *dn = dirname(f); + DEBUGF_W("Parent of %s is %s\n", f, dn); + if ((dn != NULL) && (stat(dn, &res) == 0)) + { + if (S_ISDIR(res.st_mode)) + { + // HERE: Parent directory exists. + DEBUGF_W("1-mkdir(%s)\n", f); + if (mkdir_agressive(f, mode) != 0) + rv = -1; + goto done; + } + } + + // HERE: Neither directory nor parent directory exist. Start from + // the left and create entire hirachy.... + + // No need to check '/' which always exists. char *ptr = f; + while (*ptr == '/') + ptr++; + + // Remove '/' from the right (tmp/foo///// -> tmp/foo) + char *end = ptr + strlen(ptr); + while (end > ptr) + { + end--; + if (*end != '/') + break; + *end = '\0'; + } + + // Empty string. + if (ptr == end) + goto done; + + // HERE: Create all directory starting from left + // '/tmp' -> '/tmp/foo' -> '/tmp/foo/bar' -> ... while (1) { ptr = index(ptr, '/'); - if (ptr == NULL) - break; - *ptr = '\0'; - if (*f != 0) + if (ptr != NULL) + *ptr = '\0'; + DEBUGF_W("2-mkdir(%s)\n", f); + if (mkdir_agressive(f, mode) != 0) { - DEBUGF_W("mkdir(%s)\n", f); - mkdir(f, 0755); + rv = -1; + goto done; } + if (ptr == NULL) + break; *ptr = '/'; ptr += 1; } - free(f); +done: + XFREE(f); + return rv; } -// SERVER -void -GS_FT_switch(GS_FT *ft, uint32_t id, int64_t offset) +static void +gs_ft_switch(GS_FT *ft, uint32_t id, int64_t fz_remote, struct _gs_ft_file **active, GS_LIST *fsource) { GS_LIST_ITEM *li; - li = GS_LIST_by_id(&ft->freceiving, id); + li = GS_LIST_by_id(fsource, id); if (li == NULL) { DEBUGF_R("Unknown file id %u\n", id); @@ -404,25 +859,25 @@ GS_FT_switch(GS_FT *ft, uint32_t id, int64_t offset) // Switch from active receiving file to new file struct _gs_ft_file *new = (struct _gs_ft_file *)li->data; - DEBUGF_W("Switching to id %u '%s' (got %"PRId64", want %"PRId64")\n", id, new->name, new->offset, new->fsize); + DEBUGF_W("Switching to id %u '%s' (got %"PRId64", want %"PRId64")\n", id, new->name, new->fz_local, new->fz_remote); - if ((ft->active_receiving != NULL) && (ft->active_receiving != new)) + if ((*active != NULL) && (*active != new)) { - fclose(ft->active_receiving->fp); + fclose((*active)->fp); } - ft->active_receiving = NULL; + *active = NULL; // Server: Existing file is larger. Overwrite. - if (new->offset > new->fsize) + if (new->fz_local > new->fz_remote) { DEBUGF_W("File larger. Overwritting...\n"); - XASSERT(offset == 0, "OFFSET not 0 but server's file is larger\n"); - new->offset = 0; - offset = 0; + XASSERT(fz_remote == 0, "fz_remote not 0 but server's file is larger\n"); + new->fz_local = 0; + fz_remote = 0; } - if ((new->offset == new->fsize) && (new->fsize != 0)) + if ((new->fz_remote == new->fz_local) && (new->fz_local != 0)) { // FIXME: currently _not_ updating mtime/fperm if file is already on peer side. // Do we want this? (if so then move this code block after fopen().) @@ -430,19 +885,30 @@ GS_FT_switch(GS_FT *ft, uint32_t id, int64_t offset) return; } - // new->fsize = fsize; - if (offset == 0) + if (fz_remote == 0) { - DEBUGF_G("New file (%s)\n", new->name); - mkdirp(new->name); + DEBUGF_G("New file (%s) %s\n", new->realname, new->name); + + char *ptr = new->realname; + // mkdir(): Remove leading './///' in './////foo/bar' (when globbing './') + if (*ptr == '.') + { + ptr++; + while (*ptr == '/') + ptr++; + } + ptr = dirname(ptr); + // put(test1k.dat) must not modify the permission of parent directory. + mkdirp(ptr, 0 /*do not update permission on existing directory*/); + new->fp = fopen(new->realname, "w"); } else { // Check fsize of local file. - DEBUGF_G("Appending file\n"); + DEBUGF_G("Appending to file\n"); struct stat res; if (stat(new->realname, &res) != 0) goto err; - if (res.st_size != offset) + if (res.st_size != new->fz_local) { // Size changed do_error(ft, new->li, GS_FT_ERR_BADFSIZE, NULL); @@ -457,20 +923,32 @@ GS_FT_switch(GS_FT *ft, uint32_t id, int64_t offset) goto err; } - if (new->fsize == 0) + if (new->fz_remote == 0) { // Zero sized file. Completed. do_complete(ft, new); return; } - ft->active_receiving = new; + *active = new; return; - err: do_error(ft, new->li, GS_FT_ERR_PERM, NULL); } +void +GS_FT_switch(GS_FT *ft, uint32_t id, int64_t offset) +{ + if (ft->is_server) + { + // Client is uploading (client send SWITCH command) + gs_ft_switch(ft, id, offset, &ft->active_put_file, &ft->freceiving); + } else { + // Client is downloading (server sent SWITCH command) + gs_ft_switch(ft, id, offset, &ft->active_dl_file, &ft->fdl_waiting); + } +} + static void file_free(struct _gs_ft_file *f) { @@ -520,29 +998,36 @@ mk_bps(char *str, size_t sz, uint64_t duration, uint64_t amount, int err) // Generate stats per file and call call-back. static void -mk_stats(GS_FT *ft, uint32_t id, struct _gs_ft_file *f, int err) +mk_stats(GS_FT *ft, uint32_t id, struct _gs_ft_file *f, const char *fname, int err) { struct _gs_ft_stats s; memset(&s, 0, sizeof s); s.id = id; - s.f = f; - s.xfer_amount = f->xfer_amount; - if (f->usec_start > f->usec_end) - f->usec_end = GS_usec(); + if (fname != NULL) + s.fname = fname; + else if (f != NULL) + s.fname = f->name; - if (f->usec_suspend_start != 0) + if (f != NULL) { - DEBUGF_R("Oops, Reporting stats on a suspended file\n"); - f->usec_suspend_duration += (GS_usec() - f->usec_suspend_start); + s.xfer_amount = f->xfer_amount; + if (f->usec_start > f->usec_end) + f->usec_end = GS_usec(); + + if (f->usec_suspend_start != 0) + { + DEBUGF_R("Oops, Reporting stats on a suspended file\n"); + f->usec_suspend_duration += (GS_usec() - f->usec_suspend_start); + } + s.xfer_duration = (f->usec_end - f->usec_start) - f->usec_suspend_duration; } - s.xfer_duration = (f->usec_end - f->usec_start) - f->usec_suspend_duration; - mk_bps(s.speed_str, sizeof s.speed_str, s.xfer_duration, f->xfer_amount, err); + mk_bps(s.speed_str, sizeof s.speed_str, s.xfer_duration, s.xfer_amount, err); // Global stats for all files ft->stats_total.xfer_duration += s.xfer_duration; - ft->stats_total.xfer_amount += f->xfer_amount; + ft->stats_total.xfer_amount += s.xfer_amount; if (err == 0) ft->stats_total.n_files_success += 1; else @@ -572,28 +1057,42 @@ GS_FT_status(GS_FT *ft, uint32_t id, uint8_t code, const char *err_str, size_t l { GS_LIST_ITEM *li; int err = 1; + int is_make_stats = 1; if (err_str[len] != '\0') return; // protocol error. Not 0 terminated. - DEBUGF_R("#%u STATUS: %u (%s)\n", id, code, err_str); + DEBUGF_R("#%u STATUS: code=%u (%s)\n", id, code, err_str); + + // There can not be an error in fqueue or plistreq or flist as those are + // local lists. Here we only care about lists that send a request + // to the server and are waiting for a reply. + // li can be one of two structures only: + // _gs_ft_file or gs_ft_list_pattern li = GS_LIST_by_id(&ft->fcompleted, id); if (li == NULL) { - li = GS_LIST_by_id(&ft->fqueue, id); + li = GS_LIST_by_id(&ft->fputs, id); if (li == NULL) { - li = GS_LIST_by_id(&ft->fputs, id); + li = GS_LIST_by_id(&ft->fadded, id); if (li == NULL) { - li = GS_LIST_by_id(&ft->fadded, id); + li = GS_LIST_by_id(&ft->freceiving, id); if (li == NULL) { - li = GS_LIST_by_id(&ft->freceiving, id); + li = GS_LIST_by_id(&ft->fdl_waiting, id); if (li == NULL) { - DEBUGF_R("id %u not found\n", id); - return; // not found + li = GS_LIST_by_id(&ft->plistreq_waiting, id); + if (li == NULL) + { + DEBUGF_R("id %u not found\n", id); + return; // not found + } + // This 'li' does not hold a _gs_ft_file structure and thus + // must not generate stats or try to free a _gs_ft_file when it is not. + is_make_stats = 0; } } } @@ -604,24 +1103,32 @@ GS_FT_status(GS_FT *ft, uint32_t id, uint8_t code, const char *err_str, size_t l err = 0; } - // Make status + // Report (error-)status to caller struct _gs_ft_status s; memset(&s, 0, sizeof s); s.code = code; s.file = li->data; // FIXME: sanitize error string snprintf(s.err_str, sizeof s.err_str, "%s", err_str); - if (ft->func_stats != NULL) + if (ft->func_status != NULL) (*ft->func_status)(ft, &s); - // Report stats to caller - mk_stats(ft, id, li->data, err); - if (li->data == ft->active_put_file) - ft->active_put_file = NULL; + if (is_make_stats) + { + // Report stats to caller + mk_stats(ft, id, li->data, NULL, err); + + if (li->data == ft->active_put_file) + ft->active_put_file = NULL; + + ft_done(ft); + ft_del(li); + } else { + ft_done(ft); + GS_LIST_del(li); + } - ft_done(ft); - ft_del(li); } /* @@ -667,6 +1174,116 @@ ft_mk_error(GS_FT *ft, void *dst, size_t len, int *pkt_type, GS_LIST_ITEM *li, u return sz; } +static void +update_stats(struct _gs_ft_file *f, size_t sz) +{ + // -----BEGIN Log statistics----- + if (f->usec_start == 0) + { + f->usec_start = GS_usec(); + } + + if (f->usec_suspend_start != 0) + { + f->usec_suspend_duration += (GS_usec() - f->usec_suspend_start); + f->usec_suspend_start = 0; + } + f->xfer_amount += sz; + // -----END Log statistics----- +} + +static size_t +mk_xfer_data(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fcompleted, int *pkt_type, void *dst, size_t len, int is_log_stats) +{ + struct _gs_ft_file *f = *active; + size_t sz; + *pkt_type = GS_FT_TYPE_DATA; + + sz = fread(dst, 1, len, f->fp); + + if (sz <= 0) + { + *active = NULL; + return ft_mk_error(ft, dst, len, pkt_type, f->li, GS_FT_ERR_BADF, NULL); + } + f->fz_remote += sz; + + if (f->fz_remote >= f->fz_local) + { + GS_LIST_move(fcompleted, f->li); + *active = NULL; + } + + if (is_log_stats) + { + update_stats(f, sz); + } + + return sz; +} + +static size_t +mk_switch(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fsource, GS_LIST *fcompleted, int *pkt_type, void *dst, size_t len) +{ + GS_LIST_ITEM *li; + struct _gs_ft_file *f; + int ret; + + li = GS_LIST_next(fsource, NULL); + f = (struct _gs_ft_file *)li->data; + + f->fp = fopen(f->realname, "r"); + if (f->fp == NULL) + { + DEBUGF("Could not open file %s\n", f->realname); + return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_PERM, NULL); + } + + ret = fseek(f->fp, 0, SEEK_END); + if (ret != 0) + return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_BADF, NULL); + f->fz_local = ftell(f->fp); + + // Peer already has this file. + // Overwrite if remote size is smaller _or_ larger. + if ((f->fz_local == f->fz_remote) && (f->fz_local != 0)) + { + DEBUGF("#%u Skipping %s (already on peer)\n", (unsigned int)f->li->id, f->name); + mk_stats(ft, li->id, f, NULL, 0 /*success*/); + return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_NODATA, NULL); + } + + // Remote size is larger. Overwrite from beginning. + if (f->fz_local < f->fz_remote) + f->fz_remote = 0; + + // Remote size is smaller. Restart transmission. + ret = fseek(f->fp, f->fz_remote, SEEK_SET); + if (ret != 0) + return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_BADF, NULL); + + struct _gs_ft_switch sw; + memset(&sw, 0, sizeof sw); + sw.id = htonl(li->id); + sw.offset = htonll(f->fz_remote); + memcpy(dst, &sw, sizeof sw); + + // Handle zero size files + if (f->fz_local == 0) + { + GS_LIST_move(fcompleted, f->li); + *active = NULL; + } else { + // HERE: fsize is not zero. + *active = f; + } + + *pkt_type = GS_FT_TYPE_SWITCH; + DEBUGF_W("#%"PRIu64" Switch to %s (fz_local=%"PRIu64", fz_remote=%"PRIu64")\n", li->id, f->name, f->fz_local, f->fz_remote); + + return sizeof sw; +} + /* * Create a data packet (from job/queue that need attention). * Return the length. @@ -690,7 +1307,6 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) DEBUGF_R("Does this ever happen?\n"); return 0; } - // XASSERT(len >= GS_FT_MIN_BUF_SIZE, "len to small\n"); // Check if any queue'd errors needs sending. // Server & Client @@ -703,6 +1319,7 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) sz = mk_error(dst, len, qerr->id, qerr->code, qerr->str); *pkt_type = GS_FT_TYPE_ERROR; + qerr_free(qerr); GS_LIST_del(li); return sz; } @@ -722,8 +1339,11 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) memset(&put, 0, sizeof put); put.fperm = htonl(GS_mode2fperm(f->mode)); put.id = htonl(li->id); - put.fsize = htonll(f->fsize); put.mtime = htonl(f->mtime); + if (S_ISDIR(f->mode)) + put.flags |= GS_FT_FL_ISDIR; + else + put.fsize = htonll(f->fz_local); memcpy(dst, &put, sizeof put); sz = MIN(len - sizeof put, strlen(f->name) + 1); @@ -731,12 +1351,105 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) memcpy(p->name, f->name, sz - 1); p->name[sz - 1] = '\0'; - GS_LIST_move(&ft->fputs, li); + if (S_ISDIR(f->mode)) + { + ft_done(ft); + ft_del(li); + } else { + GS_LIST_move(&ft->fputs, li); + } *pkt_type = GS_FT_TYPE_PUT; return sizeof put + sz; } + // Client + if (ft->plistreq.n_items > 0) + { + GS_LIST_ITEM *li = NULL; + li = GS_LIST_next(&ft->plistreq, NULL); + struct _gs_ft_list_pattern *p = li->data; + + DEBUGF("%d LIST-REQ '%s' in queue (LIST-REQ to be send)\n", ft->plistreq.n_items, p->pattern); + + struct _gs_ft_list_request list_req; + struct _gs_ft_list_request *lr = (struct _gs_ft_list_request *)dst; + memset(&list_req, 0, sizeof list_req); + list_req.globbing_id = htonl(p->globbing_id); + memcpy(dst, &list_req, sizeof list_req); + + sz = MIN(len - sizeof list_req, strlen(p->pattern) + 1); + memcpy(lr->pattern, p->pattern, sz - 1); + lr->pattern[sz - 1] = '\0'; + + GS_LIST_move(&ft->plistreq_waiting, li); + *pkt_type = GS_FT_TYPE_LISTREQUEST; + return sizeof list_req + sz; + } + // Server + if (ft->flistreply.n_items > 0) + { + // FIXME: do stats here on the list of files?! No need to keep this all in memory. + DEBUGF("%d LIST-REPLY in queue (LIST-REPLY to be send)\n", ft->flistreply.n_items); + GS_LIST_ITEM *li = NULL; + li = GS_LIST_next(&ft->flistreply, NULL); + f = (struct _gs_ft_file *)li->data; + + struct _gs_ft_list_reply list_reply; + struct _gs_ft_list_reply *lr = (struct _gs_ft_list_reply *)dst; + memset(&list_reply, 0, sizeof list_reply); + + list_reply.fperm = htonl(GS_mode2fperm(f->mode)); + list_reply.globbing_id = htonl(f->globbing_id); + list_reply.fsize = htonll(f->fz_local); + list_reply.mtime = htonl(f->mtime); + if (S_ISDIR(f->mode)) + list_reply.flags |= GS_FT_FL_ISDIR; + // Check if this is the last entry for this globbing-id + GS_LIST_ITEM *nli = GS_LIST_next(&ft->flistreply, li); + if ((nli == NULL) || (f->globbing_id != ((struct _gs_ft_file *)nli->data)->globbing_id)) + { + DEBUGF_C("Last entry for this globbing G#%u\n", f->globbing_id); + list_reply.flags |= GS_FT_LISTREPLY_FL_LAST; + } + memcpy(dst, &list_reply, sizeof list_reply); + + sz = MIN(len - sizeof list_reply, strlen(f->name) + 1); + memcpy(lr->name, f->name, sz - 1); + lr->name[sz - 1] = '\0'; + + file_free(f); + GS_LIST_del(li); + *pkt_type = GS_FT_TYPE_LISTREPLY; + return sizeof list_reply + sz; + } + + // Client, download + if (ft->flist.n_items > 0) + { + DEBUGF("%d files in list. Download them...\n", ft->flist.n_items); + GS_LIST_ITEM *li; + li = GS_LIST_next(&ft->flist, NULL); + f = (struct _gs_ft_file *)li->data; + + struct _gs_ft_dl dl; + struct _gs_ft_dl *d = (struct _gs_ft_dl *)dst; + memset(&dl, 0, sizeof dl); + dl.id = htonl(li->id); + dl.offset = htonll(f->fz_local); + memcpy(dst, &dl, sizeof dl); + + sz = MIN(len - sizeof dl, strlen(f->name) + 1); + memcpy(d->name, f->name, sz - 1); + d->name[sz - 1] = '\0'; + + GS_LIST_move(&ft->fdl_waiting, li); + *pkt_type = GS_FT_TYPE_DL; + return sizeof dl + sz; + + } + + // Server, PUT (upload), make accept message if (ft->fadded.n_items > 0) { GS_LIST_ITEM *li = NULL; @@ -746,7 +1459,7 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) struct _gs_ft_accept acc; memset(&acc, 0, sizeof acc); acc.id = htonl(li->id); - acc.offset_dst = htonll(f->offset); + acc.offset_dst = htonll(f->fz_local); memcpy(dst, &acc, sizeof acc); // HERE: Server. Inform client that we accepted. @@ -755,117 +1468,44 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) return sizeof acc; } - // Client - if (ft->faccepted.n_items > 0) + // Server - download to client + if (ft->fdl.n_items > 0) { - if (ft->active_put_file == NULL) + DEBUGF("Files to send: %d\n", ft->fdl.n_items); + if (ft->active_dl_file == NULL) { - // HERE: Currently not transmitting any file data => Select new file. - GS_LIST_ITEM *li = NULL; - int ret; - - // FIXME: find file with least amount of outstanding data - li = GS_LIST_next(&ft->faccepted, NULL); - f = (struct _gs_ft_file *)li->data; - - // Open file and seek to location - f->fp = fopen(f->realname, "r"); - if (f->fp == NULL) - return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_PERM, NULL); - - ret = fseek(f->fp, 0, SEEK_END); - if (ret != 0) - return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_BADF, NULL); - f->fsize = ftell(f->fp); - - // Peer already has this file. - // Overwrite if remote size is smaller _or_ larger. - // f->fsize == local size, f->offset == remote size - if ((f->fsize == f->offset) && (f->fsize != 0)) - { - DEBUGF("#%u Skipping %s (already on peer)\n", (unsigned int)f->li->id, f->name); - mk_stats(ft, li->id, f, 0 /*success*/); - return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_NODATA, NULL); - } - - // Remote size is larger. Overwrite from beginning. - if (f->fsize < f->offset) - f->offset = 0; - - // Remote size is smaller. Restart transmission. - ret = fseek(f->fp, f->offset, SEEK_SET); - if (ret != 0) - return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_BADF, NULL); - - struct _gs_ft_switch sw; - memset(&sw, 0, sizeof sw); - sw.id = htonl(li->id); - sw.offset = htonll(f->offset); - memcpy(dst, &sw, sizeof sw); - - // Handle zero size files - if (f->fsize == 0) - { - GS_LIST_move(&ft->fcompleted, f->li); - ft->active_put_file = NULL; - } else { - // HERE: fsize is not zero. - ft->active_put_file = f; - } - - *pkt_type = GS_FT_TYPE_SWITCH; - DEBUGF_W("#%"PRIu64" Switch to %s (%"PRIu64")\n", li->id, f->name, f->fsize); - - return (sizeof sw); + DEBUGF("Switching to new file\n"); + return mk_switch(ft, &ft->active_dl_file, &ft->fdl, &ft->fdl_completed, pkt_type, dst, len); } if (ft->is_paused_data) - { - DEBUGF_W("IS-PAUSED-DATA==1. Not sending file data....\n"); return 0; - } - // HERE: active file - *pkt_type = GS_FT_TYPE_DATA; - f = ft->active_put_file; - - sz = fread(dst, 1, len, f->fp); - - if (sz <= 0) - { - // HERE: Read error or file may have shrunk. - ft->active_put_file = NULL; - return ft_mk_error(ft, dst, len, pkt_type, f->li, GS_FT_ERR_BADF, NULL); - } - f->offset += sz; + return mk_xfer_data(ft, &ft->active_dl_file, &ft->fdl_completed, pkt_type, dst, len, 0 /*is_log_stats*/); + } - // -----BEGIN Log statistics----- - if (f->usec_start == 0) + // Client + if (ft->faccepted.n_items > 0) + { + if (ft->active_put_file == NULL) { - f->usec_start = GS_usec(); + return mk_switch(ft, &ft->active_put_file, &ft->faccepted, &ft->fcompleted, pkt_type, dst, len); } - if (f->usec_suspend_start != 0) + if (ft->is_paused_data) { - f->usec_suspend_duration += (GS_usec() - f->usec_suspend_start); - f->usec_suspend_start = 0; + DEBUGF_W("IS-PAUSED-DATA==1. Not sending file data....\n"); + return 0; } - f->xfer_amount += sz; - // -----END Log statistics----- - // File completed. No more data. - if (f->offset >= f->fsize) - { - GS_LIST_move(&ft->fcompleted, f->li); - ft->active_put_file = NULL; - } - return sz; + return mk_xfer_data(ft, &ft->active_put_file, &ft->fcompleted, pkt_type, dst, len, 1 /*is_log_stats*/); } if (ft->is_server == 0) { // CLIENT - if (ft->n_files_waiting == 0) + DEBUGF_R("n_files_waiting=%d, plist.n_items=%d\n", ft->n_files_waiting, ft->plistreq_waiting.n_items); + if ((ft->n_files_waiting == 0) && (ft->plistreq_waiting.n_items == 0)) { *pkt_type = GS_FT_TYPE_DONE; return 0; @@ -889,6 +1529,26 @@ free_gsl(GS_LIST *gsl) } } +static void +free_get_li(GS_LIST_ITEM *li) +{ + struct _gs_ft_list_pattern *p = (struct _gs_ft_list_pattern *)li->data; + XFREE(p->pattern); + XFREE(p->wdir); + GS_LIST_del(li); +} + +static void +free_get_gsl(GS_LIST *gsl) +{ + GS_LIST_ITEM *li = GS_LIST_next(gsl, NULL); + + for (; li != NULL; li = GS_LIST_next(gsl, li)) + { + free_get_li(li); + } +} + void GS_FT_free(GS_FT *ft) { @@ -900,6 +1560,16 @@ GS_FT_free(GS_FT *ft) free_gsl(&ft->fadded); free_gsl(&ft->freceiving); + free_get_gsl(&ft->plistreq); + free_get_gsl(&ft->plistreq_waiting); + + free_gsl(&ft->flist); + free_gsl(&ft->fdl_waiting); + + // GS_LIST_del_all(&ft->flistreply, 1 /*deep*/); FIXME: why is thsi deep and not free_gsl???? + free_gsl(&ft->flistreply); + free_gsl(&ft->fdl); + GS_LIST_ITEM *li; for (li = GS_LIST_next(&ft->qerrs, NULL); li != NULL; li = GS_LIST_next(&ft->qerrs, li)) { @@ -909,6 +1579,7 @@ GS_FT_free(GS_FT *ft) } ft->active_put_file = NULL; + ft->active_dl_file = NULL; } void diff --git a/tools/filetransfer.h b/tools/filetransfer.h index 8e1517ee..b1ffdc19 100644 --- a/tools/filetransfer.h +++ b/tools/filetransfer.h @@ -1,12 +1,14 @@ #ifndef __GS_FILETRANSFER_H__ #define __GS_FILETRANSFER_H__ 1 -#define GS_FT_CHN_PUT (0) // 128 -#define GS_FT_CHN_ACCEPT (1) // 129 - -#define GS_FT_CHN_DATA (3) // 131 0x83 -#define GS_FT_CHN_ERROR (4) // 132 -#define GS_FT_CHN_SWITCH (5) // 133 0x85 +#define GS_FT_CHN_PUT (0) // 128 +#define GS_FT_CHN_ACCEPT (1) // 129 +#define GS_FT_CHN_LIST_REQUEST (2) // 130 0x82 +#define GS_FT_CHN_DATA (3) // 131 0x83 +#define GS_FT_CHN_ERROR (4) // 132 +#define GS_FT_CHN_SWITCH (5) // 133 0x85 +#define GS_FT_CHN_LIST_REPLY (6) // 134 0x86 +#define GS_FT_CHN_DL (7) // 135 0x87 // Number of bytes needed for largest message (could be data) #define GS_FT_MIN_BUF_SIZE (64) @@ -19,8 +21,9 @@ struct _gs_ft_file mode_t mode; time_t mtime; FILE *fp; - off_t offset; // Offset to read(=client)/write(=server) - off_t fsize; // Total file size (from client) + off_t fz_remote; //offset; // Offset to read(=client)/write(=server) + off_t fz_local; // Total file size (from client) + uint32_t globbing_id; // statistics // Note: xfer might be suspended by 'switch' to another file. @@ -32,10 +35,19 @@ struct _gs_ft_file int64_t xfer_amount; // Actual data on the wire }; +// Client structure to keep outstanding 'list' request in a GS-LIST +struct _gs_ft_list_pattern +{ + uint32_t globbing_id; + char *pattern; + char *wdir; // working directory +}; + struct _gs_ft_stats { uint32_t id; - struct _gs_ft_file *f; + // struct _gs_ft_file *f; + const char *fname; uint64_t xfer_duration; // Actual transfer time (without suspension) uint64_t xfer_amount; // Actual data transfered char speed_str[8]; // Speed (bps). Human readable string. @@ -70,19 +82,32 @@ struct _gs_ft_status char err_str[128]; // 0-terminated error string }; typedef void (*gsft_cb_status_t)(void *ft_ptr, struct _gs_ft_status *s); - + typedef struct { + // PUT (upload) - Client Side GS_LIST fqueue; // Client List of files to be transfered GS_LIST fputs; // Client list of files we requested transfer (put sent) GS_LIST faccepted; // Client List of accepted files GS_LIST fcompleted; // Completed. Waiting for 'ERR_COMPLETED' - + // PUT (upload) - Server Side GS_LIST fadded; // Server Side list of ready files GS_LIST freceiving; // Server Side list of receiving files - int g_id; - struct _gs_ft_file *active_put_file; // Current active file - struct _gs_ft_file *active_receiving; // + + // GET (download) - Client Side + GS_LIST plistreq; // Client List of files to be requested (globbing) + GS_LIST plistreq_waiting; // 'list' request sent. awaiting positive reply or error. + GS_LIST flist; // List received from server of files (with names) + GS_LIST fdl_waiting; // List of files to download ('gs_ft_dl' was send for these). + // GET (download) - Server Side + GS_LIST flistreply; // Server list of file to be send to client. + GS_LIST fdl; // List of files ready to send (switch to). + GS_LIST fdl_completed; // Waiting for ERR_COMPLETED + + int g_id; // global request ID (unique) to match error-replies to requests + int g_globbing_id; // global globbing id + struct _gs_ft_file *active_put_file; // Current active upload file + struct _gs_ft_file *active_dl_file; // Current active download file gsft_cb_stats_t func_stats; gsft_cb_status_t func_status; @@ -90,23 +115,61 @@ typedef struct int is_server; int is_paused_data; // write() blocked. Queue control data. Pause sending file data - int n_files_waiting; // Files waiting for completion or error + int n_files_waiting; // Files waiting for completion or error FIXME: This should be n_requests_waiting + // ..and be a counter of all outstanding requests we are awaiting an answer for.... + // int n_listreply_waiting; // Statistics total (all files) GS_FT_stats_total stats_total; } GS_FT; -// CLIENT -> Server: put a file to server. +// CLIENT -> Server: upload a file to server. +// Server replies with 'gs_ft_accept' struct _gs_ft_put { uint32_t id; uint32_t fperm; int64_t fsize; uint32_t mtime; - uint8_t reserved[32 - 4 - 4 - 4 - 8]; + uint8_t flags; + uint8_t reserved[32 - 4 - 4 - 4 - 8 - 1]; uint8_t name[0]; // 0-terminated file name } __attribute__((__packed__)); +// CLIENT -> Server: request a file from server +struct _gs_ft_list_request +{ + uint32_t globbing_id; // ID of _this_ LIST request + uint8_t pattern[0]; // 0-terminated file name (globbing allowed: ~/*.[ch]) +} __attribute__((__packed__)); + +// SERVER -> Client: Push a file to server. +struct _gs_ft_list_reply +{ + uint32_t globbing_id; // Matching ID of get-request. + uint32_t file_idNOTUSED; // NBO: 0..file_max + uint32_t file_maxNOTUSED; // NBO: max number of files this globbing resolved into + uint32_t fperm; + int64_t fsize; + uint32_t mtime; + uint8_t flags; + uint8_t reserved[32 - 4 - 4 - 4 - 8 - 1]; + uint8_t name[0]; // 0-terminated file name +} __attribute__((__packed__)); +#define GS_FT_LISTREPLY_FL_LAST (1) +#define GS_FT_FL_ISDIR (2) + + +// CLIENT -> Server: request file for download +struct _gs_ft_dl +{ + uint32_t id; + uint8_t res[4]; + int64_t offset; // Offset to start from + uint8_t res2[8]; + uint8_t name[0]; // 0-terminated file name +} __attribute__((__packed__)); + // SERVER -> Client: Accept file. (reply to PUT) struct _gs_ft_accept { @@ -118,6 +181,7 @@ struct _gs_ft_accept } __attribute__((__packed__)); // CLIENT -> Server: Following data is for this file id. +// SERVER -> Client: Following data is for this file id. struct _gs_ft_switch { uint32_t id; @@ -144,26 +208,32 @@ struct _gs_ft_error void GS_FT_init(GS_FT *ft, gsft_cb_stats_t func_stats, gsft_cb_status_t func_status, int is_server); void GS_FT_free(GS_FT *ft); -int GS_FT_add_file(GS_FT *ft, uint32_t id, const char *fname, size_t len, int64_t fsize, uint32_t mtime, uint32_t fperm); +int GS_FT_add_file(GS_FT *ft, uint32_t id, const char *fname, size_t len, int64_t fsize, uint32_t mtime, uint32_t fperm, uint8_t flags); +int GS_FT_dl_add_file(GS_FT *ft, uint32_t id, const char *fname, size_t len, int64_t fsize); +int GS_FT_list_add_files(GS_FT *ft, uint32_t get_id, const char *pattern, size_t len); +int GS_FT_list_add(GS_FT *ft, uint32_t globbing_id, const char *fname, size_t len, int64_t fsize, uint32_t mtime, uint32_t fperm, uint8_t flags); int GS_FT_put(GS_FT *ft, const char *fname); -void GS_FT_switch(GS_FT *ft, uint32_t id, /*int64_t fsize, */int64_t offset); +int GS_FT_get(GS_FT *ft, const char *pattern); +void GS_FT_switch(GS_FT *ft, uint32_t id, int64_t offset); void GS_FT_accept(GS_FT *ft, uint32_t id, int64_t offset); void GS_FT_data(GS_FT *ft, const void *data, size_t len); void GS_FT_status(GS_FT *ft, uint32_t id, uint8_t code, const char *err_str, size_t len); -// void GS_FT_del_file(GS_FT *ft, uint32_t id); size_t GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type); void GS_FT_pause_data(GS_FT *ft); void GS_FT_unpause_data(GS_FT *ft); int GS_FT_WANT_WRITE(GS_FT *ft); // Packet types -#define GS_FT_TYPE_NONE (0) -#define GS_FT_TYPE_SWITCH (1) -#define GS_FT_TYPE_DATA (2) -#define GS_FT_TYPE_ERROR (3) -#define GS_FT_TYPE_PUT (4) -#define GS_FT_TYPE_DONE (5) // Inform caller that stack is done -#define GS_FT_TYPE_ACCEPT (6) +#define GS_FT_TYPE_NONE (0) +#define GS_FT_TYPE_SWITCH (1) +#define GS_FT_TYPE_DATA (2) +#define GS_FT_TYPE_ERROR (3) +#define GS_FT_TYPE_PUT (4) +#define GS_FT_TYPE_DONE (5) // Inform caller that stack is done +#define GS_FT_TYPE_ACCEPT (6) +#define GS_FT_TYPE_LISTREQUEST (7) // CLIENT -> Server (globbing pattern) +#define GS_FT_TYPE_LISTREPLY (8) // SERVER -> Client +#define GS_FT_TYPE_DL (9) // CLIENT -> Server (request file by name) // Packet Bulding functions //uint16_t GS_FT_mk_put(GS_FT *ft, void *dst, size_t len, const char *fname); diff --git a/tools/run_ft_tests.sh b/tools/run_ft_tests.sh index e91b0ee1..d6df8bdb 100755 --- a/tools/run_ft_tests.sh +++ b/tools/run_ft_tests.sh @@ -22,11 +22,12 @@ mk_dummy test8k.dat 8 rm -rf "${IODIRSRC}/foo" mkdir -p "${IODIRSRC}/foo/bar" -# mkdir -p "${IODIRSRC}/foo/dir_empty" // FIXME: Empty directory not yet supported -cp test1k.dat "${IODIRSRC}/foo/bar/test1k.dat" -cp test1k.dat "${IODIRSRC}/foo/.rcfile1" +mkdir -p "${IODIRSRC}/foo/dir_empty" +cp test1k.dat "${IODIRSRC}/foo/bar/" cp test1k.dat "${IODIRSRC}/foo/.rcfile1" +cp test4k.dat "${IODIRSRC}/foo/" cp test4k.dat "${IODIRSRC}/foo/bar/.rcfile2" +cp test8k.dat "${IODIRSRC}/" test_start() { @@ -47,6 +48,14 @@ md5fail() [[ "$(MD5 ${2})" != "$(MD5 ${3})" ]] && fail $1; } +fail_file_count() +{ + # Do not quote so that globbing takes effect. + nf_src=$(find -x $2 -type f -o -type d | wc -l) + nf_dst=$(find -x $3 -type f -o -type d | wc -l) + [[ $nf_src -eq $nf_dst ]] || fail $1 +} + run_put() { # set -f disabled globbing @@ -55,6 +64,17 @@ run_put() socat SYSTEM:"set -f && ${BIN} c $* 2>${LOGDIR}/client.log" SYSTEM:"(cd ${IODIR}; ${BIN} s 2>${LOGDIR}/server.log)" } +run_get() +{ + # set -f disabled globbing + socat SYSTEM:"(cd ${IODIR}; set -f && ${BIN} C $* 2>${LOGDIR}/client.log)" SYSTEM:"(cd ${IODIRSRC}; ${BIN} s 2>${LOGDIR}/server.log)" +} + +run_get2() +{ + # set -f disabled globbing + socat SYSTEM:"(cd ${IODIR}; set -f && ${BIN} C $* 2>${LOGDIR}/client.log)" SYSTEM:"(cd ${IODIRSRC}/foo; ${BIN} s 2>${LOGDIR}/server.log)" +} tests="1.0 " tests+="1.1 " @@ -62,11 +82,13 @@ tests+="1.2 " tests+="1.3 " tests+="2.1 2.2 " tests+="2.3 " -tests+="3.1 3.2 3.3 " +tests+="3.1 3.2 " +tests+="3.3 " tests+="3.4 " tests+="4.1 " tests+="4.2 " tests+="4.3 " +tests+="4.4 " tests+="5.1 " tests+="5.2 " tests+="5.3 " @@ -74,6 +96,64 @@ tests+="5.4 " tests+="5.5 " tests+="5.6 " +tests+="8.1 " +tests+="8.2 " +tests+="8.3 " +tests+="8.4 " +tests+="8.5 " +tests+="8.6 " + + +if [[ "$tests" =~ '8.1 ' ]]; then +test_start -n "Running #8.1 (get, 2 files)................................" +run_get test8k.dat foo/bar/.rcfile2 +md5fail 1 test8k.dat "${IODIR}/test8k.dat" +md5fail 2 test4k.dat "${IODIR}/.rcfile2" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '8.2 ' ]]; then +test_start -n "Running #8.2 (get, 2 files /./ test)......................." +run_get ./foo/bar/test1k.dat ./foo/./bar/test1k.dat +md5fail 1 test1k.dat "${IODIR}/test1k.dat" +md5fail 2 test1k.dat "${IODIR}/bar/test1k.dat" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '8.3 ' ]]; then +test_start -n "Running #8.3 (directory test).............................." +run_get foo/bar +md5fail 1 "${IODIRSRC}/foo/bar/test1k.dat" "${IODIR}/bar/test1k.dat" +md5fail 2 "${IODIRSRC}/foo/bar/.rcfile2" "${IODIR}/bar/.rcfile2" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '8.4 ' ]]; then +test_start -n "Running #8.4 (get, non-exist).............................." +run_get not-exists.dat foobar*noexist[1234].d[ab]t +[[ -f "${IODIR}/not-exists.dat" ]] && fail 1 +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '8.5 ' ]]; then +test_start -n "Running #8.5 (get, ../test8k.dat).........................." +run_get2 ../test8k.dat ../foo ../foo/./bar +md5fail 1 "${IODIRSRC}/test8k.dat" "${IODIR}/test8k.dat" +fail_file_count 2 "${IODIRSRC}/foo" "${IODIR}/foo" +fail_file_count 3 "${IODIRSRC}/foo/bar" "${IODIR}/bar" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '8.6 ' ]]; then +test_start -n "Running #8.6 (get, /etc/hosts)............................." +run_get /etc/hosts /etc/./ssh/ssh_config /./etc/ssh/ssh_config +# run_get /etc/./ssh/ssh_config +md5fail 1 "/etc/hosts" "${IODIR}/hosts" +md5fail 2 "/etc/ssh/ssh_config" "${IODIR}/ssh/ssh_config" +md5fail 3 "/etc/ssh/ssh_config" "${IODIR}/etc/ssh/ssh_config" +$ECHO "${OK}" +fi + if [[ "$tests" =~ '1.0 ' ]]; then test_start -n "Running #1.0 (put 1 file).................................." run_put test1k.dat @@ -107,7 +187,6 @@ run_put_fail() if [[ "$tests" =~ '1.3 ' ]]; then test_start -n "Running #1.3 (absolute file)..............................." - run_put_fail 1 "${IODIRSRC}/foo/bar/test1k.dat" "${IODIR}/test1k.dat" run_put_fail 2 "${IODIRSRC}/foo/bar/./test1k.dat" "${IODIR}/test1k.dat" run_put_fail 3 "./${IODIRSRC}/foo/bar/test1k.dat" "${IODIR}/test1k.dat" @@ -116,8 +195,7 @@ run_put_fail 5 "././${IODIRSRC}/foo/bar/test1k.dat" "${IODIR}/${IODIRSRC}/foo/ba run_put_fail 6 "${IODIRSRC}/foo/../foo/bar/test1k.dat" "${IODIR}/test1k.dat" run_put_fail 7 "${IODIRSRC}/foo/../foo/./bar/test1k.dat" "${IODIR}/bar/test1k.dat" run_put_fail 8 "${PWD}/${IODIRSRC}/foo/bar/test1k.dat" "${IODIR}/test1k.dat" -# run_put foo/./../foo/../foo/bar/test1k.dat # escape. wanted behavior (?). -[[ -f "${IODIR}/etc/hosts" ]] && fail 1 +### run_put foo/./../foo/../foo/bar/test1k.dat # escape. wanted behavior (?). $ECHO "${OK}" fi @@ -206,13 +284,19 @@ run_put zero.dat $ECHO "${OK}" fi -fail_file_count() -{ - # Do not quote so that globbing takes effect. - nf_src=$(find -x $2 -type f -o -type d | wc -l) - nf_dst=$(find -x $3 -type f -o -type d | wc -l) - [[ $nf_src -eq $nf_dst ]] || fail $1 -} +if [[ "$tests" =~ '4.4 ' ]]; then +test_start -n "Running #4.4 (put, empty directory)........................" +touch "${IODIR}/foo" # Place a file in its way (should be overwritten) +run_put "${IODIRSRC}/./foo/dir_empty" +[[ -d "${IODIR}/foo/dir_empty" ]] || fail 1 +touch "${IODIR}/dir_empty" # Place a file in its way (should be overwritten) +run_put "${IODIRSRC}/foo/dir_empty" +[[ -d "${IODIR}/dir_empty" ]] || fail 2 +rmdir "${IODIR}/dir_empty" +run_put "${IODIRSRC}/foo/dir_empty" +[[ -d "${IODIR}/dir_empty" ]] || fail 3 +$ECHO "${OK}" +fi if [[ "$tests" =~ '5.1 ' ]]; then test_start -n "Running #5.1 (Globbing ./*)................................" From 948de4f4416e695ba8b4126cd951a86fce6f9147 Mon Sep 17 00:00:00 2001 From: rootTHC Date: Wed, 27 Jan 2021 14:08:23 +0000 Subject: [PATCH 11/21] almost there! --- tools/filetransfer-test.c | 1 + tools/filetransfer.c | 263 +++++++++++++++++++++++++++----------- tools/filetransfer.h | 5 +- tools/run_ft_tests.sh | 220 +++++++++++++++++++++++-------- 4 files changed, 363 insertions(+), 126 deletions(-) diff --git a/tools/filetransfer-test.c b/tools/filetransfer-test.c index 4d56c83c..886f03fc 100644 --- a/tools/filetransfer-test.c +++ b/tools/filetransfer-test.c @@ -289,6 +289,7 @@ main(int arc, char *argv[]) GS_library_init(stderr, stderr); gopt.err_fp = stderr; gopt.log_fp = stderr; + // gopt.err_fp = NULL; // DISABLE DEBUG if (*argv[1] == 'G') do_globtest(argv[2]); diff --git a/tools/filetransfer.c b/tools/filetransfer.c index f8be3294..b5690307 100644 --- a/tools/filetransfer.c +++ b/tools/filetransfer.c @@ -15,9 +15,10 @@ static uint32_t GS_mode2fperm(mode_t m); static void update_stats(struct _gs_ft_file *f, size_t sz); static const char *str_dotslash(const char *src); static const char *str_stripslash(const char *src); -static int mkdirp(const char *dir, mode_t mode); - - +static int mkdirp(const char *dir, mode_t mode, uint32_t mtime); +static void dir_restore_mtime(const char *path, struct timespec *mtime); +static void dir_save_mtime(const char *path, struct timespec *mtime); +static void set_mode_mtime(const char *path, mode_t mode, uint32_t mtime); @@ -32,11 +33,12 @@ static int mkdirp(const char *dir, mode_t mode); (add files until it does not fit. Return to caller the ID that failed and let caller decide if he likes to remove that ID from our list or just let caller try again until it fits...) is there a limt? -- empty directories - fnmatch for receiving files (hehe. yes please). STOP HERE: - Test '/usr/./share/' downloads etc - deal with server sending /etc/hosts or ../../ shit (strchr?) +- do a HUGE get download. +- do a HUGE put upload TEST CASES: 1. pathname + filename 4096 long @@ -116,7 +118,7 @@ gs_ft_add_file(GS_FT *ft, GS_LIST *gsl, uint32_t id, const char *fname, uint32_t if (flags & GS_FT_FL_ISDIR) { // mkdirp() will remove any ordinary file that is in its way - if (mkdirp(fname, GS_fperm2mode(fperm)) != 0) + if (mkdirp(fname, GS_fperm2mode(fperm), mtime) != 0) { DEBUGF_R("mkdir(%s): %s\n", fname, strerror(errno)); qerr_add(ft, id, GS_FT_ERR_PERM, NULL); @@ -274,7 +276,7 @@ GS_FT_list_add(GS_FT *ft, uint32_t globbing_id, const char *fname, size_t len, i { DEBUGF_C("Directory: %s\n", fn_local); // mkdirp() will remove any ordinary file that is in its way - if (mkdirp(fn_local, GS_fperm2mode(fperm)) != 0) + if (mkdirp(fn_local, GS_fperm2mode(fperm), mtime) != 0) { DEBUGF_R("mkdir(%s): %s\n", fn_local, strerror(errno)); } @@ -587,6 +589,7 @@ GS_FT_get(GS_FT *ft, const char *pattern) return 0; } +// Add an error message to (outgoing) queue. static void qerr_add(GS_FT *ft, uint32_t id, uint8_t code, const char *str) { @@ -603,28 +606,81 @@ qerr_add(GS_FT *ft, uint32_t id, uint8_t code, const char *str) GS_LIST_add(&ft->qerrs, NULL, qerr, ft->qerrs.add_count); } +// Send status to peer (error-msg) and free the file structure. static void -do_error(GS_FT *ft, GS_LIST_ITEM *li, uint32_t code, const char *str) +send_status_and_del(GS_FT *ft, GS_LIST_ITEM *li, uint32_t code, const char *str) { qerr_add(ft, li->id, code, str); ft_del(li); } +// Error while receiving data (called from GS_FT_switch() or GS_FT_data() +// Stop peer sending any further data for this id. static void -do_complete(GS_FT *ft, struct _gs_ft_file *f) +do_recv_error(GS_FT *ft, GS_LIST_ITEM *li, uint32_t code, const char *str) { - if (f->fp != NULL) + struct _gs_ft_file *f = (struct _gs_ft_file *)li->data; + + if (ft->is_server == 0) + mk_stats(ft, f->li->id, f->li->data, NULL, 1 /*err*/); + + send_status_and_del(ft, li, code, str); +} + +// No more data to be send. +// Called by 'sending' party (either from mk_switch() or mk_xfer_data() +// when all data has been send. +// CLIENT: wait for COMPLETE error +// SERVER: Close immediately. +static void +sending_complete(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fcompleted) +{ + struct _gs_ft_file *f = *active; + + if (ft->is_server == 0) { - fflush(f->fp); - fchmod(fileno(f->fp), f->mode & ~S_IFMT); - if (f->mtime != 0) - { - DEBUGF_B("Setting time to %ld\n", f->mtime); - struct timeval t[] = {{f->mtime, 0}, {f->mtime, 0}}; - futimes(fileno(f->fp), t); - } + XASSERT(fcompleted != NULL, "Client has no complete-queue??\n"); + XFCLOSE(f->fp); + GS_LIST_move(fcompleted, f->li); + } else { + // SERVER + XASSERT(fcompleted == NULL, "Server has a complete-queue??\n"); + ft_del(f->li); + } + + *active = NULL; +} + +// Expecting no more data. Called on receiving side +// Called by receiving party (from GS_FT_switch() or GS_FT_data()) +// CLIENT: output stats, set time +static void +receiving_complete(GS_FT *ft, struct _gs_ft_file *f) +{ + char *s = strdup(f->realname); + char *dn = dirname(s); + + if (ft->is_server == 0) + { + // CLIENT (get, download: All data received) + mk_stats(ft, f->li->id, f->li->data, NULL, 0 /*err*/); + XFCLOSE(f->fp); // Must close before setting mtime + set_mode_mtime(f->realname, f->mode, f->mtime); + dir_restore_mtime(dn, &f->dir_mtime); + + ft_done(ft); + ft_del(f->li); + } else { + // SERVER (put, upload: all data received) + XFCLOSE(f->fp); + set_mode_mtime(f->realname, f->mode, f->mtime); + dir_restore_mtime(dn, &f->dir_mtime); + + // close FP, free file, return COMPLETE message to client. + send_status_and_del(ft, f->li, GS_FT_ERR_COMPLETED, NULL); } - do_error(ft, f->li, GS_FT_ERR_COMPLETED, NULL); + + XFREE(s); } static void @@ -669,10 +725,7 @@ GS_FT_data(GS_FT *ft, const void *data, size_t len) if (sz != len) { - if (ft->is_server == 0) - mk_stats(ft, f->li->id, f->li->data, NULL, 1 /*err*/); - - do_error(ft, f->li, GS_FT_ERR_BADF, NULL); + do_recv_error(ft, f->li, GS_FT_ERR_BADF, NULL); *active = NULL; return; } @@ -687,16 +740,7 @@ GS_FT_data(GS_FT *ft, const void *data, size_t len) return; // still data missing... DEBUGF_B("All data received (%"PRIu64" of %"PRIu64")\n", f->fz_local, f->fz_remote); - if (ft->is_server == 0) - { - // CLIENT, get (download) - mk_stats(ft, f->li->id, f->li->data, NULL, 0 /*err*/); - ft_done(ft); - ft_del(f->li); - } else { - // SERVER, put (upload) - do_complete(ft, f); - } + receiving_complete(ft, f); *active = NULL; } @@ -722,11 +766,22 @@ GS_FT_accept(GS_FT *ft, uint32_t id, int64_t fz_remote) f->fz_remote = fz_remote; } +static void +set_mode_mtime(const char *path, mode_t mode, uint32_t mtime) +{ + chmod(path, mode & ~S_IFMT); + struct timeval t[] = {{mtime, 0}, {mtime, 0}}; + + if (mtime != 0) + utimes(path, t); +} + // unlink any file in our way and create directory. static int -mkdir_agressive(const char *path, mode_t mode) +mkdir_agressive(const char *path, mode_t mode, uint32_t mtime) { struct stat res; + if (stat(path, &res) == 0) { if (S_ISDIR(res.st_mode)) @@ -737,11 +792,19 @@ mkdir_agressive(const char *path, mode_t mode) unlink(path); } - if (mkdir(path, mode) != 0) + struct timespec ts; + char *s = strdup(path); + char *parent_dir = dirname(s); + dir_save_mtime(parent_dir, &ts); + if (mkdir(path, 0755 /*use chmod() to bypass umask-value*/) != 0) { DEBUGF_R("mkdir(%s) failed: %s\n", path, strerror(errno)); + XFREE(s); return -1; } + set_mode_mtime(path, mode, mtime); // bypass umask-value + dir_restore_mtime(parent_dir, &ts); + XFREE(s); return 0; } @@ -754,7 +817,7 @@ mkdir_agressive(const char *path, mode_t mode) * ///// would return success ('/' always exists) */ static int -mkdirp(const char *path, mode_t mode) +mkdirp(const char *path, mode_t mode, uint32_t mtime) { int rv = 0; char *f = strdup(path); @@ -772,9 +835,9 @@ mkdirp(const char *path, mode_t mode) // if (access(path, W_OK) != 0) goto done; // Done have access } else { - chmod(f, mode); + set_mode_mtime(f, mode, mtime); } - DEBUGF_W("%s already exists\n", f); + // DEBUGF_W("%s already exists\n", f); goto done; } } @@ -785,14 +848,13 @@ mkdirp(const char *path, mode_t mode) // If parent directory exist then only create last directory. char *dn = dirname(f); - DEBUGF_W("Parent of %s is %s\n", f, dn); if ((dn != NULL) && (stat(dn, &res) == 0)) { if (S_ISDIR(res.st_mode)) { // HERE: Parent directory exists. DEBUGF_W("1-mkdir(%s)\n", f); - if (mkdir_agressive(f, mode) != 0) + if (mkdir_agressive(f, mode, mtime) != 0) rv = -1; goto done; } @@ -822,13 +884,17 @@ mkdirp(const char *path, mode_t mode) // HERE: Create all directory starting from left // '/tmp' -> '/tmp/foo' -> '/tmp/foo/bar' -> ... + // and set mode for last directory. + mode_t new_mode = 0755; while (1) { ptr = index(ptr, '/'); if (ptr != NULL) *ptr = '\0'; - DEBUGF_W("2-mkdir(%s)\n", f); - if (mkdir_agressive(f, mode) != 0) + else + new_mode = mode; // last part of directory + DEBUGF_W("2-mkdir(%s, 0%o)\n", f, new_mode); + if (mkdir_agressive(f, new_mode, mtime) != 0) { rv = -1; goto done; @@ -844,6 +910,31 @@ mkdirp(const char *path, mode_t mode) return rv; } +static void +dir_save_mtime(const char *path, struct timespec *mtime) +{ + struct stat res; + + if (stat(path, &res) != 0) + return; + + *mtime = res.st_mtimespec; + // DEBUGF_W("SAVE Dir-TimeStamp %lu.%lu %s\n", mtime->tv_sec, mtime->tv_nsec, path); +} + +static void +dir_restore_mtime(const char *path, struct timespec *mtime) +{ + // DEBUGF_W("RESTORE Dir-TimeStamp %lu %s\n", mtime->tv_sec, path); + + if (mtime->tv_sec == 0) + return; + + struct timeval t[] = {{mtime->tv_sec, mtime->tv_nsec / 1000}, {mtime->tv_sec, mtime->tv_nsec / 1000}}; + utimes(path, t); +} + +// SWITCH message received. static void gs_ft_switch(GS_FT *ft, uint32_t id, int64_t fz_remote, struct _gs_ft_file **active, GS_LIST *fsource) { @@ -879,9 +970,7 @@ gs_ft_switch(GS_FT *ft, uint32_t id, int64_t fz_remote, struct _gs_ft_file **act if ((new->fz_remote == new->fz_local) && (new->fz_local != 0)) { - // FIXME: currently _not_ updating mtime/fperm if file is already on peer side. - // Do we want this? (if so then move this code block after fopen().) - do_complete(ft, new); + receiving_complete(ft, new); return; } @@ -898,8 +987,9 @@ gs_ft_switch(GS_FT *ft, uint32_t id, int64_t fz_remote, struct _gs_ft_file **act ptr++; } ptr = dirname(ptr); + dir_save_mtime(ptr, &new->dir_mtime); // put(test1k.dat) must not modify the permission of parent directory. - mkdirp(ptr, 0 /*do not update permission on existing directory*/); + mkdirp(ptr, 0 /*do not update permission on existing directory*/, 0 /*do not update mtime*/); new->fp = fopen(new->realname, "w"); } else { @@ -911,7 +1001,7 @@ gs_ft_switch(GS_FT *ft, uint32_t id, int64_t fz_remote, struct _gs_ft_file **act if (res.st_size != new->fz_local) { // Size changed - do_error(ft, new->li, GS_FT_ERR_BADFSIZE, NULL); + do_recv_error(ft, new->li, GS_FT_ERR_BADFSIZE, NULL); return; } new->fp = fopen(new->realname, "a"); @@ -925,15 +1015,15 @@ gs_ft_switch(GS_FT *ft, uint32_t id, int64_t fz_remote, struct _gs_ft_file **act if (new->fz_remote == 0) { - // Zero sized file. Completed. - do_complete(ft, new); + // Zero File Size + receiving_complete(ft, new); return; } *active = new; return; err: - do_error(ft, new->li, GS_FT_ERR_PERM, NULL); + do_recv_error(ft, new->li, GS_FT_ERR_PERM, NULL); } void @@ -961,6 +1051,14 @@ file_free(struct _gs_ft_file *f) static void ft_done(GS_FT *ft) { + if (ft->is_server != 0) + { + DEBUGF_R("WARN: DOES NOT HAPPEN\n"); + return; + } + + // Only client does requests. Server only answers. Thus only client keeps + // track of number of outstanding requests: if (ft->n_files_waiting > 0) ft->n_files_waiting -= 1; else @@ -975,7 +1073,10 @@ ft_del(GS_LIST_ITEM *li) { struct _gs_ft_file *f = (struct _gs_ft_file *)li->data; - XFCLOSE(f->fp); + // on client this should be closed already but on server this might + // still be open. + if (f->fp != NULL) + XFCLOSE(f->fp); file_free(f); GS_LIST_del(li); @@ -1125,7 +1226,8 @@ GS_FT_status(GS_FT *ft, uint32_t id, uint8_t code, const char *err_str, size_t l ft_done(ft); ft_del(li); } else { - ft_done(ft); + // From a PATTERN request (LISTREQ, Client, get, download). + // File structure is not available. GS_LIST_del(li); } @@ -1193,7 +1295,7 @@ update_stats(struct _gs_ft_file *f, size_t sz) } static size_t -mk_xfer_data(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fcompleted, int *pkt_type, void *dst, size_t len, int is_log_stats) +mk_xfer_data(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fcompleted, int *pkt_type, void *dst, size_t len) { struct _gs_ft_file *f = *active; size_t sz; @@ -1208,20 +1310,21 @@ mk_xfer_data(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fcompleted, int *p } f->fz_remote += sz; - if (f->fz_remote >= f->fz_local) + if (ft->is_server == 0) { - GS_LIST_move(fcompleted, f->li); - *active = NULL; + update_stats(f, sz); } - if (is_log_stats) + if (f->fz_remote >= f->fz_local) { - update_stats(f, sz); + // Immediatley close f->fp to free file descriptor. + sending_complete(ft, active, fcompleted); } + return sz; } - + static size_t mk_switch(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fsource, GS_LIST *fcompleted, int *pkt_type, void *dst, size_t len) { @@ -1229,13 +1332,15 @@ mk_switch(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fsource, GS_LIST *fco struct _gs_ft_file *f; int ret; + XASSERT(active != NULL, "active file is NULL\n"); + li = GS_LIST_next(fsource, NULL); f = (struct _gs_ft_file *)li->data; f->fp = fopen(f->realname, "r"); if (f->fp == NULL) { - DEBUGF("Could not open file %s\n", f->realname); + DEBUGF("fopen(%s): %s\n", f->realname, strerror(errno)); return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_PERM, NULL); } @@ -1249,7 +1354,8 @@ mk_switch(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fsource, GS_LIST *fco if ((f->fz_local == f->fz_remote) && (f->fz_local != 0)) { DEBUGF("#%u Skipping %s (already on peer)\n", (unsigned int)f->li->id, f->name); - mk_stats(ft, li->id, f, NULL, 0 /*success*/); + if (ft->is_server == 0) + mk_stats(ft, li->id, f, NULL, 0 /*success*/); return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_NODATA, NULL); } @@ -1268,18 +1374,17 @@ mk_switch(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fsource, GS_LIST *fco sw.offset = htonll(f->fz_remote); memcpy(dst, &sw, sizeof sw); + DEBUGF_W("#%"PRIu64" Switch to %s (fz_local=%"PRIu64", fz_remote=%"PRIu64")\n", li->id, f->name, f->fz_local, f->fz_remote); + // Handle zero size files + *active = f; if (f->fz_local == 0) { - GS_LIST_move(fcompleted, f->li); - *active = NULL; - } else { - // HERE: fsize is not zero. - *active = f; + // For SERVER (get, download) the fcompleted is NULL + sending_complete(ft, active, fcompleted); } *pkt_type = GS_FT_TYPE_SWITCH; - DEBUGF_W("#%"PRIu64" Switch to %s (fz_local=%"PRIu64", fz_remote=%"PRIu64")\n", li->id, f->name, f->fz_local, f->fz_remote); return sizeof sw; } @@ -1468,20 +1573,22 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) return sizeof acc; } - // Server - download to client + // Server - get (download to client) if (ft->fdl.n_items > 0) { - DEBUGF("Files to send: %d\n", ft->fdl.n_items); if (ft->active_dl_file == NULL) { - DEBUGF("Switching to new file\n"); - return mk_switch(ft, &ft->active_dl_file, &ft->fdl, &ft->fdl_completed, pkt_type, dst, len); + DEBUGF("Files to send: %d\n", ft->fdl.n_items); + return mk_switch(ft, &ft->active_dl_file, &ft->fdl, NULL, pkt_type, dst, len); } if (ft->is_paused_data) + { + DEBUGF_W("IS-PAUSED-DATA==1. Not sending file data....\n"); return 0; + } - return mk_xfer_data(ft, &ft->active_dl_file, &ft->fdl_completed, pkt_type, dst, len, 0 /*is_log_stats*/); + return mk_xfer_data(ft, &ft->active_dl_file, NULL, pkt_type, dst, len); } // Client @@ -1489,6 +1596,7 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) { if (ft->active_put_file == NULL) { + DEBUGF("Files to send: %d\n", ft->faccepted.n_items); return mk_switch(ft, &ft->active_put_file, &ft->faccepted, &ft->fcompleted, pkt_type, dst, len); } @@ -1498,13 +1606,22 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) return 0; } - return mk_xfer_data(ft, &ft->active_put_file, &ft->fcompleted, pkt_type, dst, len, 1 /*is_log_stats*/); + return mk_xfer_data(ft, &ft->active_put_file, &ft->fcompleted, pkt_type, dst, len); } if (ft->is_server == 0) { // CLIENT - DEBUGF_R("n_files_waiting=%d, plist.n_items=%d\n", ft->n_files_waiting, ft->plistreq_waiting.n_items); +#ifdef DEBUG + static int old_waiting; + static int old_pl_waiting; + if ((ft->n_files_waiting != old_waiting) || (ft->plistreq_waiting.n_items != old_pl_waiting)) + { + DEBUGF_R("n_files_waiting=%d, plist.n_items=%d\n", ft->n_files_waiting, ft->plistreq_waiting.n_items); + old_waiting = ft->n_files_waiting; + old_pl_waiting = ft->plistreq_waiting.n_items; + } +#endif if ((ft->n_files_waiting == 0) && (ft->plistreq_waiting.n_items == 0)) { *pkt_type = GS_FT_TYPE_DONE; diff --git a/tools/filetransfer.h b/tools/filetransfer.h index b1ffdc19..a4fe66b6 100644 --- a/tools/filetransfer.h +++ b/tools/filetransfer.h @@ -17,7 +17,7 @@ struct _gs_ft_file { GS_LIST_ITEM *li; char *name; - char *realname; // realpath() resolved + char *realname; // local file name (absolute) mode_t mode; time_t mtime; FILE *fp; @@ -25,6 +25,7 @@ struct _gs_ft_file off_t fz_local; // Total file size (from client) uint32_t globbing_id; + struct timespec dir_mtime; // statistics // Note: xfer might be suspended by 'switch' to another file. // Log suspended time in usec_suspend. @@ -102,7 +103,7 @@ typedef struct // GET (download) - Server Side GS_LIST flistreply; // Server list of file to be send to client. GS_LIST fdl; // List of files ready to send (switch to). - GS_LIST fdl_completed; // Waiting for ERR_COMPLETED + // GS_LIST fdl_completed; // Waiting for ERR_COMPLETED int g_id; // global request ID (unique) to match error-replies to requests int g_globbing_id; // global globbing id diff --git a/tools/run_ft_tests.sh b/tools/run_ft_tests.sh index d6df8bdb..ec3e7a40 100755 --- a/tools/run_ft_tests.sh +++ b/tools/run_ft_tests.sh @@ -28,6 +28,7 @@ cp test1k.dat "${IODIRSRC}/foo/.rcfile1" cp test4k.dat "${IODIRSRC}/foo/" cp test4k.dat "${IODIRSRC}/foo/bar/.rcfile2" cp test8k.dat "${IODIRSRC}/" +touch "${IODIRSRC}/zero.dat" test_start() { @@ -56,6 +57,34 @@ fail_file_count() [[ $nf_src -eq $nf_dst ]] || fail $1 } +fail_file_bypipe() +{ + while read f; do + if [ $(stat -f%A-%m-%z "${2}/${f}") != $(stat -f%A-%m-%z "${3}/${f}") ]; then + echo "${f} not equal"; + fail $1 + fi + done +} + +fail_dir_bypipe() +{ + while read f; do + if [ $(stat -f%A-%m "${2}/${f}") != $(stat -f%A-%m "${3}/${f}") ]; then + echo "${f} not equal"; + fail $1 + fi + done +} + +# Recursively compare st_mode and mtime and fail if different +fail_dir_compare() +{ + (cd "$2"; find -x "$4" -type d) | fail_dir_bypipe "$1" "$2" "$3" + (cd "$2"; find -x "$4" -type f) | fail_file_bypipe "$1" "$2" "$3" +} + + run_put() { # set -f disabled globbing @@ -64,6 +93,12 @@ run_put() socat SYSTEM:"set -f && ${BIN} c $* 2>${LOGDIR}/client.log" SYSTEM:"(cd ${IODIR}; ${BIN} s 2>${LOGDIR}/server.log)" } +# put with command +run_putc() +{ + socat SYSTEM:"set -f && ${BIN} c \'$*\' 2>${LOGDIR}/client.log" SYSTEM:"(cd ${IODIR}; ${BIN} s 2>${LOGDIR}/server.log)" +} + run_get() { # set -f disabled globbing @@ -76,6 +111,11 @@ run_get2() socat SYSTEM:"(cd ${IODIR}; set -f && ${BIN} C $* 2>${LOGDIR}/client.log)" SYSTEM:"(cd ${IODIRSRC}/foo; ${BIN} s 2>${LOGDIR}/server.log)" } +run_getc() +{ + socat SYSTEM:"(cd ${IODIR}; set -f && ${BIN} C \'$*\' 2>${LOGDIR}/client.log)" SYSTEM:"(cd ${IODIRSRC}; ${BIN} s 2>${LOGDIR}/server.log)" +} + tests="1.0 " tests+="1.1 " tests+="1.2 " @@ -95,6 +135,8 @@ tests+="5.3 " tests+="5.4 " tests+="5.5 " tests+="5.6 " +tests+="5.7 " +tests+="5.8 " tests+="8.1 " tests+="8.2 " @@ -102,57 +144,12 @@ tests+="8.3 " tests+="8.4 " tests+="8.5 " tests+="8.6 " +tests+="8.7 " +tests+="8.8 " +tests+="9.1 " +tests+="9.2 " -if [[ "$tests" =~ '8.1 ' ]]; then -test_start -n "Running #8.1 (get, 2 files)................................" -run_get test8k.dat foo/bar/.rcfile2 -md5fail 1 test8k.dat "${IODIR}/test8k.dat" -md5fail 2 test4k.dat "${IODIR}/.rcfile2" -$ECHO "${OK}" -fi - -if [[ "$tests" =~ '8.2 ' ]]; then -test_start -n "Running #8.2 (get, 2 files /./ test)......................." -run_get ./foo/bar/test1k.dat ./foo/./bar/test1k.dat -md5fail 1 test1k.dat "${IODIR}/test1k.dat" -md5fail 2 test1k.dat "${IODIR}/bar/test1k.dat" -$ECHO "${OK}" -fi - -if [[ "$tests" =~ '8.3 ' ]]; then -test_start -n "Running #8.3 (directory test).............................." -run_get foo/bar -md5fail 1 "${IODIRSRC}/foo/bar/test1k.dat" "${IODIR}/bar/test1k.dat" -md5fail 2 "${IODIRSRC}/foo/bar/.rcfile2" "${IODIR}/bar/.rcfile2" -$ECHO "${OK}" -fi - -if [[ "$tests" =~ '8.4 ' ]]; then -test_start -n "Running #8.4 (get, non-exist).............................." -run_get not-exists.dat foobar*noexist[1234].d[ab]t -[[ -f "${IODIR}/not-exists.dat" ]] && fail 1 -$ECHO "${OK}" -fi - -if [[ "$tests" =~ '8.5 ' ]]; then -test_start -n "Running #8.5 (get, ../test8k.dat).........................." -run_get2 ../test8k.dat ../foo ../foo/./bar -md5fail 1 "${IODIRSRC}/test8k.dat" "${IODIR}/test8k.dat" -fail_file_count 2 "${IODIRSRC}/foo" "${IODIR}/foo" -fail_file_count 3 "${IODIRSRC}/foo/bar" "${IODIR}/bar" -$ECHO "${OK}" -fi - -if [[ "$tests" =~ '8.6 ' ]]; then -test_start -n "Running #8.6 (get, /etc/hosts)............................." -run_get /etc/hosts /etc/./ssh/ssh_config /./etc/ssh/ssh_config -# run_get /etc/./ssh/ssh_config -md5fail 1 "/etc/hosts" "${IODIR}/hosts" -md5fail 2 "/etc/ssh/ssh_config" "${IODIR}/ssh/ssh_config" -md5fail 3 "/etc/ssh/ssh_config" "${IODIR}/etc/ssh/ssh_config" -$ECHO "${OK}" -fi if [[ "$tests" =~ '1.0 ' ]]; then test_start -n "Running #1.0 (put 1 file).................................." @@ -272,7 +269,7 @@ if [[ "$tests" =~ '4.2 ' ]]; then test_start -n "Running #4.2 (mtime)......................................." touch -r /etc/hosts test4k.dat run_put test4k.dat -[[ x`stat -f%a "test4k.dat"` = x`stat -f%a "${IODIR}/test4k.dat"` ]] || fail 1 +[[ x`stat -f%m "test4k.dat"` = x`stat -f%m "${IODIR}/test4k.dat"` ]] || fail 1 $ECHO "${OK}" fi @@ -280,7 +277,7 @@ if [[ "$tests" =~ '4.3 ' ]]; then test_start -n "Running #4.3 (zero-size, mtime)............................" touch -r /etc/hosts zero.dat run_put zero.dat -[[ x`stat -f%a "zero.dat"` = x`stat -f%a "${IODIR}/zero.dat"` ]] || fail 1 +[[ x`stat -f%m "zero.dat"` = x`stat -f%m "${IODIR}/zero.dat"` ]] || fail 1 $ECHO "${OK}" fi @@ -339,3 +336,124 @@ test_start -n "Running #5.6 (Globbing foo/)..............................." fail_file_count 1 "${IODIRSRC}/foo/" "${IODIR}/" $ECHO "${OK}" fi + +if [[ "$tests" =~ '5.7 ' ]]; then +test_start -n "Running #5.7 (put, globbing \$(find...*.dat)................" +# (cd "${IODIRSRC}" && run_putc '\$(echo *.dat)') +(cd "${IODIRSRC}" && run_putc '\$(find . -type f -name \\*.dat)') +(cd "${IODIRSRC}" && find . -type f -name '*.dat') | fail_file_bypipe 1 "${IODIRSRC}" "${IODIR}" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '5.8 ' ]]; then +test_start -n "Running #5.8 (get, globbing \$(find...*.dat)................" +run_getc '\$(find . -type f -name \\*.dat)' +(cd "${IODIRSRC}" && find . -type f -name '*.dat') | fail_file_bypipe 1 "${IODIRSRC}" "${IODIR}" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '8.1 ' ]]; then +test_start -n "Running #8.1 (get, 2 files)................................" +run_get test8k.dat foo/bar/.rcfile2 +md5fail 1 test8k.dat "${IODIR}/test8k.dat" +md5fail 2 test4k.dat "${IODIR}/.rcfile2" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '8.2 ' ]]; then +test_start -n "Running #8.2 (get, 2 files /./ test)......................." +run_get ./foo/bar/test1k.dat ./foo/./bar/test1k.dat +md5fail 1 test1k.dat "${IODIR}/test1k.dat" +md5fail 2 test1k.dat "${IODIR}/bar/test1k.dat" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '8.3 ' ]]; then +test_start -n "Running #8.3 (directory test).............................." +run_get foo/bar +md5fail 1 "${IODIRSRC}/foo/bar/test1k.dat" "${IODIR}/bar/test1k.dat" +md5fail 2 "${IODIRSRC}/foo/bar/.rcfile2" "${IODIR}/bar/.rcfile2" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '8.4 ' ]]; then +test_start -n "Running #8.4 (get, non-exist).............................." +run_get not-exists.dat foobar*noexist[1234].d[ab]t +[[ -f "${IODIR}/not-exists.dat" ]] && fail 1 +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '8.5 ' ]]; then +test_start -n "Running #8.5 (get, ../test8k.dat).........................." +run_get2 ../test8k.dat ../foo ../foo/./bar +md5fail 1 "${IODIRSRC}/test8k.dat" "${IODIR}/test8k.dat" +fail_file_count 2 "${IODIRSRC}/foo" "${IODIR}/foo" +fail_file_count 3 "${IODIRSRC}/foo/bar" "${IODIR}/bar" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '8.6 ' ]]; then +test_start -n "Running #8.6 (get, /etc/hosts)............................." +run_get /etc/hosts /etc/./ssh/ssh_config /./etc/ssh/ssh_config +md5fail 1 "/etc/hosts" "${IODIR}/hosts" +md5fail 2 "/etc/ssh/ssh_config" "${IODIR}/ssh/ssh_config" +md5fail 3 "/etc/ssh/ssh_config" "${IODIR}/etc/ssh/ssh_config" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '8.7 ' ]]; then +test_start -n "Running #8.7 (get, permission, mtime, zero)................" +chmod 462 "${IODIRSRC}/test1k.dat" +chmod 624 "${IODIRSRC}/zero.dat" +chmod 3751 "${IODIRSRC}/foo/dir_empty" +touch -r /etc/hosts "${IODIRSRC}/test1k.dat" "${IODIRSRC}/zero.dat" "${IODIRSRC}/foo/dir_empty" +touch -r /etc "${IODIRSRC}/foo" +touch -r /etc "${IODIR}" +run_get test1k.dat zero.dat ././foo/dir_empty +[[ $(stat -f%A-%m-%z "${IODIRSRC}/test1k.dat") = $(stat -f%A-%m-%z "${IODIR}/test1k.dat") ]] || fail 1 +[[ $(stat -f%A-%m-%z "${IODIRSRC}/zero.dat") = $(stat -f%A-%m-%z "${IODIR}/zero.dat") ]] || fail 2 +[[ $(stat -f%A-%m "${IODIRSRC}/foo/dir_empty") = $(stat -f%A-%m "${IODIR}/foo/dir_empty") ]] || fail 3 +[[ $(stat -L -f%A-%m "/etc") = $(stat -f%A-%m "${IODIRSRC}/foo") ]] || fail 4 +[[ $(stat -L -f%A-%m "/etc") = $(stat -f%A-%m "${IODIR}") ]] || fail 5 +chmod 644 "${IODIRSRC}/test1k.dat" "${IODIRSRC}/zero.dat" +chmod 755 "${IODIRSRC}/foo/dir_empty" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '8.8 ' ]]; then +test_start -n "Running #8.8 (get restart: dst is larger, smaller & zero).." +dd bs=1k count=5 if="${IODIRSRC}/test8k.dat" of="${IODIR}/test8k.dat" &>/dev/null +cp "${IODIRSRC}/test8k.dat" "${IODIR}/test1k.dat" +touch "${IODIR}/test4k.dat" +run_get test1k.dat test8k.dat foo/test4k.dat +md5fail 1 "${IODIRSRC}/test8k.dat" "${IODIR}/test8k.dat" +md5fail 2 "${IODIRSRC}/test1k.dat" "${IODIR}/test1k.dat" +md5fail 2 "${IODIRSRC}/foo/test4k.dat" "${IODIR}/test4k.dat" +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '9.1 ' ]]; then +test_start -n "Running #9.1 (HUGE put)..........................." +## Small +# run_put /usr/./share/man/man4 +# $ECHO -n "verify..." +# fail_dir_compare 1 /usr/share/man "${IODIR}/share/man" man4 +## HUGE +run_put /usr/./share/man +$ECHO -n "verify..." +fail_dir_compare 1 /usr/share "${IODIR}/share" man +$ECHO "${OK}" +fi + +if [[ "$tests" =~ '9.2 ' ]]; then +test_start -n "Running #9.2 (HUGE get)..........................." +## Small +# run_get /usr/./share/man/man4 +# $ECHO -n "verify..." +# fail_dir_compare 1 /usr/share/man "${IODIR}/share/man" man4 +## HUGE +run_get /usr/./share/man +$ECHO -n "verify..." +fail_dir_compare 1 /usr/share "${IODIR}/share" man +$ECHO "${OK}" +fi From d1b75c6aed9ef1d62279431716194eacf85a05ef Mon Sep 17 00:00:00 2001 From: rootTHC Date: Tue, 9 Feb 2021 12:14:53 +0000 Subject: [PATCH 12/21] almost! --- configure.ac | 2 +- include/gsocket/gsocket.h | 3 +- lib/gs-common.h | 1 + lib/gs-readline.c | 5 +- lib/gsocket-engine.c | 8 +- lib/gsocket-engine.h | 2 +- lib/gsocket-select.c | 6 +- lib/gsocket-util.c | 38 +++- tools/4_gs-netcat.c | 202 ++++++++++------- tools/Makefile.am | 4 +- tools/common.h | 8 + tools/console.c | 211 ++++++++++-------- tools/console_display.c | 1 + tools/console_display.h | 1 - tools/event_mgr.c | 2 +- tools/filetransfer-test.c | 48 ++-- tools/filetransfer.c | 458 ++++++++++++++++++++++++++++---------- tools/filetransfer.h | 61 ++--- tools/gs-netcat.h | 1 + tools/ids.c | 24 -- tools/pkt_mgr.c | 93 ++++++++ tools/pkt_mgr.h | 20 +- tools/run_ft_tests.sh | 23 +- tools/utils.c | 20 +- tools/utils.h | 3 +- 25 files changed, 834 insertions(+), 411 deletions(-) diff --git a/configure.ac b/configure.ac index 34152508..0c57676b 100755 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ dnl Process this File with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([gsocket], 1.4.23) +AC_INIT([gsocket], 1.4.23-ft1) dnl AC_CONFIG_AUX_DIR(config-x86_64-apple-darwin19.6.0) AC_CONFIG_AUX_DIR(config) AC_CANONICAL_SYSTEM diff --git a/include/gsocket/gsocket.h b/include/gsocket/gsocket.h index 67e73135..80b2833c 100644 --- a/include/gsocket/gsocket.h +++ b/include/gsocket/gsocket.h @@ -373,7 +373,8 @@ void GS_SELECT_FD_SET_W(GS *gs); void GS_daemonize(FILE *logfp); uint64_t GS_usec(void); -void GS_format_bps(char *dst, size_t size, int64_t bytes); +void GS_format_bps(char *dst, size_t size, int64_t bytes, const char *suffix); +char *GS_getpidwd(pid_t pid); const char *GS_gen_secret(void); const char *GS_user_secret(GS_CTX *ctx, const char *file, const char *sec_str); diff --git a/lib/gs-common.h b/lib/gs-common.h index 137b9ac8..f0b129dc 100755 --- a/lib/gs-common.h +++ b/lib/gs-common.h @@ -36,6 +36,7 @@ #include #include #include /* basename() */ +#include // getpidwd(pid_t) #include #include #include diff --git a/lib/gs-readline.c b/lib/gs-readline.c index fa55c904..8d18b6de 100644 --- a/lib/gs-readline.c +++ b/lib/gs-readline.c @@ -90,7 +90,10 @@ visible_create(GS_RL_CTX *rl, int row, int col, uint8_t key) char *ptr = rl->esc_data; if (rl->is_need_redraw) + { + DEBUGF_Y("moving to %d;%df\n", row, col); SXPRINTF(ptr, d_end - ptr, "\x1B[%d;%df", row, col); + } size_t len; len = MIN(s_end - src, d_end - ptr); @@ -158,7 +161,7 @@ GS_RL_add(GS_RL_CTX *rl, uint8_t c, uint8_t *key, int row, int col) // rl->is_need_redraw = 1; // DEFAULT: FIXME-PERFORMANCE // ^A or ^OA - DEBUGF_W("esc=%d c=0x%02x\n", rl->is_in_esc, c); + DEBUGF_W("esc=%d c=0x%02x r%d,c%d\n", rl->is_in_esc, c, row, col); if (rl->is_in_esc) { if ((rl->is_in_esc == 1) && (c == 'O')) diff --git a/lib/gsocket-engine.c b/lib/gsocket-engine.c index fd64509e..76b0c24f 100755 --- a/lib/gsocket-engine.c +++ b/lib/gsocket-engine.c @@ -1885,7 +1885,7 @@ GS_read(GS *gsocket, void *buf, size_t count) /* Mark if there is still data in the input buffer so another cb is done */ #ifdef WITH_GSOCKET_SSL if ((gsocket->ssl) && (SSL_pending(gsocket->ssl) > 0)) - gs_select_set_rdata_pending(gsocket->ctx->gselect_ctx, gsocket->fd); + gs_select_set_rdata_pending(gsocket->ctx->gselect_ctx, gsocket->fd, SSL_pending(gsocket->ssl)); #endif } @@ -2080,12 +2080,6 @@ GS_usecstr(char *buf, size_t len, uint64_t usec) sec -= min * 60; *ptr = 0; -#if 0 - if (hr != 0) - snprintf(ptr, len, "%d:%02d:%02d.%03d", hr, min, sec, msec); - else - snprintf(ptr, len, "%02d:%02d.%03d", min, sec, msec); -#endif if (hr != 0) snprintf(ptr, len, "%dhrs %2dmin %2d.%03dsec", hr, min, sec, msec); diff --git a/lib/gsocket-engine.h b/lib/gsocket-engine.h index 98064b69..595cdd1a 100755 --- a/lib/gsocket-engine.h +++ b/lib/gsocket-engine.h @@ -10,7 +10,7 @@ int gs_ssl_shutdown(GS *gsocket); int gs_srp_init(GS *gsocket); void gs_select_rw_save_state(GS_SELECT_CTX *ctx, int fd, char *idstr); void gs_select_rw_restore_state(GS_SELECT_CTX *ctx, int fd, char *idstr); -void gs_select_set_rdata_pending(GS_SELECT_CTX *ctx, int fd); +void gs_select_set_rdata_pending(GS_SELECT_CTX *ctx, int fd, int len); void gs_fds_out(fd_set *fdset, int max, char id); void gs_fds_out_rwfd(GS_SELECT_CTX *ctx); diff --git a/lib/gsocket-select.c b/lib/gsocket-select.c index 0ead49e9..91ae6862 100755 --- a/lib/gsocket-select.c +++ b/lib/gsocket-select.c @@ -96,10 +96,10 @@ gs_select_rw_restore_state(GS_SELECT_CTX *ctx, int fd, char *idstr) } void -gs_select_set_rdata_pending(GS_SELECT_CTX *ctx, int fd) +gs_select_set_rdata_pending(GS_SELECT_CTX *ctx, int fd, int len) { ctx->rdata_pending_count++; - ctx->rdata_pending[fd] = 1; + ctx->rdata_pending[fd] = len; } static void @@ -154,7 +154,7 @@ GS_select(GS_SELECT_CTX *ctx) /* HERE: Call to GS_read() needed because there is still * data in the input buffer (not i/o buffer). */ - DEBUGF_Y("Pending data in read input buffer :>\n"); + DEBUGF_Y("fd=%d Pending data in SSL read input buffer (len=%d):>\n", i, ctx->rdata_pending[i]); ctx->rdata_pending_count--; call_item(ctx, &ctx->mgr_r[i], i); } diff --git a/lib/gsocket-util.c b/lib/gsocket-util.c index e4a07d20..fa7e29f2 100755 --- a/lib/gsocket-util.c +++ b/lib/gsocket-util.c @@ -181,27 +181,55 @@ GS_usec(void) return GS_TV_TO_USEC(&tv); } -// 7 readable characters + 0 +// 7 readable characters + suffix + 0 static const char unit[] = "BKMGT"; void -GS_format_bps(char *dst, size_t size, int64_t bytes) +GS_format_bps(char *dst, size_t size, int64_t bytes, const char *suffix) { int i; + if (suffix == NULL) + suffix = ""; + if (bytes < 1000) { - snprintf(dst, size, "%3d.0 B", (int)bytes); + snprintf(dst, size, "%3d.0 B%s", (int)bytes, suffix); return; } bytes *= 100; for (i = 0; bytes >= 100*1000 && unit[i] != 'T'; i++) bytes = (bytes + 512) / 1024; - snprintf(dst, size, "%3lld.%1lld%c%s", + snprintf(dst, size, "%3lld.%1lld%c%s%s", (long long) (bytes + 5) / 100, (long long) (bytes + 5) / 10 % 10, unit[i], - i ? "B" : " "); + i ? "B" : " ", suffix); +} + +// Get Working Directory of process with id pid or if this fails then current cwd +// of this process. +char * +GS_getpidwd(pid_t pid) +{ + int ret; + char *wd = NULL; + + if (pid <= 0) + goto err; + + struct proc_vnodepathinfo vpi; + ret = proc_pidinfo(pid, PROC_PIDVNODEPATHINFO, 0, &vpi, sizeof vpi); + if (ret <= 0) + goto err; + + wd = strdup(vpi.pvi_cdir.vip_path); + +err: + if (wd == NULL) + wd = getwd(NULL); + DEBUGF_W("PID %d CWD=%s\n", pid, wd); + return wd; } /* diff --git a/tools/4_gs-netcat.c b/tools/4_gs-netcat.c index c038554d..18c2ae1d 100755 --- a/tools/4_gs-netcat.c +++ b/tools/4_gs-netcat.c @@ -42,6 +42,7 @@ #include "pkt_mgr.h" #include "ids.h" #include "gs-netcat.h" +#include "filetransfer_mgr.h" #include "man_gs-netcat.h" /* All connected gs-peers indexed by gs->fd */ @@ -96,6 +97,7 @@ peer_free(GS_SELECT_CTX *ctx, struct _peer *p) int is_stdin_forward = p->is_stdin_forward; int fd; + DEBUGF_W("PEER FREE\n"); /* Reset console/stty before outputting stats */ if (is_stdin_forward) { @@ -107,14 +109,18 @@ peer_free(GS_SELECT_CTX *ctx, struct _peer *p) DEBUGF_R("GS_get_fd() == %d\n", GS_get_fd(gs)); XASSERT(peers[fd] == p, "Oops, %p != %p on fd = %d, cmd_fd = %d\n", peers[fd], p, fd, p->fd_in); - ids_gs_logout(p); + ids_gs_logout(p); // Signal all other interactive _clients_ that we are leaving (IDS notification) GS_EVENT_del(&gopt.event_ping); GS_EVENT_del(&gopt.event_bps); GS_PKT_close(&p->pkt); + // Free Filetransfer subsystem (-i) + GS_FTM_free(p); + // Free all pending log files and their data GS_LIST_del_all(&p->logs, 1); - GS_LIST_del(p->ids_li); // Server only + GS_LIST_del(p->ids_li); // Server only. Remvoe this client from IDS + DEBUGF("IDS Peers remaining: %d\n", gopt.ids_peers.n_items); p->ids_li = NULL; /* Exit() if we were reading from stdin/stdout. No other clients * in this case. @@ -199,6 +205,7 @@ read_esc(int fd, uint8_t *buf, size_t len, uint8_t *key) while (n < len) { sz = read(fd, &c, 1); + // DEBUGF_M("read() = %zd (0x%02x\n", sz, c); if (sz <= 0) { /* EOF or ERROR */ @@ -211,15 +218,15 @@ read_esc(int fd, uint8_t *buf, size_t len, uint8_t *key) /* Check if this is an ESCAPE and stop reading */ ret = CONSOLE_check_esc(c, &c); // DEBUGF_G("con = %d\n", ret); - if (ret != -1) + if (ret > 0) { - if (ret > 0) - { - *key = ret; - break; - } + // HERE: Was an escaped character. 'ret' contains the escaped character + // e.g. when reading ^E+c then ret==c + *key = ret; + break; + } + if (ret >= 0) continue; - } buf[n] = c; n++; @@ -230,15 +237,6 @@ read_esc(int fd, uint8_t *buf, size_t len, uint8_t *key) return n; } -// static ssize_t -// read_noesc(int fd, uint8_t *buf, size_t len) -// { -// ssize_t sz = read(fd, buf, len); -// if (sz == 0) -// sz = -1; -// return sz; -// } - static int cb_read_fd(GS_SELECT_CTX *ctx, int fd, void *arg, int val) { @@ -393,7 +391,9 @@ cb_read_gs(GS_SELECT_CTX *ctx, int fd, void *arg, int val) ssize_t len; int ret; - // XASSERT(p->rlen <= 0, "Already data in input buffer (%zd)\n", p->rlen); + if (p->rlen > 0) + DEBUGF_C("fd=%d Pending data in nc-read input buffer (len=%zd)\n", fd, p->rlen); + XASSERT(p->rlen < p->r_max, "rlen=%zd larger than buffer\n", p->rlen); len = GS_read(gs, p->rbuf + p->rlen, p->r_max - p->rlen); // DEBUGF_G("GS_read(fd = %d) == %zd\n", gs->fd, len); @@ -456,43 +456,83 @@ cb_read_gs(GS_SELECT_CTX *ctx, int fd, void *arg, int val) } } - write_fd(ctx, p); + // See if any app data is there for the FD + // (Could have been that GS_PKT_decode() consumed all data for channels). + if (p->rlen > 0) + write_fd(ctx, p); } return GS_SUCCESS; } +// Attempt to write and set write flag if busy. Non-Recursive. +// Return len on success. +// +// Return 0: SUCCESS (or no data written because p->wlen was empty) +// Return -1 : Busy (caller to return ECALLAGAIN) +// Return <-1: Fatal (exit) int -write_gs(GS_SELECT_CTX *ctx, struct _peer *p, int *killed) +write_gs_atomic(GS_SELECT_CTX *ctx, struct _peer *p) { - GS *gs = p->gs; - int len = 0; + int len; + + if (p->wlen <= 0) + return 0; - if (p->wlen > 0) + len = GS_write(p->gs, p->wbuf, p->wlen); + // DEBUGF_R("GS_write(fd==%d), len=%d\n", p->gs->fd, len); + if (len == 0) { - len = GS_write(gs, p->wbuf, p->wlen); - // DEBUGF_R("GS_write(fd==%d) = %d\n", gs->fd, len); - if (len == 0) - { - /* GS_write() would block. */ - FD_CLR(p->fd_in, ctx->rfd); // Stop reading from input - return GS_ECALLAGAIN; - } + /* GS_write() would block. */ + // DEBUGF_M("Pause reading from fd_in=%d\n", p->fd_in); + FD_CLR(p->fd_in, ctx->rfd); // Pause reading from input + GS_FT_pause_data(&p->ft); // Pause FileTransfers + return -1; // BUSY + + // STOP HERE: Oops. we then still read from stdin and then try to write - thats bad. + // but why does XASSERT not catch this? + // does gs-select go back into select() or does it process rfds after EBUSY return? + } + + if (len > 0) + { + // Always unpause FileTransfer (even if not paused). + GS_FT_unpause_data(&p->ft); + + // Check if FileTransfer still needs to xfer more data + if (GS_FT_WANT_WRITE(&p->ft)) + GS_SELECT_FD_SET_W(p->gs); } + return len; +} + +int +write_gs(GS_SELECT_CTX *ctx, struct _peer *p, int *killed) +{ + int len; + + len = write_gs_atomic(ctx, p); + if (len == -1) + return GS_ECALLAGAIN; + if (len == p->wlen) { - /* GS_write() was a success or p->wlen == 0 */ + // SUCCESS of GS_write() (or p->wlen was 0) p->wlen = 0; if (p->is_fd_connected) { /* SOCKS subsystem calls write_gs() before p->fd_in is connected * Make sure XFD_SET() is only called on a connected() socket. */ + // DEBUGF_M("Start reading from fd_in(=%d) again\n", p->fd_in); FD_CLR(p->gs->fd, ctx->wfd); XFD_SET(p->fd_in, ctx->rfd); // Start reading from input again } - /* Check if there is any other data we like to write... */ + // FIXME-PERFORMANCE: Could change this to 'make' packets and append them to + // p->wbuf and do one large atomic write instead of recursive calls.... + + // Check if there is any other data we like to write... if (gopt.is_win_resized) { get_winsize(); @@ -516,14 +556,35 @@ write_gs(GS_SELECT_CTX *ctx, struct _peer *p, int *killed) gopt.is_want_ping = 0; return pkt_app_send_ping(ctx, p); } + if (gopt.is_want_pwd) + { + gopt.is_want_pwd = 0; + return pkt_app_send_pwdrequest(ctx, p); + } + if (gopt.is_pwdreply_pending) + { + gopt.is_pwdreply_pending = 0; + return pkt_app_send_pwdreply(ctx, p); + } + if (gopt.is_want_ids_on) + { + gopt.is_want_ids_on = 0; + return pkt_app_send_ids(ctx, p); + } if (p->is_pending_logs) { return pkt_app_send_all_log(ctx, p); } - return GS_SUCCESS; - } + // Last: Send data from FileTransfer subsystem. + int ret; + ret = pkt_app_send_ft(ctx, p); + if ((ret == GS_ERROR) || (ret == GS_ERR_FATAL)) + goto err; + return ret; // ECALLAGAIN==1 || ESUCCESS==0 + } +err: /* HERE: ERROR on GS_write() */ peer_free(ctx, p); if (killed != NULL) @@ -633,11 +694,7 @@ peer_new_init(GS *gs) GS_EVENT_add_by_ts(&gs->ctx->gselect_ctx->emgr, &gopt.event_bps, 0, GS_APP_BPSFREQ, cbe_bps, p, 0); } else { /* SERVER, interactive */ - ids_gs_login(p); - // Let all others know what we have logged in: - // p->ids_li = GS_LIST_add(&gopt.ids_peers, NULL, p, 0); - // if (gopt.event_ids == NULL) - // gopt.event_ids = GS_EVENT_add_by_ts(&gs->ctx->gselect_ctx->emgr, NULL, 0, GS_APP_IDSFREQ, cbe_ids, NULL, 0); + ids_gs_login(p); // Let all others know what we have logged in: } } DEBUGF_M("[ID=%d] (fd=%d) Number of connected gs-peers: %d\n", p->id, fd, gopt.peer_count); @@ -702,18 +759,12 @@ peer_new(GS_SELECT_CTX *ctx, GS *gs) p = peer_new_init(gs); - if (gopt.is_interactive) - { - /* SERVER */ - GS_PKT_assign_msg(&p->pkt, PKT_MSG_WSIZE, pkt_app_cb_wsize, p); - GS_PKT_assign_msg(&p->pkt, PKT_MSG_PING, pkt_app_cb_ping, p); - GS_PKT_assign_msg(&p->pkt, PKT_MSG_IDS, pkt_app_cb_ids, p); - } /* Create a new fd to relay gs-traffic to/from */ if ((gopt.cmd != NULL) || (gopt.is_interactive)) { - p->fd_in = fd_cmd(gopt.cmd);// Forward to forked process stdin/stdout + p->fd_in = fd_cmd(gopt.cmd, &p->pid);// Forward to forked process stdin/stdout + DEBUGF_W("pid =%d\n", p->pid); p->fd_out = p->fd_in; p->is_app_forward = 1; } else if (gopt.port != 0) { @@ -736,6 +787,17 @@ peer_new(GS_SELECT_CTX *ctx, GS *gs) ERREXIT("Cant create forward...%s\n", GS_strerror(gs)); } + if (gopt.is_interactive) + { + /* SERVER */ + GS_PKT_assign_msg(&p->pkt, PKT_MSG_WSIZE, pkt_app_cb_wsize, p); + GS_PKT_assign_msg(&p->pkt, PKT_MSG_PING, pkt_app_cb_ping, p); + GS_PKT_assign_msg(&p->pkt, PKT_MSG_IDS, pkt_app_cb_ids, p); + GS_PKT_assign_msg(&p->pkt, PKT_MSG_PWD, pkt_app_cb_pwdrequest, p); + + GS_FTM_init(p, 1); + } + if (p->is_network_forward == 0) { /* STDIN/STDOUT or app-fd always complete immediately */ @@ -909,9 +971,14 @@ cb_connect_client(GS_SELECT_CTX *ctx, int fd_notused, void *arg, int val) // } if (gopt.is_interactive) { - pkt_app_send_wsize(ctx, p, gopt.winsize.ws_row); + // pkt_app_send_wsize(ctx, p, gopt.winsize.ws_row); + gopt.is_win_resized = 1; // Trigger: Send new window size to peer + GS_SELECT_FD_SET_W(p->gs); GS_PKT_assign_msg(&p->pkt, PKT_MSG_PONG, pkt_app_cb_pong, p); GS_PKT_assign_msg(&p->pkt, PKT_MSG_LOG, pkt_app_cb_log, p); + GS_PKT_assign_chn(&p->pkt, GS_CHN_PWD, pkt_app_cb_pwdreply, p); // Channel + + GS_FTM_init(p, 0 /*client*/); } return GS_SUCCESS; @@ -1142,40 +1209,6 @@ my_test(void) int idle; char *user; - while (1) - { - GS_IDS_utmp(&new_login, &new_active, &user, &idle); - if (new_login.n_items > 0) - { - li = NULL; - while (1) - { - li = GS_LIST_next(&new_login, li); - if (li == NULL) - break; - DEBUGF_G("New User Login: %s\n", (char *)li->data); - } - GS_LIST_del_all(&new_login, 0); - } - - if (new_active.n_items > 0) - { - li = NULL; - while (1) - { - li = GS_LIST_next(&new_active, li); - if (li == NULL) - break; - DEBUGF("Newly active: %s\n", (char *)li->data); - } - GS_LIST_del_all(&new_active, 0); - } - if (user != NULL) - { - DEBUGF_C("Least Idle: %s (%d)\n", user, idle); - } - sleep(1); - } // double load; // int ret = getloadavg(&load, 1); @@ -1231,6 +1264,7 @@ my_test(void) } #endif + int main(int argc, char *argv[]) { diff --git a/tools/Makefile.am b/tools/Makefile.am index e7e306a4..59c0eb13 100755 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -11,7 +11,7 @@ gs_pipe_LDADD = ../lib/libgsocket.a gs_full_pipe_SOURCES = 3_gs-full-pipe.c utils.c gs_full_pipe_LDADD = ../lib/libgsocket.a -gs_netcat_SOURCES = 4_gs-netcat.c utils.c socks.c console.c ids.c event_mgr.c pkt_mgr.c console_display.c filetransfer.c globbing.c +gs_netcat_SOURCES = 4_gs-netcat.c utils.c socks.c console.c ids.c event_mgr.c pkt_mgr.c console_display.c filetransfer.c globbing.c filetransfer_mgr.c gs_netcat_LDADD = ../lib/libgsocket.a dist_bin_SCRIPTS = blitz gs-sftp gs-mount gs_funcs @@ -33,5 +33,5 @@ filetransfer_test_SOURCES = filetransfer.c filetransfer-test.c utils.c globbing. filetransfer_test_LDADD = ../lib/libgsocket.a EXTRA_DIST = run_all_tests.sh -noinst_HEADERS = common.h utils.h socks.h console.h ids.h event_mgr.h pkt_mgr.h gs-netcat.h console_display.h filetransfer.h man_gs-netcat.h globbing.h +noinst_HEADERS = common.h utils.h socks.h console.h ids.h event_mgr.h pkt_mgr.h gs-netcat.h console_display.h filetransfer.h man_gs-netcat.h globbing.h filetransfer_mgr.h AM_CFLAGS = -I../include diff --git a/tools/common.h b/tools/common.h index 574cce4d..25a0abdd 100755 --- a/tools/common.h +++ b/tools/common.h @@ -43,6 +43,7 @@ #include /* basename() */ #include #include +#include #ifdef HAVE_UTMPX_H # include #endif @@ -65,6 +66,7 @@ #include #include #include +#include "filetransfer.h" #ifndef O_NOCTTY # warning "O_NOCTTY not defined. Using 0." @@ -105,6 +107,10 @@ struct _gopt int is_console; // console is being displayed int is_pong_pending; // Server: Answer to PING waiting to be send int is_want_ping; // Client: Wants to send a ping + int is_want_pwd; // Client: Wants server to send cwd + int is_pwdreply_pending; // Server: Answer to pwd-request + int is_want_chdir; + int is_want_ids_on; uint64_t ts_ping_sent; // TimeStamp ping sent fd_set rfd, r; fd_set wfd, w; @@ -166,9 +172,11 @@ struct _peer int id; /* Stats: assign an ID to each pere */ struct _socks socks; GS_PKT pkt; // In-band data for interactive shell (-i) + GS_FT ft; // Filetransfer (-i) GS_LIST logs; // Queue for log messages from Server to Client (-i) int is_pending_logs; // Log files need to be send to peer. GS_LIST_ITEM *ids_li; // Peer is interested in global IDS logs + pid_t pid; }; #define GSC_FL_IS_SERVER (0x01) diff --git a/tools/console.c b/tools/console.c index 5d86804e..26017d4e 100644 --- a/tools/console.c +++ b/tools/console.c @@ -31,7 +31,6 @@ static void console_start(void); static void console_stop(void); static int console_command(struct _peer *p, const char *cmd); -// static void get_cursor_pos(int *row, int *col); static uint8_t chr_last; static int tty_fd = -1; @@ -63,10 +62,13 @@ struct _console_info int64_t last_pos; double bps; double last_bps; + + float ft_last_perc; + float ft_perc; // FileTransfer percent completion }; struct _console_info ci; -GS_CONDIS gs_condis; +GS_CONDIS gs_condis; // ConsoleDisplay static double get_usec(void) @@ -122,61 +124,6 @@ console_init(int fd) return; } -#if 0 -static int -readstring(int fd, char *buf, size_t sz, const char *str) -{ - unsigned char last; - unsigned char c; - int n; - int rv = -1; - char *end = buf + sz; - - if (fd < 0) - return -1; - - // signal(SIGALRM, resize_timeout); - // alarm(10); - n = read(fd, &c, 1); - if (n <= 0) - goto err; - - if (c == 0233) - { /* meta-escape, CSI */ - *buf++ = ESCAPE("")[0]; - *buf++ = '['; - } else { - *buf++ = (char) c; - } - if (c != *str) - goto err; - - last = str[strlen(str) - 1]; // R - while (1) - { - n = read(fd, &c, 1); - if (n <= 0) - goto err; - *buf++ = c; - if (c == last) - break; - if (buf >= end) - goto err; - } - - alarm(0); - *buf = 0; - rv = 0; -err: - // if (rv != 0) - // { - // signal(SIGALRM, SIG_DFL); - // alarm(0); // CANCEL alarm - // } - return rv; -} -#endif - static ssize_t tty_write(void *src, size_t len) { @@ -206,9 +153,12 @@ console_cursor_on(void) char *end = buf + sizeof (buf); char *ptr = buf; - DEBUGF_W("Console Cursor ON\n"); + int row = gopt.winsize.ws_row; + int col = 1 + GS_CONSOLE_PROMPT_LEN + MIN(rl.pos, rl.visible_len); + + DEBUGF_W("Console Cursor ON (%d:%df)\n", row, col); // ESC[?2004l = Reset bracketed paste mode - SXPRINTF(ptr, end - ptr, "\x1B[%d;%zuf", gopt.winsize.ws_row, 1 + GS_CONSOLE_PROMPT_LEN + MIN(rl.pos, rl.visible_len)); + SXPRINTF(ptr, end - ptr, "\x1B[%d;%df", row, col); if (is_console_cursor_needs_reset) { SXPRINTF(ptr, end - ptr, "\x1B[?2004l"); @@ -271,9 +221,19 @@ mk_statusbar(void) VSADDF(ptr, end, vc, "]"); // BYTES/sec - char buf[8]; - GS_format_bps(buf, sizeof buf, (int64_t)ci.bps); - VSADDF(ptr, end, vc, "[%s/s]", buf); + char buf[GS_FT_SPEEDSTR_MAXSIZE]; + GS_format_bps(buf, sizeof buf, (int64_t)ci.bps, "/s"); + VSADDF(ptr, end, vc, "[%s]", buf); + + // Percent of FileTransfer completed [99.2%] or [ 0.4%] or [-----] + if (ci.ft_perc <= 0) + { + VSADDF(ptr, end, vc, "[-----]"); + } else { + char perc[5]; + snprintf(perc, sizeof perc, "%1.1f", ci.ft_perc); + VSADDF(ptr, end, vc, "[%4s%%]", perc); + } // Fill until end size_t v_left = gopt.winsize.ws_col - vc; @@ -308,6 +268,24 @@ update_bps(struct _peer *p) ci.last_usec = now_usec; ci.last_pos = cur_pos; + + if (ci.last_bps != ci.bps) + ci.is_sb_redraw_needed += 1; + + // Percentage of File Transfer + ci.ft_last_perc = ci.ft_perc; + GS_FT *ft = &p->ft; + GS_FT_stats *s = &ft->stats; + + if (s->xfer_amount_scheduled == 0) + ci.ft_perc = 0; + else { + float f = ((float)(s->xfer_amount * 100)/ s->xfer_amount_scheduled); + ci.ft_perc = MIN(f, 99.9); + } + + if (ci.ft_last_perc != ci.ft_perc) + ci.is_sb_redraw_needed += 1; } void @@ -401,7 +379,7 @@ CONSOLE_update_bps(struct _peer *p) return; // Only redraw if there was a change - if (ci.last_bps != ci.bps) + if (ci.is_sb_redraw_needed) { mk_statusbar(); console_draw(p->fd_out, 0); @@ -539,6 +517,11 @@ check_arrow(int *esc, uint8_t c) * 6. ^E + == submit=other. Send ^E + submit. Return -2 * ==> behavior screen like (see *#1* below) * 6. ^E + == do not submit. Return 0 + * + * Return 0 : Caller not to process received character (more data required). + * Return -1: Caller to process character in *submit + * Return -2: not used. + * Return >0: Escaped character. */ int CONSOLE_check_esc(uint8_t c, uint8_t *submit) @@ -910,7 +893,7 @@ ansi_write(int fd, void *data, size_t len, int *cls_code) if (write(fd, data, amount) != amount) return -1; #ifdef DEBUG - ansi_output(data, amount); + // ansi_output(data, amount); #endif if (amount < len) @@ -962,7 +945,7 @@ CONSOLE_write(int fd, void *data, size_t len) /* Move cursor to upper tier if cursor inside console */ if (is_cursor_in_console) - tty_write("\x1B""8", 2); // Restore cursor + tty_write("\x1B""8", 2); // Restore cursor to upper tier ssize_t sz; sz = ansi_write(fd, data, len, &is_detected_clearscreen); @@ -976,7 +959,7 @@ CONSOLE_write(int fd, void *data, size_t len) // HEXDUMP(data, MIN(16, len)); if (is_cursor_in_console) - tty_write("\x1B""7", 2); // Save cursor position + tty_write("\x1B""7", 2); // Save new cursor position after writing to upper tier // Now check if console needs to be re-drawn @@ -1014,7 +997,10 @@ CONSOLE_write(int fd, void *data, size_t len) console_draw(fd, 1 /*force*/); if (is_cursor_in_console) + { + DEBUGF("is_cursor_in_console is true\n"); console_cursor_on(); + } return sz; } @@ -1041,7 +1027,7 @@ CONSOLE_readline(struct _peer *p, void *data, size_t len) for (; src < s_end; src++) { - rv = GS_RL_add(&rl, *src, &key, GS_CONSOLE_INPUT_LEN, 1 + GS_CONSOLE_PROMPT_LEN); + rv = GS_RL_add(&rl, *src, &key, gopt.winsize.ws_row, 1 + GS_CONSOLE_PROMPT_LEN); // HEXDUMP(rl.esc_data, rl.esc_len); if (write(fd, rl.esc_data, rl.esc_len) != rl.esc_len) ERREXIT("write()\n"); @@ -1077,25 +1063,6 @@ CONSOLE_readline(struct _peer *p, void *data, size_t len) return 1; } -#if 0 -static void -get_cursor_pos(int *row, int *col) -{ - *row = -1; - *col = -1; - int rv; - char buf[64]; - - tty_write("\x1b" "[6n", 4); - rv = readstring(tty_fd, buf, sizeof buf, PTY_SIZE_STR); - - if (rv == 0) - sscanf(buf, PTY_SIZE_STR, row, col); - - DEBUGF_G("Current Cursor row=%d col=%d\n", *row, *col); -} -#endif - /* * Set up terminal to display console (e.g. scroll upper tier up * and save cursor location of upper tier). @@ -1180,8 +1147,8 @@ CONSOLE_action(struct _peer *p, uint8_t key) if (key == 'c') { - /* Trigger: Send new window size to peer */ - gopt.is_win_resized = 1; + gopt.is_win_resized = 1; // Trigger: Send new window size to peer + gopt.is_want_ids_on = 1; GS_SELECT_FD_SET_W(p->gs); if (gopt.is_console == 1) @@ -1195,8 +1162,6 @@ CONSOLE_action(struct _peer *p, uint8_t key) console_start(); gopt.is_console = 1; - pkt_app_send_ids(p->gs->ctx->gselect_ctx, p); - GS_condis_pos(&gs_condis, (gopt.winsize.ws_row - GS_CONSOLE_ROWS) + 1 + 1, gopt.winsize.ws_col); if (is_console_welcome_msg == 0) { @@ -1205,7 +1170,7 @@ CONSOLE_action(struct _peer *p, uint8_t key) GS_condis_add(&gs_condis, 0, "Press Ctrl-e + UP to leave the console."); is_console_welcome_msg = 1; } - // Draw console neede? Resizing remote will trigger a CLEAR (=> re-draw) + // Draw console needed? Resizing remote will trigger a CLEAR (=> re-draw) mk_statusbar(); console_draw(p->fd_out, 1); } @@ -1222,6 +1187,41 @@ cmd_help(int fd) GS_condis_draw(&gs_condis, 1); } +// Use wordexp(3) to resolve path name with ~/ and variable substitution +static int +path_resolve(const char *pattern, char *dst, size_t len) +{ + wordexp_t p; + int ret; + + if (len <= 0) + return -1; + + dst[0] = '\0'; + // On failure return 'pattern' as path + snprintf(dst, len, "%s", pattern); + + signal(SIGCHLD, SIG_DFL); + ret = wordexp(pattern, &p, WRDE_NOCMD); + signal(SIGCHLD, SIG_IGN); + if (ret != 0) + { + DEBUGF_R("wordexp(%s) error: %d\n", pattern, ret); + return -1; + } + + if (p.we_wordc <= 0) + { + wordfree(&p); + return -1; + } + + snprintf(dst, len, "%s", p.we_wordv[0]); + wordfree(&p); + + return 0; +} + static int console_command(struct _peer *p, const char *cmd) { @@ -1231,6 +1231,9 @@ console_command(struct _peer *p, const char *cmd) char *ptr; int row = gopt.winsize.ws_row - (GS_CONSOLE_ROWS - 1); + if (strlen(cmd) <= 0) + return 0; + if (memcmp(cmd, "help", 4) == 0) { cmd_help(fd); @@ -1238,17 +1241,31 @@ console_command(struct _peer *p, const char *cmd) cmd_ping(p); } else if (memcmp(cmd, "quit", 4) == 0) { hard_quit(); + } else if (memcmp(cmd, "pwd", 3) == 0) { + cmd_pwd(p); } else if (memcmp(cmd, "clear", 5) == 0) { GS_condis_clear(&gs_condis); GS_condis_draw(&gs_condis, 1); - } else if (memcmp(cmd, "put", 3) == 0) { - GS_condis_add(&gs_condis, 0, "Not yet implemented."); + } else if (memcmp(cmd, "put ", 4) == 0) { + GS_FT_put(&p->ft, cmd+4); + GS_SELECT_FD_SET_W(p->gs); + } else if (memcmp(cmd, "get ", 4) == 0) { + GS_FT_get(&p->ft, cmd+4); + GS_SELECT_FD_SET_W(p->gs); + } else if (memcmp(cmd, "xaitax", 6) == 0) { + GS_condis_add(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, "Thanks xaitax for testing!"); GS_condis_draw(&gs_condis, 1); - } else if (memcmp(cmd, "get", 3) == 0) { - GS_condis_add(&gs_condis, 0, "Not yet implemented."); + } else if (strncmp(cmd, "lpwd", 4) == 0) { + GS_condis_add(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, getwd(NULL)); GS_condis_draw(&gs_condis, 1); - } else if (memcmp(cmd, "xaitax", 6) == 0) { - GS_condis_add(&gs_condis, 0, "Thanks xaitax for testing!"); + } else if (strncmp(cmd, "lcd ", 4) == 0) { + char path[PATH_MAX]; + path_resolve(cmd + 4, path, sizeof path); + if (chdir(path) != 0) + snprintf(buf, sizeof buf, "%s: %s", strerror(errno), path); + else + snprintf(buf, sizeof buf, "%s", getwd(NULL)); + GS_condis_add(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, buf); GS_condis_draw(&gs_condis, 1); } else { snprintf(buf, sizeof buf, "Command not known: '%s'", cmd); diff --git a/tools/console_display.c b/tools/console_display.c index 75529a2f..d3d9ecef 100644 --- a/tools/console_display.c +++ b/tools/console_display.c @@ -156,6 +156,7 @@ GS_condis_draw(GS_CONDIS *cd, int force) char *end = buf + sizeof (buf) - 1; // Space for \n char *ptr = buf; + DEBUGF("Moving cursor to %d:1f\n", cd->y); SXPRINTF(ptr, end - ptr, "\x1B[%d;1f", cd->y); cd_write(cd->fd, buf, ptr - buf); diff --git a/tools/console_display.h b/tools/console_display.h index 2b60f034..d1198ff5 100644 --- a/tools/console_display.h +++ b/tools/console_display.h @@ -6,7 +6,6 @@ struct condis_line { - // int type; const char *color_str; char line[CONDIS_LINE_MAX_LEN]; }; diff --git a/tools/event_mgr.c b/tools/event_mgr.c index afcc17de..110a13ad 100644 --- a/tools/event_mgr.c +++ b/tools/event_mgr.c @@ -94,7 +94,7 @@ cbe_ids(void *ptrNOTUSED) gopt.ids_idle = 0; // treat anything below 15 as fully active (0) // DEBUGF_C("Least Idle: %s (%d)\n", gopt.ids_active_user, gopt.ids_idle); - DEBUGF_W("Login: %d, active %d, Total %d\n", new_login.n_items, new_active.n_items, gopt.n_users); + // DEBUGF_W("Login: %d, active %d, Total %d\n", new_login.n_items, new_active.n_items, gopt.n_users); /* Search through all peers that want IDS messages */ GS_LIST_ITEM *li = NULL; diff --git a/tools/filetransfer-test.c b/tools/filetransfer-test.c index 886f03fc..bf2df2af 100644 --- a/tools/filetransfer-test.c +++ b/tools/filetransfer-test.c @@ -29,6 +29,7 @@ for x in test4k.dat test8k.dat; do [[ $(md5 -q "./${x}" == $(md5 -q "test/${x}" #include "filetransfer.h" #include "utils.h" #include "globbing.h" +#include "pkt_mgr.h" // For channel numbers #define BUF_LEN (GS_PKT_MAX_SIZE) @@ -145,7 +146,7 @@ pkt_cb_error(uint8_t chn, const uint8_t *data, size_t len, void *argNOTUSED) // Output total stats static void -stats_total(GS_FT_stats_total *st) +print_stats_ft(GS_FT_stats *st) { DEBUGF_Y("Speed : %s\n", st->speed_str); DEBUGF_Y("Amount : %"PRIu64"/%"PRIu64"usec\n", st->xfer_amount, st->xfer_duration); @@ -155,21 +156,21 @@ stats_total(GS_FT_stats_total *st) // On file transfer completion (for each file) static void -cb_stats(struct _gs_ft_stats *s) +cb_stats(struct _gs_ft_stats_file *s, void *arg) { DEBUGF_C("%u stats: %s\n", s->id, s->fname); DEBUGF_C("Speed: %s\n", s->speed_str); - stats_total(&ft.stats_total); + print_stats_ft(&ft.stats); } // Status and Error messages static void -cb_status(void *ft_ptr, struct _gs_ft_status *s) +cb_status(void *ft_ptr, struct _gs_ft_status *s, void *arg) { DEBUGF_M("Status: %u\n", s->code); DEBUGF_M("error : '%s'\n", s->err_str); - DEBUGF_M("File : %s\n", s->file->name); + DEBUGF_M("File : %s\n", s->fname); } /* @@ -245,19 +246,6 @@ mk_packet(void) return 0; } - -// Client, put (uploading) -// FIXME: move this internal to GS_FT (do globbing internally and make GS_FT_put() accept wildcards) -static void -glob_cb(GS_GL *res) -{ - DEBUGF("Inside Globbing CB %s\n", res->name); - // PUT (upload) - if (GS_FT_put(&ft, res->name) != 0) - DEBUGF_Y("Not valid: %s\n", res->name); // not found or directory -} - - static void glob_cb_test(GS_GL *res) { @@ -272,10 +260,8 @@ do_globtest(const char *exp) exit(0); } -// static uint32_t globbing_id; - int -main(int arc, char *argv[]) +main(int arc, const char *argv[]) { int is_extra_puts = 0; uint8_t src[BUF_LEN]; @@ -289,7 +275,7 @@ main(int arc, char *argv[]) GS_library_init(stderr, stderr); gopt.err_fp = stderr; gopt.log_fp = stderr; - // gopt.err_fp = NULL; // DISABLE DEBUG + // gopt.err_fp = NULL; // un-comment to DISABLE DEBUG if (*argv[1] == 'G') do_globtest(argv[2]); @@ -303,7 +289,7 @@ main(int arc, char *argv[]) if ((*argv[1] == 'c') || (*argv[1] == 'C')) { // CLIENT - GS_FT_init(&ft, cb_stats, cb_status, 0); + GS_FT_init(&ft, cb_stats, cb_status, 0, NULL, 0); if (*argv[1] == 'c') @@ -311,13 +297,9 @@ main(int arc, char *argv[]) // Test PUT (upload) GS_PKT_assign_chn(&pkt, GS_FT_CHN_ACCEPT, pkt_cb_accept, NULL); // Add files to queue... - // char **ptr = &argv[2]; - // int globbing_id = 0; int i; for (i = 2; argv[i] != NULL; i++) - { - GS_GLOBBING(glob_cb, argv[i], i, &ft, 0); - } + GS_FT_put(&ft, argv[i]); } else { // Test GET (download) is_get = 1; @@ -326,17 +308,15 @@ main(int arc, char *argv[]) GS_PKT_assign_chn(&pkt, GS_FT_CHN_SWITCH, pkt_cb_switch, NULL); int i; - // GS_FT_get(&ft, "test[14]k.dat"); - // GS_FT_get(&ft, "test8k.dat"); for (i = 2; argv[i] != NULL; i++) - { GS_FT_get(&ft, argv[i]); - } } } else { // SERVER - GS_FT_init(&ft, NULL, cb_status, 1); + GS_FT_init_tests(&argv[2]); // Test [8.9] + + GS_FT_init(&ft, NULL, cb_status, 0, NULL, 1); GS_PKT_assign_chn(&pkt, GS_FT_CHN_PUT, pkt_cb_put, NULL); GS_PKT_assign_chn(&pkt, GS_FT_CHN_DATA, pkt_cb_data, NULL); GS_PKT_assign_chn(&pkt, GS_FT_CHN_SWITCH, pkt_cb_switch, NULL); @@ -436,7 +416,7 @@ main(int arc, char *argv[]) } } - // stats_total(&ft.stats_total); + // print_stats_ft(&ft.stats); GS_FT_free(&ft); GS_BUF_free(&gsb); diff --git a/tools/filetransfer.c b/tools/filetransfer.c index b5690307..316350a8 100644 --- a/tools/filetransfer.c +++ b/tools/filetransfer.c @@ -8,17 +8,20 @@ static void ft_del(GS_LIST_ITEM *li); static void free_get_li(GS_LIST_ITEM *li); static void ft_done(GS_FT *ft); static void qerr_add(GS_FT *ft, uint32_t id, uint8_t code, const char *str); -static void mk_stats_total(GS_FT *ft); -static void mk_stats(GS_FT *ft, uint32_t id, struct _gs_ft_file *f, const char *fname, int err); +static void mk_stats_ft(GS_FT *ft); +static void mk_stats_file(GS_FT *ft, uint32_t id, struct _gs_ft_file *f, const char *fname, int err); static mode_t GS_fperm2mode(uint32_t u); static uint32_t GS_mode2fperm(mode_t m); -static void update_stats(struct _gs_ft_file *f, size_t sz); +static void update_stats(GS_FT *ft, struct _gs_ft_file *f, size_t sz); +static void init_xfer_stats(GS_FT *ft, struct _gs_ft_file *f, int64_t sz); static const char *str_dotslash(const char *src); static const char *str_stripslash(const char *src); static int mkdirp(const char *dir, mode_t mode, uint32_t mtime); static void dir_restore_mtime(const char *path, struct timespec *mtime); static void dir_save_mtime(const char *path, struct timespec *mtime); static void set_mode_mtime(const char *path, mode_t mode, uint32_t mtime); +static void call_status_cb(GS_FT *ft, const char *fname, uint8_t code, const char *err_str); + @@ -49,40 +52,61 @@ TEST CASES: #6. write to symlink - as per unix (follow symlinks) #7. zero file size #8. retain timestamp + #endif void -GS_FT_init(GS_FT *ft, gsft_cb_stats_t func_stats, gsft_cb_status_t func_status, int is_server) +GS_FT_init(GS_FT *ft, gsft_cb_stats_t func_stats, gsft_cb_status_t func_status, pid_t pid, void *arg, int is_server) { memset(ft, 0, sizeof *ft); - // PUT (upload) - Client side - GS_LIST_init(&ft->fqueue, 0); - GS_LIST_init(&ft->fputs, 0); - GS_LIST_init(&ft->faccepted, 0); - GS_LIST_init(&ft->fcompleted, 0); - // PUT (upload) - Server Side - GS_LIST_init(&ft->fadded, 0); - GS_LIST_init(&ft->freceiving, 0); - - // GET (download) - Client side - GS_LIST_init(&ft->plistreq, 0); - GS_LIST_init(&ft->plistreq_waiting, 0); - GS_LIST_init(&ft->flist, 0); - GS_LIST_init(&ft->fdl_waiting, 0); - // GET (download) - Server side - GS_LIST_init(&ft->flistreply, 0); - GS_LIST_init(&ft->fdl, 0); + if (is_server == 0) + { + // CLIENT + // PUT (upload) - Client side + GS_LIST_init(&ft->fqueue, 0); + GS_LIST_init(&ft->fputs, 0); + GS_LIST_init(&ft->faccepted, 0); + GS_LIST_init(&ft->fcompleted, 0); + // GET (download) - Client side + GS_LIST_init(&ft->plistreq, 0); + GS_LIST_init(&ft->plistreq_waiting, 0); + GS_LIST_init(&ft->flist, 0); + GS_LIST_init(&ft->fdl_waiting, 0); + } else { + // SERVER + // PUT (upload) - Server Side + GS_LIST_init(&ft->fadded, 0); + GS_LIST_init(&ft->freceiving, 0); + // GET (download) - Server side + GS_LIST_init(&ft->flistreply, 0); + GS_LIST_init(&ft->fdl, 0); + } GS_LIST_init(&ft->qerrs, 0); + ft->pid = pid; + ft->func_arg = arg; ft->func_stats = func_stats; ft->func_status = func_status; ft->is_server = is_server; } +#ifdef DEBUG +static const char **g_filesv; +static int g_filesc; +static const char *g_true_fname; +// For test 8.9: Server is a bad actor and sends ../../0wned.dat etc as list-reply for any request. +void +GS_FT_init_tests(const char **argv) +{ + g_filesv = argv; + g_filesc = 0; +} +#endif + static struct _gs_ft_file * -file_new(const char *fname, const char *fn_local, int64_t fz_local, int64_t fz_remote, uint32_t mtime, uint32_t fperm) +file_new(const char *fname, const char *fn_local, const char *fn_relative, int64_t fz_local, int64_t fz_remote, uint32_t mtime, uint32_t fperm) { struct _gs_ft_file *f; f = calloc(1, sizeof *f); @@ -92,13 +116,16 @@ file_new(const char *fname, const char *fn_local, int64_t fz_local, int64_t fz_r f->fz_local = fz_local; f->fz_remote = fz_remote; + if (fn_relative != NULL) + f->fn_relative = strdup(fn_relative); + if (fn_local != NULL) { // CLIENT - f->realname = strdup(fn_local); + f->fn_local = strdup(fn_local); // absolute file name } else { // SERVER (put & get) - f->realname = strdup(fname); + f->fn_local = strdup(fname); } f->name = strdup(fname); @@ -162,11 +189,14 @@ gs_ft_add_file(GS_FT *ft, GS_LIST *gsl, uint32_t id, const char *fname, uint32_t // HERE: File (not directory) struct _gs_ft_file *f; - f = file_new(fname, NULL, fz, fz_remote, mtime, GS_fperm2mode(fperm)); + f = file_new(fname, NULL, fname, fz, fz_remote, mtime, GS_fperm2mode(fperm)); f->li = GS_LIST_add(gsl, NULL, f, id); + ft->is_want_write = 1; return 0; } + + /* * SERVER, PUT (upload) * Return < 0 on error. @@ -182,7 +212,7 @@ GS_FT_add_file(GS_FT *ft, uint32_t id, const char *fname, size_t len, int64_t fs DEBUGF_Y("#%u ADD-FILE - size %"PRIu64", fperm 0%o, '%s' mtime=%d flags=0x%02x\n", id, fsize, fperm, fname, mtime, flags); char fn_local[4096]; - char *wdir = getwd(NULL); + char *wdir = GS_getpidwd(ft->pid); snprintf(fn_local, sizeof fn_local, "%s/%s", wdir, fname); XFREE(wdir); @@ -201,6 +231,14 @@ GS_FT_dl_add_file(GS_FT *ft, uint32_t id, const char *fname, size_t len, int64_t return -1; // protocol error. Not 0 terminated. DEBUGF_Y("#%u DL-ADD-FILE - '%s' fz_remote=%"PRId64"\n", id, fname, fz_remote); +#ifdef DEBUGF + // Test [8.9] + if (g_true_fname != NULL) + { + fname = g_true_fname; + DEBUGF_B("TESTING: Fake filename. Using data from %s\n", fname); + } +#endif struct stat res; int ret; @@ -244,28 +282,30 @@ GS_FT_list_add(GS_FT *ft, uint32_t globbing_id, const char *fname, size_t len, i GS_LIST_ITEM *li = GS_LIST_by_id(&ft->plistreq_waiting, globbing_id); if (li == NULL) { - DEBUGF_R("Oops. Received G#%u but no file requested?\n", globbing_id); + DEBUGF_R("Oops. Received G#%u but no file requested? (listreq_waiting=%d)\n", globbing_id, ft->plistreq_waiting.n_items); return -1; } p = (struct _gs_ft_list_pattern *)li->data; char fn_local[4096]; - const char *ptr; - ptr = str_dotslash(fname); - if (ptr == NULL) + const char *fn_relative; + fn_relative = str_dotslash(fname); + if (fn_relative == NULL) { - ptr = str_stripslash(fname); + fn_relative = str_stripslash(fname); } - snprintf(fn_local, sizeof fn_local, "%s/%s", p->wdir, ptr); + snprintf(fn_local, sizeof fn_local, "%s/%s", p->wdir, fn_relative); +#if 0 + // DISABLED check. Not possible with wordexp() support. if (fnmatch(p->pattern, fn_local, 0) != 0) { DEBUGF_R("filename does not match request (%s != %s)\n", fname, p->pattern); // goto done; // FIXME: C 'foo/*' would return 'bar/x.txt' but should return 'foo/bar/x.txt' } +#endif - // FIXME: Check for '..' and '/' and that realpath matches when send the globbing - // request to remote. Oops, this should be done when requesting file name? + // Sanity-check for '..' and deny if server sends reply containing '..' if (strstr(fn_local, "..") != NULL) { DEBUGF_R("Bad file name (%s)...\n", fn_local); @@ -293,7 +333,7 @@ GS_FT_list_add(GS_FT *ft, uint32_t globbing_id, const char *fname, size_t len, i if (res.st_size == fz_remote) { DEBUGF_R("File of equal size already exists (%s)\n", fn_local); - mk_stats(ft, -1, NULL, fn_local, 0 /*success*/); + mk_stats_file(ft, -1, NULL, fn_relative, 0 /*success*/); goto done; } @@ -302,12 +342,16 @@ GS_FT_list_add(GS_FT *ft, uint32_t globbing_id, const char *fname, size_t len, i } struct _gs_ft_file *f; - f = file_new(fname, fn_local, fz_local, fz_remote, mtime, GS_fperm2mode(fperm)); + f = file_new(fname, fn_local, fn_relative, fz_local, fz_remote, mtime, GS_fperm2mode(fperm)); f->li = GS_LIST_add(&ft->flist, NULL, f, ft->g_id); ft->g_id += 1; + // Update global stats + init_xfer_stats(ft, f, f->fz_remote - f->fz_local); + ft->n_files_waiting += 1; + ft->is_want_write = 1; ret = 0; done: @@ -452,19 +496,35 @@ dotslash_filename(const char *src) // CLIENT, put (upload) // SERVER, get (download), after globbing. static int -add_file_to_list(GS_FT *ft, GS_LIST *gsl, const char *fname, uint32_t globbing_id, int is_get) +add_file_to_list(GS_FT *ft, GS_LIST *gsl, const char *fname, uint32_t globbing_id, int is_server) { struct _gs_ft_file *f; f = calloc(1, sizeof *f); f->globbing_id = globbing_id; - if (is_get == 1) + if (is_server == 1) { // Server, GET (download). +#ifdef DEBUG + const char *ff = fname; + if ((g_filesv != NULL) && (g_filesv[g_filesc] != NULL)) + { + ff = g_filesv[g_filesc]; + + g_filesc++; + if (g_filesv[g_filesc] == NULL) + g_filesc = 0; + g_true_fname = strdup(fname); + } + + f->name = strdup(ff); +#else f->name = strdup(fname); +#endif } else { // Client, PUT (upload). f->name = dotslash_filename(fname); + f->fn_relative = strdup(f->name); } DEBUGF_Y("#%u name = %s\n", ft->g_id, f->name); @@ -491,7 +551,7 @@ add_file_to_list(GS_FT *ft, GS_LIST *gsl, const char *fname, uint32_t globbing_i if (realfname == NULL) return -3; - f->realname = realfname; + f->fn_local = realfname; f->fz_local = res.st_size; f->mode = res.st_mode; f->mtime = res.st_mtime; @@ -502,12 +562,12 @@ add_file_to_list(GS_FT *ft, GS_LIST *gsl, const char *fname, uint32_t globbing_i /* * CLIENT: Add this file (not directory) to queue. */ -int -GS_FT_put(GS_FT *ft, const char *fname) +static int +gs_ft_put(GS_FT *ft, const char *fname) { int ret; - ret = add_file_to_list(ft, &ft->fqueue, fname, 0 /*unused*/, 0 /*is_get*/); + ret = add_file_to_list(ft, &ft->fqueue, fname, 0 /*unused*/, 0 /*is_client*/); if (ret != 0) return ret; ft->n_files_waiting += 1; @@ -515,6 +575,35 @@ GS_FT_put(GS_FT *ft, const char *fname) return 0; } +// CLIENT +static void +glob_cb_client(GS_GL *res) +{ + GS_FT *ft = (GS_FT *)res->arg_ptr; + + gs_ft_put(ft, res->name); +} + +// CLIENT +int +GS_FT_put(GS_FT *ft, const char *pattern) +{ + int n_found; + + n_found = GS_GLOBBING(glob_cb_client, pattern, -1, ft, 0); + + if (n_found <= 0) + { + // Globbing error + DEBUGF_R("NOT FOUND: %s\n", pattern); + char buf[128]; + snprintf(buf, sizeof buf, "%s: %s", GS_FT_strerror(GS_FT_ERR_INVAL), pattern); + call_status_cb(ft, pattern, GS_FT_ERR_INVAL, buf); + return -1; + } + + return 0; +} // SERVER: Add a single file static int @@ -525,7 +614,7 @@ get_add_file(GS_FT *ft, const char *fname, uint32_t globbing_id) DEBUGF_G("Adding %s\n", fname); // Retrieve all info for the file. - ret = add_file_to_list(ft, &ft->flistreply, fname, globbing_id, 1 /*is_get*/); + ret = add_file_to_list(ft, &ft->flistreply, fname, globbing_id, 1 /*is_server*/); if (ret != 0) { return ret; @@ -554,6 +643,9 @@ GS_FT_list_add_files(GS_FT *ft, uint32_t globbing_id, const char *pattern, size_ return -1; // protocol error. Not 0-terminated. DEBUGF_Y("G#%u GET-ADD-FILE: %s\n", globbing_id, pattern); + char *ptr = GS_getpidwd(ft->pid); + chdir(ptr); + XFREE(ptr); n_found = GS_GLOBBING(glob_cb, pattern, globbing_id, ft, 0); @@ -565,6 +657,7 @@ GS_FT_list_add_files(GS_FT *ft, uint32_t globbing_id, const char *pattern, size_ qerr_add(ft, globbing_id, GS_FT_ERR_NOENT, err); } + ft->is_want_write = 1; return n_found; } @@ -580,8 +673,7 @@ GS_FT_get(GS_FT *ft, const char *pattern) p = calloc(1, sizeof *p); p->pattern = strdup(pattern); p->wdir = getwd(NULL); - p->globbing_id = ft->g_globbing_id; - ft->g_globbing_id++; + p->globbing_id = ft->g_id; GS_LIST_add(&ft->plistreq, NULL, p, ft->g_id); ft->g_id += 1; @@ -604,6 +696,7 @@ qerr_add(GS_FT *ft, uint32_t id, uint8_t code, const char *str) // Must add in sequence of occurance (add_count) GS_LIST_add(&ft->qerrs, NULL, qerr, ft->qerrs.add_count); + ft->is_want_write = 1; } // Send status to peer (error-msg) and free the file structure. @@ -622,7 +715,7 @@ do_recv_error(GS_FT *ft, GS_LIST_ITEM *li, uint32_t code, const char *str) struct _gs_ft_file *f = (struct _gs_ft_file *)li->data; if (ft->is_server == 0) - mk_stats(ft, f->li->id, f->li->data, NULL, 1 /*err*/); + mk_stats_file(ft, li->id, f, NULL, 1 /*err*/); send_status_and_del(ft, li, code, str); } @@ -657,15 +750,15 @@ sending_complete(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fcompleted) static void receiving_complete(GS_FT *ft, struct _gs_ft_file *f) { - char *s = strdup(f->realname); + char *s = strdup(f->fn_local); char *dn = dirname(s); if (ft->is_server == 0) { // CLIENT (get, download: All data received) - mk_stats(ft, f->li->id, f->li->data, NULL, 0 /*err*/); + mk_stats_file(ft, f->li->id, f, NULL, 0 /*err*/); XFCLOSE(f->fp); // Must close before setting mtime - set_mode_mtime(f->realname, f->mode, f->mtime); + set_mode_mtime(f->fn_local, f->mode, f->mtime); dir_restore_mtime(dn, &f->dir_mtime); ft_done(ft); @@ -673,7 +766,7 @@ receiving_complete(GS_FT *ft, struct _gs_ft_file *f) } else { // SERVER (put, upload: all data received) XFCLOSE(f->fp); - set_mode_mtime(f->realname, f->mode, f->mtime); + set_mode_mtime(f->fn_local, f->mode, f->mtime); dir_restore_mtime(dn, &f->dir_mtime); // close FP, free file, return COMPLETE message to client. @@ -733,7 +826,8 @@ GS_FT_data(GS_FT *ft, const void *data, size_t len) if (ft->is_server == 0) { // CLIENT, get (download); - update_stats(f, sz); + update_stats(ft, f, sz); + DEBUGF_W("xfer %"PRId64"/%"PRId64"\n", ft->stats.xfer_amount, ft->stats.xfer_amount_scheduled); } if (f->fz_local < f->fz_remote) @@ -746,6 +840,18 @@ GS_FT_data(GS_FT *ft, const void *data, size_t len) } // CLIENT +static void +init_xfer_stats(GS_FT *ft, struct _gs_ft_file *f, int64_t sz) +{ + if (sz <= 0) + return; + + f->xfer_amount_scheduled = sz; + ft->stats.xfer_amount_scheduled += f->xfer_amount_scheduled; + DEBUGF_W("Xfer Bytes Scheduled: %"PRId64", (added=%"PRId64")\n", ft->stats.xfer_amount_scheduled, f->xfer_amount_scheduled); +} + +// CLIENT, put (upload) void GS_FT_accept(GS_FT *ft, uint32_t id, int64_t fz_remote) { @@ -764,6 +870,11 @@ GS_FT_accept(GS_FT *ft, uint32_t id, int64_t fz_remote) struct _gs_ft_file *f = (struct _gs_ft_file *)li->data; f->fz_remote = fz_remote; + + // Update global stats + init_xfer_stats(ft, f, f->fz_local - f->fz_remote); + + ft->is_want_write = 1; } static void @@ -976,9 +1087,9 @@ gs_ft_switch(GS_FT *ft, uint32_t id, int64_t fz_remote, struct _gs_ft_file **act if (fz_remote == 0) { - DEBUGF_G("New file (%s) %s\n", new->realname, new->name); + DEBUGF_G("New file (%s) %s\n", new->fn_local, new->name); - char *ptr = new->realname; + char *ptr = new->fn_local; // mkdir(): Remove leading './///' in './////foo/bar' (when globbing './') if (*ptr == '.') { @@ -991,12 +1102,12 @@ gs_ft_switch(GS_FT *ft, uint32_t id, int64_t fz_remote, struct _gs_ft_file **act // put(test1k.dat) must not modify the permission of parent directory. mkdirp(ptr, 0 /*do not update permission on existing directory*/, 0 /*do not update mtime*/); - new->fp = fopen(new->realname, "w"); + new->fp = fopen(new->fn_local, "w"); } else { // Check fsize of local file. DEBUGF_G("Appending to file\n"); struct stat res; - if (stat(new->realname, &res) != 0) + if (stat(new->fn_local, &res) != 0) goto err; if (res.st_size != new->fz_local) { @@ -1004,12 +1115,12 @@ gs_ft_switch(GS_FT *ft, uint32_t id, int64_t fz_remote, struct _gs_ft_file **act do_recv_error(ft, new->li, GS_FT_ERR_BADFSIZE, NULL); return; } - new->fp = fopen(new->realname, "a"); + new->fp = fopen(new->fn_local, "a"); } if (new->fp == NULL) { - DEBUGF("fopen(%s) failed: %s\n", new->realname, strerror(errno)); + DEBUGF("fopen(%s) failed: %s\n", new->fn_local, strerror(errno)); goto err; } @@ -1043,7 +1154,8 @@ static void file_free(struct _gs_ft_file *f) { XFREE(f->name); - XFREE(f->realname); + XFREE(f->fn_local); + XFREE(f->fn_relative); XFREE(f); } @@ -1053,6 +1165,7 @@ ft_done(GS_FT *ft) { if (ft->is_server != 0) { + // SERVER DEBUGF_R("WARN: DOES NOT HAPPEN\n"); return; } @@ -1092,23 +1205,27 @@ mk_bps(char *str, size_t sz, uint64_t duration, uint64_t amount, int err) return; } if (duration > 0) - GS_format_bps(str, sz, (amount * 1000000 / duration)); + GS_format_bps(str, sz, (amount * 1000000 / duration), "/s"); else snprintf(str, sz, "SKIPPED"); } // Generate stats per file and call call-back. static void -mk_stats(GS_FT *ft, uint32_t id, struct _gs_ft_file *f, const char *fname, int err) +mk_stats_file(GS_FT *ft, uint32_t id, struct _gs_ft_file *f, const char *name, int err) { - struct _gs_ft_stats s; + struct _gs_ft_stats_file s; memset(&s, 0, sizeof s); s.id = id; - if (fname != NULL) - s.fname = fname; - else if (f != NULL) - s.fname = f->name; + if (name != NULL) + { + s.fname = name; + } else if (f != NULL) { + s.fname = str_dotslash(f->name); + if (s.fname == NULL) + s.fname = f->name; + } if (f != NULL) { @@ -1121,34 +1238,74 @@ mk_stats(GS_FT *ft, uint32_t id, struct _gs_ft_file *f, const char *fname, int e DEBUGF_R("Oops, Reporting stats on a suspended file\n"); f->usec_suspend_duration += (GS_usec() - f->usec_suspend_start); } + DEBUGF_W("end=%"PRIu64", start=%"PRIu64", suspend=%"PRIu64"\n", f->usec_end, f->usec_start, f->usec_suspend_duration); + s.xfer_duration = (f->usec_end - f->usec_start) - f->usec_suspend_duration; } - mk_bps(s.speed_str, sizeof s.speed_str, s.xfer_duration, s.xfer_amount, err); // Global stats for all files - ft->stats_total.xfer_duration += s.xfer_duration; - ft->stats_total.xfer_amount += s.xfer_amount; + ft->stats.xfer_duration += s.xfer_duration; if (err == 0) - ft->stats_total.n_files_success += 1; - else - ft->stats_total.n_files_error += 1; - mk_stats_total(ft); - - // Call call-back - if (ft->func_stats != NULL) - (*ft->func_stats)(&s); + ft->stats.n_files_success += 1; + else { + ft->stats.n_files_error += 1; + // Encountered an error. Reduce the 'scheduled' amount by what + // we could not transfer. + if (f != NULL) + { + int64_t diff = MAX(0, f->xfer_amount_scheduled - f->xfer_amount); + int64_t allx = MAX(0, ft->stats.xfer_amount_scheduled - diff); + ft->stats.xfer_amount_scheduled = allx; + } + } + mk_stats_ft(ft); + + // Call call-back (but not if it is an error) + if ((ft->func_stats != NULL) && (err == 0)) + (*ft->func_stats)(&s, ft->func_arg); } // Generate total stats static void -mk_stats_total(GS_FT *ft) +mk_stats_ft(GS_FT *ft) { - GS_FT_stats_total *st = &ft->stats_total; + GS_FT_stats *st = &ft->stats; mk_bps(st->speed_str, sizeof st->speed_str, st->xfer_duration, st->xfer_amount, st->n_files_success==0?1:0); } +void +GS_FT_stats_reset(GS_FT *ft) +{ + memset(&ft->stats, 0, sizeof ft->stats); +} + +// Report error to caller +static void +call_status_cb(GS_FT *ft, const char *fname, uint8_t code, const char *err_str) +{ + if (ft->func_status == NULL) + return; + + char buf[128]; + if (err_str == NULL) + { + // Create error string if none provided + snprintf(buf, sizeof buf, "%s: %s", GS_FT_strerror(code), fname); + err_str = buf; + DEBUGF_R("string1='%s'\n", err_str); + } + struct _gs_ft_status s; + memset(&s, 0, sizeof s); + s.fname = fname; + s.code = code; + DEBUGF_R("string2='%s'\n", err_str); + + s.err_str = err_str; + (*ft->func_status)(ft, &s, ft->func_arg); +} + /* * Error received from Server or Client. * Remove and free item. @@ -1158,7 +1315,8 @@ GS_FT_status(GS_FT *ft, uint32_t id, uint8_t code, const char *err_str, size_t l { GS_LIST_ITEM *li; int err = 1; - int is_make_stats = 1; + struct _gs_ft_file *f = NULL; + struct _gs_ft_list_pattern *lp = NULL; if (err_str[len] != '\0') return; // protocol error. Not 0 terminated. @@ -1170,31 +1328,32 @@ GS_FT_status(GS_FT *ft, uint32_t id, uint8_t code, const char *err_str, size_t l // li can be one of two structures only: // _gs_ft_file or gs_ft_list_pattern - li = GS_LIST_by_id(&ft->fcompleted, id); + li = GS_LIST_by_id(&ft->fcompleted, id); // CLIENT if (li == NULL) { - li = GS_LIST_by_id(&ft->fputs, id); + li = GS_LIST_by_id(&ft->fputs, id); // CLIENT if (li == NULL) { - li = GS_LIST_by_id(&ft->fadded, id); + li = GS_LIST_by_id(&ft->fdl_waiting, id); // CLIENT (get) if (li == NULL) { - li = GS_LIST_by_id(&ft->freceiving, id); + // This 'li' does not hold a _gs_ft_file structure and thus + // must not generate stats or try to free a _gs_ft_file when it is not. + li = GS_LIST_by_id(&ft->plistreq_waiting, id); // CLIENT (get) if (li == NULL) { - li = GS_LIST_by_id(&ft->fdl_waiting, id); + li = GS_LIST_by_id(&ft->fadded, id); // SERVER (put) if (li == NULL) { - li = GS_LIST_by_id(&ft->plistreq_waiting, id); + li = GS_LIST_by_id(&ft->freceiving, id); // SERVER (put) if (li == NULL) { DEBUGF_R("id %u not found\n", id); return; // not found } - // This 'li' does not hold a _gs_ft_file structure and thus - // must not generate stats or try to free a _gs_ft_file when it is not. - is_make_stats = 0; } + } else { + lp = (struct _gs_ft_list_pattern *)li->data; } } } @@ -1204,22 +1363,35 @@ GS_FT_status(GS_FT *ft, uint32_t id, uint8_t code, const char *err_str, size_t l err = 0; } - // Report (error-)status to caller - struct _gs_ft_status s; - memset(&s, 0, sizeof s); - s.code = code; - s.file = li->data; - // FIXME: sanitize error string - snprintf(s.err_str, sizeof s.err_str, "%s", err_str); - if (ft->func_status != NULL) - (*ft->func_status)(ft, &s); - + const char *name; + if (lp == NULL) + { + // A *f file (not a globbing pattern) + f = (struct _gs_ft_file *)li->data; + name = str_dotslash(f->name); + if (name == NULL) + name = f->name; + DEBUGF_R("FILE: %s\n", name); + } else { + name = lp->pattern; + } - if (is_make_stats) + if (ft->is_server == 0) { - // Report stats to caller - mk_stats(ft, id, li->data, NULL, err); + // CLIENT + // Report (error-)status to caller + // Dont trust 'err_str' from remote. Passing 'null' will make sure we generate our own. + call_status_cb(ft, name, code, NULL /*err_str*/); + + if (f != NULL) + { + // Report stats to caller + mk_stats_file(ft, id, f, name, err); + } + } + if (f != NULL) + { if (li->data == ft->active_put_file) ft->active_put_file = NULL; @@ -1228,6 +1400,7 @@ GS_FT_status(GS_FT *ft, uint32_t id, uint8_t code, const char *err_str, size_t l } else { // From a PATTERN request (LISTREQ, Client, get, download). // File structure is not available. + // Or SERVER list GS_LIST_del(li); } @@ -1259,14 +1432,68 @@ mk_error(void *dst, size_t len, uint32_t id, uint8_t code, const char *str) return sizeof err + sz; } +static uint8_t +errno2code(int eno /*errno*/, uint8_t default_code) +{ + DEBUGF_Y("errno = %d\n", eno); + switch (eno) + { + case EACCES: + case EPERM: + return GS_FT_ERR_PERM; + case ENOENT: + case EFAULT: + return GS_FT_ERR_NOENT; + case EBADF: + return GS_FT_ERR_BADF; + case EINVAL: + return GS_FT_ERR_INVAL; + } + + return default_code; +} + +const char * +GS_FT_strerror(uint8_t code) +{ + switch (code) + { + case GS_FT_ERR_PERM: + return "Permission denied"; + case GS_FT_ERR_NOENT: + return "Not found"; + case GS_FT_ERR_BADFSIZE: + return "Bad file size"; + case GS_FT_ERR_BADF: + return "Bad file descriptor"; + case GS_FT_ERR_NODATA: + return "No Data"; + case GS_FT_ERR_INVAL: + return "Bad Value"; + case GS_FT_ERR_COMPLETED: + return "Completed"; + } + + return "UNKNONW"; +} + /* * Make an error packet. Remove errornous file from queue */ static size_t -ft_mk_error(GS_FT *ft, void *dst, size_t len, int *pkt_type, GS_LIST_ITEM *li, uint8_t code, const char *str) +ft_mk_error(GS_FT *ft, struct _gs_ft_file *f, void *dst, size_t len, int *pkt_type, GS_LIST_ITEM *li, uint8_t code, const char *str) { size_t sz; + // On CLIENT side report this error to caller: + if (ft->is_server == 0) + { + // CLIENT + char buf[128]; + DEBUGF_B("fn-rel: %s\n", f->fn_relative); + snprintf(buf, sizeof buf, "%s: %s", GS_FT_strerror(code), f->fn_relative); + call_status_cb(ft, f->fn_relative, code, buf); + } *pkt_type = GS_FT_TYPE_ERROR; sz = mk_error(dst, len, li->id, code, str); @@ -1277,7 +1504,7 @@ ft_mk_error(GS_FT *ft, void *dst, size_t len, int *pkt_type, GS_LIST_ITEM *li, u } static void -update_stats(struct _gs_ft_file *f, size_t sz) +update_stats(GS_FT *ft, struct _gs_ft_file *f, size_t sz) { // -----BEGIN Log statistics----- if (f->usec_start == 0) @@ -1291,6 +1518,7 @@ update_stats(struct _gs_ft_file *f, size_t sz) f->usec_suspend_start = 0; } f->xfer_amount += sz; + ft->stats.xfer_amount += sz; // -----END Log statistics----- } @@ -1306,13 +1534,14 @@ mk_xfer_data(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fcompleted, int *p if (sz <= 0) { *active = NULL; - return ft_mk_error(ft, dst, len, pkt_type, f->li, GS_FT_ERR_BADF, NULL); + return ft_mk_error(ft, f, dst, len, pkt_type, f->li, errno2code(errno, GS_FT_ERR_BADF), NULL); } f->fz_remote += sz; if (ft->is_server == 0) { - update_stats(f, sz); + update_stats(ft, f, sz); + DEBUGF_W("xfer %"PRId64"/%"PRId64"\n", ft->stats.xfer_amount, ft->stats.xfer_amount_scheduled); } if (f->fz_remote >= f->fz_local) @@ -1337,16 +1566,16 @@ mk_switch(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fsource, GS_LIST *fco li = GS_LIST_next(fsource, NULL); f = (struct _gs_ft_file *)li->data; - f->fp = fopen(f->realname, "r"); + f->fp = fopen(f->fn_local, "r"); if (f->fp == NULL) { - DEBUGF("fopen(%s): %s\n", f->realname, strerror(errno)); - return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_PERM, NULL); + DEBUGF("fopen(%s): %s\n", f->fn_local, strerror(errno)); + return ft_mk_error(ft, f, dst, len, pkt_type, li, errno2code(errno, GS_FT_ERR_PERM), NULL); } ret = fseek(f->fp, 0, SEEK_END); if (ret != 0) - return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_BADF, NULL); + return ft_mk_error(ft, f, dst, len, pkt_type, li, errno2code(errno, GS_FT_ERR_BADF), NULL); f->fz_local = ftell(f->fp); // Peer already has this file. @@ -1355,8 +1584,8 @@ mk_switch(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fsource, GS_LIST *fco { DEBUGF("#%u Skipping %s (already on peer)\n", (unsigned int)f->li->id, f->name); if (ft->is_server == 0) - mk_stats(ft, li->id, f, NULL, 0 /*success*/); - return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_NODATA, NULL); + mk_stats_file(ft, li->id, f, NULL, 0 /*success*/); + return ft_mk_error(ft, f, dst, len, pkt_type, li, GS_FT_ERR_NODATA, NULL); } // Remote size is larger. Overwrite from beginning. @@ -1366,7 +1595,7 @@ mk_switch(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fsource, GS_LIST *fco // Remote size is smaller. Restart transmission. ret = fseek(f->fp, f->fz_remote, SEEK_SET); if (ret != 0) - return ft_mk_error(ft, dst, len, pkt_type, li, GS_FT_ERR_BADF, NULL); + return ft_mk_error(ft, f, dst, len, pkt_type, li, errno2code(errno, GS_FT_ERR_BADF), NULL); struct _gs_ft_switch sw; memset(&sw, 0, sizeof sw); @@ -1474,7 +1703,7 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) li = GS_LIST_next(&ft->plistreq, NULL); struct _gs_ft_list_pattern *p = li->data; - DEBUGF("%d LIST-REQ '%s' in queue (LIST-REQ to be send)\n", ft->plistreq.n_items, p->pattern); + DEBUGF("%d LIST-REQ '%s' in queue (LIST-REQ to be send) [Using glob-id=%d]\n", ft->plistreq.n_items, p->pattern, p->globbing_id); struct _gs_ft_list_request list_req; struct _gs_ft_list_request *lr = (struct _gs_ft_list_request *)dst; @@ -1624,10 +1853,13 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) #endif if ((ft->n_files_waiting == 0) && (ft->plistreq_waiting.n_items == 0)) { + ft->is_want_write = 0; *pkt_type = GS_FT_TYPE_DONE; return 0; } } + if (*pkt_type == GS_FT_TYPE_NONE) + ft->is_want_write = 0; return 0; } diff --git a/tools/filetransfer.h b/tools/filetransfer.h index a4fe66b6..4db595e0 100644 --- a/tools/filetransfer.h +++ b/tools/filetransfer.h @@ -1,23 +1,16 @@ #ifndef __GS_FILETRANSFER_H__ #define __GS_FILETRANSFER_H__ 1 -#define GS_FT_CHN_PUT (0) // 128 -#define GS_FT_CHN_ACCEPT (1) // 129 -#define GS_FT_CHN_LIST_REQUEST (2) // 130 0x82 -#define GS_FT_CHN_DATA (3) // 131 0x83 -#define GS_FT_CHN_ERROR (4) // 132 -#define GS_FT_CHN_SWITCH (5) // 133 0x85 -#define GS_FT_CHN_LIST_REPLY (6) // 134 0x86 -#define GS_FT_CHN_DL (7) // 135 0x87 - // Number of bytes needed for largest message (could be data) #define GS_FT_MIN_BUF_SIZE (64) +#define GS_FT_SPEEDSTR_MAXSIZE (7 + 2 + 1) // "123.4MB" + "/s" + \0 struct _gs_ft_file { GS_LIST_ITEM *li; - char *name; - char *realname; // local file name (absolute) + char *name; // requested name + char *fn_local; // local file name (absolute) + char *fn_relative; // Client: last part after '/./' mode_t mode; time_t mtime; FILE *fp; @@ -34,6 +27,7 @@ struct _gs_ft_file uint64_t usec_suspend_start; uint64_t usec_suspend_duration; int64_t xfer_amount; // Actual data on the wire + int64_t xfer_amount_scheduled; }; // Client structure to keep outstanding 'list' request in a GS-LIST @@ -44,27 +38,27 @@ struct _gs_ft_list_pattern char *wdir; // working directory }; -struct _gs_ft_stats +struct _gs_ft_stats_file { uint32_t id; - // struct _gs_ft_file *f; const char *fname; uint64_t xfer_duration; // Actual transfer time (without suspension) uint64_t xfer_amount; // Actual data transfered - char speed_str[8]; // Speed (bps). Human readable string. + char speed_str[GS_FT_SPEEDSTR_MAXSIZE]; // Speed (bps). Human readable string. }; -typedef void (*gsft_cb_stats_t)(struct _gs_ft_stats *s); +typedef void (*gsft_cb_stats_t)(struct _gs_ft_stats_file *s, void *arg); // Updated after each file completion typedef struct { - uint64_t xfer_duration; - uint64_t xfer_amount; - char speed_str[8]; + uint64_t xfer_duration; // + uint64_t xfer_amount; // bytes actually transfered so far + char speed_str[GS_FT_SPEEDSTR_MAXSIZE]; // Overall bps (updated after each file transfer) int n_files_success; // transferred or skipped so far int n_files_error; -} GS_FT_stats_total; + int64_t xfer_amount_scheduled; // Bytes scheduled (so far) for transfer +} GS_FT_stats; /* * Queue'ed error's that need to be send to peer. @@ -79,10 +73,11 @@ struct _gs_ft_qerr struct _gs_ft_status { uint8_t code; - struct _gs_ft_file *file; - char err_str[128]; // 0-terminated error string + // struct _gs_ft_file *file; + const char *fname; + const char *err_str; }; -typedef void (*gsft_cb_status_t)(void *ft_ptr, struct _gs_ft_status *s); +typedef void (*gsft_cb_status_t)(void *ft_ptr, struct _gs_ft_status *s, void *arg); typedef struct { @@ -105,25 +100,29 @@ typedef struct GS_LIST fdl; // List of files ready to send (switch to). // GS_LIST fdl_completed; // Waiting for ERR_COMPLETED + pid_t pid; // Server only: Use the CWD of this process for uploads/downloadds int g_id; // global request ID (unique) to match error-replies to requests int g_globbing_id; // global globbing id struct _gs_ft_file *active_put_file; // Current active upload file struct _gs_ft_file *active_dl_file; // Current active download file gsft_cb_stats_t func_stats; gsft_cb_status_t func_status; + void *func_arg; GS_LIST qerrs; // queue'd errors int is_server; int is_paused_data; // write() blocked. Queue control data. Pause sending file data int n_files_waiting; // Files waiting for completion or error FIXME: This should be n_requests_waiting + int is_want_write; // ..and be a counter of all outstanding requests we are awaiting an answer for.... // int n_listreply_waiting; // Statistics total (all files) - GS_FT_stats_total stats_total; + GS_FT_stats stats; } GS_FT; + // CLIENT -> Server: upload a file to server. // Server replies with 'gs_ft_accept' struct _gs_ft_put @@ -161,7 +160,7 @@ struct _gs_ft_list_reply #define GS_FT_FL_ISDIR (2) -// CLIENT -> Server: request file for download +// CLIENT -> Server: GET (downlaod) - request file struct _gs_ft_dl { uint32_t id; @@ -171,7 +170,7 @@ struct _gs_ft_dl uint8_t name[0]; // 0-terminated file name } __attribute__((__packed__)); -// SERVER -> Client: Accept file. (reply to PUT) +// SERVER -> Client: PUT (upload) - accept file struct _gs_ft_accept { uint32_t id; @@ -205,24 +204,30 @@ struct _gs_ft_error #define GS_FT_ERR_BADFSIZE (3) // Size on server is larger #define GS_FT_ERR_BADF (9) #define GS_FT_ERR_NODATA (10) +#define GS_FT_ERR_INVAL (11) // wordexp(3) error #define GS_FT_ERR_COMPLETED (128) // All data written successfully -void GS_FT_init(GS_FT *ft, gsft_cb_stats_t func_stats, gsft_cb_status_t func_status, int is_server); +void GS_FT_init(GS_FT *ft, gsft_cb_stats_t func_stats, gsft_cb_status_t func_status, pid_t pid, void *arg, int is_server); void GS_FT_free(GS_FT *ft); int GS_FT_add_file(GS_FT *ft, uint32_t id, const char *fname, size_t len, int64_t fsize, uint32_t mtime, uint32_t fperm, uint8_t flags); int GS_FT_dl_add_file(GS_FT *ft, uint32_t id, const char *fname, size_t len, int64_t fsize); int GS_FT_list_add_files(GS_FT *ft, uint32_t get_id, const char *pattern, size_t len); int GS_FT_list_add(GS_FT *ft, uint32_t globbing_id, const char *fname, size_t len, int64_t fsize, uint32_t mtime, uint32_t fperm, uint8_t flags); -int GS_FT_put(GS_FT *ft, const char *fname); +int GS_FT_put(GS_FT *ft, const char *pattern); int GS_FT_get(GS_FT *ft, const char *pattern); void GS_FT_switch(GS_FT *ft, uint32_t id, int64_t offset); void GS_FT_accept(GS_FT *ft, uint32_t id, int64_t offset); void GS_FT_data(GS_FT *ft, const void *data, size_t len); void GS_FT_status(GS_FT *ft, uint32_t id, uint8_t code, const char *err_str, size_t len); +void GS_FT_stats_reset(GS_FT *ft); +const char *GS_FT_strerror(uint8_t code); size_t GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type); void GS_FT_pause_data(GS_FT *ft); void GS_FT_unpause_data(GS_FT *ft); -int GS_FT_WANT_WRITE(GS_FT *ft); +#define GS_FT_WANT_WRITE(xft) (xft)->is_want_write +#ifdef DEBUG +void GS_FT_init_tests(const char **argv); +#endif // Packet types #define GS_FT_TYPE_NONE (0) diff --git a/tools/gs-netcat.h b/tools/gs-netcat.h index 22dae94c..38c6548b 100644 --- a/tools/gs-netcat.h +++ b/tools/gs-netcat.h @@ -2,5 +2,6 @@ #define __GS_NETCAT_H__ 1 int write_gs(GS_SELECT_CTX *ctx, struct _peer *p, int *killed); +int write_gs_atomic(GS_SELECT_CTX *ctx, struct _peer *p); #endif /* !__GS_NETCAT_H__ */ diff --git a/tools/ids.c b/tools/ids.c index f8955b80..e5339cc8 100644 --- a/tools/ids.c +++ b/tools/ids.c @@ -19,30 +19,6 @@ struct utmp_db_user int token; }; -// struct GS_BUF -// { -// void *data; -// size_t last; -// size_t total_len; -// }; - -// int -// GS_BUF_add(struct GS_BUF *gsb, void *data, size_t len) -// { -// if (gsb->total_len - gsb->last < len) -// { -// gsb->total_len += len * 10; -// gsb->data = realloc(gsb->data, gsb->total_len); -// } - -// memcpy((uint8_t *)gsb->data + gsb->last, data, len); -// gsb->last += len; - -// return 0; -// } - -// struct GS_BUF mon_db - GS_LIST udb; static int is_udb_init; diff --git a/tools/pkt_mgr.c b/tools/pkt_mgr.c index 23458e30..883dcb4a 100644 --- a/tools/pkt_mgr.c +++ b/tools/pkt_mgr.c @@ -5,6 +5,7 @@ #include "console_display.h" #include "utils.h" #include "gs-netcat.h" +#include "filetransfer_mgr.h" extern GS_CONDIS gs_condis; // defined in console.c @@ -102,6 +103,29 @@ pkt_app_cb_ids(uint8_t msg, const uint8_t *data, size_t len, void *ptr) } } +// SERVER +void +pkt_app_cb_pwdrequest(uint8_t msg, const uint8_t *dataUNUSED, size_t lenUNUSED, void *ptr) +{ + struct _peer *p = (struct _peer *)ptr; + + gopt.is_pwdreply_pending = 1; + GS_SELECT_FD_SET_W(p->gs); +} + +// CLIENT +void +pkt_app_cb_pwdreply(uint8_t chn, const uint8_t *data, size_t len, void *ptr) +{ + if (len <= 0) + return; + + if (data[len - 1] != '\0') + return; // protocol error. + + DEBUGF_B("REMOTE WD=%s\n", data); + GS_condis_add(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, (char *)data); +} int pkt_app_send_wsize(GS_SELECT_CTX *ctx, struct _peer *p, int row) @@ -179,6 +203,75 @@ pkt_app_send_ids(GS_SELECT_CTX *ctx, struct _peer *p) return write_gs(ctx, p, NULL); } +int +pkt_app_send_pwdrequest(GS_SELECT_CTX *ctx, struct _peer *p) +{ + p->wbuf[0] = GS_PKT_ESC; + p->wbuf[1] = PKT_MSG_PWD; + p->wlen = 2 + GS_PKT_MSG_size_by_type(PKT_MSG_PWD); + return write_gs(ctx, p, NULL); +} + +int +pkt_app_send_pwdreply(GS_SELECT_CTX *ctx, struct _peer *p) +{ + struct gs_pkt_chn_hdr *hdr = (struct gs_pkt_chn_hdr *)p->wbuf; + + hdr->esc = GS_PKT_ESC; + hdr->type = GS_PKT_CHN2TYPE(GS_CHN_PWD); + + char *wd = GS_getpidwd(p->pid); + snprintf((char *)p->wbuf + sizeof *hdr, sizeof p->wbuf - sizeof *hdr, "%s", wd); + size_t sz = strlen(wd) + 1; // including \0 + XFREE(wd); + + uint16_t len = htons(sz); + memcpy(&hdr->len, &len, sizeof len); + + p->wlen = sizeof *hdr + sz; + return write_gs(ctx, p, NULL); +} + + +// Loop until all FileTransfer data is written +// or the socket would block. +int +pkt_app_send_ft(GS_SELECT_CTX *ctx, struct _peer *p) +{ + ssize_t sz; + int len; + + while (1) + { + sz = GS_FTM_mk_packet(&p->ft, p->wbuf, sizeof p->wbuf); + if (sz == 0) + return GS_SUCCESS; // No data available. + if (sz == -1) + return GS_SUCCESS; // All files have been transfered. + if (sz < 0) // Catch All (-2 mostly/always) + return GS_ERR_FATAL; // Not enough space. + + // Got data to write. + p->wlen = sz; + len = write_gs_atomic(ctx, p); + if (len == -1) + return GS_ECALLAGAIN; + if (len != p->wlen) + return GS_ERROR; + p->wlen = 0; // SUCCESS. + // Do a single write only. This function returns and enters the select() loop + // again to check if there is any data on stdin. + // Otherwise the FileTransfer subsystem will keep sending data until write() would block + // and then keep data in p->wbuf without the STDIN ever being checked for input until + // the FileTransfer has completed. We like to check STDIN... + // FIXME-PERFORMANCE: Could write() here until would-block but then do not + // leave data in p->wbuf and instead use an internal buffer. This way select() is not + // called for every write() from FileTransfer subsystem. + return GS_SUCCESS; + } + + return GS_SUCCESS; // NOT REACHED +} static int send_log(GS_SELECT_CTX *ctx, struct _peer *p, struct _pkt_app_log *log) diff --git a/tools/pkt_mgr.h b/tools/pkt_mgr.h index d788b697..cb054f65 100644 --- a/tools/pkt_mgr.h +++ b/tools/pkt_mgr.h @@ -1,12 +1,25 @@ #ifndef __GS_PKT_MGR_H__ #define __GS_PKT_MGR_H__ 1 +// Message Numbers #define PKT_MSG_WSIZE (1) #define PKT_MSG_IDS (2) +#define PKT_MSG_PWD (3) // pwd request #define PKT_MSG_PING (16) #define PKT_MSG_PONG (16) #define PKT_MSG_LOG (32) // max 64 bytes long +// Channel Numbers +#define GS_FT_CHN_PUT (0) // 128 +#define GS_FT_CHN_ACCEPT (1) // 129 +#define GS_FT_CHN_LIST_REQUEST (2) // 130 0x82 +#define GS_FT_CHN_DATA (3) // 131 0x83 +#define GS_FT_CHN_ERROR (4) // 132 +#define GS_FT_CHN_SWITCH (5) // 133 0x85 +#define GS_FT_CHN_LIST_REPLY (6) // 134 0x86 +#define GS_FT_CHN_DL (7) // 135 0x87 +#define GS_CHN_PWD (8) // Result of pwd-request (server to client) + struct _pkt_app_ping { uint8_t flags; @@ -42,19 +55,22 @@ struct _pkt_app_log #define GS_PKT_APP_LOG_TYPE_INFO (0x03) // green #define GS_PKT_APP_LOG_TYPE_MAX (0x03) // set to highest color code - - void pkt_app_cb_wsize(uint8_t msg, const uint8_t *data, size_t len, void *ptr); void pkt_app_cb_ping(uint8_t msg, const uint8_t *data, size_t len, void *ptr); void pkt_app_cb_pong(uint8_t msg, const uint8_t *data, size_t len, void *ptr); void pkt_app_cb_ids(uint8_t msg, const uint8_t *data, size_t len, void *ptr); void pkt_app_cb_log(uint8_t msg, const uint8_t *data, size_t len, void *ptr); +void pkt_app_cb_pwdrequest(uint8_t msg, const uint8_t *data, size_t len, void *ptr); +void pkt_app_cb_pwdreply(uint8_t msg, const uint8_t *data, size_t len, void *ptr); int pkt_app_send_wsize(GS_SELECT_CTX *ctx, struct _peer *p, int row); int pkt_app_send_pong(GS_SELECT_CTX *ctx, struct _peer *p); int pkt_app_send_ping(GS_SELECT_CTX *ctx, struct _peer *p); int pkt_app_send_ids(GS_SELECT_CTX *ctx, struct _peer *p); int pkt_app_send_all_log(GS_SELECT_CTX *ctx, struct _peer *p); +int pkt_app_send_ft(GS_SELECT_CTX *ctx, struct _peer *p); +int pkt_app_send_pwdrequest(GS_SELECT_CTX *ctx, struct _peer *p); +int pkt_app_send_pwdreply(GS_SELECT_CTX *ctx, struct _peer *p); #endif /* !__GS_PKT_MGR_H__ */ \ No newline at end of file diff --git a/tools/run_ft_tests.sh b/tools/run_ft_tests.sh index ec3e7a40..4008a9fc 100755 --- a/tools/run_ft_tests.sh +++ b/tools/run_ft_tests.sh @@ -111,6 +111,13 @@ run_get2() socat SYSTEM:"(cd ${IODIR}; set -f && ${BIN} C $* 2>${LOGDIR}/client.log)" SYSTEM:"(cd ${IODIRSRC}/foo; ${BIN} s 2>${LOGDIR}/server.log)" } +# Server is a bad actor and send ../../../shit.dat as reply for any request +run_get_dst() +{ + # set -f disabled globbing + socat SYSTEM:"(cd ${IODIR}/foo; set -f && ${BIN} C test4k.dat 2>${LOGDIR}/client.log)" SYSTEM:"(cd ${IODIRSRC}/foo; ${BIN} s $* 2>${LOGDIR}/server.log)" +} + run_getc() { socat SYSTEM:"(cd ${IODIR}; set -f && ${BIN} C \'$*\' 2>${LOGDIR}/client.log)" SYSTEM:"(cd ${IODIRSRC}; ${BIN} s 2>${LOGDIR}/server.log)" @@ -146,9 +153,10 @@ tests+="8.5 " tests+="8.6 " tests+="8.7 " tests+="8.8 " +tests+="8.9 " -tests+="9.1 " -tests+="9.2 " +# tests+="9.1 " +# tests+="9.2 " if [[ "$tests" =~ '1.0 ' ]]; then @@ -432,6 +440,17 @@ md5fail 2 "${IODIRSRC}/foo/test4k.dat" "${IODIR}/test4k.dat" $ECHO "${OK}" fi +if [[ "$tests" =~ '8.9 ' ]]; then +test_start -n "Running #8.9 (get, Server sending ../../../shit............" +mkdir ${IODIR}/foo +run_get_dst /tmp/0wned.dat +run_get_dst ./../../../../../../../../../../../../../tmp/0wned.dat +run_get_dst ../0wned.dat +[[ -e /tmp/0wned.dat ]] && fail 1 +[[ -e "${IODIR}/0wned.dat" ]] && fail 2 +$ECHO "${OK}" +fi + if [[ "$tests" =~ '9.1 ' ]]; then test_start -n "Running #9.1 (HUGE put)..........................." ## Small diff --git a/tools/utils.c b/tools/utils.c index 223c7e52..6d854601 100755 --- a/tools/utils.c +++ b/tools/utils.c @@ -744,7 +744,7 @@ forkpty(int *fd, void *a, void *b, void *c) #endif /* HAVE_FORKPTY */ int -pty_cmd(const char *cmd) +pty_cmd(const char *cmd, pid_t *pidptr) { pid_t pid; int fd; @@ -819,6 +819,9 @@ pty_cmd(const char *cmd) } /* HERE: Parent */ + if (pidptr) + *pidptr = pid; + return fd; } @@ -826,7 +829,7 @@ pty_cmd(const char *cmd) * Spawn a cmd and return fd. */ int -fd_cmd(const char *cmd) +fd_cmd(const char *cmd, pid_t *pidptr) { pid_t pid; int fds[2]; @@ -836,7 +839,7 @@ fd_cmd(const char *cmd) if (gopt.is_interactive) { - return pty_cmd(cmd); + return pty_cmd(cmd, pidptr); } ret = socketpair(AF_UNIX, SOCK_STREAM, 0, fds); @@ -860,6 +863,8 @@ fd_cmd(const char *cmd) } /* HERE: Parent process */ + if (pidptr) + *pidptr = pid; close(fds[0]); return fds[1]; @@ -979,6 +984,15 @@ cmd_ping(struct _peer *p) GS_SELECT_FD_SET_W(p->gs); } +void +cmd_pwd(struct _peer *p) +{ + if (gopt.is_want_pwd != 0) + return; + + gopt.is_want_pwd = 1; + GS_SELECT_FD_SET_W(p->gs); +} const char fname_valid_char[] = "" "................" diff --git a/tools/utils.h b/tools/utils.h index 9c36937f..e9ab1c6a 100755 --- a/tools/utils.h +++ b/tools/utils.h @@ -8,7 +8,7 @@ void init_vars(void); GS *gs_create(void); void do_getopt(int argc, char *argv[]); void usage(const char *params); -int fd_cmd(const char *cmd); +int fd_cmd(const char *cmd, pid_t *pidptr); int fd_new_socket(void); int fd_net_listen(int fd, uint16_t port); int fd_net_accept(int listen_fd); @@ -19,6 +19,7 @@ void stty_check_esc(GS *gs, char c); char **mk_env(char **blacklist, char **addlist); void get_winsize(void); void cmd_ping(struct _peer *p); +void cmd_pwd(struct _peer *p); // void sanitze_name_to_string(uint8_t *str, size_t len); void sanitize_fname_to_str(uint8_t *str, size_t len); void format_bps(char *buf, size_t size, int64_t bytes); From 87d5678f0f22c6ae0a6f58d1ded19feb5856d11b Mon Sep 17 00:00:00 2001 From: rootTHC Date: Wed, 10 Feb 2021 09:25:22 +0000 Subject: [PATCH 13/21] compiles on all platforms. --- bootstrap | 2 +- configure.ac | 3 +- lib/gs-common.h | 19 +++++++- lib/gsocket-util.c | 47 ++++++++++++++++++- tools/4_gs-netcat.c | 55 ++++++++++++++++++++++- tools/common.h | 30 +++++++++++++ tools/console.c | 23 ++++++---- tools/filetransfer.c | 105 ++++++++++++++++++++++++++++--------------- 8 files changed, 232 insertions(+), 52 deletions(-) diff --git a/bootstrap b/bootstrap index 28aa7a61..284e7998 100755 --- a/bootstrap +++ b/bootstrap @@ -39,7 +39,7 @@ if test $? -ne 0; then exit 1 fi echo "automake --foreign --add-missing -Wno-syntax" -automake --foreign --add-missing -Wno-syntax +automake --foreign --copy --add-missing -Wno-syntax if test $? -ne 0; then exit 1 fi diff --git a/configure.ac b/configure.ac index 0c57676b..1fba8376 100755 --- a/configure.ac +++ b/configure.ac @@ -88,13 +88,14 @@ dnl AC_CHECK_LIB([m], [sqrt]) dnl AC_CHECK_LIB([net], [libnet_name_resolve], [AC_MSG_ERROR([libnet 1.0.x found. Requires libnet 1.1 or newer])]) dnl AC_CHECK_LIB([net], [libnet_init], ,[AC_MSG_ERROR([libnet 1.1.x not found])]) AC_CHECK_LIB(dl, dlopen) +AC_CHECK_LIB(procstat, procstat_close) AC_CHECK_LIB([crypto], [ENGINE_init], [], [AC_MSG_ERROR([libcrypto not found])]) AC_CHECK_LIB([ssl], [SRP_VBASE_get1_by_user], [], [AC_MSG_ERROR([SRP not supported. Please upgrade OpenSSL lib])]) dnl Checks for header files. AC_HEADER_STDC AC_HEADER_SYS_WAIT -AC_CHECK_HEADERS(sys/time.h unistd.h fnmatch.h string.h utmp.h utmpx.h pty.h openssl/srp.h util.h libutil.h netinet/in_systm.h sys/loadavg.h) +AC_CHECK_HEADERS(sys/time.h sys/endian.h unistd.h fnmatch.h string.h utmp.h utmpx.h pty.h openssl/srp.h util.h libutil.h netinet/in_systm.h sys/loadavg.h libproc.h) AC_CHECK_HEADER(openssl/srp.h, [], [AC_MSG_ERROR([openssl/srp.h not found. Update OpenSSL?])]) diff --git a/lib/gs-common.h b/lib/gs-common.h index f0b129dc..1626e54c 100755 --- a/lib/gs-common.h +++ b/lib/gs-common.h @@ -11,8 +11,13 @@ #include #include #include -#include #include +#if defined(__FREEBSD__) +# include +# include +# include +# include +#endif // __FREEBSD__ #include #ifdef HAVE_NETINET_IN_SYSTM_H # include @@ -36,7 +41,17 @@ #include #include #include /* basename() */ -#include // getpidwd(pid_t) +#if defined(__APPLE__) && defined(HAVE_LIBPROC_H) +# include // getpidwd(pid_t) +#endif +#if defined(__FREEBSD__) +// FIXME: Please tell me where this is defined? fbsd 12-1 complains: +// /usr/include/libprocstat.h:122:15: error: field 'fs_cap_rights' has incomplete type +# ifndef cap_rights_t +typedef struct cap_rights cap_rights_t; +# endif +# include +#endif #include #include #include diff --git a/lib/gsocket-util.c b/lib/gsocket-util.c index fa7e29f2..c420f6c6 100755 --- a/lib/gsocket-util.c +++ b/lib/gsocket-util.c @@ -212,22 +212,65 @@ GS_format_bps(char *dst, size_t size, int64_t bytes, const char *suffix) char * GS_getpidwd(pid_t pid) { - int ret; char *wd = NULL; if (pid <= 0) goto err; +#if defined(__APPLE__) && defined(HAVE_LIBPROC_H) + // OSX (and others?) + int ret; struct proc_vnodepathinfo vpi; ret = proc_pidinfo(pid, PROC_PIDVNODEPATHINFO, 0, &vpi, sizeof vpi); if (ret <= 0) goto err; wd = strdup(vpi.pvi_cdir.vip_path); +#elif __FREEBSD__ + struct procstat *procstat; + struct kinfo_proc *kipp; + struct filestat_list *head; + struct filestat *fst; + unsigned int cnt; + + procstat = procstat_open_sysctl(); + if (procstat == NULL) + goto err; + + kipp = procstat_getprocs(procstat, KERN_PROC_PID, pid, &cnt); + if ((kipp == NULL) || (cnt <= 0)) + goto err; + + head = procstat_getfiles(procstat, kipp, 0); + if (head == NULL) + goto err; + + STAILQ_FOREACH(fst, head, next) + { + if (!(fst->fs_uflags & PS_FST_UFLAG_CDIR)) + continue; + if (fst->fs_path == NULL) + continue; + wd = strdup(fst->fs_path); + break; + } + procstat_freefiles(procstat, head); +#else + // Linux + char buf[1024]; + char res[PATH_MAX + 1]; + ssize_t sz; + + snprintf(buf, sizeof buf, "/proc/%d/cwd", pid); + sz = readlink(buf, res, sizeof res - 1); + if (sz < 0) + goto err; + wd = strdup(res); +#endif err: if (wd == NULL) - wd = getwd(NULL); + wd = getcwd(NULL, 0); DEBUGF_W("PID %d CWD=%s\n", pid, wd); return wd; } diff --git a/tools/4_gs-netcat.c b/tools/4_gs-netcat.c index 18c2ae1d..f111631e 100755 --- a/tools/4_gs-netcat.c +++ b/tools/4_gs-netcat.c @@ -1206,8 +1206,6 @@ my_test(void) GS_LIST_init(&new_login, 0); GS_LIST_init(&new_active, 0); - int idle; - char *user; // double load; @@ -1260,6 +1258,59 @@ my_test(void) // DEBUGF("[% 4.02f]\n", (float)load / 100); // DEBUGF("[%4.02f]\n", (float)load / 100); // DEBUGF("[% -4.02f]\n", (float)load / 100); + +#if 0 + // FBSD getppidcwd() test to find out cwd of parent pid + #include + #include + #include + #include + #include + #include + #ifndef cap_rights_t + typedef struct cap_rights cap_rights_t; + #endif + #include + + chdir("/tmp"); + + char *wd = NULL; + struct procstat *procstat; + struct kinfo_proc *kipp; + struct filestat_list *head; + struct filestat *fst; + pid_t pid; + unsigned int cnt; + + procstat = procstat_open_sysctl(); + if (procstat == NULL) + goto done; + + pid = getppid(); + + kipp = procstat_getprocs(procstat, KERN_PROC_PID, pid, &cnt); + if ((kipp == NULL) || (cnt <= 0)) + goto done; + + head = procstat_getfiles(procstat, kipp, 0); + if (head == NULL) + goto done; + + STAILQ_FOREACH(fst, head, next) + { + if (!(fst->fs_uflags & PS_FST_UFLAG_CDIR)) + continue; + if (fst->fs_path == NULL) + continue; + wd = strdup(fst->fs_path); + break; + printf("cwd %-18s\n", fst->fs_path != NULL ? fst->fs_path : "-"); + } + + procstat_freefiles(procstat, head); +done: + printf("wd='%s'\n", wd); +#endif exit(0); } #endif diff --git a/tools/common.h b/tools/common.h index 25a0abdd..9162cb61 100755 --- a/tools/common.h +++ b/tools/common.h @@ -15,6 +15,9 @@ #ifdef HAVE_SYS_LOADAVG_H # include // Solaris11 #endif +#ifdef HAVE_SYS_ENDIAN_H +# include +#endif #include #ifdef HAVE_NETINET_IN_SYSTM_H # include @@ -34,6 +37,7 @@ #endif #include #include +#include // Solaris11 #include #include #include @@ -78,6 +82,29 @@ # define UT_NAMESIZE 32 #endif +#if defined(__sun) +# if !defined(be64toh) // Solaris11 +# define be64toh(x) ntohll(x) +# define htobe64(x) htonll(x) +# endif +# if !defined(htonll) // Solaris10 +# if __BIG_ENDIAN__ +# define htonll(x) (x) +# define ntohll(x) (x) +# else +# define htonll(x) ((uint64_t)htonl((x) & 0xFFFFFFFF) << 32) | htonl((uint64_t)(x) >> 32) +# define ntohll(x) ((uint64_t)ntohl((x) & 0xFFFFFFFF) << 32) | ntohl((uint64_t)(x) >> 32) +# endif +# endif +#endif + +#ifndef htonll +# define htonll(x) htobe64(x) +#endif +#ifndef ntohll +# define ntohll(x) be64toh(x) +#endif + struct _gopt { GS_CTX gs_ctx; @@ -231,6 +258,9 @@ extern struct _gopt gopt; ptr += MIN(n, len); \ } while(0) +// Overcome GCC warning for truncation. Abort() if truncation happen. +#define SNPRINTF_ABORT(...) (snprintf(__VA_ARGS__) < 0 ? abort() : (void)0) + #define VOUT(level, a...) do { \ if (level > gopt.verboselevel) \ break; \ diff --git a/tools/console.c b/tools/console.c index 26017d4e..a14bb0b8 100644 --- a/tools/console.c +++ b/tools/console.c @@ -436,14 +436,14 @@ console_draw(int fd, int force) if (gopt.is_console == 0) return; - // DEBUGF_R("CONSOLE DRAW (force=%d)\n", force); - int cursor_to_prompt = 0; cursor_to_prompt += ci.is_sb_redraw_needed; cursor_to_prompt += gs_condis.is_redraw_needed; + cursor_to_prompt += ci.is_prompt_redraw_needed; + // DEBUGF_W("CONSOLE DRAW (force=%d, cursor-to-prompt=%d)\n", force, cursor_to_prompt); if (is_cursor_in_console == 0) - tty_write("\x1B""7", 2); + tty_write("\x1B""7", 2); // Save position (upper tier) // Status Bar (Normally black on blue) GS_sb_draw(force); @@ -457,7 +457,7 @@ console_draw(int fd, int force) // Restore cursor position if (is_cursor_in_console == 0) { - tty_write("\x1B""8", 2); + tty_write("\x1B""8", 2); // Restore position (upper tier) } else { if (cursor_to_prompt) GS_prompt_cursor(); @@ -1227,6 +1227,7 @@ console_command(struct _peer *p, const char *cmd) { int fd = p->fd_out; char buf[GS_CONSOLE_BUF_SIZE]; + char path[PATH_MAX + 1]; char *end = buf + sizeof (buf); char *ptr; int row = gopt.winsize.ws_row - (GS_CONSOLE_ROWS - 1); @@ -1256,15 +1257,19 @@ console_command(struct _peer *p, const char *cmd) GS_condis_add(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, "Thanks xaitax for testing!"); GS_condis_draw(&gs_condis, 1); } else if (strncmp(cmd, "lpwd", 4) == 0) { - GS_condis_add(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, getwd(NULL)); + char *cwd = getcwd(NULL, 0); + GS_condis_add(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, cwd); + XFREE(cwd); GS_condis_draw(&gs_condis, 1); } else if (strncmp(cmd, "lcd ", 4) == 0) { - char path[PATH_MAX]; path_resolve(cmd + 4, path, sizeof path); if (chdir(path) != 0) - snprintf(buf, sizeof buf, "%s: %s", strerror(errno), path); - else - snprintf(buf, sizeof buf, "%s", getwd(NULL)); + snprintf(buf, sizeof buf, "%s: %.512s", strerror(errno), path); + else { + char *cwd = getcwd(NULL, 0); + snprintf(buf, sizeof buf, "%s", cwd); + XFREE(cwd); + } GS_condis_add(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, buf); GS_condis_draw(&gs_condis, 1); } else { diff --git a/tools/filetransfer.c b/tools/filetransfer.c index 316350a8..35b8f9b4 100644 --- a/tools/filetransfer.c +++ b/tools/filetransfer.c @@ -16,13 +16,13 @@ static void update_stats(GS_FT *ft, struct _gs_ft_file *f, size_t sz); static void init_xfer_stats(GS_FT *ft, struct _gs_ft_file *f, int64_t sz); static const char *str_dotslash(const char *src); static const char *str_stripslash(const char *src); -static int mkdirp(const char *dir, mode_t mode, uint32_t mtime); +static int mkdirpm(const char *dir, mode_t mode, uint32_t mtime); static void dir_restore_mtime(const char *path, struct timespec *mtime); static void dir_save_mtime(const char *path, struct timespec *mtime); static void set_mode_mtime(const char *path, mode_t mode, uint32_t mtime); static void call_status_cb(GS_FT *ft, const char *fname, uint8_t code, const char *err_str); - - +static void status_report_error(GS_FT *ft, const char *name, uint8_t code); +static uint8_t errno2code(int eno /*errno*/, uint8_t default_code); #if 0 @@ -144,8 +144,8 @@ gs_ft_add_file(GS_FT *ft, GS_LIST *gsl, uint32_t id, const char *fname, uint32_t // First: Directory struture. if (flags & GS_FT_FL_ISDIR) { - // mkdirp() will remove any ordinary file that is in its way - if (mkdirp(fname, GS_fperm2mode(fperm), mtime) != 0) + // mkdirpm() will remove any ordinary file that is in its way + if (mkdirpm(fname, GS_fperm2mode(fperm), mtime) != 0) { DEBUGF_R("mkdir(%s): %s\n", fname, strerror(errno)); qerr_add(ft, id, GS_FT_ERR_PERM, NULL); @@ -231,7 +231,7 @@ GS_FT_dl_add_file(GS_FT *ft, uint32_t id, const char *fname, size_t len, int64_t return -1; // protocol error. Not 0 terminated. DEBUGF_Y("#%u DL-ADD-FILE - '%s' fz_remote=%"PRId64"\n", id, fname, fz_remote); -#ifdef DEBUGF +#ifdef DEBUG // Test [8.9] if (g_true_fname != NULL) { @@ -315,8 +315,8 @@ GS_FT_list_add(GS_FT *ft, uint32_t globbing_id, const char *fname, size_t len, i if (flags & GS_FT_FL_ISDIR) { DEBUGF_C("Directory: %s\n", fn_local); - // mkdirp() will remove any ordinary file that is in its way - if (mkdirp(fn_local, GS_fperm2mode(fperm), mtime) != 0) + // mkdirpm() will remove any ordinary file that is in its way + if (mkdirpm(fn_local, GS_fperm2mode(fperm), mtime) != 0) { DEBUGF_R("mkdir(%s): %s\n", fn_local, strerror(errno)); } @@ -498,6 +498,28 @@ dotslash_filename(const char *src) static int add_file_to_list(GS_FT *ft, GS_LIST *gsl, const char *fname, uint32_t globbing_id, int is_server) { + // SERVER, get: Even if it does not exist then we still need to add it to the LISTREPLY list + // Client will request it and only then will we send an error. + // This can happen when client requests "foo[123].notexist[233].da*" and globbing + // fails. + // CLIENT: Immediatly return. + int ret; + int is_notfound = 0; + struct stat res; + ret = stat(fname, &res); + if (ret != 0) + { + DEBUGF_R("%s NOT FOUND\n", fname); + if (is_server == 0) + { + char *name = dotslash_filename(fname); + status_report_error(ft, name, GS_FT_ERR_NOENT); + XFREE(name); + return -1; + } + is_notfound = 1; // Server, get (download) + } + struct _gs_ft_file *f; f = calloc(1, sizeof *f); f->globbing_id = globbing_id; @@ -531,18 +553,9 @@ add_file_to_list(GS_FT *ft, GS_LIST *gsl, const char *fname, uint32_t globbing_i f->li = GS_LIST_add(gsl, NULL, f, ft->g_id); ft->g_id += 1; - // Even if it does not exist then we still need to add it to the LISTREPLY list - // Client will request it and only then will we send an error. - // This can happen when client requests "foo[123].notexist[233].da*" and globbing - // fails. - int ret; - struct stat res; - ret = stat(fname, &res); - if (ret != 0) - { - DEBUGF_R("%s NOT FOUND\n", fname); + // stat() failed. + if (is_notfound) return -1; - } // Get absolute and real path as CWD may change before // upload starts. @@ -637,28 +650,38 @@ glob_cb(GS_GL *res) int GS_FT_list_add_files(GS_FT *ft, uint32_t globbing_id, const char *pattern, size_t len) { - int n_found; + int ret; + char err[128]; if (pattern[len] != '\0') return -1; // protocol error. Not 0-terminated. DEBUGF_Y("G#%u GET-ADD-FILE: %s\n", globbing_id, pattern); char *ptr = GS_getpidwd(ft->pid); - chdir(ptr); - XFREE(ptr); + ret = chdir(ptr); + + if (ret != 0) + { + DEBUGF_R("chrdir(%s): %s\n", ptr, strerror(errno)); + snprintf(err, sizeof err, "chdir(%s): %s\n", ptr, strerror(errno)); + qerr_add(ft, globbing_id, errno2code(errno, GS_FT_ERR_NOENT), err); + + goto done; + } - n_found = GS_GLOBBING(glob_cb, pattern, globbing_id, ft, 0); + ret = GS_GLOBBING(glob_cb, pattern, globbing_id, ft, 0); - if (n_found <= 0) + if (ret <= 0) { DEBUGF_R("NOT FOUND: %s\n", pattern); - char err[128]; snprintf(err, sizeof err, "Not found: %s", pattern); qerr_add(ft, globbing_id, GS_FT_ERR_NOENT, err); } +done: + XFREE(ptr); ft->is_want_write = 1; - return n_found; + return ret; } @@ -672,7 +695,7 @@ GS_FT_get(GS_FT *ft, const char *pattern) p = calloc(1, sizeof *p); p->pattern = strdup(pattern); - p->wdir = getwd(NULL); + p->wdir = getcwd(NULL, 0); p->globbing_id = ft->g_id; GS_LIST_add(&ft->plistreq, NULL, p, ft->g_id); @@ -921,14 +944,14 @@ mkdir_agressive(const char *path, mode_t mode, uint32_t mtime) } /* - * Create all directories recursively. + * Create all directories recursively. Remove any file that is in our way. * * /tmp/foo/bar/test.dat would create /tmp/foo/bar/test.dat [directory] * /tmp/foo/bar/test.dat/ would create /tmp/foo/bar/test.dat/ * ///// would return success ('/' always exists) */ static int -mkdirp(const char *path, mode_t mode, uint32_t mtime) +mkdirpm(const char *path, mode_t mode, uint32_t mtime) { int rv = 0; char *f = strdup(path); @@ -1026,10 +1049,15 @@ dir_save_mtime(const char *path, struct timespec *mtime) { struct stat res; + memset(mtime, 0, sizeof *mtime); if (stat(path, &res) != 0) return; +#ifdef __APPLE__ *mtime = res.st_mtimespec; +#else + *mtime = res.st_mtim; +#endif // DEBUGF_W("SAVE Dir-TimeStamp %lu.%lu %s\n", mtime->tv_sec, mtime->tv_nsec, path); } @@ -1100,7 +1128,7 @@ gs_ft_switch(GS_FT *ft, uint32_t id, int64_t fz_remote, struct _gs_ft_file **act ptr = dirname(ptr); dir_save_mtime(ptr, &new->dir_mtime); // put(test1k.dat) must not modify the permission of parent directory. - mkdirp(ptr, 0 /*do not update permission on existing directory*/, 0 /*do not update mtime*/); + mkdirpm(ptr, 0 /*do not update permission on existing directory*/, 0 /*do not update mtime*/); new->fp = fopen(new->fn_local, "w"); } else { @@ -1477,6 +1505,16 @@ GS_FT_strerror(uint8_t code) return "UNKNONW"; } +// CLIENT: Create error string and report to caller (via callback) +static void +status_report_error(GS_FT *ft, const char *name, uint8_t code) +{ + char buf[128]; + DEBUGF_B("fn-rel: %s\n", name); + snprintf(buf, sizeof buf, "%s: %s", GS_FT_strerror(code), name); + call_status_cb(ft, name, code, buf); +} + /* * Make an error packet. Remove errornous file from queue */ @@ -1489,10 +1527,7 @@ ft_mk_error(GS_FT *ft, struct _gs_ft_file *f, void *dst, size_t len, int *pkt_ty if (ft->is_server == 0) { // CLIENT - char buf[128]; - DEBUGF_B("fn-rel: %s\n", f->fn_relative); - snprintf(buf, sizeof buf, "%s: %s", GS_FT_strerror(code), f->fn_relative); - call_status_cb(ft, f->fn_relative, code, buf); + status_report_error(ft, f->fn_relative, code); } *pkt_type = GS_FT_TYPE_ERROR; sz = mk_error(dst, len, li->id, code, str); @@ -1541,7 +1576,7 @@ mk_xfer_data(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fcompleted, int *p if (ft->is_server == 0) { update_stats(ft, f, sz); - DEBUGF_W("xfer %"PRId64"/%"PRId64"\n", ft->stats.xfer_amount, ft->stats.xfer_amount_scheduled); + // DEBUGF_W("xfer %"PRId64"/%"PRId64"\n", ft->stats.xfer_amount, ft->stats.xfer_amount_scheduled); } if (f->fz_remote >= f->fz_local) From d459b7138cb94970e9d0e8d0d4f56606b70e9d14 Mon Sep 17 00:00:00 2001 From: rootTHC Date: Thu, 11 Feb 2021 16:03:45 +0000 Subject: [PATCH 14/21] all compile but solaris10.... --- Makefile.am | 2 +- configure.ac | 28 ++- lib/Makefile.am | 1 - tests/Makefile | 6 + tests/run_all_tests.sh | 3 + {tools => tests}/run_ft_tests.sh | 206 +++++++++++------- .../run_all_tests.sh => tests/run_gs_tests.sh | 174 ++++++++------- tools/4_gs-netcat.c | 2 +- tools/Makefile.am | 2 - tools/filetransfer-test.c | 26 +-- tools/filetransfer.c | 64 +++--- tools/filetransfer.h | 4 +- 12 files changed, 299 insertions(+), 219 deletions(-) create mode 100644 tests/Makefile create mode 100755 tests/run_all_tests.sh rename {tools => tests}/run_ft_tests.sh (76%) rename tools/run_all_tests.sh => tests/run_gs_tests.sh (70%) diff --git a/Makefile.am b/Makefile.am index 510983df..87796331 100755 --- a/Makefile.am +++ b/Makefile.am @@ -1,3 +1,3 @@ SUBDIRS = lib tools man -EXTRA_DIST = README.md config bootstrap include/gsocket LICENSE +EXTRA_DIST = README.md config bootstrap tests/Makefile tests/run_all_tests.sh tests/run_gs_tests.sh tests/run_ft_tests.sh include/gsocket LICENSE diff --git a/configure.ac b/configure.ac index 1fba8376..f9a4f34d 100755 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ -dnl Process this File with autoconf to produce a configure script. +${EXEEXT}dnl Process this File with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([gsocket], 1.4.23-ft1) +AC_INIT([gsocket], 1.4.23-ft2) dnl AC_CONFIG_AUX_DIR(config-x86_64-apple-darwin19.6.0) AC_CONFIG_AUX_DIR(config) AC_CANONICAL_SYSTEM @@ -49,7 +49,7 @@ for xincdir in $trydir_i ; do INCLUDES="$INCLUDES -I${xincdir}"; fi done -CPPFLAGS="${INCLUDES} $CPPFLAGS" +CPPFLAGS="-I../include ${INCLUDES} $CPPFLAGS" dnl Try library paths... trydir_l="${trydir_l} /usr/local/opt/openssl/lib" @@ -116,18 +116,24 @@ AC_TYPE_UID_T AC_CHECK_FUNCS(gettimeofday memcpy strchr strlcat forkpty openpty getline stat64 open64 statvfs64) AC_ARG_ENABLE([31337], - AS_HELP_STRING([--enable-31337], - [Enable experimental features.]), + AS_HELP_STRING([--enable-31337], [Enable experimental features.]), AC_DEFINE(D31337, 1, [Expermental feature]) ) -AC_ARG_ENABLE(debug, -[ --enable-debug Enable debug information], - AC_DEFINE(DEBUG, 1, [Debug infos]) +AC_ARG_ENABLE([debug], + AS_HELP_STRING([--enable-debug], [Enable debug information.]), + [debug=true AC_DEFINE(DEBUG, 1, [Debug infos])] +) + +AC_ARG_ENABLE([tests], + AS_HELP_STRING([--enable-tests], [Enable self-tests.]), + [selftests=true] ) AS_IF([test x$enable_debug = xyes], AC_DEFINE(D31337, 1, [Expermental feature])) -AS_IF([test x$enable_debug = xyes], [debug=true]) +AS_IF([test x$enable_debug = xyes], [selftests=true]) + +AS_IF([test x$selftests = xtrue], AC_DEFINE(SELFTESTS, 1, [Self Tests])) AC_ARG_ENABLE(dist, [ --enable-dist Enable distribution mode, Use own libraries.], @@ -143,8 +149,8 @@ if test x"$STATIC" = x"yes"; then CFLAGS="-static ${CFLAGS}" fi -AS_IF([test x$debug = xtrue], AC_SUBST(PROGRAMS_TEST_LIB, "list-test event-test")) -AS_IF([test x$debug = xtrue], AC_SUBST(PROGRAMS_TEST_TOOLS, "packet-test readline-test console_display-test filetransfer-test")) +AS_IF([test x$selftests = xtrue], AC_SUBST(PROGRAMS_TEST_LIB, "list-test${EXEEXT} event-test${EXEEXT}")) +AS_IF([test x$selftests = xtrue], AC_SUBST(PROGRAMS_TEST_TOOLS, "packet-test${EXEEXT} readline-test${EXEEXT} console_display-test${EXEEXT} filetransfer-test${EXEEXT}")) AC_OUTPUT([Makefile lib/Makefile tools/Makefile man/Makefile]) diff --git a/lib/Makefile.am b/lib/Makefile.am index a1bea783..a9149065 100755 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -11,4 +11,3 @@ event_test_SOURCES = event-test.c event_test_LDADD = libgsocket.a noinst_HEADERS = gsocket-sha256.h gs-common.h gs-externs.h gsocket-engine.h -AM_CFLAGS = -I../include diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 00000000..0694792b --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,6 @@ + + +clean: + rm -rf id_sec.txt gs_nc test*.dat client_err*.txt server_err*.txt client_out*.dat server_out*.dat ft_test_dst ft_test_src + >server.logclient.log/dev/null 2>&1 && MD5(){ md5 -q "${1}";} -command -v md5sum >/dev/null 2>&1 && MD5() { md5sum "${1}" | cut -f1 -d' ';} +command -v md5sum >/dev/null 2>&1 && MD5(){ md5sum "${1}" | cut -f1 -d' ';} + +if [[ $(uname) =~ Darwin ]]; then + FSIZE(){ stat -f%z "$1";} + FACCESS(){ stat -f%A "$1";} + FMTIME(){ stat -f%m "$1";} + FSTAT(){ stat -f%A-%m-%z "$1";} + DSTAT(){ stat -L -f%A-%m "$1";} +elif [[ $(uname) =~ FreeBSD ]]; then + FSIZE(){ stat -f%z "$1";} + FACCESS(){ stat -f%p "$1";} + FMTIME(){ stat -f%m "$1";} + FSTAT(){ stat -f%p-%m-%z "$1";} + DSTAT(){ stat -L -f%p-%m "$1";} +else + FSIZE(){ stat --format=%s "$1";} + FACCESS(){ stat --format=%a "$1";} + FMTIME(){ stat --format=%Y "$1";} + FSTAT(){ stat --format=%a-%Y-%s "$1";} + DSTAT(){ stat -L --format=%a-%Y "$1";} +fi IODIR="${PWD}/ft_test_dst" IODIRSRC="ft_test_src" LOGDIR="${PWD}" OK="....[\033[1;32mOK\033[0m]" FAIL="[\033[1;31mFAILED\033[0m]" +SKIP="[\033[1;33mskipping\033[0m]" +BINDIR="${PWD}/../tools" ECHO="echo -e" -BIN="${PWD}/filetransfer-test" +BIN="${BINDIR}/filetransfer-test" + +[[ -f "${BIN}" ]] || { echo "${BIN} not found. Try ./configure --enable-tests"; exit 255; } + +command -v socat >/dev/null 2>&1 || { echo >&2 "socat not installed. ${SKIP}"; exit 0; } + mk_dummy() { @@ -23,6 +50,7 @@ mk_dummy test8k.dat 8 rm -rf "${IODIRSRC}/foo" mkdir -p "${IODIRSRC}/foo/bar" mkdir -p "${IODIRSRC}/foo/dir_empty" +cp test1k.dat "${IODIRSRC}/" cp test1k.dat "${IODIRSRC}/foo/bar/" cp test1k.dat "${IODIRSRC}/foo/.rcfile1" cp test4k.dat "${IODIRSRC}/foo/" @@ -52,15 +80,15 @@ md5fail() fail_file_count() { # Do not quote so that globbing takes effect. - nf_src=$(find -x $2 -type f -o -type d | wc -l) - nf_dst=$(find -x $3 -type f -o -type d | wc -l) + nf_src=$(find $2 -xdev -type f -o -type d | wc -l) + nf_dst=$(find $3 -xdev -type f -o -type d | wc -l) [[ $nf_src -eq $nf_dst ]] || fail $1 } fail_file_bypipe() { while read f; do - if [ $(stat -f%A-%m-%z "${2}/${f}") != $(stat -f%A-%m-%z "${3}/${f}") ]; then + if [ $(FSTAT "${2}/${f}") != $(FSTAT "${3}/${f}") ]; then echo "${f} not equal"; fail $1 fi @@ -70,7 +98,7 @@ fail_file_bypipe() fail_dir_bypipe() { while read f; do - if [ $(stat -f%A-%m "${2}/${f}") != $(stat -f%A-%m "${3}/${f}") ]; then + if [ $(DSTAT "${2}/${f}") != $(DSTAT "${3}/${f}") ]; then echo "${f} not equal"; fail $1 fi @@ -80,8 +108,8 @@ fail_dir_bypipe() # Recursively compare st_mode and mtime and fail if different fail_dir_compare() { - (cd "$2"; find -x "$4" -type d) | fail_dir_bypipe "$1" "$2" "$3" - (cd "$2"; find -x "$4" -type f) | fail_file_bypipe "$1" "$2" "$3" + (cd "$2"; find "$4" -xdev -type d) | fail_dir_bypipe "$1" "$2" "$3" + (cd "$2"; find "$4" -xdev -type f) | fail_file_bypipe "$1" "$2" "$3" } @@ -155,19 +183,22 @@ tests+="8.7 " tests+="8.8 " tests+="8.9 " -# tests+="9.1 " -# tests+="9.2 " +tests+="9.1 " +tests+="9.2 " +if [ x"$1" != x ]; then + tests="$@ " +fi if [[ "$tests" =~ '1.0 ' ]]; then -test_start -n "Running #1.0 (put 1 file).................................." +test_start -n "Running #1.0 (put 1 file)................................." run_put test1k.dat md5fail 1 test1k.dat "${IODIR}/test1k.dat" $ECHO "${OK}" fi if [[ "$tests" =~ '1.1 ' ]]; then -test_start -n "Running #1.1 (put 2 files)................................." +test_start -n "Running #1.1 (put 2 files)................................" run_put test4k.dat test8k.dat md5fail 1 test4k.dat "${IODIR}/test4k.dat" md5fail 2 test8k.dat "${IODIR}/test8k.dat" @@ -175,7 +206,7 @@ $ECHO "${OK}" fi if [[ "$tests" =~ '1.2 ' ]]; then -test_start -n "Running #1.2 (non-exist)..................................." +test_start -n "Running #1.2 (non-exist).................................." run_put not-exists.dat [[ -f "${IODIR}/not-exists.dat" ]] && fail 1 $ECHO "${OK}" @@ -191,7 +222,7 @@ run_put_fail() } if [[ "$tests" =~ '1.3 ' ]]; then -test_start -n "Running #1.3 (absolute file)..............................." +test_start -n "Running #1.3 (absolute file).............................." run_put_fail 1 "${IODIRSRC}/foo/bar/test1k.dat" "${IODIR}/test1k.dat" run_put_fail 2 "${IODIRSRC}/foo/bar/./test1k.dat" "${IODIR}/test1k.dat" run_put_fail 3 "./${IODIRSRC}/foo/bar/test1k.dat" "${IODIR}/test1k.dat" @@ -205,7 +236,7 @@ $ECHO "${OK}" fi if [[ "$tests" =~ '2.1 ' ]]; then -test_start -n "Running #2.1 (src is larger, restart)......................" +test_start -n "Running #2.1 (src is larger, restart)....................." dd bs=1k count=5 if=test8k.dat of="${IODIR}/test8k.dat" &>/dev/null run_put test4k.dat test8k.dat md5fail 1 test8k.dat "${IODIR}/test8k.dat" @@ -213,7 +244,7 @@ $ECHO "${OK}" fi if [[ "$tests" =~ '2.2 ' ]]; then -test_start -n "Running #2.2 (dst is larger, overwrite)...................." +test_start -n "Running #2.2 (dst is larger, overwrite)..................." cp test8k.dat "${IODIR}/test4k.dat" run_put test4k.dat md5fail 1 test4k.dat "${IODIR}/test4k.dat" @@ -221,76 +252,93 @@ $ECHO "${OK}" fi if [[ "$tests" =~ '2.3 ' ]]; then -test_start -n "Running #2.3 (zero src size)..............................." +test_start -n "Running #2.3 (zero src size).............................." touch zero.dat run_put zero.dat md5fail 1 zero.dat "${IODIR}/zero.dat" $ECHO "${OK}" fi + if [[ "$tests" =~ '3.1 ' ]]; then -test_start -n "Running #3.2 (write-error 0-sized dst)....................." +test_start -n "Running #3.1 (write-error 0-sized dst)...................." touch "${IODIR}/test4k.dat" -chmod 400 "${IODIR}/test4k.dat" +if [[ $(uname) =~ CYGWIN ]]; then + chattr +r "${IODIR}/test4k.dat" +else + chmod 400 "${IODIR}/test4k.dat" +fi run_put test4k.dat -[[ x`stat -f%z "${IODIR}/test4k.dat"` = x0 ]] || fail 1 +[[ x`FSIZE "${IODIR}/test4k.dat"` = x0 ]] || fail 1 $ECHO "${OK}" fi if [[ "$tests" =~ '3.2 ' ]]; then -test_start -n "Running #3.2 (write-error partial)........................." +test_start -n "Running #3.2 (write-error partial)........................" cp test4k.dat "${IODIR}/test8k.dat" -chmod 400 "${IODIR}/test8k.dat" +if [[ $(uname) =~ CYGWIN ]]; then + chattr +r "${IODIR}/test8k.dat" +else + chmod 400 "${IODIR}/test8k.dat" +fi run_put test8k.dat md5fail 1 test4k.dat "${IODIR}/test8k.dat" $ECHO "${OK}" fi if [[ "$tests" =~ '3.3 ' ]]; then -test_start -n "Running #3.3 (dir not writeable)..........................." +test_start -n "Running #3.3 (dir not writeable).........................." +if [[ $(uname) =~ CYGWIN ]]; then + $ECHO "${SKIP}" +else chmod a-w "${IODIR}" run_put test4k.dat [[ -f "${IODIR}/test4k.dat" ]] && fail 1 $ECHO "${OK}" fi +fi if [[ "$tests" =~ '3.4 ' ]]; then -test_start -n "Running #3.4 (src not readable)............................" -chmod a-r test4k.dat -run_put test4k.dat -[[ -f "${IODIR}/test4k.dat" ]] && fail 1 -chmod a+r test4k.dat +test_start -n "Running #3.4 (src not readable)..........................." +if [[ $(uname) =~ CYGWIN ]]; then + $ECHO "${SKIP}" +else +chmod a-r "${IODIRSRC}/test1k.dat" +run_put "${IODIRSRC}/test1k.dat" +[[ -f "${IODIR}/test1k.dat" ]] && fail 1 +chmod a+r "${IODIRSRC}/test1k.dat" $ECHO "${OK}" fi +fi if [[ "$tests" =~ '4.1 ' ]]; then -test_start -n "Running #4.1 (permission).................................." +test_start -n "Running #4.1 (permission)................................." chmod 462 test4k.dat # chmod u+s test4k.dat # On MacOS our own app can not set +s... run_put test4k.dat -[[ x`stat -f%A "test4k.dat"` = x`stat -f%A "${IODIR}/test4k.dat"` ]] || fail 1 +[[ x`FACCESS "test4k.dat"` = x`FACCESS "${IODIR}/test4k.dat"` ]] || fail 1 chmod 644 test4k.dat $ECHO "${OK}" fi if [[ "$tests" =~ '4.2 ' ]]; then -test_start -n "Running #4.2 (mtime)......................................." +test_start -n "Running #4.2 (mtime)......................................" touch -r /etc/hosts test4k.dat run_put test4k.dat -[[ x`stat -f%m "test4k.dat"` = x`stat -f%m "${IODIR}/test4k.dat"` ]] || fail 1 +[[ x`FMTIME "test4k.dat"` = x`FMTIME "${IODIR}/test4k.dat"` ]] || fail 1 $ECHO "${OK}" fi if [[ "$tests" =~ '4.3 ' ]]; then -test_start -n "Running #4.3 (zero-size, mtime)............................" +test_start -n "Running #4.3 (zero-size, mtime)..........................." touch -r /etc/hosts zero.dat run_put zero.dat -[[ x`stat -f%m "zero.dat"` = x`stat -f%m "${IODIR}/zero.dat"` ]] || fail 1 +[[ x`FMTIME "zero.dat"` = x`FMTIME "${IODIR}/zero.dat"` ]] || fail 1 $ECHO "${OK}" fi if [[ "$tests" =~ '4.4 ' ]]; then -test_start -n "Running #4.4 (put, empty directory)........................" +test_start -n "Running #4.4 (put, empty directory)......................." touch "${IODIR}/foo" # Place a file in its way (should be overwritten) run_put "${IODIRSRC}/./foo/dir_empty" [[ -d "${IODIR}/foo/dir_empty" ]] || fail 1 @@ -304,49 +352,49 @@ $ECHO "${OK}" fi if [[ "$tests" =~ '5.1 ' ]]; then -test_start -n "Running #5.1 (Globbing ./*)................................" +test_start -n "Running #5.1 (Globbing ./*)..............................." run_put "${IODIRSRC}/*" fail_file_count 1 "${IODIRSRC}/*" "${IODIR}/*" $ECHO "${OK}" fi if [[ "$tests" =~ '5.2 ' ]]; then -test_start -n "Running #5.2 (Globbing ./foo/.*)..........................." +test_start -n "Running #5.2 (Globbing ./foo/.*).........................." run_put "${IODIRSRC}/./foo/.*" [[ $(find ${IODIR}/ -type f -o -type d | wc -l) -eq 3 ]] || fail 1 $ECHO "${OK}" fi if [[ "$tests" =~ '5.3 ' ]]; then -test_start -n "Running #5.3 (Globbing .*)................................." +test_start -n "Running #5.3 (Globbing .*)................................" (cd "${IODIRSRC}/foo" && run_put ".*") [[ $(find ${IODIR}/ -type f -o -type d | wc -l) -eq 2 ]] || fail 1 $ECHO "${OK}" fi if [[ "$tests" =~ '5.4 ' ]]; then -test_start -n "Running #5.4 (Globbing foo)................................" +test_start -n "Running #5.4 (Globbing foo)..............................." (cd "${IODIRSRC}" && run_put "foo") fail_file_count 1 "${IODIRSRC}/foo" "${IODIR}/foo" $ECHO "${OK}" fi if [[ "$tests" =~ '5.5 ' ]]; then -test_start -n "Running #5.5 (Globbing .).................................." +test_start -n "Running #5.5 (Globbing .)................................." (cd "${IODIRSRC}" && run_put ".") fail_file_count 1 "${IODIRSRC}/" "${IODIR}/" $ECHO "${OK}" fi if [[ "$tests" =~ '5.6 ' ]]; then -test_start -n "Running #5.6 (Globbing foo/)..............................." +test_start -n "Running #5.6 (Globbing foo/).............................." (cd "${IODIRSRC}" && run_put "foo/") fail_file_count 1 "${IODIRSRC}/foo/" "${IODIR}/" $ECHO "${OK}" fi if [[ "$tests" =~ '5.7 ' ]]; then -test_start -n "Running #5.7 (put, globbing \$(find...*.dat)................" +test_start -n "Running #5.7 (put, globbing \$(find...*.dat)..............." # (cd "${IODIRSRC}" && run_putc '\$(echo *.dat)') (cd "${IODIRSRC}" && run_putc '\$(find . -type f -name \\*.dat)') (cd "${IODIRSRC}" && find . -type f -name '*.dat') | fail_file_bypipe 1 "${IODIRSRC}" "${IODIR}" @@ -354,14 +402,14 @@ $ECHO "${OK}" fi if [[ "$tests" =~ '5.8 ' ]]; then -test_start -n "Running #5.8 (get, globbing \$(find...*.dat)................" +test_start -n "Running #5.8 (get, globbing \$(find...*.dat)..............." run_getc '\$(find . -type f -name \\*.dat)' (cd "${IODIRSRC}" && find . -type f -name '*.dat') | fail_file_bypipe 1 "${IODIRSRC}" "${IODIR}" $ECHO "${OK}" fi if [[ "$tests" =~ '8.1 ' ]]; then -test_start -n "Running #8.1 (get, 2 files)................................" +test_start -n "Running #8.1 (get, 2 files)..............................." run_get test8k.dat foo/bar/.rcfile2 md5fail 1 test8k.dat "${IODIR}/test8k.dat" md5fail 2 test4k.dat "${IODIR}/.rcfile2" @@ -369,7 +417,7 @@ $ECHO "${OK}" fi if [[ "$tests" =~ '8.2 ' ]]; then -test_start -n "Running #8.2 (get, 2 files /./ test)......................." +test_start -n "Running #8.2 (get, 2 files /./ test)......................" run_get ./foo/bar/test1k.dat ./foo/./bar/test1k.dat md5fail 1 test1k.dat "${IODIR}/test1k.dat" md5fail 2 test1k.dat "${IODIR}/bar/test1k.dat" @@ -377,7 +425,7 @@ $ECHO "${OK}" fi if [[ "$tests" =~ '8.3 ' ]]; then -test_start -n "Running #8.3 (directory test).............................." +test_start -n "Running #8.3 (directory test)............................." run_get foo/bar md5fail 1 "${IODIRSRC}/foo/bar/test1k.dat" "${IODIR}/bar/test1k.dat" md5fail 2 "${IODIRSRC}/foo/bar/.rcfile2" "${IODIR}/bar/.rcfile2" @@ -385,14 +433,14 @@ $ECHO "${OK}" fi if [[ "$tests" =~ '8.4 ' ]]; then -test_start -n "Running #8.4 (get, non-exist).............................." +test_start -n "Running #8.4 (get, non-exist)............................." run_get not-exists.dat foobar*noexist[1234].d[ab]t [[ -f "${IODIR}/not-exists.dat" ]] && fail 1 $ECHO "${OK}" fi if [[ "$tests" =~ '8.5 ' ]]; then -test_start -n "Running #8.5 (get, ../test8k.dat).........................." +test_start -n "Running #8.5 (get, ../test8k.dat)........................." run_get2 ../test8k.dat ../foo ../foo/./bar md5fail 1 "${IODIRSRC}/test8k.dat" "${IODIR}/test8k.dat" fail_file_count 2 "${IODIRSRC}/foo" "${IODIR}/foo" @@ -401,35 +449,42 @@ $ECHO "${OK}" fi if [[ "$tests" =~ '8.6 ' ]]; then -test_start -n "Running #8.6 (get, /etc/hosts)............................." +test_start -n "Running #8.6 (get, /etc/hosts)............................" +if [[ $(uname) =~ CYGWIN ]]; then +run_get /etc/hosts /etc/./pki/tls/cert.pem /./etc/pki/tls/cert.pem +md5fail 1 "/etc/hosts" "${IODIR}/hosts" +md5fail 2 "/etc/pki/tls/cert.pem" "${IODIR}/pki/tls/cert.pem" +md5fail 3 "/etc/pki/tls/cert.pem" "${IODIR}/etc/pki/tls/cert.pem" +else run_get /etc/hosts /etc/./ssh/ssh_config /./etc/ssh/ssh_config md5fail 1 "/etc/hosts" "${IODIR}/hosts" md5fail 2 "/etc/ssh/ssh_config" "${IODIR}/ssh/ssh_config" md5fail 3 "/etc/ssh/ssh_config" "${IODIR}/etc/ssh/ssh_config" +fi $ECHO "${OK}" fi if [[ "$tests" =~ '8.7 ' ]]; then -test_start -n "Running #8.7 (get, permission, mtime, zero)................" +test_start -n "Running #8.7 (get, permission, mtime, zero)..............." chmod 462 "${IODIRSRC}/test1k.dat" chmod 624 "${IODIRSRC}/zero.dat" chmod 3751 "${IODIRSRC}/foo/dir_empty" touch -r /etc/hosts "${IODIRSRC}/test1k.dat" "${IODIRSRC}/zero.dat" "${IODIRSRC}/foo/dir_empty" touch -r /etc "${IODIRSRC}/foo" -touch -r /etc "${IODIR}" +touch -r /etc "${IODIR}" "${IODIRSRC}" run_get test1k.dat zero.dat ././foo/dir_empty -[[ $(stat -f%A-%m-%z "${IODIRSRC}/test1k.dat") = $(stat -f%A-%m-%z "${IODIR}/test1k.dat") ]] || fail 1 -[[ $(stat -f%A-%m-%z "${IODIRSRC}/zero.dat") = $(stat -f%A-%m-%z "${IODIR}/zero.dat") ]] || fail 2 -[[ $(stat -f%A-%m "${IODIRSRC}/foo/dir_empty") = $(stat -f%A-%m "${IODIR}/foo/dir_empty") ]] || fail 3 -[[ $(stat -L -f%A-%m "/etc") = $(stat -f%A-%m "${IODIRSRC}/foo") ]] || fail 4 -[[ $(stat -L -f%A-%m "/etc") = $(stat -f%A-%m "${IODIR}") ]] || fail 5 +[[ $(FSTAT "${IODIRSRC}/test1k.dat") = $(FSTAT "${IODIR}/test1k.dat") ]] || fail 1 +[[ $(FSTAT "${IODIRSRC}/zero.dat") = $(FSTAT "${IODIR}/zero.dat") ]] || fail 2 +[[ $(DSTAT "${IODIRSRC}/foo/dir_empty") = $(DSTAT "${IODIR}/foo/dir_empty") ]] || fail 3 +[[ $(DSTAT "${IODIRSRC}/foo") = $(DSTAT "${IODIRSRC}/foo") ]] || fail 4 +[[ $(DSTAT "${IODIRSRC}") = $(DSTAT "${IODIR}") ]] || fail 5 chmod 644 "${IODIRSRC}/test1k.dat" "${IODIRSRC}/zero.dat" chmod 755 "${IODIRSRC}/foo/dir_empty" $ECHO "${OK}" fi if [[ "$tests" =~ '8.8 ' ]]; then -test_start -n "Running #8.8 (get restart: dst is larger, smaller & zero).." +test_start -n "Running #8.8 (get restart: dst is larger, smaller & zero)." dd bs=1k count=5 if="${IODIRSRC}/test8k.dat" of="${IODIR}/test8k.dat" &>/dev/null cp "${IODIRSRC}/test8k.dat" "${IODIR}/test1k.dat" touch "${IODIR}/test4k.dat" @@ -441,7 +496,7 @@ $ECHO "${OK}" fi if [[ "$tests" =~ '8.9 ' ]]; then -test_start -n "Running #8.9 (get, Server sending ../../../shit............" +test_start -n "Running #8.9 (get, Server sending ../../../shit..........." mkdir ${IODIR}/foo run_get_dst /tmp/0wned.dat run_get_dst ./../../../../../../../../../../../../../tmp/0wned.dat @@ -451,28 +506,31 @@ run_get_dst ../0wned.dat $ECHO "${OK}" fi +bigdir="share/man" # HUGE +[[ -n "$QUICK" ]] && bigdir="share/man/man4" # Less huge +[[ -n "$QUICK" ]] && [[ $(uname) =~ CYGWIN ]] && bigdir="share/man/man8" # Less huge +[[ -n "$QUICK" ]] && [[ $(uname) =~ FreeBSD ]] && bigdir="share/man/man6" # Less huge +[[ -n "$QUICK" ]] && [[ $(uname) =~ SunOS ]] && bigdir="share/man/man8s" # Less huge + if [[ "$tests" =~ '9.1 ' ]]; then -test_start -n "Running #9.1 (HUGE put)..........................." -## Small -# run_put /usr/./share/man/man4 -# $ECHO -n "verify..." -# fail_dir_compare 1 /usr/share/man "${IODIR}/share/man" man4 -## HUGE -run_put /usr/./share/man +test_start -n "Running #9.1 (HUGE put).........................." +run_put "/usr/./${bigdir}" $ECHO -n "verify..." -fail_dir_compare 1 /usr/share "${IODIR}/share" man +fail_dir_compare 1 "/usr/${bigdir}" "${IODIR}/${bigdir}" . $ECHO "${OK}" fi if [[ "$tests" =~ '9.2 ' ]]; then -test_start -n "Running #9.2 (HUGE get)..........................." -## Small -# run_get /usr/./share/man/man4 -# $ECHO -n "verify..." -# fail_dir_compare 1 /usr/share/man "${IODIR}/share/man" man4 -## HUGE -run_get /usr/./share/man +test_start -n "Running #9.2 (HUGE get).........................." +run_get "/usr/./${bigdir}" $ECHO -n "verify..." -fail_dir_compare 1 /usr/share "${IODIR}/share" man +fail_dir_compare 1 "/usr/${bigdir}" "${IODIR}/${bigdir}" . $ECHO "${OK}" fi + + +if [ x"$1" == x ]; then + rm -rf "${IODIRSRC}" "${IODIR}" +fi + + diff --git a/tools/run_all_tests.sh b/tests/run_gs_tests.sh similarity index 70% rename from tools/run_all_tests.sh rename to tests/run_gs_tests.sh index 98dc3ef2..94e78b21 100755 --- a/tools/run_all_tests.sh +++ b/tests/run_gs_tests.sh @@ -55,6 +55,7 @@ OK="....[\033[1;32mOK\033[0m]" FAIL="[\033[1;31mFAILED\033[0m]" SKIP="[\033[1;33mskipping\033[0m]" ECHO="echo -e" + NETSTATTCP(){ netstat -ant;} [[ x"$OSTYPE" == "xsolaris"* ]] && NETSTATTCP(){ netstat -an -f inet; } [[ x"$OSTYPE" == *BSD* ]] && NETSTATTCP(){ netstat -an -f inet; } @@ -85,17 +86,14 @@ mk_dummy test1k.dat 1 mk_dummy test4k.dat 4 mk_dummy test50k.dat 50 mk_dummy test1M.dat 1024 -mk_dummy test5M.dat 5120 +mk_dummy test50Mx.dat 51200 if [[ -n "$QUICK" ]]; then - rm -rf test50M.dat &>/dev/null - mk_dummy test50M.dat 15 + cp test50k.dat test50M.dat else - rm -rf test50M.dat &>/dev/null - mk_dummy test50M.dat 51200 + cp test50Mx.dat test50M.dat fi echo "Fubar" >>test50M.dat # Make it an odd length MD50MB="$(MD5 test50M.dat)" -MD5MB="$(MD5 test5M.dat)" MD1MB="$(MD5 test1M.dat)" MDHELLOW="$(echo "Hello World" | MD5 /dev/stdin)" @@ -103,7 +101,7 @@ test_start() { rm -f client_out.dat server_out.dat server_err.txt client_err.txt server[123]_out.dat client[12]_out.dat server[123]_err.txt client[12]_err.txt nc[123]_out.dat nc[123]_err.txt [[ x"$1" != x ]] && $ECHO $* - new_id + [[ -s id_sec.txt ]] || new_id } fail() @@ -215,19 +213,19 @@ sleep_ct() new_id() { # Create a random secret for all tests - ./gs-helloworld -g 2>/dev/null >id_sec.txt + ../tools/gs-helloworld -g 2>/dev/null >id_sec.txt } # killall -9 gs-helloworld gs-pipe gs-full-pipe gs-netcat &>/dev/null -# new_id +# [[ -f id_sec.txt ]] || new_id if [[ "$tests" =~ '1.1 ' ]]; then ### 1 - Hello World test_start -n "Running: Hello World #1.1 ................................" -GSPID="$(sh -c './gs-helloworld -k id_sec.txt -l 2>server_err.txt >server_out.dat & echo ${!}')" +GSPID="$(sh -c '../tools/gs-helloworld -k id_sec.txt -l 2>server_err.txt >server_out.dat & echo ${!}')" # sleep 0.5 required or otherwise kernel will send both strings in single # tcp and that would result in a single read() call on other side. -sleep_ct && (echo "Hello World"; sleep 1; echo "That's the end") | ./gs-helloworld -k id_sec.txt 2>client_err.txt >client_out.dat +sleep_ct && (echo "Hello World"; sleep 1; echo "That's the end") | ../tools/gs-helloworld -k id_sec.txt 2>client_err.txt >client_out.dat waitk $GSPID if [ "$(MD5 client_out.dat)" != "628eca04c4cb6c8f539381be1c5cd325" ]; then fail 1; fi $ECHO "${OK}" @@ -237,8 +235,8 @@ if [[ "$tests" =~ '2.1 ' ]]; then ### 2 - Pipe # Normal (server listening, client connecting) test_start -n "Running: pipe #2.1 ......................................." -GSPID="$(sh -c './gs-pipe -k id_sec.txt -l 2>server_err.txt >server_out.dat & echo ${!}')" -sleep_ct && ./gs-pipe -k id_sec.txt client_err.txt >client_out.dat +GSPID="$(sh -c '../tools/gs-pipe -k id_sec.txt -l 2>server_err.txt >server_out.dat & echo ${!}')" +sleep_ct && ../tools/gs-pipe -k id_sec.txt client_err.txt >client_out.dat waitk $GSPID if [ "$(MD5 test50k.dat)" != "$(MD5 server_out.dat)" ]; then fail 1; fi $ECHO "${OK}" @@ -247,8 +245,8 @@ fi if [[ "$tests" =~ '2.2 ' ]]; then ### Waiting client test test_start -n "Running: pipe #2.2 (waiting for server)..................." -GSPID="$(sh -c './gs-pipe -k id_sec.txt -w client_err.txt >client_out.dat & echo ${!}')" -sleep_ct && ./gs-pipe -k id_sec.txt -l 2>server_err.txt >server_out.dat +GSPID="$(sh -c '../tools/gs-pipe -k id_sec.txt -w client_err.txt >client_out.dat & echo ${!}')" +sleep_ct && ../tools/gs-pipe -k id_sec.txt -l 2>server_err.txt >server_out.dat waitk $GSPID if [ "$(MD5 test50k.dat)" != "$(MD5 server_out.dat)" ]; then fail 1; fi $ECHO "${OK}" @@ -257,16 +255,16 @@ fi if [[ "$tests" =~ '3.1 ' ]]; then ### Impersonate 'listen' test_start -n "Running: pipe #3.1 (auth token)..........................." -GS1PID="$(sh -c './gs-pipe -k id_sec.txt -l -a player-alice 2>server1_err.txt >server1_out.dat & echo ${!}')" -GS2PID="$(sh -c './gs-pipe -k id_sec.txt -l -a player-alice 2>server2_err.txt >server2_out.dat & echo ${!}')" +GS1PID="$(sh -c '../tools/gs-pipe -k id_sec.txt -l -a player-alice 2>server1_err.txt >server1_out.dat & echo ${!}')" +GS2PID="$(sh -c '../tools/gs-pipe -k id_sec.txt -l -a player-alice 2>server2_err.txt >server2_out.dat & echo ${!}')" # Next server should not be allowed to listen (wrong -a key) sleep_ct -./gs-pipe -k id_sec.txt -l -a player-mallory 2>server3_err.txt >server3_out.dat +../tools/gs-pipe -k id_sec.txt -l -a player-mallory 2>server3_err.txt >server3_out.dat RET=$? if [ $RET -ne 255 ]; then fail 1; fi # Here: Two servers are still running... -./gs-pipe -k id_sec.txt client_err.txt >client_out.dat -./gs-pipe -k id_sec.txt client_err.txt >client_out.dat +../tools/gs-pipe -k id_sec.txt client_err.txt >client_out.dat +../tools/gs-pipe -k id_sec.txt client_err.txt >client_out.dat waitk $GS1PID $GS2PID &>/dev/null if [ "$(MD5 test50k.dat)" != "$(MD5 server1_out.dat)" ]; then fail 2; fi if [ "$(MD5 test50k.dat)" != "$(MD5 server2_out.dat)" ]; then fail 3; fi @@ -276,9 +274,9 @@ fi if [[ "$tests" =~ '4.1' ]]; then ### Client to become a server if no server is listening test_start -n "Running: pipe #4.1 (become server if possible)............" -GSPID="$(sh -c './gs-pipe -k id_sec.txt -A server_err.txt >server_out.dat & echo ${!}')" +GSPID="$(sh -c '../tools/gs-pipe -k id_sec.txt -A server_err.txt >server_out.dat & echo ${!}')" sleep_ct -./gs-pipe -k id_sec.txt -A client_err.txt >client_out.dat +../tools/gs-pipe -k id_sec.txt -A client_err.txt >client_out.dat waitk $GSPID FC=0 [[ "$(MD5 test50k.dat)" != "$(MD5 server_out.dat)" ]] && FC=$((FX+1)) @@ -290,9 +288,9 @@ fi if [[ "$tests" =~ '4.2' ]]; then # Client already waiting. 2nd client to become server (if no server available) test_start -n "Running: pipe #4.2 (..while client waiting)..............." -GSPID="$(sh -c './gs-pipe -k id_sec.txt -w client_err.txt >client_out.dat & echo ${!}')" +GSPID="$(sh -c '../tools/gs-pipe -k id_sec.txt -w client_err.txt >client_out.dat & echo ${!}')" sleep_ct -./gs-pipe -k id_sec.txt -A 2>server_err.txt >server_out.dat +../tools/gs-pipe -k id_sec.txt -A 2>server_err.txt >server_out.dat waitk $GSPID if [ "$(MD5 test50k.dat)" != "$(MD5 server_out.dat)" ]; then fail 1; fi $ECHO "${OK}" @@ -300,9 +298,9 @@ fi if [[ "$tests" =~ '5.1' ]]; then test_start -n "Running: full-pipe #5.1..................................." -GSPID="$(sh -c './gs-full-pipe -k id_sec.txt -A client_err.txt >client_out.dat & echo ${!}')" +GSPID="$(sh -c '../tools/gs-full-pipe -k id_sec.txt -A client_err.txt >client_out.dat & echo ${!}')" sleep_ct -./gs-full-pipe -k id_sec.txt -A server_err.txt >server_out.dat +../tools/gs-full-pipe -k id_sec.txt -A server_err.txt >server_out.dat waitk $GSPID if [ "$(MD5 test50k.dat)" != "$(MD5 server_out.dat)" ]; then fail 1; fi if [ "$(MD5 test50k.dat)" != "$(MD5 client_out.dat)" ]; then fail 2; fi @@ -311,9 +309,9 @@ fi if [[ "$tests" =~ '5.2' ]]; then test_start -n "Running: full-pipe #5.2 (50MB)............................" -GSPID="$(sh -c './gs-full-pipe -k id_sec.txt -A client_err.txt >client_out.dat & echo ${!}')" +GSPID="$(sh -c '../tools/gs-full-pipe -k id_sec.txt -A client_err.txt >client_out.dat & echo ${!}')" sleep_ct -./gs-full-pipe -k id_sec.txt -A server_err.txt >server_out.dat +../tools/gs-full-pipe -k id_sec.txt -A server_err.txt >server_out.dat waitk $GSPID if [ "$MD50MB" != "$(MD5 server_out.dat)" ]; then fail 1; fi if [ "$MD50MB" != "$(MD5 client_out.dat)" ]; then fail 2; fi @@ -322,10 +320,10 @@ fi if [[ "$tests" =~ '5.3' ]]; then test_start -n "Running: full-pipe #5.3 (assym sizes, large server)......." -GSPID="$(sh -c './gs-full-pipe -k id_sec.txt -A server_err.txt >server_out.dat & echo ${!}')" +GSPID="$(sh -c '../tools/gs-full-pipe -k id_sec.txt -A server_err.txt >server_out.dat & echo ${!}')" sleep_ct sleep 1 -./gs-full-pipe -A -k id_sec.txt client_err.txt >client_out.dat +../tools/gs-full-pipe -A -k id_sec.txt client_err.txt >client_out.dat waitk $GSPID &>/dev/null # if [ "$MD50MB" != "$(MD5 server_out.dat)" ]; then fail 1; fi if [ "$(MD5 test1M.dat)" != "$(MD5 client_out.dat)" ]; then fail 1; fi @@ -335,10 +333,10 @@ fi if [[ "$tests" =~ '5.4' ]]; then test_start -n "Running: full-pipe #5.4 (assym sizes, small server)......." -GSPID="$(sh -c './gs-full-pipe -k id_sec.txt -A server_err.txt >server_out.dat & echo ${!}')" +GSPID="$(sh -c '../tools/gs-full-pipe -k id_sec.txt -A server_err.txt >server_out.dat & echo ${!}')" sleep_ct sleep 1 -./gs-full-pipe -A -k id_sec.txt client_err.txt >client_out.dat +../tools/gs-full-pipe -A -k id_sec.txt client_err.txt >client_out.dat waitk $GSPID &>/dev/null # if [ "$MD50MB" != "$(MD5 server_out.dat)" ]; then fail 1; fi if [ "$(MD5 test1M.dat)" != "$(MD5 server_out.dat)" ]; then fail 1; fi @@ -348,9 +346,9 @@ fi if [[ "$tests" =~ '5.5' ]]; then test_start -n "Running: full-pipe #5.5 (assymetric sizes, clear)........." -GSPID="$(sh -c './gs-full-pipe -k id_sec.txt -AC server_err.txt >server_out.dat & echo ${!}')" +GSPID="$(sh -c '../tools/gs-full-pipe -k id_sec.txt -AC server_err.txt >server_out.dat & echo ${!}')" sleep_ct -./gs-full-pipe -k id_sec.txt -AC client_err.txt >client_out.dat +../tools/gs-full-pipe -k id_sec.txt -AC client_err.txt >client_out.dat waitk $GSPID if [ "$MD1MB" != "$(MD5 client_out.dat)" ]; then fail 1; fi if [ "$(MD5 test50k.dat)" != "$(MD5 server_out.dat)" ]; then fail 2; fi @@ -359,9 +357,9 @@ fi if [[ "$tests" =~ '6.1' ]]; then test_start -n "Running: netcat #6.1 (stdin, 1MB)........................." -GSPID="$(sh -c './gs-netcat -k id_sec.txt -w client_err.txt >client_out.dat & echo ${!}')" +GSPID="$(sh -c '../tools/gs-netcat -k id_sec.txt -w client_err.txt >client_out.dat & echo ${!}')" sleep_ct -./gs-netcat -k id_sec.txt -l server_err.txt >server_out.dat +../tools/gs-netcat -k id_sec.txt -l server_err.txt >server_out.dat waitk $GSPID if [ "$MD1MB" != "$(MD5 server_out.dat)" ]; then fail 1; fi if [ "$MD1MB" != "$(MD5 client_out.dat)" ]; then fail 2; fi @@ -370,9 +368,9 @@ fi if [[ "$tests" =~ '6.2' ]]; then test_start -n "Running: netcat #6.2 (stdin, assymetric sizes)............" -GSPID="$(sh -c './gs-netcat -k id_sec.txt -w client_err.txt >client_out.dat & echo ${!}')" +GSPID="$(sh -c '../tools/gs-netcat -k id_sec.txt -w client_err.txt >client_out.dat & echo ${!}')" sleep_ct -./gs-netcat -k id_sec.txt -l server_err.txt >server_out.dat +../tools/gs-netcat -k id_sec.txt -l server_err.txt >server_out.dat waitk $GSPID if [ "$MD1MB" != "$(MD5 server_out.dat)" ]; then fail 1; fi if [ "$(MD5 test50k.dat)" != "$(MD5 client_out.dat)" ]; then fail 2; fi @@ -381,8 +379,8 @@ fi if [[ "$tests" =~ '6.3' ]]; then test_start -n "Running: netcat #6.3 (stdin, assym sizes, kill client)...." -GSPID1="$(sh -c '(cat test4k.dat; sleep 30) | ./gs-netcat -k id_sec.txt -w 2>client_err.txt >client_out.dat & echo ${!}')" -GSPID2="$(sh -c '(cat test1k.dat; sleep 30) | ./gs-netcat -k id_sec.txt -l 2>server_err.txt >server_out.dat & echo ${!}')" +GSPID1="$(sh -c '(cat test4k.dat; sleep 30) | ../tools/gs-netcat -k id_sec.txt -w 2>client_err.txt >client_out.dat & echo ${!}')" +GSPID2="$(sh -c '(cat test1k.dat; sleep 30) | ../tools/gs-netcat -k id_sec.txt -l 2>server_err.txt >server_out.dat & echo ${!}')" # sleep_ct waitf test4k.dat server_out.dat waitf test1k.dat client_out.dat @@ -395,8 +393,8 @@ fi if [[ "$tests" =~ '6.4' ]]; then test_start -n "Running: netcat #6.4 (stdin, assym sizes, kill server)...." -GSPID1="$(sh -c '(cat test4k.dat; sleep 30) | ./gs-netcat -k id_sec.txt -w 2>client_err.txt >client_out.dat & echo ${!}')" -GSPID2="$(sh -c '(cat test1k.dat; sleep 30) | ./gs-netcat -k id_sec.txt -l 2>server_err.txt >server_out.dat & echo ${!}')" +GSPID1="$(sh -c '(cat test4k.dat; sleep 30) | ../tools/gs-netcat -k id_sec.txt -w 2>client_err.txt >client_out.dat & echo ${!}')" +GSPID2="$(sh -c '(cat test1k.dat; sleep 30) | ../tools/gs-netcat -k id_sec.txt -l 2>server_err.txt >server_out.dat & echo ${!}')" waitf test4k.dat server_out.dat waitf test1k.dat client_out.dat kill -9 $GSPID2 &>/dev/null @@ -408,9 +406,9 @@ fi if [[ "$tests" =~ '6.5' ]]; then test_start -n "Running: netcat #6.5 (/dev/null C2S)......................" -GSPID="$(sh -c './gs-netcat -k id_sec.txt -w client_err.txt >client_out.dat & echo ${!}')" +GSPID="$(sh -c '../tools/gs-netcat -k id_sec.txt -w client_err.txt >client_out.dat & echo ${!}')" sleep_ct -./gs-netcat -k id_sec.txt -l server_err.txt >server_out.dat +../tools/gs-netcat -k id_sec.txt -l server_err.txt >server_out.dat waitk $GSPID if [ -s server_out.dat ]; then fail 1; fi if [ "$(MD5 test4k.dat)" != "$(MD5 client_out.dat)" ]; then fail 2; fi @@ -419,9 +417,9 @@ fi if [[ "$tests" =~ '6.6' ]]; then test_start -n "Running: netcat #6.6 (/dev/null S2C)......................" -GSPID="$(sh -c './gs-netcat -k id_sec.txt -w client_err.txt >client_out.dat & echo ${!}')" +GSPID="$(sh -c '../tools/gs-netcat -k id_sec.txt -w client_err.txt >client_out.dat & echo ${!}')" sleep_ct -./gs-netcat -k id_sec.txt -l server_err.txt >server_out.dat +../tools/gs-netcat -k id_sec.txt -l server_err.txt >server_out.dat waitk $GSPID if [ -s client_out.dat ]; then fail 1; fi if [ "$(MD5 test4k.dat)" != "$(MD5 server_out.dat)" ]; then fail 2; fi @@ -430,9 +428,9 @@ fi if [[ "$tests" =~ '6.7' ]]; then test_start -n "Running: netcat #6.7 (stdin, assymetric sizes, clear)....." -GSPID="$(sh -c './gs-netcat -k id_sec.txt -wC client_err.txt >client_out.dat & echo ${!}')" +GSPID="$(sh -c '../tools/gs-netcat -k id_sec.txt -wC client_err.txt >client_out.dat & echo ${!}')" sleep_ct -./gs-netcat -k id_sec.txt -lC server_err.txt >server_out.dat +../tools/gs-netcat -k id_sec.txt -lC server_err.txt >server_out.dat waitk $GSPID if [ "$MD1MB" != "$(MD5 server_out.dat)" ]; then fail 1; fi if [ "$(MD5 test50k.dat)" != "$(MD5 client_out.dat)" ]; then fail 2; fi @@ -445,9 +443,9 @@ NETSTATTCP 2>/dev/null | grep LISTEN | grep 9050 &>/dev/null if [ $? -ne 0 ]; then skip "(no TOR)" else - GSPID="$(sh -c './gs-netcat -k id_sec.txt -wT client_err.txt >client_out.dat & echo ${!}')" + GSPID="$(sh -c '../tools/gs-netcat -k id_sec.txt -wT client_err.txt >client_out.dat & echo ${!}')" sleep_ct - ./gs-netcat -k id_sec.txt -l server_err.txt >server_out.dat + ../tools/gs-netcat -k id_sec.txt -l server_err.txt >server_out.dat waitk $GSPID if [ "$(MD5 test4k.dat)" != "$(MD5 server_out.dat)" ]; then fail 1; fi if [ "$(MD5 test50k.dat)" != "$(MD5 client_out.dat)" ]; then fail 2; fi @@ -457,10 +455,10 @@ fi if [[ "$tests" =~ '7.1' ]]; then test_start -n "Running: netcat #7.1 (cmd, multi connect)................." -GSPID1="$(sh -c './gs-netcat -k id_sec.txt -l -e "echo Hello World" 2>server_err.txt >server_out.dat & echo ${!}')" -GSPID2="$(sh -c './gs-netcat -k id_sec.txt -w client2_err.txt >client2_out.dat & echo ${!}')" -GSPID3="$(sh -c './gs-netcat -k id_sec.txt -w client3_err.txt >client3_out.dat & echo ${!}')" -./gs-netcat -k id_sec.txt -w client_err.txt >client_out.dat +GSPID1="$(sh -c '../tools/gs-netcat -k id_sec.txt -l -e "echo Hello World" 2>server_err.txt >server_out.dat & echo ${!}')" +GSPID2="$(sh -c '../tools/gs-netcat -k id_sec.txt -w client2_err.txt >client2_out.dat & echo ${!}')" +GSPID3="$(sh -c '../tools/gs-netcat -k id_sec.txt -w client3_err.txt >client3_out.dat & echo ${!}')" +../tools//gs-netcat -k id_sec.txt -w client_err.txt >client_out.dat waitk $GSPID2 $GSPID3 kill -9 $GSPID1 &>/dev/null if [ "${MDHELLOW}" != "$(MD5 client_out.dat)" ]; then fail 1; fi @@ -471,19 +469,27 @@ fi if [[ "$tests" =~ '7.2' ]]; then test_start -n "Running: netcat #7.2 (shell, exit)........................" -GSPID1="$(sh -c './gs-netcat -k id_sec.txt -l -e /bin/sh 2>server_err.txt >server_out.dat & echo ${!}')" -echo "date; echo Hello World; exit" | ./gs-netcat -k id_sec.txt -w 2>client_err.txt >client_out.dat +GSPID1="$(sh -c '../tools/gs-netcat -k id_sec.txt -l -e /bin/sh 2>server_err.txt >server_out.dat & echo ${!}')" +echo "date; echo Hello World; exit" | ../tools/gs-netcat -k id_sec.txt -w 2>client_err.txt >client_out.dat sleep_ct kill $GSPID1 if [ "${MDHELLOW}" != "$(tail -1 client_out.dat | MD5 /dev/stdin)" ]; then fail 1; fi $ECHO "${OK}" fi +# FreeBSD sends ansi-request to determine screen size. Wait for it to timeout and +# then send our command string. +XCMD="printf \"date && echo Hello World && exit\n\"" +if [[ "$OSTYPE" == *"BSD"* ]]; then + XCMD="sleep 3; ${XCMD}" +fi + if [[ "$tests" =~ '7.3' ]]; then test_start -n "Running: netcat #7.3 (pty shell, exit)...................." -GSPID1="$(sh -c './gs-netcat -k id_sec.txt -l -i 2>server_err.txt >server_out.dat & echo ${!}')" +GSPID1="$(sh -c '../tools/gs-netcat -k id_sec.txt -l -i 2>server_err.txt >server_out.dat & echo ${!}')" # Can not start Client with -i because it's not a tty. Must 'fake' the terminal response. -(printf "date; echo Hello World; exit\n") | ./gs-netcat -k id_sec.txt -w 2>client_err.txt >client_out.dat +sh -c "$XCMD" | ../tools/gs-netcat -k id_sec.txt -w 2>client_err.txt >client_out.dat +# (printf "date; echo Hello World; exit\n") | ../tools/gs-netcat -k id_sec.txt -w 2>client_err.txt >client_out.dat sleep_ct kill $GSPID1 tail -2 client_out.dat | grep 'Hello World' &>/dev/null @@ -493,10 +499,10 @@ fi if [[ "$tests" =~ '7.4' ]]; then test_start -n "Running: netcat #7.4 (multi pty shell, exit).............." -GSPID1="$(sh -c './gs-netcat -k id_sec.txt -l -i 2>server_err.txt >server_out.dat & echo ${!}')" -GSPID2=$(sh -c "(printf \"date && echo Hello World && exit\n\") | ./gs-netcat -k id_sec.txt -w 2>client1_err.txt >client1_out.dat & echo \${!}") -GSPID3=$(sh -c "(printf \"date && echo Hello World && exit\n\") | ./gs-netcat -k id_sec.txt -w 2>client2_err.txt >client2_out.dat & echo \${!}") -GSPID4=$(sh -c "(printf \"date && echo Hello World && exit\n\") | ./gs-netcat -k id_sec.txt -w 2>client3_err.txt >client3_out.dat & echo \${!}") +GSPID1="$(sh -c '../tools/gs-netcat -k id_sec.txt -l -i 2>server_err.txt >server_out.dat & echo ${!}')" +GSPID2=$(sh -c "($XCMD) | ../tools/gs-netcat -k id_sec.txt -w 2>client1_err.txt >client1_out.dat & echo \${!}") +GSPID3=$(sh -c "($XCMD) | ../tools/gs-netcat -k id_sec.txt -w 2>client2_err.txt >client2_out.dat & echo \${!}") +GSPID4=$(sh -c "($XCMD) | ../tools/gs-netcat -k id_sec.txt -w 2>client3_err.txt >client3_out.dat & echo \${!}") waitk $GSPID2 $GSPID3 $GSPID4 kill $GSPID1 if [ x"$(tail -2 client1_out.dat | grep 'Hello World')" == x ]; then fail 1; fi @@ -507,10 +513,10 @@ fi if [[ "$tests" =~ '8.1' ]]; then test_start -n "Running: netcat #8.1 (port forward server side)..........." -GSPID1="$(sh -c './gs-netcat -k id_sec.txt -l -d 127.0.0.1 -p 12345 2>server_err.txt >server_out.dat & echo ${!}')" +GSPID1="$(sh -c '../tools/gs-netcat -k id_sec.txt -l -d 127.0.0.1 -p 12345 2>server_err.txt >server_out.dat & echo ${!}')" GSPID2="$(sh -c '(sleep 10) | $NC_LISTEN 12345 >nc1_out.dat 2>nc1_err.txt & echo ${!}')" waittcp 12345 -GSPID3="$(sh -c './gs-netcat -k id_sec.txt -w client_err.txt >client_out.dat & echo ${!}')" +GSPID3="$(sh -c '../tools/gs-netcat -k id_sec.txt -w client_err.txt >client_out.dat & echo ${!}')" waitf test50k.dat nc1_out.dat kill -9 $GSPID1 $GSPID2 $GSPID3 &>/dev/null if [ "$(MD5 test50k.dat)" != "$(MD5 nc1_out.dat)" ]; then fail 1; fi @@ -520,9 +526,9 @@ fi if [[ "$tests" =~ '8.2' ]]; then # nc -> Port 12344 -> GS-NET -> Port 12345 -> nc -ln test_start -n "Running: netcat #8.2 (port forward both sides)............" -GSPID1="$(sh -c './gs-netcat -k id_sec.txt -l -d 127.0.0.1 -p 12345 2>server_err.txt >server_out.dat & echo ${!}')" +GSPID1="$(sh -c '../tools/gs-netcat -k id_sec.txt -l -d 127.0.0.1 -p 12345 2>server_err.txt >server_out.dat & echo ${!}')" GSPID2="$(sh -c '(sleep 10) | $NC_LISTEN 12345 >nc1_out.dat 2>nc1_err.txt & echo ${!}')" -GSPID3="$(sh -c './gs-netcat -k id_sec.txt -w -p 12344 2>server_err.txt >server_out.dat & echo ${!}')" +GSPID3="$(sh -c '../tools/gs-netcat -k id_sec.txt -w -p 12344 2>server_err.txt >server_out.dat & echo ${!}')" waittcp 12344 waittcp 12345 GSPID4="$(sh -c '(cat test50k.dat; sleep 15) | $NC -vn 127.0.0.1 12344 >nc2_out.dat 2>nc2_err.txt & echo ${!}')" @@ -536,9 +542,9 @@ if [[ "$tests" =~ '8.3' ]]; then # nc -> Port 12344 -> GS-NET -> Port 12345 -> nc -ln # Bi-Directional test_start -n "Running: netcat #8.3 (port forward both sides, bi-dir)...." -GSPID1="$(sh -c './gs-netcat -k id_sec.txt -l -d 127.0.0.1 -p 12345 2>server1_err.txt >server1_out.dat & echo ${!}')" +GSPID1="$(sh -c '../tools/gs-netcat -k id_sec.txt -l -d 127.0.0.1 -p 12345 2>server1_err.txt >server1_out.dat & echo ${!}')" GSPID2="$(sh -c '(cat test4k.dat; sleep 15) | $NC_LISTEN 12345 >nc1_out.dat 2>nc1_err.txt & echo ${!}')" -GSPID3="$(sh -c './gs-netcat -k id_sec.txt -w -p 12344 2>client_err.txt >client_out.dat & echo ${!}')" +GSPID3="$(sh -c '../tools/gs-netcat -k id_sec.txt -w -p 12344 2>client_err.txt >client_out.dat & echo ${!}')" waittcp 12344 waittcp 12345 GSPID4="$(sh -c '(cat test50k.dat; sleep 15) | $NC -vn 127.0.0.1 12344 >nc2_out.dat 2>nc2_err.txt & echo ${!}')" @@ -557,9 +563,9 @@ socat -h 2>/dev/null | grep socks5 &>/dev/null if [ $? -ne 0 ]; then skip "(no socat2)" else - GSPID1="$(sh -c './gs-netcat -k id_sec.txt -lS 2>server1_err.txt >server1_out.dat & echo ${!}')" + GSPID1="$(sh -c '../tools/gs-netcat -k id_sec.txt -lS 2>server1_err.txt >server1_out.dat & echo ${!}')" GSPID2="$(sh -c '(cat test4k.dat; sleep 15) | $NC_LISTEN 12345 >nc1_out.dat 2>nc1_err.txt & echo ${!}')" - GSPID3="$(sh -c './gs-netcat -k id_sec.txt -w -p 1085 2>client_err.txt >client_out.dat & echo ${!}')" + GSPID3="$(sh -c '../tools/gs-netcat -k id_sec.txt -w -p 1085 2>client_err.txt >client_out.dat & echo ${!}')" waittcp 1085 waittcp 12345 GSPID4="$(sh -c '(cat test50k.dat; sleep 15) | socat - "SOCKS5:localhost:12345 | TCP:127.1:1085" >nc2_out.dat 2>nc2_err.txt & echo ${!}')" @@ -579,9 +585,9 @@ socat -h 2>/dev/null | grep socks4 &>/dev/null if [ $? -ne 0 ]; then skip "(no socat)" else - GSPID1="$(sh -c './gs-netcat -k id_sec.txt -lS 2>server1_err.txt >server1_out.dat & echo ${!}')" + GSPID1="$(sh -c '../tools/gs-netcat -k id_sec.txt -lS 2>server1_err.txt >server1_out.dat & echo ${!}')" GSPID2="$(sh -c '(cat test4k.dat; sleep 15) | $NC_LISTEN 12345 >nc1_out.dat 2>nc1_err.txt & echo ${!}')" - GSPID3="$(sh -c './gs-netcat -k id_sec.txt -p 1085 2>client_err.txt >client_out.dat & echo ${!}')" + GSPID3="$(sh -c '../tools/gs-netcat -k id_sec.txt -p 1085 2>client_err.txt >client_out.dat & echo ${!}')" waittcp 1085 waittcp 12345 GSPID4="$(sh -c '(cat test50k.dat; sleep 15) | socat - "SOCKS4:127.0.0.1:127.0.0.1:12345,socksport=1085" >nc2_out.dat 2>nc2_err.txt & echo ${!}')" @@ -601,9 +607,9 @@ socat -h 2>/dev/null | grep socks4 &>/dev/null if [ $? -ne 0 ]; then skip "(no socat)" else - GSPID1="$(sh -c './gs-netcat -k id_sec.txt -lS 2>server1_err.txt >server1_out.dat & echo ${!}')" + GSPID1="$(sh -c '../tools/gs-netcat -k id_sec.txt -lS 2>server1_err.txt >server1_out.dat & echo ${!}')" GSPID2="$(sh -c '(cat test4k.dat; sleep 15) | $NC_LISTEN 12345 >nc1_out.dat 2>nc1_err.txt & echo ${!}')" - GSPID3="$(sh -c './gs-netcat -k id_sec.txt -p 1085 2>client_err.txt >client_out.dat & echo ${!}')" + GSPID3="$(sh -c '../tools/gs-netcat -k id_sec.txt -p 1085 2>client_err.txt >client_out.dat & echo ${!}')" waittcp 1085 waittcp 12345 GSPID4="$(sh -c '(cat test50k.dat; sleep 15) | socat - "SOCKS4a:127.0.0.1:127.0.0.1:12345,socksport=1085" >nc2_out.dat 2>nc2_err.txt & echo ${!}')" @@ -623,8 +629,8 @@ curl --help all 2>/dev/null | grep socks5-hostname &>/dev/null if [ $? -ne 0 ]; then skip "(no curl)" else - GSPID1="$(sh -c './gs-netcat -k id_sec.txt -lS 2>server1_err.txt >server1_out.dat & echo ${!}')" - GSPID3="$(sh -c './gs-netcat -k id_sec.txt -p 1085 2>client_err.txt >client_out.dat & echo ${!}')" + GSPID1="$(sh -c '../tools/gs-netcat -k id_sec.txt -lS 2>server1_err.txt >server1_out.dat & echo ${!}')" + GSPID3="$(sh -c '../tools/gs-netcat -k id_sec.txt -p 1085 2>client_err.txt >client_out.dat & echo ${!}')" waittcp 1085 touch testmp3.dat testmp3-2.dat GSPID4="$(sh -c 'curl --socks5-hostname 127.0.0.1:1085 --output testmp3.dat https://raw.githubusercontent.com/hackerschoice/thc-art/master/deep-phreakin.mp3 >client1_out.dat 2>client1_err.txt & echo ${!}')" @@ -648,9 +654,9 @@ mkfifo test_client/fifo.io ln -s foo/bar/test4k.dat test_client/test4k.dat ln -s /etc/hosts test_client/etc-hosts ln -s /dev/zero test_client/zero -GSPID1="$(sh -c './blitz -k id_sec.txt -w -o "RSOPT=--bwlimit=100 -v" test_client/./ 2>client1_err.txt >client1_out.dat & echo ${!}')" +GSPID1="$(sh -c '../tools/blitz -k id_sec.txt -w -o "RSOPT=--bwlimit=100 -v" test_client/./ 2>client1_err.txt >client1_out.dat & echo ${!}')" cd test_server -GSPID2="$(sh -c '../blitz -k ../id_sec.txt -l 2>../server1_err.txt >../server1_out.dat & echo ${!}')" +GSPID2="$(sh -c '../../tools/blitz -k ../id_sec.txt -l 2>../server1_err.txt >../server1_out.dat & echo ${!}')" cd .. waitk $GSPID1 kill $GSPID2 @@ -669,9 +675,9 @@ if [[ "${tests}" =~ '10.2' ]]; then test_start -n "Running: blitz #10.2 (stdin).............................." rm -rf test_client mkdir -p test_client -GSPID1="$(sh -c '(echo test1k.dat; echo test4k.dat) | ./blitz -k id_sec.txt -w -o "RSOPT=--bwlimit=100 -v" -f - 2>client1_err.txt >client1_out.dat & echo ${!}')" +GSPID1="$(sh -c '(echo test1k.dat; echo test4k.dat) | ../tools/blitz -k id_sec.txt -w -o "RSOPT=--bwlimit=100 -v" -f - 2>client1_err.txt >client1_out.dat & echo ${!}')" cd test_client -GSPID2="$(sh -c '../blitz -k ../id_sec.txt -l 2>../server1_err.txt >../server1_out.dat & echo ${!}')" +GSPID2="$(sh -c '../../tools/blitz -k ../id_sec.txt -l 2>../server1_err.txt >../server1_out.dat & echo ${!}')" cd .. waitk $GSPID1 kill $GSPID2 @@ -685,8 +691,8 @@ if [[ "${tests}" =~ '10.3' ]]; then test_start -n "Running: gs-sftp #10.3 ..................................." rm -rf test_client mkdir -p test_client -GSPID1="$(bash -c '(echo -en "lcd test_client\nget test4k.dat\nlcd ..\ncd test_client\nput test1k.dat\nls\nquit\n") | ./gs-sftp -k id_sec.txt -w 2>client1_err.txt >client1_out.dat & echo ${!}')" -GSPID2="$(sh -c './gs-sftp -k id_sec.txt -l 2>server1_err.txt >server1_out.dat & echo ${!}')" +GSPID1="$(bash -c '(echo -en "lcd test_client\nget test4k.dat\nlcd ..\ncd test_client\nput test1k.dat\nls\nquit\n") | ../tools/gs-sftp -k id_sec.txt -w 2>client1_err.txt >client1_out.dat & echo ${!}')" +GSPID2="$(sh -c '../tools/gs-sftp -k id_sec.txt -l 2>server1_err.txt >server1_out.dat & echo ${!}')" waitk $GSPID1 kill $GSPID2 md5fail 1 test1k.dat test_client/test1k.dat @@ -705,8 +711,8 @@ else rmdir test_mnt &>/dev/null mkdir -p test_client test_mnt &>/dev/null cp test1k.dat test4k.dat test_client - GSPID1="$(sh -c './gs-mount -k id_sec.txt -w test_mnt 2>client1_err.txt >client1_out.dat & echo ${!}')" - GSPID2="$(sh -c 'cd test_client; ../gs-mount -k ../id_sec.txt -l 2>../server1_err.txt >../server1_out.dat & echo ${!}')" + GSPID1="$(sh -c '../tools/gs-mount -k id_sec.txt -w test_mnt 2>client1_err.txt >client1_out.dat & echo ${!}')" + GSPID2="$(sh -c 'cd test_client; ../../tools/gs-mount -k ../id_sec.txt -l 2>../server1_err.txt >../server1_out.dat & echo ${!}')" waitk $GSPID1 md5fail 1 test_mnt/test1k.dat test_client/test1k.dat md5fail 2 test_mnt/test4k.dat test_client/test4k.dat diff --git a/tools/4_gs-netcat.c b/tools/4_gs-netcat.c index f111631e..63d6f138 100755 --- a/tools/4_gs-netcat.c +++ b/tools/4_gs-netcat.c @@ -175,7 +175,7 @@ cb_atexit(void) { CONSOLE_reset(); stty_reset(); - printf("\n[Bye]\n"); + fprintf(stderr, "\n[Bye]\n"); // stdout must be clean for pipe & gs-netcat } /* *********************** FD READ / WRITE ******************************/ diff --git a/tools/Makefile.am b/tools/Makefile.am index 59c0eb13..d9891013 100755 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -32,6 +32,4 @@ console_display_test_LDADD = ../lib/libgsocket.a filetransfer_test_SOURCES = filetransfer.c filetransfer-test.c utils.c globbing.c filetransfer_test_LDADD = ../lib/libgsocket.a -EXTRA_DIST = run_all_tests.sh noinst_HEADERS = common.h utils.h socks.h console.h ids.h event_mgr.h pkt_mgr.h gs-netcat.h console_display.h filetransfer.h man_gs-netcat.h globbing.h filetransfer_mgr.h -AM_CFLAGS = -I../include diff --git a/tools/filetransfer-test.c b/tools/filetransfer-test.c index bf2df2af..4ad3e443 100644 --- a/tools/filetransfer-test.c +++ b/tools/filetransfer-test.c @@ -261,22 +261,27 @@ do_globtest(const char *exp) } int -main(int arc, const char *argv[]) +main(int argc, const char *argv[]) { - int is_extra_puts = 0; uint8_t src[BUF_LEN]; uint8_t dst[2 * sizeof src]; ssize_t sz; size_t dsz; int ret; int n; - int is_get = 0; + GS_library_init(stderr, stderr); gopt.err_fp = stderr; gopt.log_fp = stderr; // gopt.err_fp = NULL; // un-comment to DISABLE DEBUG + if (argc < 2) + { + fprintf(stderr, "filetransfer-test [scCG] \n"); + exit(255); + } + if (*argv[1] == 'G') do_globtest(argv[2]); @@ -302,7 +307,6 @@ main(int arc, const char *argv[]) GS_FT_put(&ft, argv[i]); } else { // Test GET (download) - is_get = 1; GS_PKT_assign_chn(&pkt, GS_FT_CHN_LIST_REPLY, pkt_cb_listreply, NULL); GS_PKT_assign_chn(&pkt, GS_FT_CHN_DATA, pkt_cb_data, NULL); GS_PKT_assign_chn(&pkt, GS_FT_CHN_SWITCH, pkt_cb_switch, NULL); @@ -333,19 +337,7 @@ main(int arc, const char *argv[]) { // No more files to transfer // (All data send. Not waiting for any reply). - break; - #if 0 - // HERE: test adding files after transfer completed... - if (is_extra_puts >= 1) - break; - is_extra_puts++; - if (GS_FT_put(&ft, "test1k-extra1.dat") != 0) - DEBUGF_Y("Not found: test1k-extra1.dat\n"); - if (GS_FT_put(&ft, "test1k-extra2.dat") != 0) - DEBUGF_Y("Not found: test1k-extra2.dat\n"); - continue; - #endif } // If there is data to write then write data first. @@ -403,7 +395,7 @@ main(int arc, const char *argv[]) sz = read(0, src, sizeof src); // DEBUGF_G("read() == %zu\n", sz); if (sz <= 0) - ERREXIT("read()\n"); + break; ret = GS_PKT_decode(&pkt, src, sz, dst, &dsz); if (ret != 0) ERREXIT("GS_PKT_decode()\n"); diff --git a/tools/filetransfer.c b/tools/filetransfer.c index 35b8f9b4..b45470a8 100644 --- a/tools/filetransfer.c +++ b/tools/filetransfer.c @@ -92,7 +92,7 @@ GS_FT_init(GS_FT *ft, gsft_cb_stats_t func_stats, gsft_cb_status_t func_status, ft->is_server = is_server; } -#ifdef DEBUG +#ifdef SELFTESTS static const char **g_filesv; static int g_filesc; static const char *g_true_fname; @@ -231,7 +231,7 @@ GS_FT_dl_add_file(GS_FT *ft, uint32_t id, const char *fname, size_t len, int64_t return -1; // protocol error. Not 0 terminated. DEBUGF_Y("#%u DL-ADD-FILE - '%s' fz_remote=%"PRId64"\n", id, fname, fz_remote); -#ifdef DEBUG +#ifdef SELFTESTS // Test [8.9] if (g_true_fname != NULL) { @@ -527,7 +527,7 @@ add_file_to_list(GS_FT *ft, GS_LIST *gsl, const char *fname, uint32_t globbing_i if (is_server == 1) { // Server, GET (download). -#ifdef DEBUG +#ifdef SELFTESTS const char *ff = fname; if ((g_filesv != NULL) && (g_filesv[g_filesc] != NULL)) { @@ -954,11 +954,12 @@ static int mkdirpm(const char *path, mode_t mode, uint32_t mtime) { int rv = 0; - char *f = strdup(path); struct stat res; + char *f = NULL; + DEBUGF("mkdirpm(%s)\n", path); // Return 0 if directory already exist - if (stat(f, &res) == 0) + if (stat(path, &res) == 0) { if (S_ISDIR(res.st_mode)) { @@ -969,7 +970,7 @@ mkdirpm(const char *path, mode_t mode, uint32_t mtime) // if (access(path, W_OK) != 0) goto done; // Done have access } else { - set_mode_mtime(f, mode, mtime); + set_mode_mtime(path, mode, mtime); } // DEBUGF_W("%s already exists\n", f); goto done; @@ -979,20 +980,22 @@ mkdirpm(const char *path, mode_t mode, uint32_t mtime) // Directory does not exist. Use default permission to create it. if (mode == 0) mode = 0755; - + f = strdup(path); // If parent directory exist then only create last directory. - char *dn = dirname(f); - if ((dn != NULL) && (stat(dn, &res) == 0)) + char *parent = dirname(f); + if ((parent != NULL) && (stat(parent, &res) == 0)) { if (S_ISDIR(res.st_mode)) { // HERE: Parent directory exists. - DEBUGF_W("1-mkdir(%s)\n", f); - if (mkdir_agressive(f, mode, mtime) != 0) + DEBUGF_W("1-mkdir(%s)\n", path); + if (mkdir_agressive(path, mode, mtime) != 0) rv = -1; goto done; } } + XFREE(f); // dirname() may have modified it + f = strdup(path); // HERE: Neither directory nor parent directory exist. Start from // the left and create entire hirachy.... @@ -1125,10 +1128,15 @@ gs_ft_switch(GS_FT *ft, uint32_t id, int64_t fz_remote, struct _gs_ft_file **act while (*ptr == '/') ptr++; } - ptr = dirname(ptr); + char *dir = strdup(ptr); // direname() may modify dir + // ptr points to an internal buffer. strdup() it because mkdirpm() calls dirname() as well + // and would overwrite this structure.. + ptr = strdup(dirname(dir)); dir_save_mtime(ptr, &new->dir_mtime); // put(test1k.dat) must not modify the permission of parent directory. mkdirpm(ptr, 0 /*do not update permission on existing directory*/, 0 /*do not update mtime*/); + XFREE(dir); + XFREE(ptr); new->fp = fopen(new->fn_local, "w"); } else { @@ -1356,32 +1364,36 @@ GS_FT_status(GS_FT *ft, uint32_t id, uint8_t code, const char *err_str, size_t l // li can be one of two structures only: // _gs_ft_file or gs_ft_list_pattern - li = GS_LIST_by_id(&ft->fcompleted, id); // CLIENT + li = GS_LIST_by_id(&ft->faccepted, id); // CLIENT if (li == NULL) { - li = GS_LIST_by_id(&ft->fputs, id); // CLIENT + li = GS_LIST_by_id(&ft->fcompleted, id); // CLIENT if (li == NULL) { - li = GS_LIST_by_id(&ft->fdl_waiting, id); // CLIENT (get) + li = GS_LIST_by_id(&ft->fputs, id); // CLIENT if (li == NULL) { - // This 'li' does not hold a _gs_ft_file structure and thus - // must not generate stats or try to free a _gs_ft_file when it is not. - li = GS_LIST_by_id(&ft->plistreq_waiting, id); // CLIENT (get) + li = GS_LIST_by_id(&ft->fdl_waiting, id); // CLIENT (get) if (li == NULL) { - li = GS_LIST_by_id(&ft->fadded, id); // SERVER (put) + // This 'li' does not hold a _gs_ft_file structure and thus + // must not generate stats or try to free a _gs_ft_file when it is not. + li = GS_LIST_by_id(&ft->plistreq_waiting, id); // CLIENT (get) if (li == NULL) { - li = GS_LIST_by_id(&ft->freceiving, id); // SERVER (put) + li = GS_LIST_by_id(&ft->fadded, id); // SERVER (put) if (li == NULL) { - DEBUGF_R("id %u not found\n", id); - return; // not found + li = GS_LIST_by_id(&ft->freceiving, id); // SERVER (put) + if (li == NULL) + { + DEBUGF_R("id %u not found\n", id); + return; // not found + } } + } else { + lp = (struct _gs_ft_list_pattern *)li->data; } - } else { - lp = (struct _gs_ft_list_pattern *)li->data; } } } @@ -1842,7 +1854,7 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) { if (ft->active_dl_file == NULL) { - DEBUGF("Files to send: %d\n", ft->fdl.n_items); + DEBUGF("S Files to send: %d\n", ft->fdl.n_items); return mk_switch(ft, &ft->active_dl_file, &ft->fdl, NULL, pkt_type, dst, len); } @@ -1860,7 +1872,7 @@ GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type) { if (ft->active_put_file == NULL) { - DEBUGF("Files to send: %d\n", ft->faccepted.n_items); + DEBUGF("C Files to send: %d\n", ft->faccepted.n_items); return mk_switch(ft, &ft->active_put_file, &ft->faccepted, &ft->fcompleted, pkt_type, dst, len); } diff --git a/tools/filetransfer.h b/tools/filetransfer.h index 4db595e0..ded622c6 100644 --- a/tools/filetransfer.h +++ b/tools/filetransfer.h @@ -84,7 +84,7 @@ typedef struct // PUT (upload) - Client Side GS_LIST fqueue; // Client List of files to be transfered GS_LIST fputs; // Client list of files we requested transfer (put sent) - GS_LIST faccepted; // Client List of accepted files + GS_LIST faccepted; // Client List of file server has accepted (now can mk_switch() to any of those files) GS_LIST fcompleted; // Completed. Waiting for 'ERR_COMPLETED' // PUT (upload) - Server Side GS_LIST fadded; // Server Side list of ready files @@ -225,7 +225,7 @@ size_t GS_FT_packet(GS_FT *ft, void *dst, size_t len, int *pkt_type); void GS_FT_pause_data(GS_FT *ft); void GS_FT_unpause_data(GS_FT *ft); #define GS_FT_WANT_WRITE(xft) (xft)->is_want_write -#ifdef DEBUG +#ifdef SELFTESTS void GS_FT_init_tests(const char **argv); #endif From 4803f3131a0914c3b43f11f3f287d634317a7969 Mon Sep 17 00:00:00 2001 From: rootTHC Date: Thu, 11 Feb 2021 16:23:00 +0000 Subject: [PATCH 15/21] initial# --- tools/filetransfer_mgr.c | 329 ++++++++++++++++++++++++++++++++ tools/filetransfer_mgr.h | 8 + tools/globbing.c | 393 +++++++++++++++++++++++++++++++++++++++ tools/globbing.h | 24 +++ 4 files changed, 754 insertions(+) create mode 100644 tools/filetransfer_mgr.c create mode 100644 tools/filetransfer_mgr.h create mode 100644 tools/globbing.c create mode 100644 tools/globbing.h diff --git a/tools/filetransfer_mgr.c b/tools/filetransfer_mgr.c new file mode 100644 index 00000000..41b8c673 --- /dev/null +++ b/tools/filetransfer_mgr.c @@ -0,0 +1,329 @@ +/* + * Filetransfer Manager - Initialize callbacks for filetransfer.h + * + * Used by gs-netcat and put/get file transfer + */ + +#include "common.h" +#include "filetransfer.h" +#include "filetransfer_mgr.h" +#include "pkt_mgr.h" +#include "console.h" +#include "console_display.h" + +extern GS_CONDIS gs_condis; // defined in console.c + + +// SERVER receiving PUT from client +static void +pkt_cb_put(uint8_t chn, const uint8_t *data, size_t len, void *arg) +{ + struct _gs_ft_put *p = (struct _gs_ft_put *)data; + struct _gs_ft_put hdr; + struct _peer *peer = (struct _peer *)arg; + GS_FT *ft = &peer->ft; + + if (len < sizeof hdr + 1) + return; // protocol error + + memcpy(&hdr, data, sizeof hdr); + GS_FT_add_file(ft, ntohl(hdr.id), (char *)p->name, len - sizeof hdr - 1, ntohll(hdr.fsize), ntohl(hdr.mtime), ntohl(hdr.fperm), hdr.flags); + if (GS_FT_WANT_WRITE(ft)) + GS_SELECT_FD_SET_W(peer->gs); +} + +// SERVER receiving DL from client +static void +pkt_cb_dl(uint8_t chn, const uint8_t *data, size_t len, void *arg) +{ + struct _gs_ft_dl *d = (struct _gs_ft_dl *)data; + struct _gs_ft_dl hdr; + struct _peer *peer = (struct _peer *)arg; + GS_FT *ft = &peer->ft; + + if (len < sizeof hdr + 1) + return; // protocol error + + memcpy(&hdr, data, sizeof hdr); + GS_FT_dl_add_file(ft, ntohl(hdr.id), (char *)d->name, len - sizeof hdr - 1, ntohll(hdr.offset)); + if (GS_FT_WANT_WRITE(ft)) + GS_SELECT_FD_SET_W(peer->gs); +} + + +// SERVER receiving a LIST request from client +static void +pkt_cb_listrequest(uint8_t chn, const uint8_t *data, size_t len, void *arg) +{ + DEBUGF_B("LIST-REQUEST received!\n"); + struct _gs_ft_list_request *lr = (struct _gs_ft_list_request *)data; + struct _gs_ft_list_request hdr; + struct _peer *peer = (struct _peer *)arg; + GS_FT *ft = &peer->ft; + + if (len < sizeof hdr + 1) + return; // protocol error + + memcpy(&hdr, data, sizeof hdr); + GS_FT_list_add_files(ft, ntohl(hdr.globbing_id), (char *)lr->pattern, len - sizeof hdr - 1); + if (GS_FT_WANT_WRITE(ft)) + GS_SELECT_FD_SET_W(peer->gs); +} + +// CLIENT receiving answer to LIST request +static void +pkt_cb_listreply(uint8_t chn, const uint8_t *data, size_t len, void *arg) +{ + // DEBUGF_B("LIST-REPLY received\n"); + struct _gs_ft_list_reply *lr = (struct _gs_ft_list_reply *)data; + struct _gs_ft_list_reply hdr; + struct _peer *peer = (struct _peer *)arg; + GS_FT *ft = &peer->ft; + + if (len < sizeof hdr + 1) + return; + + memcpy(&hdr, data, sizeof hdr); + GS_FT_list_add(ft, ntohl(hdr.globbing_id), (char *)lr->name, len - sizeof hdr - 1, ntohll(hdr.fsize), ntohl(hdr.mtime), ntohl(hdr.fperm), hdr.flags); + if (GS_FT_WANT_WRITE(ft)) + GS_SELECT_FD_SET_W(peer->gs); +} + +static void +pkt_cb_switch(uint8_t chn, const uint8_t *data, size_t len, void *arg) +{ + struct _gs_ft_switch hdr; + struct _peer *peer = (struct _peer *)arg; + GS_FT *ft = &peer->ft; + + if (len < sizeof hdr) + return; // protocol error + + memcpy(&hdr, data, sizeof hdr); + GS_FT_switch(ft, ntohl(hdr.id), ntohll(hdr.offset)); +} + +/* SERVER receiving DATA from client */ +static void +pkt_cb_data(uint8_t chn, const uint8_t *data, size_t len, void *arg) +{ + struct _peer *peer = (struct _peer *)arg; + GS_FT *ft = &peer->ft; + + // DEBUGF("Data chn %d, len %zu\n", chn, len); + GS_FT_data(ft, data, len); + if (GS_FT_WANT_WRITE(ft)) + GS_SELECT_FD_SET_W(peer->gs); +} + +/* PUT, CLIENT receiving ACCEPT from server */ +static void +pkt_cb_accept(uint8_t chn, const uint8_t *data, size_t len, void *arg) +{ + struct _gs_ft_accept hdr; + struct _peer *peer = (struct _peer *)arg; + GS_FT *ft = &peer->ft; + + if (len < sizeof hdr) + return; // protocol error + + memcpy(&hdr, data, sizeof hdr); + GS_FT_accept(ft, ntohl(hdr.id), ntohll(hdr.offset_dst)); + if (GS_FT_WANT_WRITE(ft)) + GS_SELECT_FD_SET_W(peer->gs); +} + +// Client & Server +static void +pkt_cb_error(uint8_t chn, const uint8_t *data, size_t len, void *arg) +{ + struct _gs_ft_error *p = (struct _gs_ft_error *)data; + struct _gs_ft_error hdr; + struct _peer *peer = (struct _peer *)arg; + GS_FT *ft = &peer->ft; + + if (len < sizeof hdr + 1) + return; // protocol error + + memcpy(&hdr, data, sizeof hdr); + GS_FT_status(ft, ntohl(hdr.id), hdr.code, (char *)p->str, len - sizeof hdr - 1); +} + +// Output total stats +static void +print_stats_ft(GS_FT_stats *st) +{ + DEBUGF_Y("Speed : %s\n", st->speed_str); + DEBUGF_Y("Amount : %"PRIu64"/%"PRIu64"usec\n", st->xfer_amount, st->xfer_duration); + DEBUGF_Y("Success: %d\n", st->n_files_success); + DEBUGF_Y("Errors : %d\n", st->n_files_error); +} + +// On file transfer completion (for each file) +static void +cb_stats(struct _gs_ft_stats_file *s, void *arg) +{ + struct _peer *p = (struct _peer *)arg; + + DEBUGF_C("%u stats: %s\n", s->id, s->fname); + DEBUGF_C("Speed: %s\n", s->speed_str); + print_stats_ft(&p->ft.stats); + + char buf[256]; + snprintf(buf, sizeof buf, "[%s] %s", s->speed_str, s->fname); + GS_condis_log(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, buf); + GS_condis_draw(&gs_condis, 1); +} + +// Status and Error messages +static void +cb_errors(void *ft_ptr, struct _gs_ft_status *s, void *arg) +{ + // struct _peer *p = (struct _peer *)arg; + + DEBUGF_M("Code : %u\n", s->code); + DEBUGF_M("Error : '%s'\n", s->err_str); + DEBUGF_M("File : %s\n", s->fname); + + // COMPLETE messages are not displayed as errors. + if (s->code == GS_FT_ERR_COMPLETED) + return; + + // char buf[256]; + // snprintf(buf, sizeof buf, "(%u)%s", s->code, s->err_str); + GS_condis_log(&gs_condis, GS_PKT_APP_LOG_TYPE_NOTICE, s->err_str); + GS_condis_draw(&gs_condis, 1); +} + +// Return length of packet created. +// Return 0 on if not packet available. +// Return -1 if all files have been transfered (client only. Server never stops +// as server is always waiting for instructions by client to get/put more files). +// Return -2 if not enough space. +ssize_t +GS_FTM_mk_packet(GS_FT *ft, uint8_t *dst, size_t dlen) +{ + struct gs_pkt_chn_hdr *hdr = (struct gs_pkt_chn_hdr *)dst; + int pkt_type; + size_t max_len; + char buf[256]; + GS_FT_stats *st = &ft->stats; + + if (dlen <= sizeof *hdr) + return -2; // Insuficient space. + + max_len = MIN(GS_PKT_MAX_SIZE, dlen - sizeof *hdr); + + ssize_t sz; + sz = GS_FT_packet(ft, dst + sizeof *hdr, max_len, &pkt_type); + + switch (pkt_type) + { + case GS_FT_TYPE_NONE: + // Nothing to write...waiting for peer's reply. + // DEBUGF_G("TYPE NONE\n"); + return 0; + case GS_FT_TYPE_DONE: + // CLIENT only: done with all files. + // FIXME: for a 'get' request this is triggered very late and not as soon + // as the filetransfer is done (because select() only waits for reading + // and while there is no data to write then GS_FTM_mk_packet is only called after + // select() returns after 1sec idle. + if (st->n_files_success + st->n_files_error >= 2) // Summary for Multi-file transfers only + { + // FIXME: turn /sec into h/min/sec etc (e.g. 99999.8h or 99.4 minutes or 99.5 sec or 0.102 sec) + if (st->n_files_success > 0) + { + float ms = (float)st->xfer_duration / 1000; + snprintf(buf, sizeof buf, "OK: %d/%d [%s] (%"PRIu64"/%1.03fsec)", st->n_files_success, st->n_files_success + st->n_files_error, st->speed_str, st->xfer_amount, ms / 1000); + GS_condis_log(&gs_condis, GS_PKT_APP_LOG_TYPE_INFO, buf); + } + if (st->n_files_error > 0) + { + snprintf(buf, sizeof buf, "FAILED: %d", st->n_files_error); + GS_condis_log(&gs_condis, GS_PKT_APP_LOG_TYPE_ALERT, buf); + } + GS_condis_draw(&gs_condis, 1); + } + if (st->n_files_success + st->n_files_error > 0) + GS_FT_stats_reset(ft); + + // DEBUGF_G("All done (%d/%d)\n", st->n_files_success, st->n_files_error); + return -1; + case GS_FT_TYPE_PUT: + hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_PUT); + break; + case GS_FT_TYPE_ERROR: + hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_ERROR); + break; + case GS_FT_TYPE_SWITCH: + hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_SWITCH); + break; + case GS_FT_TYPE_ACCEPT: + hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_ACCEPT); + break; + case GS_FT_TYPE_DATA: + hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_DATA); + break; + case GS_FT_TYPE_LISTREQUEST: + // GET (download), CLIENT + hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_LIST_REQUEST); + break; + case GS_FT_TYPE_LISTREPLY: + // SERVER (reply to 'get' (list request)) + hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_LIST_REPLY); + break; + case GS_FT_TYPE_DL: + // CLIENT - download by filename + hdr->type = GS_PKT_CHN2TYPE(GS_FT_CHN_DL); + break; + default: + ERREXIT("unknown type %d\n", pkt_type); + } + + hdr->esc = GS_PKT_ESC; + uint16_t len = htons(sz); + memcpy(&hdr->len, &len, sizeof len); + + return sz + sizeof *hdr; +} + +void +GS_FTM_init(struct _peer *p, int is_server) +{ + GS_PKT_assign_chn(&p->pkt, GS_FT_CHN_ERROR, pkt_cb_error, p); + if (is_server == 0) + { + // CLIENT + GS_FT_init(&p->ft, cb_stats, cb_errors, 0 /*pid, unused*/, p, 0); + + // PUT (upload) + GS_PKT_assign_chn(&p->pkt, GS_FT_CHN_ACCEPT, pkt_cb_accept, p); + + // GET (download) + GS_PKT_assign_chn(&p->pkt, GS_FT_CHN_LIST_REPLY, pkt_cb_listreply, p); + GS_PKT_assign_chn(&p->pkt, GS_FT_CHN_DATA, pkt_cb_data, p); + GS_PKT_assign_chn(&p->pkt, GS_FT_CHN_SWITCH, pkt_cb_switch, p); + } else { + // SERVER + GS_FT_init(&p->ft, NULL, cb_errors, p->pid, p, 1); + + // PUT (upload) + GS_PKT_assign_chn(&p->pkt, GS_FT_CHN_PUT, pkt_cb_put, p); + GS_PKT_assign_chn(&p->pkt, GS_FT_CHN_DATA, pkt_cb_data, p); + GS_PKT_assign_chn(&p->pkt, GS_FT_CHN_SWITCH, pkt_cb_switch, p); + + // GET (download) + GS_PKT_assign_chn(&p->pkt, GS_FT_CHN_LIST_REQUEST, pkt_cb_listrequest, p); + GS_PKT_assign_chn(&p->pkt, GS_FT_CHN_DL, pkt_cb_dl, p); + } +} + +void +GS_FTM_free(struct _peer *p) +{ + GS_FT_free(&p->ft); +} + + + diff --git a/tools/filetransfer_mgr.h b/tools/filetransfer_mgr.h new file mode 100644 index 00000000..099701bc --- /dev/null +++ b/tools/filetransfer_mgr.h @@ -0,0 +1,8 @@ +#ifndef __GS_FILETRANSFER_MGR_H__ +#define __GS_FILETRANSFER_MGR_H__ 1 + +void GS_FTM_init(struct _peer *p, int is_server); +void GS_FTM_free(struct _peer *p); +ssize_t GS_FTM_mk_packet(GS_FT *ft, uint8_t *dst, size_t dlen); + +#endif // __GS_FILETRANSFER_MGR_H__ \ No newline at end of file diff --git a/tools/globbing.c b/tools/globbing.c new file mode 100644 index 00000000..90981db8 --- /dev/null +++ b/tools/globbing.c @@ -0,0 +1,393 @@ +/* + * Create a file list. Used by filetransfer.c (which is used by + * gs-netcat.c) + * + * Used 'wordexp(3)' where available. Alernatively use 'sh -c echo $str' + * (not implemented) + */ + +#include "common.h" +#include +#include +#include "utils.h" +#include "globbing.h" + +static void +gl_add_file(gsglobbing_cb_t func, GS_GL *res) +{ + (*func)(res); +} + +static void +gl_add_dir(gsglobbing_cb_t func, GS_GL *res) +{ + (*func)(res); +} + +static void +gl_dir(gsglobbing_cb_t func, const char *path, GS_GL *res) +{ + DIR *d; + + d = opendir(path); + if (d == NULL) + return; + + struct dirent *entry; + for (entry = readdir(d); entry != NULL; entry = readdir(d)) + { + char fullname[4096]; + int is_dir = 0; + int is_reg = 0; + res->name = fullname; + snprintf(fullname, sizeof fullname, "%s/%s", path, entry->d_name); +#ifdef __sun + // Solaris does not support d_type. Use stat() + struct stat sr; + + if (stat(fullname, &sr) == 0) + { + if (S_ISDIR(sr.st_mode)) + is_dir = 1; + else if (S_ISREG(sr.st_mode)) + is_reg = 1; + } else { + DEBUGF_R("FAIL %s\n", entry->d_name); + continue; + } +#else + if (entry->d_type == DT_DIR) + is_dir = 1; + else if (entry->d_type == DT_REG) + is_reg = 1; +#endif + + if (is_dir) + { + if ((strlen(entry->d_name) == 1) && (entry->d_name[0] == '.')) + continue; + if ((strlen(entry->d_name) == 2) && (memcmp(entry->d_name, "..", 2) == 0)) + continue; + gl_add_dir(func, res); + gl_dir(func, fullname, res); // recursive call + } else if (is_reg) { + gl_add_file(func, res); + } else { + DEBUGF_R("FAIL (is_reg=%d, is_dir=%d) %s\n", is_reg, is_dir, entry->d_name); + } + } + + closedir(d); +} + +static void +gs_gl(gsglobbing_cb_t func, const char *path, GS_GL *res) +{ + DEBUGF("Glob(%s)\n", path); + + int rv; + struct stat s; + rv = stat(path, &s); + // if stat() fails then still send this result back to caller. + // This can happen when caller requests "notexist.dat". + + res->name = path; + // If file then call callback. + if ((rv != 0) || S_ISREG(s.st_mode)) + { + gl_add_file(func, res); + return; + } + + // If directory then call callback, enter directory and traverse + if (S_ISDIR(s.st_mode)) + { + gl_add_dir(func, res); + gl_dir(func, path, res); + return; + } + + return; +} + + +// GET (download) +// assume CWD==/tmp +// get /tmp/f* should create foo/bar/test4k.dat [file /tmp/./foo/bar/test4k.dat] +// get foo/*.dat should create test1k.dat [file /tmp/foo/./test1k.dat] +// get foo/./bar/test4k.dat should create bar/test4k.dat [file /tmp/foo/bar/test4k.dat] +// get $(find /tmp/ -name *.dat) should create bar/test4k.dat and test1k.dat by using +// smallest common base (/tmp/foo) +// [file /tmp/foo/./bar/test4k.dat /tmp/foo/./test1k.dat] + +// The CWD might change between the LIST-request and actual file request. Thus any file-list returned +// to the downloading client must contain the _absolute_ path to the file (rather than relative). +// - If file is a relative request then prefix with '${CWD}/./'. +// - If file is absolute request then add '/./' (see below how to add this). + +// => Add '/./' last. + +// PUT (upload) + +// Test Cases: +// ./filetransfer-test G '$(echo dir1/foo.txt dir2)' + +// How to insert '/./' +// Wordexp() might do command substituion (replacing $(command) by the output of command). This +// needs consideration of where to add the '/./'. +// - If fname already contains '/./' then do nothing. + +int +GS_GLOBBING(gsglobbing_cb_t func, const char *exp, uint32_t glob_id, void *arg_ptr, uint32_t arg_val) +{ + // wordexp(3) to expand expressions like '*.[ch]'. + wordexp_t p; + int i; + char **w; + char buf[4096]; + int n_found; + char *wdir; + + GS_GL res; + res.globbing_id = glob_id; + res.arg_ptr = arg_ptr; + res.arg_val = arg_val; + + wdir = getcwd(NULL, 0); + + DEBUGF("G#%u GS_GLOBBING('%s') [wdir='%s']\n", glob_id, exp, wdir); + + // Special case: Globbing loop below discards '.' from globbing results + if (strcmp(exp, ".") == 0) + exp = "./"; + + // Test Setup: + // mkdir -p /tmp/foo/dir_empty /tmp/foo/dir1 + // cp /etc/hosts /tmp/foo/dir1/ + // cp /etc/passwd /tmp/foo/.rcfile + + // CASES for globbing and what file structure to create at _destination_: + // put /tmp/f* should create foo/bar/test4k.dat but not /tmp/foo/bar/test4k.dat + // put /tmp/foo should create foo/* (and .rcfile) + // put /tmp/foo/* should create ./* (but not foo/* and not .rcfile) + // put /tmp/foo/ should create ./* (and .rcfile) [[** SPECIAL CASE**]] + // put /tmp/foo/. should create ./* (and .rcfile) + // put /tmp/foo/bar/test4k.dat should create test4k.dat + // put /tmp/./foo/* should create foo/* (but not .rcfile) + // put /tmp/./foo/ should create foo/* (and .rcfile) + // put /tmp/./foo should create foo/* (and .rcfile) + // put /./tmp/foo should create ./tmp/foo/* (and .rcfile) + // cd /tmp/foo; put . should create ./* (and .rcfile) + // put $(find /tmp/ -name *.dat) should based on smallest common directory structure (base). + // -> /tmp/foo/bar/test1k.dat /tmp/foo/test4k.dat then base => /tmp/foo + // -> /tmp/test1k.dat /tmp/foo/bar/test1k.dat then base => /tmp + int ret; + signal(SIGCHLD, SIG_DFL); + ret = wordexp(exp, &p, 0); + signal(SIGCHLD, SIG_IGN); + if (ret != 0) + return 0; // error (0 found) + + w = p.we_wordv; + n_found = p.we_wordc; + + if (n_found <= 0) + goto done; + + char base_buf[4096]; + char *base = base_buf; + + + // Find the smallest common name of directory. This is needed + // when wordexp(3) does command subs such as '$(find /tmp -name \*.dat)' + // + // /tmp/foo/test1k.dat /tmp/foo/bar/test4k.dat + // base => /tmp/foo + // + // foo/bar/test4k.dat foo/test1k.dat + // base => foo + // + // foo/bar/test4k.dat foo/test1k.dat test8k.dat + // base => "" (empty) + snprintf(base_buf, sizeof base_buf, "%s", w[0]); + char *ptr; + ptr = rindex(base, '/'); + if (ptr != NULL) + { + *ptr = '\0'; + for (i = 1; i < p.we_wordc; i++) + { + int ii; + for (ii = 0; base[ii] == w[i][ii]; ii++) + { + continue; + } + + // DEBUGF("last identical at %d [%s]\n", ii, w[i]); + // if it was "/tmp/dir1" and "/tmp/dir2" then make base => /tmp + // if it was /tmp/foo/bar.txt and /tmp/foo/dir2 then make base => /tmp/foo + for (; ii > 0 && w[i][ii] != '/'; ii--) + { + continue; + } + // DEBUGF("/ at %d\n", ii); + base[ii] = '\0'; // trailing '/' or empty string (ii==0) + if (ii <= 0) + break; + } + } else { + base[0] = '\0'; + } + DEBUGF_C("base = '%s'\n", base); + + for (i = 0; i < p.we_wordc; i++) + { + // Check if '/tmp/foo/.*' was specified and ignore expansion for those cases: + // 1. /tmp/foo/.* shall not glob /tmp/foo/.. or /tmp/foo/. but only /tmp/foo/.rcfile + // 2. '.*' shall not glob '..' or '.' but only '.rcfile' + // 3. 'testfile.' shall still glob 'testfile.' [note the '.' at the end of the filename] + size_t sz = strlen(w[i]); + if (sz >= 3) + { + // Special case: put /tmp/foo/.* shall not glob /tmp/foo/.. [/tmp/] + if (memcmp(w[i] + sz - 3, "/..", 3) == 0) + continue; + } + if (sz >= 2) + { + // Special case: put /tmp/foo/.* shall not glob /tmp/foo/. but only /tmp/foo/.rcfile + if (memcmp(w[i] + sz - 2, "/.", 2) == 0) + continue; + if ((sz == 2) && memcmp(w[i], "..", 2) == 0) + continue; + } + if ((sz == 1) && (memcmp(w[i], ".", 1) == 0)) + continue; + + // Add '/./' + char *fname = w[i]; + DEBUGF_C("fname = '%s'\n", fname); + + // GET request of "./foo/bar/test1k.dat" should be treated as "foo/bar/test1k.dat" + // and create file 'foo/bar/./test1k.dat'. + + // PUT request of "././foo/bar/test1k.dat" should + // and create file '././foo/bar/test1k.dat' + while (1) + { + if ((base[0] == 0) || (base[1] == 0)) + break; + if ((base[0] == '.') && (base[1] == '/')) + base += 2; + break; + } + DEBUGF_C("BASE = %s\n", base); + + int is_absolute = 0; + ptr = fname; + while (*ptr == '/') + { + ptr++; + is_absolute = 1; // Starts with '/' + } + + if (strstr(fname, "/./") == NULL) + { + // Does NOT contain '/./'. + // $ get 'foo/*' + // wdir=/tmp, base=foo => /tmp/foo/bar/test4k.dat & /tmp/foo/test1k.dat + // base = foo + // $ get '$(find //tmp/foo -name \*.dat)' + ptr = rindex(ptr, '/'); + if (ptr == NULL) + { + // single file without directory structure + if (is_absolute) + { + // "/test[14]k.dat" + DEBUGF_Y("1\n"); + snprintf(buf, sizeof buf, "/./%s", fname); + } else { + // "test[14]k.dat" + DEBUGF_Y("2\n"); + snprintf(buf, sizeof buf, "%s/./%s", wdir, fname); + } + } else { + // HERE: fname contains '/' (e.g tmp/foo or foo/bar/test1k.dat) + if (is_absolute) + { + // "/tmp/foo/" or "/tmp/foo/*" or "/tmp/fo*" + sz = strlen(base_buf); + XASSERT(sz <= strlen(fname), "fname to short\n"); + DEBUGF_Y("3\n"); + SNPRINTF_ABORT(buf, sizeof buf, "%s/.%s", base, fname + sz); + } else { + if (base[0] == '\0') + { + // $(echo dir1/ dir2/) + DEBUGF_Y("4\n"); + snprintf(buf, sizeof buf, "%s/./%s", wdir, fname); + } else { + // "foo/*" + sz = strlen(base_buf); + XASSERT(sz <= strlen(fname), "fname to short.\n"); + DEBUGF_Y("5\n"); + SNPRINTF_ABORT(buf, sizeof buf, "%s/%s/.%s", wdir, base, fname + sz); + } + } + } + fname = buf; + } else { + // HERE: Contains '/./' + if (is_absolute) + { + // '/etc/./ssh/ssh_config' + snprintf(buf, sizeof buf, "%s", fname); // as is + } else { + // foo/./bar/test1k.dat + sz = strlen(base_buf); + XASSERT(sz <= strlen(fname), "fname to short.\n"); + DEBUGF_Y("6\n"); + SNPRINTF_ABORT(buf, sizeof buf, "%s/%s%s", wdir, base, fname + sz); + } + fname = buf; + } + + DEBUGF_C("Globb result '%s' -> '%s'\n", w[i], fname); + gs_gl(func, fname, &res); + } + +done: + XFREE(wdir); + wordfree(&p); + + return n_found; +} + +static uint32_t glob_id = 0xFFFFFFFF; + +int +GS_GLOBBING_argv(gsglobbing_cb_t func, const char *argv[], void *arg_ptr, uint32_t arg_val) +{ + const char *ptr; + + if (argv == NULL) + return -1; // Bad parameter + + for (ptr = *argv; ptr != NULL; argv++) + { + GS_GLOBBING(func, ptr, glob_id, arg_ptr, arg_val); + glob_id--; + ptr = *argv; + } + + return 0; +} + +#if 0 +int +GS_GLOBBING_free(GS_GL *ctx) +{ + return 0; +} +#endif \ No newline at end of file diff --git a/tools/globbing.h b/tools/globbing.h new file mode 100644 index 00000000..f9c5088b --- /dev/null +++ b/tools/globbing.h @@ -0,0 +1,24 @@ +#ifndef __GS_GLOBBING_H__ +#define __GS_GLOBBING_H__ 1 + +typedef struct +{ + const char *name; + mode_t mode; + uint32_t globbing_id; + void *arg_ptr; + uint32_t arg_val; +} GS_GL; + +struct _gs_gl +{ + int res; + void *func; +}; + +typedef void (gsglobbing_cb_t)(GS_GL *res); + +int GS_GLOBBING(gsglobbing_cb_t func, const char *path, uint32_t glob_id, void *arg_ptr, uint32_t arg_val); +int GS_GLOBBING_argv(gsglobbing_cb_t func, const char *argv[], void *arg_ptr, uint32_t arg_val); + +#endif /* !__GS_GLOBBING_H__ */ \ No newline at end of file From 3a14bf74dc1a22793e2fc1d570ab1cf78f04ebb0 Mon Sep 17 00:00:00 2001 From: rootTHC Date: Fri, 12 Feb 2021 08:38:03 +0000 Subject: [PATCH 16/21] solaris10 stable# --- lib/gsocket-util.c | 7 +++++-- tests/Makefile | 2 +- tests/run_ft_tests.sh | 2 +- tools/console.c | 4 ++-- tools/filetransfer.c | 13 +++++++++---- tools/globbing.c | 11 ++++++++--- tools/gs_uchroot.c | 2 +- tools/utils.c | 5 +---- 8 files changed, 28 insertions(+), 18 deletions(-) diff --git a/lib/gsocket-util.c b/lib/gsocket-util.c index c420f6c6..51c14352 100755 --- a/lib/gsocket-util.c +++ b/lib/gsocket-util.c @@ -257,7 +257,7 @@ GS_getpidwd(pid_t pid) procstat_freefiles(procstat, head); #else - // Linux + // Linux & other unix (solaris etc) char buf[1024]; char res[PATH_MAX + 1]; ssize_t sz; @@ -270,7 +270,10 @@ GS_getpidwd(pid_t pid) #endif err: if (wd == NULL) - wd = getcwd(NULL, 0); + { + wd = getcwd(NULL, PATH_MAX + 1); + XASSERT(wd != NULL, "getcwd(): %s\n", strerror(errno)); // hard fail + } DEBUGF_W("PID %d CWD=%s\n", pid, wd); return wd; } diff --git a/tests/Makefile b/tests/Makefile index 0694792b..a31d8b0e 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,6 +1,6 @@ clean: - rm -rf id_sec.txt gs_nc test*.dat client_err*.txt server_err*.txt client_out*.dat server_out*.dat ft_test_dst ft_test_src + rm -rf id_sec.txt gs_nc test*.dat client*.txt server*.txt client*.dat server*.dat ft_test_dst ft_test_src >server.logclient.logfn_local = realfname; f->fz_local = res.st_size; @@ -695,7 +700,7 @@ GS_FT_get(GS_FT *ft, const char *pattern) p = calloc(1, sizeof *p); p->pattern = strdup(pattern); - p->wdir = getcwd(NULL, 0); + p->wdir = getcwd(NULL, PATH_MAX + 1); p->globbing_id = ft->g_id; GS_LIST_add(&ft->plistreq, NULL, p, ft->g_id); diff --git a/tools/globbing.c b/tools/globbing.c index 90981db8..aac2523d 100644 --- a/tools/globbing.c +++ b/tools/globbing.c @@ -153,7 +153,13 @@ GS_GLOBBING(gsglobbing_cb_t func, const char *exp, uint32_t glob_id, void *arg_p res.arg_ptr = arg_ptr; res.arg_val = arg_val; - wdir = getcwd(NULL, 0); + wdir = getcwd(NULL, PATH_MAX + 1); + if (wdir == NULL) + { + DEBUGF_R("getcwd(): %s\n", strerror(errno)); + return 0; + } + DEBUGF("G#%u GS_GLOBBING('%s') [wdir='%s']\n", glob_id, exp, wdir); @@ -358,9 +364,8 @@ GS_GLOBBING(gsglobbing_cb_t func, const char *exp, uint32_t glob_id, void *arg_p } done: - XFREE(wdir); wordfree(&p); - + XFREE(wdir); return n_found; } diff --git a/tools/gs_uchroot.c b/tools/gs_uchroot.c index 310cb561..af178ea8 100755 --- a/tools/gs_uchroot.c +++ b/tools/gs_uchroot.c @@ -120,7 +120,7 @@ thc_init(void) /* OSX's getcwd() calls stat() */ char *ptr; - ptr = getcwd(NULL, 0); + ptr = getcwd(NULL, PATH_MAX + 1); if (ptr == NULL) exit(123); if (realpath(ptr, rp_cwd) == NULL) diff --git a/tools/utils.c b/tools/utils.c index 6d854601..9196ce26 100755 --- a/tools/utils.c +++ b/tools/utils.c @@ -67,7 +67,7 @@ add_env_argv(int *argcptr, char **argvptr[]) *argvptr = newargv; DEBUGF("Total argv[] == %d\n", newargc); int i; - for (i = 0; i < newargc + 1; i++) + for (i = 0; i < newargc; i++) DEBUGF("argv[%d] = %s\n", i, newargv[i]); } @@ -163,7 +163,6 @@ init_vars(void) if ((str != NULL) && (strlen(str) > 0)) VLOG("=Extra arguments: '%s'\n", str); - DEBUGF("sec_str %s\n", gopt.sec_str); gopt.sec_str = GS_user_secret(&gopt.gs_ctx, gopt.sec_file, gopt.sec_str); if (gopt.sec_str == NULL) ERREXIT("%s\n", GS_CTX_strerror(&gopt.gs_ctx)); @@ -835,8 +834,6 @@ fd_cmd(const char *cmd, pid_t *pidptr) int fds[2]; int ret; - DEBUGF("cmd == %s\n", cmd); - if (gopt.is_interactive) { return pty_cmd(cmd, pidptr); From 8e203aff66ffdc313f14cfd888c20c7b7cdc925d Mon Sep 17 00:00:00 2001 From: rootTHC Date: Fri, 12 Feb 2021 10:30:53 +0000 Subject: [PATCH 17/21] clnearnup# --- lib/gsocket-util.c | 1 + tools/filetransfer.c | 2 +- tools/utils.c | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/gsocket-util.c b/lib/gsocket-util.c index 51c14352..855d635e 100755 --- a/lib/gsocket-util.c +++ b/lib/gsocket-util.c @@ -266,6 +266,7 @@ GS_getpidwd(pid_t pid) sz = readlink(buf, res, sizeof res - 1); if (sz < 0) goto err; + res[sz] = '\0'; wd = strdup(res); #endif err: diff --git a/tools/filetransfer.c b/tools/filetransfer.c index 115aac72..0b260aad 100644 --- a/tools/filetransfer.c +++ b/tools/filetransfer.c @@ -667,7 +667,7 @@ GS_FT_list_add_files(GS_FT *ft, uint32_t globbing_id, const char *pattern, size_ if (ret != 0) { - DEBUGF_R("chrdir(%s): %s\n", ptr, strerror(errno)); + DEBUGF_R("chdir(%s): %s\n", ptr, strerror(errno)); snprintf(err, sizeof err, "chdir(%s): %s\n", ptr, strerror(errno)); qerr_add(ft, globbing_id, errno2code(errno, GS_FT_ERR_NOENT), err); diff --git a/tools/utils.c b/tools/utils.c index 9196ce26..ee28b3ae 100755 --- a/tools/utils.c +++ b/tools/utils.c @@ -798,7 +798,7 @@ pty_cmd(const char *cmd, pid_t *pidptr) * HISTFILE= does not work on oh-my-zsh (it sets it again) */ char *env_blacklist[] = {"STY", "GSOCKET_ARGS", "HISTFILE", NULL}; - char *env_addlist[] = {shell_env, "TERM=xterm-256color", "HISTFILE=\"\"", home_env, NULL}; + char *env_addlist[] = {shell_env, "TERM=xterm-256color", "HISTFILE=/dev/null", home_env, NULL}; char **envp = mk_env(env_blacklist, env_addlist); if (cmd != NULL) From eb23906ba349060fad5bb2042cc724e65f3a5898 Mon Sep 17 00:00:00 2001 From: rootTHC Date: Fri, 12 Feb 2021 12:09:46 +0000 Subject: [PATCH 18/21] fixed tab and other problems in readline()# --- lib/gs-readline.c | 4 ++++ tools/console.c | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/gs-readline.c b/lib/gs-readline.c index 8d18b6de..14243402 100644 --- a/lib/gs-readline.c +++ b/lib/gs-readline.c @@ -238,7 +238,10 @@ GS_RL_add(GS_RL_CTX *rl, uint8_t c, uint8_t *key, int row, int col) // Unhandled control character if ((c < 0x20) || (c > 0x7E)) + { + DEBUGF("Unhandled: 0x%02x\n", c); goto ret_unhandled; + } if (rl->pos >= GS_RL_LINE_MAX) return 1; @@ -252,6 +255,7 @@ GS_RL_add(GS_RL_CTX *rl, uint8_t c, uint8_t *key, int row, int col) return 1; ret_unhandled: + rl->esc_len = 0; *key = k; return -1; } diff --git a/tools/console.c b/tools/console.c index e069fdd0..1c3b9549 100644 --- a/tools/console.c +++ b/tools/console.c @@ -1028,7 +1028,6 @@ CONSOLE_readline(struct _peer *p, void *data, size_t len) for (; src < s_end; src++) { rv = GS_RL_add(&rl, *src, &key, gopt.winsize.ws_row, 1 + GS_CONSOLE_PROMPT_LEN); - // HEXDUMP(rl.esc_data, rl.esc_len); if (write(fd, rl.esc_data, rl.esc_len) != rl.esc_len) ERREXIT("write()\n"); From a206a1c451f9745c7baad2349ee77220dc8372ba Mon Sep 17 00:00:00 2001 From: rootTHC Date: Mon, 15 Feb 2021 21:55:24 +0000 Subject: [PATCH 19/21] static compile --- configure.ac | 69 ++++++++++++++++++++++++++++--------------- tests/run_ft_tests.sh | 5 +++- tools/Makefile.am | 12 +++++--- 3 files changed, 58 insertions(+), 28 deletions(-) diff --git a/configure.ac b/configure.ac index f9a4f34d..c41ece52 100755 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -${EXEEXT}dnl Process this File with autoconf to produce a configure script. +dnl Process this File with autoconf to produce a configure script. AC_PREREQ(2.61) AC_INIT([gsocket], 1.4.23-ft2) dnl AC_CONFIG_AUX_DIR(config-x86_64-apple-darwin19.6.0) @@ -80,18 +80,6 @@ mips-sony-bsd|mips-sony-newsos4) ;; esac -AC_CHECK_LIB(util, forkpty) -dnl AC_CHECK_LIB(bsd, main) -AC_CHECK_LIB(socket, socket) -AC_CHECK_LIB(nsl, gethostbyname) -dnl AC_CHECK_LIB([m], [sqrt]) -dnl AC_CHECK_LIB([net], [libnet_name_resolve], [AC_MSG_ERROR([libnet 1.0.x found. Requires libnet 1.1 or newer])]) -dnl AC_CHECK_LIB([net], [libnet_init], ,[AC_MSG_ERROR([libnet 1.1.x not found])]) -AC_CHECK_LIB(dl, dlopen) -AC_CHECK_LIB(procstat, procstat_close) -AC_CHECK_LIB([crypto], [ENGINE_init], [], [AC_MSG_ERROR([libcrypto not found])]) -AC_CHECK_LIB([ssl], [SRP_VBASE_get1_by_user], [], [AC_MSG_ERROR([SRP not supported. Please upgrade OpenSSL lib])]) - dnl Checks for header files. AC_HEADER_STDC AC_HEADER_SYS_WAIT @@ -113,6 +101,30 @@ AC_TYPE_SIZE_T dnl If uid_t is not defined, define uid_t to be int and gid_t to be int. AC_TYPE_UID_T +AC_ARG_ENABLE(static, +[ --enable-static Compile static binary], + [STATIC="yes"], [STATIC="no"] +) + +if test x"$STATIC" = x"yes"; then + dnl static OpenSSL lib requires pthread + dnl AC_CHECK_LIB(pthread, pthread_once) + dnl LDADD_STATIC="" dnl -static" + CFLAGS_STATIC="-static " +fi + +AC_CHECK_LIB(util, forkpty) +AC_CHECK_LIB(socket, socket) +AC_CHECK_LIB(nsl, gethostbyname) +dnl AC_CHECK_LIB([net], [libnet_name_resolve], [AC_MSG_ERROR([libnet 1.0.x found. Requires libnet 1.1 or newer])]) +dnl AC_CHECK_LIB([net], [libnet_init], ,[AC_MSG_ERROR([libnet 1.1.x not found])]) +if test x"$STATIC" = xno; then + AC_CHECK_LIB(dl, dlopen) +fi +AC_CHECK_LIB(procstat, procstat_close) +AC_CHECK_LIB([crypto], [ENGINE_init], [], [AC_MSG_ERROR([libcrypto not found])]) +AC_CHECK_LIB([ssl], [SRP_VBASE_get1_by_user], [], [AC_MSG_ERROR([SRP not supported. Please upgrade OpenSSL lib])]) + AC_CHECK_FUNCS(gettimeofday memcpy strchr strlcat forkpty openpty getline stat64 open64 statvfs64) AC_ARG_ENABLE([31337], @@ -140,18 +152,11 @@ AC_ARG_ENABLE(dist, [DIST="yes"], [DIST="no"] ) -AC_ARG_ENABLE(static, -[ --enable-static Compile static binary], - [STATIC="yes"], [STATIC="no"] -) - -if test x"$STATIC" = x"yes"; then - CFLAGS="-static ${CFLAGS}" -fi - AS_IF([test x$selftests = xtrue], AC_SUBST(PROGRAMS_TEST_LIB, "list-test${EXEEXT} event-test${EXEEXT}")) AS_IF([test x$selftests = xtrue], AC_SUBST(PROGRAMS_TEST_TOOLS, "packet-test${EXEEXT} readline-test${EXEEXT} console_display-test${EXEEXT} filetransfer-test${EXEEXT}")) +AC_SUBST(LDADD_STATIC, "${LDADD_STATIC}") +AC_SUBST(CFLAGS_STATIC, "${CFLAGS_STATIC}") AC_OUTPUT([Makefile lib/Makefile tools/Makefile man/Makefile]) echo " @@ -162,12 +167,30 @@ echo " --acpizer/United Cracking Force " +if test x"${STATIC}" = xyes; then +echo " +********************************** WARNING *********************************** +* Compiling STATICLY disables proper use of blitz, gs-sftp etc. * +* Do not do it unless you know what you are doing... * +* * +* Your system needs staticly compiled versions of OpenSSL etc. * +* If in doubt compile them first: * +* openssl-src> * +* ./Configure --prefix=\$HOME/usr no-dso no-threads no-shared linux-generic64 * +* openssl-src> mkdir -p \$HOME/usr && make all install * +* gsocket-src> ./configure --prefix=\$HOME/usr --enable-static * +* gsocket-src> make all install * +* gsocket-src> export PATH=\$HOME/usr/bin:\$PATH * +****************************************************************************** +" +fi + echo " ${PACKAGE_NAME}-${PACKAGE_VERSION} has been configured: Host..............: ${host} Compiler..........: ${CC} - Compiler flags....: ${CFLAGS} + Compiler flags....: ${CFLAGS_STATIC}${CFLAGS} Preprocessor flags: ${CPPFLAGS} Linker flags......: ${LDFLAGS} Libraries.........: ${LIBS} diff --git a/tests/run_ft_tests.sh b/tests/run_ft_tests.sh index d2df8981..5532dae1 100755 --- a/tests/run_ft_tests.sh +++ b/tests/run_ft_tests.sh @@ -35,7 +35,10 @@ BIN="${BINDIR}/filetransfer-test" [[ -f "${BIN}" ]] || { echo "${BIN} not found. Try ./configure --enable-tests"; exit 255; } -command -v socat >/dev/null 2>&1 || { echo >&2 "socat not installed. ${SKIP}"; exit 0; } +if [[ $(uname) =~ Darwin ]]; then + export PATH=$HOME/usr/bin:$PATH +fi +command -v socat >/dev/null 2>&1 || { $ECHO >&2 "socat not installed. ${SKIP}"; exit 0; } mk_dummy() diff --git a/tools/Makefile.am b/tools/Makefile.am index d9891013..5f88400d 100755 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -3,16 +3,20 @@ EXTRA_PROGRAMS = packet-test readline-test console_display-test filetransfer-tes bin_PROGRAMS = gs-netcat gs_uchroot_so gs_helloworld_SOURCES = 1_gs-helloworld.c utils.c -gs_helloworld_LDADD = ../lib/libgsocket.a +gs_helloworld_LDADD = ../lib/libgsocket.a @LDADD_STATIC@ +gs_helloworld_CFLAGS = @CFLAGS_STATIC@ gs_pipe_SOURCES = 2_gs-pipe.c utils.c -gs_pipe_LDADD = ../lib/libgsocket.a +gs_pipe_LDADD = ../lib/libgsocket.a @LDADD_STATIC@ +gs_pipe_CFLAGS = @CFLAGS_STATIC@ gs_full_pipe_SOURCES = 3_gs-full-pipe.c utils.c -gs_full_pipe_LDADD = ../lib/libgsocket.a +gs_full_pipe_LDADD = ../lib/libgsocket.a @LDADD_STATIC@ +gs_ful_pipe_CFLAGS = @CFLAGS_STATIC@ gs_netcat_SOURCES = 4_gs-netcat.c utils.c socks.c console.c ids.c event_mgr.c pkt_mgr.c console_display.c filetransfer.c globbing.c filetransfer_mgr.c -gs_netcat_LDADD = ../lib/libgsocket.a +gs_netcat_LDADD = ../lib/libgsocket.a @LDADD_STATIC@ +gs_netcat_CFLAGS = @CFLAGS_STATIC@ dist_bin_SCRIPTS = blitz gs-sftp gs-mount gs_funcs From 69237b5f48562c83110c4e1e145e695d979715ad Mon Sep 17 00:00:00 2001 From: rootTHC Date: Thu, 18 Feb 2021 10:38:00 +0000 Subject: [PATCH 20/21] terminal fixes --- tools/common.h | 1 + tools/console.c | 177 ++++++++++++++++++++++++++++++++++++--- tools/console.h | 4 +- tools/console_display.c | 7 +- tools/console_display.h | 2 +- tools/filetransfer.c | 27 +++--- tools/filetransfer_mgr.c | 6 +- tools/pkt_mgr.c | 2 + 8 files changed, 196 insertions(+), 30 deletions(-) diff --git a/tools/common.h b/tools/common.h index 9162cb61..c1dea370 100755 --- a/tools/common.h +++ b/tools/common.h @@ -66,6 +66,7 @@ #if defined __sun || defined __hpux /* Solaris, HP-UX */ # include #endif +#include #include #include #include diff --git a/tools/console.c b/tools/console.c index 1c3b9549..cebdacde 100644 --- a/tools/console.c +++ b/tools/console.c @@ -10,6 +10,8 @@ * https://www.linuxquestions.org/questions/programming-9/get-cursor-position-in-c-947833/ */ #include "common.h" +#include +#include #include "pkt_mgr.h" #include "console.h" #include "console_display.h" @@ -43,7 +45,7 @@ static int is_console_cursor_needs_reset; static const char *sb_color = "\x1B[44m\x1B[30m"; // Black on Blue #define GS_CONSOLE_BUF_SIZE (1024) -#define GS_CONDIS_ROWS (3) +#define GS_CONDIS_ROWS (GS_CONSOLE_ROWS - 2) struct _console_info { @@ -430,6 +432,12 @@ GS_prompt_cursor(void) tty_write(buf, ptr - buf); } +void +CONSOLE_draw(int fd) +{ + console_draw(fd, 0); +} + static void console_draw(int fd, int force) { @@ -1041,12 +1049,10 @@ CONSOLE_readline(struct _peer *p, void *data, size_t len) } else if (key == 'A') { DEBUGF_Y("UP\n"); GS_condis_up(&gs_condis); - GS_condis_draw(&gs_condis, 1); - console_cursor_on(); + CONSOLE_draw(gs_condis.fd); } else if (key == 'B') { GS_condis_down(&gs_condis); - GS_condis_draw(&gs_condis, 1); - console_cursor_on(); + CONSOLE_draw(gs_condis.fd); } /* Unhandled control character */ continue; @@ -1164,11 +1170,14 @@ CONSOLE_action(struct _peer *p, uint8_t key) GS_condis_pos(&gs_condis, (gopt.winsize.ws_row - GS_CONSOLE_ROWS) + 1 + 1, gopt.winsize.ws_col); if (is_console_welcome_msg == 0) { - GS_condis_add(&gs_condis, 0, "Press Ctrl-e + DOWN to enter the console. Then type 'help'."); - GS_condis_add(&gs_condis, 0, "Press Ctrl-e + c to close the console or Ctrl-e + q to quit."); + GS_condis_add(&gs_condis, 0, "Press Ctrl-e + c to close this console or Ctrl-e + q to quit."); GS_condis_add(&gs_condis, 0, "Press Ctrl-e + UP to leave the console."); + GS_condis_add(&gs_condis, 0, "Press Ctrl-e + DOWN to enter the console."); + GS_condis_add(&gs_condis, 0, "Use UP/DOWN to scroll through the console's log"); + GS_condis_add(&gs_condis, 0, "Type 'help' for a list of commands."); is_console_welcome_msg = 1; } + console_cursor_on(); // Start with cursor in console // Draw console needed? Resizing remote will trigger a CLEAR (=> re-draw) mk_statusbar(); console_draw(p->fd_out, 1); @@ -1181,8 +1190,10 @@ static void cmd_help(int fd) { GS_condis_add(&gs_condis, 0, "quit - Quit | Ctrl-e q : quit | Ctrl-e c : toggle console"); - GS_condis_add(&gs_condis, 0, "put - Upload file | Ctrl-e UP: Go Up |"); - GS_condis_add(&gs_condis, 0, "get - Download file | Ctrl-e DN: Go Down |"); + GS_condis_add(&gs_condis, 0, "ping - RTT to peer | Ctrl-e UP: Go Up | Ctrl-e DN: Go Down"); + GS_condis_add(&gs_condis, 0, "put - Upload file - Example: put /usr/./share/ma*"); + GS_condis_add(&gs_condis, 0, "get - Download file - Example: get ~/*.[ch]"); + GS_condis_add(&gs_condis, 0, "Other commands: lls, lcd, lmkdir, lpwd, pwd"); GS_condis_draw(&gs_condis, 1); } @@ -1221,6 +1232,136 @@ path_resolve(const char *pattern, char *dst, size_t len) return 0; } + +static const char * +strip_space(const char *str) +{ + while (*str == ' ') + str++; + return str; +} + +// Output single file information +static void +cmd_lls_file(const char *name) +{ + char buf[1024]; + struct stat sr; + if (stat(name, &sr) != 0) + { + // ERROR + snprintf(buf, sizeof buf, "%s: %s", strerror(errno), name); + GS_condis_add(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, buf); + return; + } +#ifdef __APPLE__ + struct timespec ts = sr.st_mtimespec; +#else + struct timespec ts = sr.st_mtim; +#endif + + struct tm tm; + localtime_r(&ts.tv_sec, &tm); + // MS-DOS style output (oldskewl) + char tmstr[32]; + strftime(tmstr, sizeof tmstr, "%Y-%m-%d %H:%M", &tm); + const char *typestr = "<\?\?\?>"; + if (S_ISDIR(sr.st_mode)) + typestr = ""; + else if (S_ISLNK(sr.st_mode)) + typestr = ""; + else if (S_ISFIFO(sr.st_mode)) + typestr = ""; + else if (S_ISBLK(sr.st_mode)) + typestr = ""; + else if (S_ISCHR(sr.st_mode)) + typestr = ""; + else if (S_ISREG(sr.st_mode)) + typestr = ""; + + snprintf(buf, sizeof buf, "%16s %5.5s %' 16lld %s", tmstr, typestr, sr.st_size, name); + GS_condis_add(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, buf); +} + +// List local files. +static void +cmd_lls_single(const char *exp) +{ + wordexp_t p; + char **w; + DIR *d = NULL; + char buf[PATH_MAX]; + + int ret; + signal(SIGCHLD, SIG_DFL); + ret = wordexp(exp, &p, 0); + signal(SIGCHLD, SIG_IGN); + if (ret != 0) + return; // error (0 found) + + setlocale(LC_NUMERIC, ""); // for printf("'%d" thausand separator + + w = p.we_wordv; + // If there is only ONE result and that result is a DIRECTORY then output the content + // of that directory instead. (e.g. 'ls .' or 'ls /tmp') + struct stat sr; + if ((p.we_wordc == 1) && (stat(w[0], &sr) == 0) && S_ISDIR(sr.st_mode)) + { + // Opendir etc.. + d = opendir(w[0]); + if (d == NULL) + { + // ERROR + snprintf(buf, sizeof buf, "%s: %s", strerror(errno), w[0]); + GS_condis_add(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, buf); + goto err; + } + + struct dirent *entry; + for (entry = readdir(d); entry != NULL; entry = readdir(d)) + { + if (memcmp(w[0], ".\0", 2) == 0) + snprintf(buf, sizeof buf, "%s", entry->d_name); + else + snprintf(buf, sizeof buf, "%s/%s", w[0], entry->d_name); + cmd_lls_file(buf); + } + } else { + int i; + for (i = 0; i < p.we_wordc; i++) + cmd_lls_file(w[i]); + } + +err: + if (d != NULL) + closedir(d); + + wordfree(&p); +} + +static void +cmd_lls(const char *str) +{ + char *orig = strdup(str); + char *next; + char *name = orig; + + + while (name != NULL) + { + next = strchr(name, ' '); + if (next != NULL) + { + *next = '\0'; + next += 1; + } + cmd_lls_single(name); + name = next; + } + + XFREE(orig); +} + static int console_command(struct _peer *p, const char *cmd) { @@ -1229,6 +1370,7 @@ console_command(struct _peer *p, const char *cmd) char path[PATH_MAX + 1]; char *end = buf + sizeof (buf); char *ptr; + const char *arg; int row = gopt.winsize.ws_row - (GS_CONSOLE_ROWS - 1); if (strlen(cmd) <= 0) @@ -1261,7 +1403,8 @@ console_command(struct _peer *p, const char *cmd) XFREE(cwd); GS_condis_draw(&gs_condis, 1); } else if (strncmp(cmd, "lcd ", 4) == 0) { - path_resolve(cmd + 4, path, sizeof path); + arg = strip_space(cmd + 4); + path_resolve(arg, path, sizeof path); if (chdir(path) != 0) snprintf(buf, sizeof buf, "%s: %.512s", strerror(errno), path); else { @@ -1271,6 +1414,20 @@ console_command(struct _peer *p, const char *cmd) } GS_condis_add(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, buf); GS_condis_draw(&gs_condis, 1); + } else if (strncmp(cmd, "lmkdir ", 7) == 0) { + arg = strip_space(cmd + 7); + if (mkdir(arg, 0777) != 0) + { + snprintf(buf, sizeof buf, "%s: %.512s", strerror(errno), arg); + GS_condis_add(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, buf); + GS_condis_draw(&gs_condis, 1); + } + } else if (strncmp(cmd, "lls", 3) == 0) { + arg = strip_space(cmd + 3); + if (*arg == 0) + arg = "."; // 'lls' should be 'lls .' (current directory) + cmd_lls(arg); + GS_condis_draw(&gs_condis, 1); } else { snprintf(buf, sizeof buf, "Command not known: '%s'", cmd); GS_condis_add(&gs_condis, 0, buf); diff --git a/tools/console.h b/tools/console.h index 7113a99f..a6036e60 100644 --- a/tools/console.h +++ b/tools/console.h @@ -14,7 +14,7 @@ #define GS_CONSOLE_ESC_LCHR 'e' #define GS_CONSOLE_ESC_STR "^E" -#define GS_CONSOLE_ROWS (5) // 1x Line + 4 real rows +#define GS_CONSOLE_ROWS (8) // 1x Line + 4 real rows ssize_t CONSOLE_write(int fd, void *data, size_t len); int CONSOLE_check_esc(uint8_t c, uint8_t *submit); @@ -23,7 +23,7 @@ int CONSOLE_command(struct _peer *p, const char *cmd); void CONSOLE_reset(void); void CONSOLE_resize(struct _peer *p); int CONSOLE_readline(struct _peer *p, void *data, size_t len); - +void CONSOLE_draw(int fd); void CONSOLE_update_pinginfo(struct _peer *p, float ms, int load, char *active_user, int sec_idle, uint8_t n_users); void CONSOLE_update_bps(struct _peer *p); diff --git a/tools/console_display.c b/tools/console_display.c index d3d9ecef..73b73f72 100644 --- a/tools/console_display.c +++ b/tools/console_display.c @@ -41,6 +41,7 @@ GS_condis_add(GS_CONDIS *cd, int level, const char *str) size_t len; struct condis_line *cdl = &cd->cdl[cd->pos_add]; + DEBUGF("-> '%s'\n", str); len = MIN(sizeof cdl->line - 1, strlen(str)); memcpy(cdl->line, str, len); cdl->line[len] = 0x00; @@ -65,6 +66,7 @@ GS_condis_add(GS_CONDIS *cd, int level, const char *str) cd->is_redraw_needed = 1; } +// Add " " to console display. void GS_condis_log(GS_CONDIS *cd, int level, const char *str) { @@ -106,7 +108,7 @@ GS_condis_up(GS_CONDIS *cd) int max_scroll; int scroll; max_scroll = cd->entries - (apos - dpos); - scroll = MIN(3, max_scroll); + scroll = MIN(cd->rows, max_scroll); cd->pos_display = (dpos + CONDIS_MAX_HISTORY - scroll) % CONDIS_MAX_HISTORY; cd->is_redraw_needed = 1; @@ -136,9 +138,12 @@ cd_write(int fd, void *buf, size_t len) if (write(fd, buf, len) != len) ERREXIT("write()\n"); } + /* * Draw the console at position and with each string * up to max_char length. Add '..' if string is longer... + * + * THIS WILL LEAVE THE CURSOR ASTRAY. Use CONSOLE_draw() to correct cursor position. */ void GS_condis_draw(GS_CONDIS *cd, int force) diff --git a/tools/console_display.h b/tools/console_display.h index d1198ff5..c2726d17 100644 --- a/tools/console_display.h +++ b/tools/console_display.h @@ -2,7 +2,7 @@ #define __CONSOLE_DISPLAY_H__ 1 #define CONDIS_LINE_MAX_LEN (125) -#define CONDIS_MAX_HISTORY (64) +#define CONDIS_MAX_HISTORY (256) struct condis_line { diff --git a/tools/filetransfer.c b/tools/filetransfer.c index 0b260aad..c3948ad9 100644 --- a/tools/filetransfer.c +++ b/tools/filetransfer.c @@ -1279,7 +1279,7 @@ mk_stats_file(GS_FT *ft, uint32_t id, struct _gs_ft_file *f, const char *name, i DEBUGF_R("Oops, Reporting stats on a suspended file\n"); f->usec_suspend_duration += (GS_usec() - f->usec_suspend_start); } - DEBUGF_W("end=%"PRIu64", start=%"PRIu64", suspend=%"PRIu64"\n", f->usec_end, f->usec_start, f->usec_suspend_duration); + DEBUGF_W("err=%d end=%"PRIu64", start=%"PRIu64", suspend=%"PRIu64"\n", err, f->usec_end, f->usec_start, f->usec_suspend_duration); s.xfer_duration = (f->usec_end - f->usec_start) - f->usec_suspend_duration; } @@ -1401,11 +1401,12 @@ GS_FT_status(GS_FT *ft, uint32_t id, uint8_t code, const char *err_str, size_t l } } } + } else { + // HERE: Client was waiting for "COMPLETED" message from server. + // Was waiting for 'complete' signal. No error if 'complete' is a success. + if (code == GS_FT_ERR_COMPLETED) + err = 0; } - } else { - // Was waiting for 'complete' signal. No error if 'complete' is a success. - if (code == GS_FT_ERR_COMPLETED) - err = 0; } const char *name; @@ -1430,7 +1431,7 @@ GS_FT_status(GS_FT *ft, uint32_t id, uint8_t code, const char *err_str, size_t l if (f != NULL) { - // Report stats to caller + // Report stats to caller. Tread ERR_COMPLETED not as an error. mk_stats_file(ft, id, f, name, err); } } @@ -1536,12 +1537,12 @@ status_report_error(GS_FT *ft, const char *name, uint8_t code) * Make an error packet. Remove errornous file from queue */ static size_t -ft_mk_error(GS_FT *ft, struct _gs_ft_file *f, void *dst, size_t len, int *pkt_type, GS_LIST_ITEM *li, uint8_t code, const char *str) +ft_mk_error(GS_FT *ft, struct _gs_ft_file *f, void *dst, size_t len, int *pkt_type, GS_LIST_ITEM *li, uint8_t code, const char *str, int is_report_local) { size_t sz; // On CLIENT side report this error to caller: - if (ft->is_server == 0) + if ((ft->is_server == 0) && (is_report_local)) { // CLIENT status_report_error(ft, f->fn_relative, code); @@ -1586,7 +1587,7 @@ mk_xfer_data(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fcompleted, int *p if (sz <= 0) { *active = NULL; - return ft_mk_error(ft, f, dst, len, pkt_type, f->li, errno2code(errno, GS_FT_ERR_BADF), NULL); + return ft_mk_error(ft, f, dst, len, pkt_type, f->li, errno2code(errno, GS_FT_ERR_BADF), NULL, 1); } f->fz_remote += sz; @@ -1622,12 +1623,12 @@ mk_switch(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fsource, GS_LIST *fco if (f->fp == NULL) { DEBUGF("fopen(%s): %s\n", f->fn_local, strerror(errno)); - return ft_mk_error(ft, f, dst, len, pkt_type, li, errno2code(errno, GS_FT_ERR_PERM), NULL); + return ft_mk_error(ft, f, dst, len, pkt_type, li, errno2code(errno, GS_FT_ERR_PERM), NULL, 1); } ret = fseek(f->fp, 0, SEEK_END); if (ret != 0) - return ft_mk_error(ft, f, dst, len, pkt_type, li, errno2code(errno, GS_FT_ERR_BADF), NULL); + return ft_mk_error(ft, f, dst, len, pkt_type, li, errno2code(errno, GS_FT_ERR_BADF), NULL, 1); f->fz_local = ftell(f->fp); // Peer already has this file. @@ -1637,7 +1638,7 @@ mk_switch(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fsource, GS_LIST *fco DEBUGF("#%u Skipping %s (already on peer)\n", (unsigned int)f->li->id, f->name); if (ft->is_server == 0) mk_stats_file(ft, li->id, f, NULL, 0 /*success*/); - return ft_mk_error(ft, f, dst, len, pkt_type, li, GS_FT_ERR_NODATA, NULL); + return ft_mk_error(ft, f, dst, len, pkt_type, li, GS_FT_ERR_NODATA, NULL, 0 /* do not report locally */); } // Remote size is larger. Overwrite from beginning. @@ -1647,7 +1648,7 @@ mk_switch(GS_FT *ft, struct _gs_ft_file **active, GS_LIST *fsource, GS_LIST *fco // Remote size is smaller. Restart transmission. ret = fseek(f->fp, f->fz_remote, SEEK_SET); if (ret != 0) - return ft_mk_error(ft, f, dst, len, pkt_type, li, errno2code(errno, GS_FT_ERR_BADF), NULL); + return ft_mk_error(ft, f, dst, len, pkt_type, li, errno2code(errno, GS_FT_ERR_BADF), NULL, 1); struct _gs_ft_switch sw; memset(&sw, 0, sizeof sw); diff --git a/tools/filetransfer_mgr.c b/tools/filetransfer_mgr.c index 41b8c673..d005ad73 100644 --- a/tools/filetransfer_mgr.c +++ b/tools/filetransfer_mgr.c @@ -172,7 +172,7 @@ cb_stats(struct _gs_ft_stats_file *s, void *arg) char buf[256]; snprintf(buf, sizeof buf, "[%s] %s", s->speed_str, s->fname); GS_condis_log(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, buf); - GS_condis_draw(&gs_condis, 1); + CONSOLE_draw(gs_condis.fd); } // Status and Error messages @@ -192,7 +192,7 @@ cb_errors(void *ft_ptr, struct _gs_ft_status *s, void *arg) // char buf[256]; // snprintf(buf, sizeof buf, "(%u)%s", s->code, s->err_str); GS_condis_log(&gs_condis, GS_PKT_APP_LOG_TYPE_NOTICE, s->err_str); - GS_condis_draw(&gs_condis, 1); + CONSOLE_draw(gs_condis.fd); } // Return length of packet created. @@ -243,7 +243,7 @@ GS_FTM_mk_packet(GS_FT *ft, uint8_t *dst, size_t dlen) snprintf(buf, sizeof buf, "FAILED: %d", st->n_files_error); GS_condis_log(&gs_condis, GS_PKT_APP_LOG_TYPE_ALERT, buf); } - GS_condis_draw(&gs_condis, 1); + CONSOLE_draw(gs_condis.fd); } if (st->n_files_success + st->n_files_error > 0) GS_FT_stats_reset(ft); diff --git a/tools/pkt_mgr.c b/tools/pkt_mgr.c index 883dcb4a..a2faf3d0 100644 --- a/tools/pkt_mgr.c +++ b/tools/pkt_mgr.c @@ -79,6 +79,7 @@ pkt_app_cb_log(uint8_t msg, const uint8_t *data, size_t len, void *ptr) sanitize_fname_to_str(log->msg, sizeof log->msg); GS_condis_log(&gs_condis, log->type, (const char *)log->msg); + CONSOLE_draw(gs_condis.fd); DEBUGF_G("LOG (%d) '%s'\n", log->type, log->msg); } @@ -125,6 +126,7 @@ pkt_app_cb_pwdreply(uint8_t chn, const uint8_t *data, size_t len, void *ptr) DEBUGF_B("REMOTE WD=%s\n", data); GS_condis_add(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, (char *)data); + CONSOLE_draw(gs_condis.fd); } int From e6f2e1e3926df876a0c474384c6321dca8ba9cf4 Mon Sep 17 00:00:00 2001 From: rootTHC Date: Thu, 18 Feb 2021 14:20:39 +0000 Subject: [PATCH 21/21] man page --- configure.ac | 11 ++++----- lib/gsocket-util.c | 9 ++++++-- man/gs-netcat.1 | 48 +++++++++++++++++++++++++++++++++++++++- tests/run_gs_tests.sh | 5 ++++- tools/console.c | 6 ++--- tools/filetransfer.c | 4 +--- tools/filetransfer_mgr.c | 2 +- tools/globbing.c | 2 +- tools/gs_uchroot.c | 8 ++++++- tools/utils.c | 11 +++++++++ tools/utils.h | 1 + 11 files changed, 87 insertions(+), 20 deletions(-) diff --git a/configure.ac b/configure.ac index c41ece52..d406e40f 100755 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ dnl Process this File with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([gsocket], 1.4.23-ft2) +AC_INIT([gsocket], 1.4.23-ft3) dnl AC_CONFIG_AUX_DIR(config-x86_64-apple-darwin19.6.0) AC_CONFIG_AUX_DIR(config) AC_CANONICAL_SYSTEM @@ -170,14 +170,11 @@ echo " if test x"${STATIC}" = xyes; then echo " ********************************** WARNING *********************************** -* Compiling STATICLY disables proper use of blitz, gs-sftp etc. * -* Do not do it unless you know what you are doing... * -* * -* Your system needs staticly compiled versions of OpenSSL etc. * -* If in doubt compile them first: * +* Your MUST compile OpenSSL like this: * * openssl-src> * * ./Configure --prefix=\$HOME/usr no-dso no-threads no-shared linux-generic64 * -* openssl-src> mkdir -p \$HOME/usr && make all install * +* mkdir -p \$HOME/usr && make all install * +* Only then compile gsocket \(using the same --prefix=\): * * gsocket-src> ./configure --prefix=\$HOME/usr --enable-static * * gsocket-src> make all install * * gsocket-src> export PATH=\$HOME/usr/bin:\$PATH * diff --git a/lib/gsocket-util.c b/lib/gsocket-util.c index 855d635e..5d8169be 100755 --- a/lib/gsocket-util.c +++ b/lib/gsocket-util.c @@ -262,7 +262,7 @@ GS_getpidwd(pid_t pid) char res[PATH_MAX + 1]; ssize_t sz; - snprintf(buf, sizeof buf, "/proc/%d/cwd", pid); + snprintf(buf, sizeof buf, "/proc/%d/cwd", (int)pid); sz = readlink(buf, res, sizeof res - 1); if (sz < 0) goto err; @@ -272,7 +272,12 @@ GS_getpidwd(pid_t pid) err: if (wd == NULL) { - wd = getcwd(NULL, PATH_MAX + 1); + #if defined(__sun) && defined(HAVE_OPEN64) + // This is solaris 10 + wd = getcwd(NULL, PATH_MAX + 1); // solaris10 segfaults if size is 0... + #else + wd = getcwd(NULL, 0); + #endif XASSERT(wd != NULL, "getcwd(): %s\n", strerror(errno)); // hard fail } DEBUGF_W("PID %d CWD=%s\n", pid, wd); diff --git a/man/gs-netcat.1 b/man/gs-netcat.1 index ecb3fa21..8e4521c9 100755 --- a/man/gs-netcat.1 +++ b/man/gs-netcat.1 @@ -78,12 +78,58 @@ IPv4 address for port forwarding. .It Fl p Ar port TCP port to listen on or to forward traffic to. .It Fl i -Interactive login shell. The server spawns a true PTY login shell. The client acts as a true PTY client (with Ctrl-C etc working). The client can terminate the session by typing \'~.\' at any time or by typing \'exit\'. The server supports multiple clients at the same time. +Interactive login shell. The server spawns a true PTY login shell. The client acts as a true PTY client (with Ctrl-C etc working). The client can terminate the session by typing \'Ctrl-e q\' at any time or by typing \'exit\'. The server supports multiple clients at the same time. .El .Pp .Ar port can be a numerical value between 1-65535. +.Sh CONSOLE +Pressing \'Ctrl-e c\' (e for EEEElite) opens the command console. The command console displays the following information: +.Pp +.Bl -bullet -offset indent -compact +.It +Latency (in milliseconds) to the remote host +.It +Warning when a user logs into the system or becomes active +.It +Data troughput +.It +File transfer logs +.El +Type \'help\' for a list of available commands. + +.Sh FILETRANSFER +File transfer is available from the command console. Files are transfered with the permission and modification timestamp unchanged. Partially transfered files are re-started where the transfer was left off. + +The 'put' command is used for uploading: + +.Dl put foobar.txt +.Dl put $HOME/foobar.txt +.Dl put /tmp/*.log +.Dl put $(find . -type f -name '*.c') + +(The above example shows Shell Variable substitution and word expansion) + +It is possible to limit the amount of path information that is sent as implied directories for each path you specify. You can insert a dot and a slash into the source path, like this: + +.Dl put /foo/./bar/baz.c + +That would create /tmp/bar/baz.c on the remote machine. + +The 'get' command is used for downloading: + +.Dl get foobar.txt +.Dl get $(find /var/./ -name '*.log') + +Transfering a directory automatically transfers all files and directories within that directory (recursively): + +.Dl get /var/log +.Dl get / +The first command transfers all directories and files in /var/log/*. The latter command transfers the entire filesystem. + +Multiple get/put commands can be scheduled at the same time. + .Sh EXAMPLES .Nm Example 1 - Listen for a new connection using the password \'MySecret\': diff --git a/tests/run_gs_tests.sh b/tests/run_gs_tests.sh index 94e78b21..894b9d90 100755 --- a/tests/run_gs_tests.sh +++ b/tests/run_gs_tests.sh @@ -28,6 +28,7 @@ SLEEP_WD=20 # Max seconds to wait for a process to finish receiving... command -v md5 >/dev/null 2>&1 && MD5(){ md5 -q "${1}";} command -v md5sum >/dev/null 2>&1 && MD5() { md5sum "${1}" | cut -f1 -d' ';} command -v bc >/dev/null 2>&1 || { echo >&2 "bc not installed. apt-get install bc."; exit 255; } +command -v rsync >/dev/null 2>&1 || { echo >&2 "rsync not installed. apt-get install rsync."; exit 255; } # Use traditional netcat that supports "netcat -nlp" for cross-platform comp. # on CentOS there is only nmap's netcat as 'nc' but we are expecting 'netcat()'. if [[ "$(nc --version 2>&1)" =~ Ncat ]]; then @@ -719,7 +720,9 @@ else if command -v fusermount >/dev/null 2>&1; then fusermount -zu test_mnt else - umount -f test_mnt + # archLinux -f flag needs superuser (bug in umount) + umount test_mnt &>/dev/null + umount -f test_mnt &>/dev/null fi kill $GSPID2 rm -rf test_client diff --git a/tools/console.c b/tools/console.c index cebdacde..292104d1 100644 --- a/tools/console.c +++ b/tools/console.c @@ -1279,7 +1279,7 @@ cmd_lls_file(const char *name) else if (S_ISREG(sr.st_mode)) typestr = ""; - snprintf(buf, sizeof buf, "%16s %5.5s %' 16lld %s", tmstr, typestr, sr.st_size, name); + snprintf(buf, sizeof buf, "%16s %5.5s %' 16"PRId64" %s", tmstr, typestr, (int64_t)sr.st_size, name); GS_condis_add(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, buf); } @@ -1398,7 +1398,7 @@ console_command(struct _peer *p, const char *cmd) GS_condis_add(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, "Thanks xaitax for testing!"); GS_condis_draw(&gs_condis, 1); } else if (strncmp(cmd, "lpwd", 4) == 0) { - char *cwd = getcwd(NULL, PATH_MAX + 1); + char *cwd = getcwdx(); GS_condis_add(&gs_condis, GS_PKT_APP_LOG_TYPE_DEFAULT, cwd); XFREE(cwd); GS_condis_draw(&gs_condis, 1); @@ -1408,7 +1408,7 @@ console_command(struct _peer *p, const char *cmd) if (chdir(path) != 0) snprintf(buf, sizeof buf, "%s: %.512s", strerror(errno), path); else { - char *cwd = getcwd(NULL, PATH_MAX + 1); + char *cwd = getcwdx(); snprintf(buf, sizeof buf, "%s", cwd); XFREE(cwd); } diff --git a/tools/filetransfer.c b/tools/filetransfer.c index c3948ad9..d2406ad5 100644 --- a/tools/filetransfer.c +++ b/tools/filetransfer.c @@ -700,7 +700,7 @@ GS_FT_get(GS_FT *ft, const char *pattern) p = calloc(1, sizeof *p); p->pattern = strdup(pattern); - p->wdir = getcwd(NULL, PATH_MAX + 1); + p->wdir = getcwdx(); p->globbing_id = ft->g_id; GS_LIST_add(&ft->plistreq, NULL, p, ft->g_id); @@ -1335,13 +1335,11 @@ call_status_cb(GS_FT *ft, const char *fname, uint8_t code, const char *err_str) // Create error string if none provided snprintf(buf, sizeof buf, "%s: %s", GS_FT_strerror(code), fname); err_str = buf; - DEBUGF_R("string1='%s'\n", err_str); } struct _gs_ft_status s; memset(&s, 0, sizeof s); s.fname = fname; s.code = code; - DEBUGF_R("string2='%s'\n", err_str); s.err_str = err_str; (*ft->func_status)(ft, &s, ft->func_arg); diff --git a/tools/filetransfer_mgr.c b/tools/filetransfer_mgr.c index d005ad73..10fd0a90 100644 --- a/tools/filetransfer_mgr.c +++ b/tools/filetransfer_mgr.c @@ -243,11 +243,11 @@ GS_FTM_mk_packet(GS_FT *ft, uint8_t *dst, size_t dlen) snprintf(buf, sizeof buf, "FAILED: %d", st->n_files_error); GS_condis_log(&gs_condis, GS_PKT_APP_LOG_TYPE_ALERT, buf); } - CONSOLE_draw(gs_condis.fd); } if (st->n_files_success + st->n_files_error > 0) GS_FT_stats_reset(ft); + CONSOLE_draw(gs_condis.fd); // DEBUGF_G("All done (%d/%d)\n", st->n_files_success, st->n_files_error); return -1; case GS_FT_TYPE_PUT: diff --git a/tools/globbing.c b/tools/globbing.c index aac2523d..05082789 100644 --- a/tools/globbing.c +++ b/tools/globbing.c @@ -153,7 +153,7 @@ GS_GLOBBING(gsglobbing_cb_t func, const char *exp, uint32_t glob_id, void *arg_p res.arg_ptr = arg_ptr; res.arg_val = arg_val; - wdir = getcwd(NULL, PATH_MAX + 1); + wdir = getcwdx(); if (wdir == NULL) { DEBUGF_R("getcwd(): %s\n", strerror(errno)); diff --git a/tools/gs_uchroot.c b/tools/gs_uchroot.c index af178ea8..fa9d49a4 100755 --- a/tools/gs_uchroot.c +++ b/tools/gs_uchroot.c @@ -120,7 +120,13 @@ thc_init(void) /* OSX's getcwd() calls stat() */ char *ptr; - ptr = getcwd(NULL, PATH_MAX + 1); +#if defined(__sun) && defined(HAVE_OPEN64) + // This is solaris 10 + ptr = getcwd(NULL, PATH_MAX + 1); // solaris10 segfaults if size is 0... +#else + ptr = getcwd(NULL, 0); +#endif + if (ptr == NULL) exit(123); if (realpath(ptr, rp_cwd) == NULL) diff --git a/tools/utils.c b/tools/utils.c index ee28b3ae..844f3978 100755 --- a/tools/utils.c +++ b/tools/utils.c @@ -238,6 +238,17 @@ zap_arg(char *str) memset(str, '*', len); } +char * +getcwdx(void) +{ +#if defined(__sun) && defined(HAVE_OPEN64) + // This is solaris 10 + return getcwd(NULL, PATH_MAX + 1); // solaris10 segfaults if size is 0... +#else + return getcwd(NULL, 0); +#endif +} + void do_getopt(int argc, char *argv[]) { diff --git a/tools/utils.h b/tools/utils.h index e9ab1c6a..4de1c956 100755 --- a/tools/utils.h +++ b/tools/utils.h @@ -23,6 +23,7 @@ void cmd_pwd(struct _peer *p); // void sanitze_name_to_string(uint8_t *str, size_t len); void sanitize_fname_to_str(uint8_t *str, size_t len); void format_bps(char *buf, size_t size, int64_t bytes); +char *getcwdx(void); #define VLOG(a...) do{if (gopt.log_fp != NULL){ fprintf(gopt.log_fp, a); fflush(gopt.log_fp); } }while(0)