diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d6f0af..dc1ded8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## v0.2.0 + +- Add Spack package for pdwfs [b6cb1f2] +- Update README.md [1e062df] +- Add an example for using pdwfs with SLURM job scheduler [b2839c1] +- Update SLURM helpers scripts [3aca9ae] +- Increase default stripe size to 50MB [9907704] +- Merge branch 'feature/custom-redigo-client' into develop [6705d0c] +- Fix performance issues on write and read [8a54135] +- Optimize write with Redis Set cmd if whole buffer is to be written (faster) [bd34304] +- Refactor tests to improve isolation and avoid need for running Redis beforehand [9ea7596] +- Refactor the C layer to move all "triage" into C layer and minimize CGo cross-layer calls [1f53f0c] + ## v0.1.2 - Refactor the inode layer to remove a lock [0b573de] diff --git a/Makefile b/Makefile index a126db0..9069284 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ BUILDDIR=build #install dir PREFIX ?= /usr/local -all: dirs pdwfslibc scripts +all: dirs pdwfslibc tools dirs: mkdir -p $(BUILDDIR) @@ -14,13 +14,17 @@ pdwfslibc: pdwfsgo pdwfsgo: make -C src/go -.PHONY: scripts -scripts: scripts/pdwfs +.PHONY: tools +tools: tools/pdwfs mkdir -p $(BUILDDIR)/bin - install scripts/pdwfs $(BUILDDIR)/bin/ - install scripts/pdwfs-redis $(BUILDDIR)/bin/ + install tools/pdwfs $(BUILDDIR)/bin/ + install tools/pdwfs-slurm $(BUILDDIR)/bin/ + install tools/redis.srun $(BUILDDIR)/bin/ + #cd tools/pv && ./configure && make # if pv needs rebuild + install tools/pv/pv $(BUILDDIR)/bin/ -test: scripts pdwfslibc + +test: tools pdwfslibc make -C src/go test make -C src/c test @@ -32,12 +36,16 @@ install: pdwfslibc install -d $(PREFIX)/lib $(PREFIX)/bin install $(BUILDDIR)/lib/libpdwfs_go.so $(PREFIX)/lib install $(BUILDDIR)/lib/pdwfs.so $(PREFIX)/lib - install scripts/pdwfs $(PREFIX)/bin - install scripts/pdwfs-redis $(PREFIX)/bin + install tools/pdwfs $(PREFIX)/bin + install tools/pdwfs-slurm $(PREFIX)/bin + install tools/redis.srun $(PREFIX)/bin chmod +x $(PREFIX)/bin/* tag: git tag -a $(TAG) -m "Version $(TAG)" dist: - scripts/create_tarballs.sh + tools/create_tarballs.sh + +bench: + make -C examples/ior_benchmark bench diff --git a/README.md b/README.md index e4d69d3..e3fd002 100644 --- a/README.md +++ b/README.md @@ -2,90 +2,106 @@ [![Build Status](https://travis-ci.org/cea-hpc/pdwfs.png?branch=master)](https://travis-ci.org/cea-hpc/pdwfs) -pdwfs (pronounced "*padawan-f-s*", see below) is a preload library implementing a minimalist filesystem in user space suitable for intercepting *bulk* I/Os typical of HPC simulations and storing data in memory in a [Redis](https://redis.io) database. +pdwfs (we like to pronounce it "*padawan-f-s*", see below) is a preload library implementing a distributed in-memory filesystem in user space suitable for intercepting *bulk* I/O workloads typical of HPC simulations. It is using [Redis](https://redis.io) as the backend memory store. -pdwfs objective is to provide a very lightweight infrastructure to execute HPC simulation workflows without writing/reading any intermediate data to/from a (parallel) filesystem. This type of approach is known as *in transit* or *loosely-coupled in situ*, see the two next sections for further details. +pdwfs objective is to provide a lightweight infrastructure to execute HPC simulation workflows without writing/reading any intermediate data to/from a (parallel) filesystem. This type of approach is known as *in transit* or *loosely-coupled in situ*, see the two next sections for further details. pdwfs is written in [Go](https://golang.org) and C and runs on Linux systems only (we provide a Dockerfile for testing and development on other systems). -Though it's a work in progress and still at an early stage of development, it can already be tested with Parallel HDF5, MPI-IO and a POSIX-based ParaView workflow. See Examples section below. +## Dependencies -## PaDaWAn project - -pdwfs is a component of the PaDaWAn project (for Parallel Data Workflow for Analysis), a [CEA](http://www.cea.fr) project that aims at providing building blocks of a lightweight and *least*-intrusive software infrastructure to facilitate *in transit* execution of HPC simulation workflows. - -The foundational work for this project was an initial version of pdwfs entierly written in Python and presented in the paper below: +pdwfs only dependencies are: -- *PaDaWAn: a Python Infrastructure for Loosely-Coupled In Situ Workflows*, J. Capul, S. Morais, J-B. Lekien, ISAV@SC (2018). +Build: +- Go version ≥ 1.11 -## In situ / in transit HPC workflows -Within the HPC community, in situ data processing is getting quite some interests as a potential enabler for future exascale-era simulations. +Runtime (and testing): +- Redis (version ≥ 5.0.3 is recommended) -The original in situ approach, also called tightly-coupled in situ, consists in executing data processing routines within the same address space as the simulation and sharing the resources with it. It requires the simulation to use a dedicated API and to link against a library embedding a processing runtime. Notable in situ frameworks are ParaView [Catalyst](https://www.paraview.org/in-situ/), VisIt [LibSim](https://wci.llnl.gov/simulation/computer-codes/visit). [SENSEI](http://sensei-insitu.org) provides a common API that can map to various in situ processing backends. -The loosely-coupled flavor of in situ, or in transit, relies on separate resources from the simulation to stage and/or process data. It requires a dedicated distributed infrastructure to extract data from the simulation and send it to a staging area or directly to consumers. Compared to the tightly-coupled in situ approach, it offers greater flexibility to adjust the resources needed by each application in the workflow (not bound to efficiently use the same resources as the simulation). It can also accommodate a larger variety of workflows, in particular those requiring memory space for data windowing (e.g. statistics, time integration). +## Installation -This latter approach, loosely-coupled in situ or in transit, is at the core of pdwfs. +### From binary distribution -## Dependencies +A binary distribution is available for Linux system and x86_64 architecture in the [releases](http://github.com/cea-hpc/pdwfs/releases) page. -pdwfs only dependencies are: -- Go version ≥ 1.11 to build pdwfs -- Redis version ≥ 5.0.3 +The following steps will install pdwfs and make it available in your PATH. +```bash +$ wget -O pdwfs.tar.gz https://github.com/cea-hpc/pdwfs/releases/download/v0.2.0/pdwfs-v0.2.0-linux-amd64.tar.gz +$ mkdir /usr/local/pdwfs +$ tar xf pdwfs.tar.gz --strip-component=1 -C /usr/local/pdwfs +$ export PATH="/usr/local/pdwfs/bin:$PATH" +``` -## Installation +### From source +A Go distribution (version ≥ 1.11) is required to build pdwfs from source (instructions can be found on the [Go download page](https://golang.org/dl/)). -To build pdwfs from source (assuming Go is installed) : -``` +To build pdwfs develop branch (default branch): +```bash $ git clone https://github.com/cea-hpc/pdwfs $ cd pdwfs -$ make +$ make PREFIX=/usr/local/pdwfs install ``` -Binary distributions are also available for Linux system and x86_64 architecture in the [releases](http://github.com/cea-hpc/pdwfs/releases) page. -To run the test suite, you will need a running Redis instance on the default host and port. Just type the following command to have an instance running in the background: -``` -$ redis-server --daemonize yes -``` -Then: +### Using Spack + +pdwfs and its dependencies (even Go) can be installed with the package manager [Spack](https://spack.io). + +A Spack package for pdwfs is not yet available in Spack upstream repository, but is available [here](https://github.com/cea-hpc/pdwfs/releases/download/v0.2.0/pdwfs-spack.py). + +To add pdwfs package to your Spack installation, you can proceed as follows: + +```bash +$ export DIR=$SPACK_ROOT/var/spack/repos/builtin/pdwfs +$ mkdir $DIR +$ wget -O $DIR/package.py https://github.com/cea-hpc/pdwfs/releases/download/v0.2.0/pdwfs-spack.py +$ spack spec pdwfs # check everything is OK ``` + +## Testing + +To run the test suite, you will need a Redis installation and make sure ```redis-server``` and ```redis-cli``` binaries are available in your PATH. Then: + +```bash $ make test ``` -To install pdwfs: -``` -$ make PREFIX=/your/installation/path install -``` -Default prefix is /usr/local. +To allow development and testing on a different OS (e.g. MacOS), we provide a development Dockerfile based on an official CentOS image from DockerHub. To build and run the container: -We also provide a development Dockerfile based on an official Go image from DockerHub. To build and run the container: -``` +```bash $ make -C docker run ``` -The working directory in the container is a mounted volume on the pdwfs repo on your host, so to build pdwfs, just use the Makefile as previously described. + +The working directory in the container is a mounted volume on the pdwfs repo on your host, so to build pdwfs inside the container, just use the Makefile as previously described (ie. ```make test```). NOTE: if you encounter permission denied issue when building pdwfs in the container that's probably because the non-root user and group IDs set in the Dockerfile do not match your UID and GID. Change the UID and GID values to yours in the Dockerfile and re-run the above command. ## Quick start First, start a default Redis instance in the background. -``` + +```bash $ redis-server --daemonize yes ``` -Then, assuming your simulation will write its data into the output/ directory, simply wrap the execution command of your simulation with pdwfs command-line script like this: -``` + +Then, assuming your simulation will write its data into the ```output``` directory, simply wrap the execution command of your simulation with pdwfs command-line script like this: + +```bash $ pdwfs -p output/ -- your_simulation_command ``` That's it ! pdwfs will transparently intercept low-level I/O calls (open, write, read, ...) on any file/directory within the output/ directory and send data to Redis, no data will be written on disk. To process the simulation data, just run your processing tool the same way: -``` + +```bash $ pdwfs -p output/ -- your_processing_command ``` -To see the data staged within Redis (keys only) and check the memory used (and to give you a hint at how sweet Redis is): -``` + +To see the data staged within Redis (keys only) and check the memory used (and to give you a hint at how nice Redis is): + +```bash $ redis-cli keys * ... $ redis-cli info memory @@ -93,17 +109,63 @@ $ redis-cli info memory ``` Finally, to stop Redis (and discard all data staged in memory !): -``` + +```bash $ redis-cli shutdown ``` +## Running pdwfs in a SLURM job + +pdwfs comes with a specialized CLI tool called ```pdwfs-slurm``` that simplifies the deployment of Redis instances in a SLURM job. + +```pdwfs-slurm``` has two subcommands: ```init``` and ```finalize```. + +The following script illustrates the use of pdwfs and ```pdwfs-slurm``` to run the I/O benchmarking tool [ior](https://github.com/hpc/ior) (the script is available in examples/ior_slurm). The script assumes that pdwfs, Redis and ior are installed and available in the PATH. + +ior_pdwfs.sh: +```bash +#!/bin/bash +# SBATCH --job-name=pdwfs +# SBATCH --nodes=12 +# SBATCH --exclusive + +# Initialize the Redis instances: +# - 32 instances distributed on 4 nodes (8 per node) +# - bind Redis servers to ib0 network interface +pdwfs-slurm init -N 4 -n 8 -i ib0 + +# pdwfs-slurm produces a session file with some environment variables to source +source pdwfs.session + +# ior command will use MPI-IO in collective mode with data blocks of 100 MBytes +IOR_CMD="ior -a MPIIO -c -t 100m -b 100m -o $SCRATCHDIR/testFile" + +# pdwfs command will forward all I/O in $SCRATCHDIR in Redis instances +WITH_PDWFS="pdwfs -p $SCRATCHDIR" + +# Execute ior benchmark on 128 tasks +srun -N 8 --ntasks-per-node 16 $WITH_PDWFS $IOR_CMD + +# gracefully shuts down Redis instances +pdwfs-slurm finalize + +# pdwfs-slurm uses srun in background to execute Redis instances +# wait for background srun to complete +wait +``` + +This script can be run with SLURM ```sbatch``` or with ```salloc``` as follows: +```bash +$ salloc -N 12 --exclusive ./ior_pdwfs.sh +``` + ## How does it work ? pdwfs used the often-called "LD_PRELOAD trick" to intercept a set of I/O-related function calls provided by the C standard library (libc). -The pdwfs command-line script execute the user command passed in argument with the LD_PRELOAD environment variable set to the installation path of the pdwfs.so shared library. +The pdwfs CLI script execute the user command passed in argument with the LD_PRELOAD environment variable set to the installation path of the pdwfs.so shared library. -Currently about 90 libc I/O calls are intercepted but only 50 of them are currently implemented by pdwfs. If your program uses an I/O call intercepted but not implemented, it will raise an error. In this case, please file an [issue](https://github.com/cea-hpc/pdwfs/issues) (or even better, send a pull request!). +Currently about 90 libc I/O calls are intercepted but only 50 of them are currently implemented by pdwfs. If your program uses an I/O call intercepted but not implemented, it will raise an error. In this case, please file an [issue](https://github.com/cea-hpc/pdwfs/issues) (or send a pull request!). The category of calls currently implemented are: - basic POSIX and C standard I/O calls (open, close, read, write, lseek, access, unlink, stat, fopen, fread, fprintf, ...) @@ -112,12 +174,11 @@ The category of calls currently implemented are: ## Performance To address the challenge of being competitive with parallel filesystems, an initial set of design choices and trade-offs have been made: -- selecting the widely used database Redis to benefit from its mix of performance, simplicity and flexibility (and performance is an important part of the mix), -- files are sharded (or stripped) accross multiple Redis instances with a predefined layout (by configuration), -- file shards are sent/fetched in parallel using Go concurrency mechanism (goroutines), - no central metadata server, metadata are distributed accross Redis instances, -- (planned) drastically limit the amount of metadata and metadata manipulations by opting not to implement typical filesystem features such as linking, renaming and timestamping, -- (planned) implement write buffers and leverages Redis pipelining feature, +- drasticaly limiting the amount of metadata and metadata manipulations by opting not to implement typical filesystem features such as linking, renaming and timestamping +- selecting the widely used database Redis to benefit from its mix of performance, simplicity and flexibility (and performance is an important part of the mix), +- files are stripped accross multiple Redis instances with a predefined layout (by configuration), no metadata query for read/write, +- being a simple infrastructure in user space allows adjustement and configuration on a per-simulation or per-workflow basis for increased efficiency. With this set of choices, we expect our infrastructure to be horizontally scalable (adding more Redis instances to accomodate higher loads) and to accomodate certain I/O loads that are known to be detrimental for parallel filesystem (many files). @@ -127,7 +188,6 @@ On the other hand, a few factors are known for impacting performance versus para Obvisouly, proper benchmarking at scale will have to be performed to assess pdwfs performances. Yet, considering these design choices and our past experience with PaDaWAn in designing in transit infrastructure, we are hopefull that we will get decent performances. -It is also noted that a significant difference with parallel filesystems is that pdwfs is a simple infrastructure in user space that can be easily adjusted on a per-simulation or per-workflow basis for efficiency. ## Validation @@ -161,6 +221,25 @@ Yep, you can go grab a second coffee... - Works only for dynamically linked executables, - Most core or shell utilities for file manipulations (e.g. ls, rm, redirections) requires particular libc calls not implemented, + +## PaDaWAn project + +pdwfs is a component of the PaDaWAn project (for Parallel Data Workflow for Analysis), a [CEA](http://www.cea.fr) project that aims at providing building blocks of a lightweight and *least*-intrusive software infrastructure to facilitate *in transit* execution of HPC simulation workflows. + +The foundational work for this project was an initial version of pdwfs entierly written in Python and presented in the paper below: + +- *PaDaWAn: a Python Infrastructure for Loosely-Coupled In Situ Workflows*, J. Capul, S. Morais, J-B. Lekien, ISAV@SC (2018). + +## In situ / in transit HPC workflows +Within the HPC community, in situ data processing is getting quite some interests as a potential enabler for future exascale-era simulations. + +The original in situ approach, also called tightly-coupled in situ, consists in executing data processing routines within the same address space as the simulation and sharing the resources with it. It requires the simulation to use a dedicated API and to link against a library embedding a processing runtime. Notable in situ frameworks are ParaView [Catalyst](https://www.paraview.org/in-situ/), VisIt [LibSim](https://wci.llnl.gov/simulation/computer-codes/visit). [SENSEI](http://sensei-insitu.org) provides a common API that can map to various in situ processing backends. + +The loosely-coupled flavor of in situ, or in transit, relies on separate resources from the simulation to stage and/or process data. It requires a dedicated distributed infrastructure to extract data from the simulation and send it to a staging area or directly to consumers. Compared to the tightly-coupled in situ approach, it offers greater flexibility to adjust the resources needed by each application in the workflow (not bound to efficiently use the same resources as the simulation). It can also accommodate a larger variety of workflows, in particular those requiring memory space for data windowing (e.g. statistics, time integration). + +This latter approach, loosely-coupled in situ or in transit, is at the core of pdwfs. + + ## License pdwfs is distributed under the Apache License (Version 2.0). diff --git a/docker/Dockerfile b/docker/Dockerfile index 7e675fb..80845d9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,20 @@ -FROM golang:1.11.5 +FROM centos:latest + +RUN yum -y update && yum -y install \ + wget \ + gcc \ + make \ + strace \ + git \ + glib2-devel \ + yum clean all + +# Go language +RUN wget -O go.tar.gz 'https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz' && \ + tar xf go.tar.gz -C /usr/local && \ + rm go.tar.gz + +ENV PATH "/usr/local/go/bin:$PATH" ENV GO111MODULE=on diff --git a/docker/Makefile b/docker/Makefile index 86a267e..9401827 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -8,6 +8,9 @@ build: run: build docker run $(DOCKER_RUN_OPT) -it --rm -v $(shell pwd)/..:/home/dev/pdwfs -w /home/dev/pdwfs --name pdwfs-dev pdwfs +connect: + docker exec -it pdwfs-dev bash + clean: docker rm $(shell docker ps -qa --no-trunc --filter "status=exited"); \ docker rmi $(shell docker images --filter "dangling=true" -q --no-trunc) diff --git a/docker/banner.sh b/docker/banner.sh index a93b01b..8bcafd9 100644 --- a/docker/banner.sh +++ b/docker/banner.sh @@ -4,7 +4,6 @@ echo "* *" echo "* To build pdwfs: *" echo "* $ make *" echo "* *" -echo "* To run the test suite you need a running Redis instance: *" -echo "* $ scripts/pdwfs-redis start *" +echo "* To run the test suite: *" echo "* $ make test *" echo "*************************************************************" diff --git a/examples/base_dockerfile/Dockerfile b/examples/base_dockerfile/Dockerfile index 74479a3..76fb48d 100644 --- a/examples/base_dockerfile/Dockerfile +++ b/examples/base_dockerfile/Dockerfile @@ -8,6 +8,7 @@ RUN yum -y update && yum -y install \ make \ strace \ git \ + glib2-devel \ yum clean all # Go language diff --git a/examples/hydroC_ParaView/Dockerfile b/examples/hydroC_ParaView/Dockerfile index b8ba91a..aa79b89 100644 --- a/examples/hydroC_ParaView/Dockerfile +++ b/examples/hydroC_ParaView/Dockerfile @@ -42,9 +42,9 @@ RUN cd src && git clone 'https://github.com/cea-hpc/pdwfs' && \ ENV PATH "${HOME}/opt/pdwfs/bin:${PATH}" -# uncomment to build pdwfs from a volume mounted on the pdwfs source directory on the host (for debug/development) -# (a modification is needed in the Makefile as well) -#ENV PATH "/pdwfs/build/bin:${PATH}" +# pdwfs bin will be first search in /pdwfs/build which is a local build directory (on the host, not in the container, /pdwfs is a mounted volume) +# if no bin is found, it will look into the container installed version checked out from the GitHub repo +ENV PATH "/pdwfs/build/bin:${PATH}" COPY banner.sh /tmp/ RUN cat /tmp/banner.sh >> ${HOME}/.bashrc diff --git a/examples/hydroC_ParaView/Makefile b/examples/hydroC_ParaView/Makefile index 15dcfec..55ec08b 100644 --- a/examples/hydroC_ParaView/Makefile +++ b/examples/hydroC_ParaView/Makefile @@ -1,7 +1,7 @@ # uncomment to mount pdwfs sources folder into the container (for debug/development) # a modification is needed in the Dockerfile as well -#DOCKER_RUN_OPT = -v $(shell pwd)/../../:/pdwfs +DOCKER_RUN_OPT = -v $(shell pwd)/../../:/pdwfs # uncomment to allow stracing in docker (for debug) #DOCKER_RUN_OPT = --security-opt seccomp:unconfined $(DOCKER_RUN_OPT) @@ -16,19 +16,6 @@ run: build connect: docker exec -it hydro-run bash - -define BROWSER_PYSCRIPT -import os, webbrowser, sys -try: - from urllib import pathname2url -except: - from urllib.request import pathname2url -webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) -endef -export BROWSER_PYSCRIPT - -BROWSER := python -c "$$BROWSER_PYSCRIPT" - watch: - @echo "" > output/video.html && $(BROWSER) output/video.html + @echo "" > output/video.html && python -m webbrowser -t "output/video.html" diff --git a/examples/hydroC_ParaView/README.md b/examples/hydroC_ParaView/README.md index 9646b08..f672277 100644 --- a/examples/hydroC_ParaView/README.md +++ b/examples/hydroC_ParaView/README.md @@ -1,17 +1,16 @@ # Workflow Example: HydroC + ParaView + FFmpeg -This example illustrates the use of pdwfs to produce movie images from a simulation without writing any simulation output data on disk. +This example illustrates the use of pdwfs to produce movie images from an [HydroC](https://github.com/HydroBench/Hydro) simulation without writing any simulation data on disk. -It can run on a laptop and only requires Docker: +The example runs on a laptop and everything you need is packaged in a Dockerfile. +We also provide a Makefile which does the plumbing for you. So, to build and run the container, just type: ``` $ git clone https://github.com/cea-hpc/pdwfs $ cd pdwfs $ make -C examples/hydroC_ParaView run ``` -The last command will build the container the first time it is executed. As it is compiling OpenMPI and downloading Go and ParaView, you'll have time to grab some coffee... - -Once the build is complete, it runs the container. Just follow the help message to run the workflow with pdwfs. +Once you are in the container, just follow the help message to run the workflow with pdwfs. Once the workflow has run, go back to your host (not the container) and type the following command to watch the movie you just produced: diff --git a/examples/hydroC_ParaView/run_on_pdwfs.sh b/examples/hydroC_ParaView/run_on_pdwfs.sh index e250791..915f2ba 100755 --- a/examples/hydroC_ParaView/run_on_pdwfs.sh +++ b/examples/hydroC_ParaView/run_on_pdwfs.sh @@ -1,7 +1,11 @@ #!/bin/bash -# Start a Redis instance in background to stage data (+deactivate Redis snapshotting) -redis-server --daemonize yes --save "" +# Start two Redis instances in background to stage data (+deactivate Redis snapshotting) +redis-server --port 6379 --daemonize yes --save "" +redis-server --port 6380 --daemonize yes --save "" + +# Export server addresses to pdwfs +export PDWFS_REDIS=localhost:6379,localhost:6380 # Wrap the simulation command with pdwfs intercepting IO in Dep/ directory # Note: redirecting stderr to /dev/null as a read error appears at each iteration @@ -16,4 +20,5 @@ pdwfs -p Dep -- ./process_all.py ffmpeg -i images/test_%02d.jpg -vcodec libx264 -pix_fmt yuv420p -profile:v baseline -level 3 /output/hydro.mp4 # No data to clean up on disk, just shutdown Redis -#redis-cli shutdown +#redis-cli -p 6379 shutdown +#redis-cli -p 6380 shutdown diff --git a/examples/ior_benchmark/Dockerfile b/examples/ior_benchmark/Dockerfile index 2c6ab62..c7e9a8e 100644 --- a/examples/ior_benchmark/Dockerfile +++ b/examples/ior_benchmark/Dockerfile @@ -40,19 +40,18 @@ RUN mkdir -p ${HOME}/run RUN git clone 'https://github.com/cea-hpc/pdwfs' && \ cd pdwfs && make PREFIX=${HOME}/opt/pdwfs install -ENV PATH "${HOME}/opt/pdwfs:${PATH}" - -# uncomment to build pdwfs from a volume mounted on the pdwfs source directory on the host (for debug/development) -# (a modification is needed in the Makefile as well) -#ENV PATH "/pdwfs/build/bin:${PATH}" - +# pdwfs bin will be first search in /pdwfs/build which is a local build directory (on the host, not in the container, /pdwfs is a mounted volume) +# if no bin is found, it will look into the container installed version checked out from the GitHub repo +ENV PATH "/pdwfs/build/bin:${HOME}/opt/pdwfs/bin:${PATH}" COPY banner.sh /tmp/ RUN cat /tmp/banner.sh >> ${HOME}/.bashrc COPY --chown=luke:rebels jupyter_notebook_config.py ${HOME}/.jupyter/ +COPY --chown=luke:rebels ior_script.jinja2 . +COPY --chown=luke:rebels utils.py . +COPY --chown=luke:rebels bench.py . COPY --chown=luke:rebels notebook/ior_example.ipynb . -COPY --chown=luke:rebels notebook/ior_script.jinja2 . CMD bash diff --git a/examples/ior_benchmark/Makefile b/examples/ior_benchmark/Makefile index 0279517..e0d1bfc 100644 --- a/examples/ior_benchmark/Makefile +++ b/examples/ior_benchmark/Makefile @@ -1,17 +1,22 @@ # uncomment to mount pdwfs sources folder into the container (for debug/development) -# a modification is needed in the Dockerfile as well DOCKER_RUN_OPT = -v $(shell pwd)/../../:/pdwfs # uncomment to allow stracing in docker (for debug) #DOCKER_RUN_OPT = --security-opt seccomp:unconfined $(DOCKER_RUN_OPT) +DOCKER_RUN = docker run $(DOCKER_RUN_OPT) -it --rm -p 8888:8888 -v $(shell pwd)/output:/output -v $(shell pwd)/notebook:/notebook --name ior-run ior + build: docker build -t pdwfs-base ../base_dockerfile docker build -t ior . run: build - docker run $(DOCKER_RUN_OPT) -it --rm -p 8888:8888 -v $(shell pwd)/output:/output -v $(shell pwd)/notebook:/notebook --name ior-run ior + $(DOCKER_RUN) jupyter-notebook + +bench: build + $(DOCKER_RUN) ./bench.py $(shell git describe) + python -m webbrowser -t "output/bench.html" connect: docker exec -it ior-run bash diff --git a/examples/ior_benchmark/README.md b/examples/ior_benchmark/README.md new file mode 100644 index 0000000..442e488 --- /dev/null +++ b/examples/ior_benchmark/README.md @@ -0,0 +1,21 @@ +# IOR Benchmark Example + +This example illustrates the use of pdwfs with the HPC I/O benchmark tool [IOR](https://github.com/hpc/ior). + +The example runs in a Jupyter notebook and everything you need is packaged in a Dockerfile. + +We also provide a Makefile which does the plumbing for you. So, to build and run the container, just type: + +``` +$ git clone https://github.com/cea-hpc/pdwfs +$ cd pdwfs +$ make -C examples/ior_benchmark run +``` + +When the Jupyter notebook server is up and running, open your browser on your host at http://localhost:8888, open the notebook *ior_example.ipynb* and follow the steps. + +This example also allows to run the benchmark in a non-interactive way and save the results in the output directory. To run the benchmark this way, just type: +``` +$ make -C examples/ior_benchmark bench +``` +Once finished it will show you the results in your browser. \ No newline at end of file diff --git a/examples/ior_benchmark/bench.py b/examples/ior_benchmark/bench.py new file mode 100755 index 0000000..c8ecf3d --- /dev/null +++ b/examples/ior_benchmark/bench.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +import os +import sys +import glob +import subprocess +from datetime import datetime + +# override the matplotlib import in utils.py +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot + +# load local utils.py module +import utils + +bench_script = """ +#!/bin/bash + +cd run + +echo " Executing the benchmark on disk" +mpirun ior -f ../ior_script > /output/ior_disk.out + +echo " Executing the benchmark with pdwfs" +# start a new Redis instance in the background +redis-cli shutdown 2> /dev/null +redis-server --daemonize yes --save "" > /dev/null + +# wrap the previous command with pdwfs -p . (indicating the current directory as the intercepted directory) +pdwfs -p . -- mpirun ior -f ../ior_script > /output/ior_pdwfs.out +""" + +def run_bench(api, version): + + bench_title = api + " IOR benchmark - pdwfs " + version + " - " + str(datetime.utcnow()) + " UTC" + print "Running:", bench_title + + read = "1" # 1: perform read benchmark + numTasks="2" # number of parallel processes + filePerProc="0" # 1: write one file per processes + collective="1" # 1: enable collective IO operations (MPIIO, HDF5 only) + segmentCount="1" # see previous schematic + transferSize = ["512k", "1m", "3m", "5m", "7m", "10m","25m","35m", "50m","60m","75m","85m", "100m","115m","125m","150m","175m","200m", "225m", "250m"] + utils.build_ior_script(api, read, numTasks, filePerProc, collective, segmentCount, transferSize) + + with open("run/bench.sh", "w") as f: + f.write(bench_script) + + subprocess.check_call(["bash", "run/bench.sh"]) + + print " Parsing and saving the results in a plot" + df_disk = utils.parse_ior_results("/output/ior_disk.out") + df_pdwfs = utils.parse_ior_results("/output/ior_pdwfs.out") + + os.rename("/output/ior_disk.out", "/output/ior_" + api + "_disk.out") + os.rename("/output/ior_pdwfs.out", "/output/ior_" + api + "_pdwfs-" + version + ".out") + + matplotlib.use('Agg') + + for readOrWrite in ["write", "read"]: + filename = readOrWrite + "_ior_" + api + "_pdwfs-" + version + ".png" + utils.plot_results( + readOrWrite, + df_disk[df_disk["Operation"] == readOrWrite], + df_pdwfs[df_pdwfs["Operation"] == readOrWrite], + title=bench_title, + filename="/output/" + filename) + with open("/output/bench.html", "a") as f: + f.write("\n") + +if __name__ == '__main__': + + for f in glob.glob("/output/*"): + os.remove(f) + + version = sys.argv[1] + run_bench("POSIX", version) + run_bench("MPIIO", version) + run_bench("HDF5", version) diff --git a/examples/ior_benchmark/ior_script.jinja2 b/examples/ior_benchmark/ior_script.jinja2 new file mode 100644 index 0000000..57f8dee --- /dev/null +++ b/examples/ior_benchmark/ior_script.jinja2 @@ -0,0 +1,16 @@ +IOR START + api={{ api }} + verbose=0 + testFile=testFile + writeFile=1 + readFile={{ read }} + numTasks={{ numTasks }} + filePerProc={{ filePerProc }} + collective={{ collective }} + segmentCount={{ segmentCount }} +{% for size in transferSize %} +RUN + blockSize={{ size }} + transferSize={{ size }} +{% endfor %} +IOR STOP \ No newline at end of file diff --git a/examples/ior_benchmark/notebook/ior_example.ipynb b/examples/ior_benchmark/notebook/ior_example.ipynb index 09438f2..589b335 100644 --- a/examples/ior_benchmark/notebook/ior_example.ipynb +++ b/examples/ior_benchmark/notebook/ior_example.ipynb @@ -18,32 +18,9 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "IOR START\n", - " api={{ api }}\n", - " verbose=0\n", - " testFile=testFile\n", - " writeFile=1\n", - " readFile={{ read }}\n", - "{% for case in cases %}\n", - "RUN\n", - " numTasks={{ case.numTasks }}\n", - " filePerProc={{ case.filePerProc }}\n", - " collective={{ case.collective }}\n", - " segmentCount={{ case.segmentCount }}\n", - " blockSize={{ case.transferSize }}\n", - " transferSize={{ case.transferSize }}\n", - "{% endfor %}\n", - "IOR STOP" - ] - } - ], + "outputs": [], "source": [ "%%bash\n", "cat ior_script.jinja2" @@ -60,62 +37,17 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Benchmark script builder" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "import itertools as it\n", - "from collections import namedtuple\n", - "from jinja2 import Template\n", - "\n", - "IORCase = namedtuple(\"IORCase\", [\"numTasks\", \"filePerProc\", \"collective\", \"segmentCount\", \"transferSize\"])\n", - "\n", - "def make_ior_script(api, numTasks, filePerProc, collective, segmentCount, transferSize, read):\n", - " matrix = list(it.product(numTasks, filePerProc, collective, segmentCount, transferSize))\n", - " cases = [IORCase(*case) for case in matrix]\n", - "\n", - " with open(\"ior_script.jinja2\", \"r\") as f:\n", - " template = Template(f.read())\n", - "\n", - " with open(\"ior_script\", \"w\") as f:\n", - " f.write(template.render(api=api, cases=cases, read=read))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Results parser (Pandas dataframe)" + "## Load utility functions\n", + "Execute the cell below twice, one to load the python script into the cell, second to execute the cell." ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "def parse_ior_results(filename):\n", - " \"\"\"\n", - " Parse IOR results in file given by filename and return a Pandas dataframe\n", - " \"\"\"\n", - " import re\n", - " import pandas\n", - "\n", - " start_line = None\n", - " end_line = None\n", - " with open(filename,'r') as f: \n", - " for i, line in enumerate(f.readlines()):\n", - " if re.search(\"Summary of all tests:\", line):\n", - " start_line = i + 1\n", - " if re.search(\"Finished\", line):\n", - " end_line = i - 1\n", - "\n", - " return pandas.read_csv(filename, sep='\\s+', skiprows=start_line, nrows=end_line-start_line)" + "%load utils.py" ] }, { @@ -134,22 +66,21 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "# Global parameters\n", - "api = \"POSIX\" # possible values: POSIX, MPIIO, HDF5 \n", - "read = \"0\"\n", + "# Parameters\n", + "api = \"MPIIO\" # possible values: POSIX, MPIIO, HDF5 \n", + "read = \"0\" # 1: perform read benchmark\n", + "numTasks=\"2\" # number of parallel processes\n", + "filePerProc=\"0\" # 1: write one file per processes\n", + "collective=\"1\" # 1: enable collective IO operations (MPIIO, HDF5 only)\n", + "segmentCount=\"1\" # see previous schematic\n", "\n", - "# Case parameters\n", - "numTasks=[\"2\"]\n", - "filePerProc=[\"0\"]\n", - "collective=[\"1\"]\n", - "segmentCount=[\"1\"]\n", - "transferSize = [\"512k\", \"1m\", \"5m\", \"10m\", \"50m\", \"100m\", \"250m\"]\n", + "transferSize = [\"512k\", \"1m\", \"3m\", \"5m\", \"7m\", \"10m\",\"25m\",\"35m\", \"50m\",\"60m\",\"75m\",\"85m\", \"100m\",\"115m\",\"125m\",\"150m\",\"175m\",\"200m\", \"225m\", \"250m\"]\n", "\n", - "make_ior_script(api, numTasks, filePerProc, collective, segmentCount, transferSize, read)\n", + "build_ior_script(api, read, numTasks, filePerProc, collective, segmentCount, transferSize)\n", "#%cat ior_script" ] }, @@ -162,7 +93,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -181,15 +112,19 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%%bash\n", "cd run\n", - "pdwfs-redis restart &> /dev/null\n", - "pdwfs -p . -- mpirun ior -f ../ior_script > ior_results_pdwfs.out\n", - "# %cat ior_results_pdwfs.out" + "\n", + "# start a new Redis instance in the background\n", + "redis-cli shutdown 2> /dev/null\n", + "redis-server --daemonize yes --save \"\" > /dev/null\n", + "\n", + "# wrap the previous command with pdwfs -p . (indicating the current directory as the intercepted directory)\n", + "pdwfs -p . -- mpirun ior -f ../ior_script > ior_results_pdwfs.out" ] }, { @@ -201,7 +136,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -211,7 +146,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -228,48 +163,12 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 57, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7wAAAGeCAYAAAC3uA0jAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJzs3Xl8nFXd9/HvSSaZSSb7JOneJqG7trQ03aQU2iKglEXEstUW2rL4ur1xRVFvl9tbFMVHFPURkCJLsYIIKMItj1KUrXSjlSp031O6ZG0ymUkyM+f5YyZpkmajTXIlk8/79corM9dcc81v0hbmm/M75xhrrQAAAAAAiDcJThcAAAAAAEBPIPACAAAAAOISgRcAAAAAEJcIvAAAAACAuETgBQAAAADEJQIvAAAAACAuEXgBAMApjDEXGGMOOV0HAABngsALAMAZMMbsM8Zc2Oz+cGPME8aYMmOM3xiz3hizsNVzbOyxGmNMiTHmJ8aYxE5eIxA7/4gx5hFjTFpPvq92ariw8zMBAOg7CLwAAHQTY0yOpNcl1Uv6kKRcSfdK+q0x5upWp59trU2TdL6kayQt6+Tyl8XOnyJpqqSvdWftAADEIwIvAADd5wuSaiQtt9YesdYGrLWrJd0l6f8YY0zrJ1hrd0l6Q9Eg2ylr7RFJLzU/3xjjNsb82BhzwBhz1BhzvzEmJfZYrjHmz8aYSmNMuTHmNWNMQuwxa4wZ3ew6jxhjvtf6NY0xj0saKen52CjzV4wxHmPMqthIdqUxZoMxZlDXf1QAAPQ8Ai8AAN3no5L+YK2NtDr+lKKBcWzrJxhjxks6T9KurryAMWa4pI+1Ov/u2LWnSBotaZikb8Ue+5KkQ5LyJA2S9HVJtmtvJ8pa+2lJBxQbZbbW/kjSUkmZkkZI8km6TVLgg1wXAICeRuAFAKD75Ep6v43j7zd7vNHbxhi/pPck/V3S/+3k2s8ZY6olHZR0TNK3JSk2anyLpC9Ya8uttdWSvi/p2tjzGiQNkTTKWttgrX3NWvuBAm87GhQNuqOttWFr7SZr7YluuC4AAN2GwAsAQPcpVTRctjak2eONzpGUpuj83ZmSvJ1c+0prbbqkCySN18nwnCcpVdKmWGtxpaS/xI5L0j2Kjgb/P2PMHmPMnR/oHbXvcUVbq39njDlsjPmRMSapm64NAEC3IPACANB9/ibpqsY5ss0sUnRkdkfzgzbqKUlrdbIFuUPW2n9IekTSj2OHShVtJf6QtTYr9pUZW+BK1tpqa+2XrLVFki6X9EVjzILYc2sVDcuNBnf00q3qaLDW/re1dqKkj0haKGlJV94DAAC9hcALAED3uVfRea0rjTGDYws7XSfpG5Lu6KCV+G5JNxtjOgqczf1U0keNMWfH5gv/WtK9xph8STLGDDPGXBy7vdAYMzrW+lwlKSypcY7xFknXG2MSjTGXKLpidHuOSipqvGOMmWeMmRTbTumEoi3OrecuAwDgKAIvAADdxFpbJmmOJI+kdyWVSfqipE9ba5/s4HlbJb0q6Y4uvs5xSY/p5KjwVxVtW37LGHNC0ZHmcbHHxsTu1yg6kvx/rbWvxB77nKTLJFVKukHScx287A8k/VesbfrLio4GP61o2H1P0j8UbXMGAKDPMN2zbgUAAAAAAH0LI7wAAAAAgLhE4AUAAAAAxCUCLwAAAAAgLhF4AQAAAABxyeV0AT0hNzfXFhQUOF0GAAAAAKAHbNq0qdRam9fZeXEZeAsKCrRx40anywAAAAAA9ABjzP6unEdLMwAAAAAgLhF4AQAAAABxicALAAAAAIhLcTmHFwAAAADiQUNDgw4dOqRgMOh0KY7weDwaPny4kpKSTuv5BF4AAAAA6KMOHTqk9PR0FRQUyBjjdDm9ylqrsrIyHTp0SIWFhad1DVqaAQAAAKCPCgaD8vl8Ay7sSpIxRj6f74xGtwm8AAAAANCHDcSw2+hM3zuBFwAAAAAQl5jDCwAAAADoku985ztKS0vTiRMnNHfuXF144YVtnnfjjTdq4cKFuvrqq3u5wpYIvAAAAAAQJ57bXKJ7Xtquw5UBDc1K0R0Xj9OVU4d1++t897vf7fZr9gRamgEAAAAgDjy3uURfe2arSioDspJKKgP62jNb9dzmkjO67l133aWxY8dqzpw52r59u6ToCO7TTz8tSbrzzjs1ceJETZ48WV/+8pdPef43v/lN3XjjjQqHw2dUx+lghBcAAAAA+oH/fv7fevfwiXYf33ygUvXhSItjgYawvvL0O1q9/kCbz5k4NEPfvuxD7V5z06ZN+t3vfqctW7YoFArpnHPO0bRp05oeLysr07PPPqtt27bJGKPKysoWz7/jjjtUXV2t3/zmN44svsUIby97bnOJzr17jQrvfEHn3r3mjH/bAgAAAACSTgm7nR3vitdee02f+MQnlJqaqoyMDF1++eUtHs/MzJTH49Hy5cv1zDPPKDU1temx//mf/1FVVZXuv/9+x1aaZoS3FzW2GAQaokP5jS0Gknqkrx4AAABA/OhoJFaSzr17jUoqA6ccH5aVoidvnd0jNblcLq1fv14vv/yynn76af3iF7/QmjVrJEnTp0/Xpk2bVF5erpycnB55/c4wwtuL7nlpe1PYbRRoCOuel7Y7VBEAAACAeHHHxeOUkpTY4lhKUqLuuHjcaV9z7ty5eu655xQIBFRdXa3nn3++xeM1NTWqqqrSxz/+cd1777365z//2fTYJZdcojvvvFOXXnqpqqurT7uGM8EIby863MZvWzo6DgAAAABd1dg12p2rNJ9zzjm65pprdPbZZys/P1/Tp09v8Xh1dbWuuOIKBYNBWWv1k5/8pMXjn/rUp1RdXa3LL79cL774olJSUk67ltNhrLW9+oK9obi42G7cuNHpMk7RUYvBG3fOd6AiAAAAAH3Ze++9pwkTJjhdhqPa+hkYYzZZa4s7ey4tzb2oJ1oMAAAAAABto6W5FzW2EnzlD++oPhTRsB7cCBoAAAAABjoCby+7cuowPbu5RJW19frjZ+c4XQ4AAAAAxC1amh2QlJig+nD8zZ0GAAAAgL6EwOsAtytB9aFw5ycCAAAAAE4bgdcBya4E1YcjTpcBAAAAAHGNwOuA5MQE1YcIvAAAAAD6v7///e9auHBhl8+/7rrrNHnyZN177709WFUUi1Y5INmVoAbm8AIAAADobu88Jb38XanqkJQ5XFrwLWnyIqeranLkyBFt2LBBu3bt6pXXY4TXAUmM8AIAAADobu88JT1/u1R1UJKNfn/+9ujx07Rv3z6NHz9eN9xwgyZMmKCrr75atbW1+stf/qLx48frnHPO0TPPPNN0/qRJk1RZWSlrrXw+nx577DFJ0pIlS/TXv/5VF110kUpKSjRlyhS99tpruu+++zRx4kRNnjxZ11577Zn+BE7BCK8Dkl0EXgAAAAAf0P/eKR3Z2v7jhzZI4bqWxxoC0h8/K216tO3nDJ4kfezuDl92+/btWrlypc4991wtW7ZMP/nJT/TAAw9ozZo1Gj16tK655pqmc88991y98cYbGjVqlIqKivTaa69pyZIlWrt2rX71q1/pT3/6kxYuXKgtW7ZIkq655hrt3btXbrdblZWVXfoxfBCM8DqgcdEqa2lrBgAAANBNWofdzo530YgRI3TuuedKkhYvXqyNGzeqsLBQY8aMkTFGixcvbjr3vPPO06uvvqpXX31Vn/nMZ7R161aVlJQoOztbXq/3lGtPnjxZN9xwg1atWiWXq/vHYxnhdYDbFf09Q0PYKtllHK4GAAAAQL/QyUis7v1wrJ25lcwR0k0vnPbLGtMys1RVVbV77ty5c/XLX/5SBw4c0F133aVnn31WTz/9tM4777w2z3/hhRf06quv6vnnn9ddd92lrVu3dmvwZYTXAUmJ0b8wbE0EAAAAoNss+JaUlNLyWFJK9PgZOHDggNauXStJ+u1vf6sLL7xQ+/bt0+7duyVJq1evbjp3xIgRKi0t1c6dO1VUVKQ5c+boxz/+sebOnXvKdSORiA4ePKh58+bphz/8oaqqqlRTU3NGtbZG4HVAcmL0x848XgAAAADdZvIi6bL7oiO6MtHvl913xqs0jxs3Tr/85S81YcIEVVRU6Atf+IIefPBBXXrppTrnnHOUn5/f4vyZM2dq7NixkqItziUlJZozZ84p1w2Hw1q8eLEmTZqkqVOn6vbbb1dWVtYZ1doaLc0OSHYlSiLwAgAAAOhmkxd1+zZELpdLq1atanHskksu0bZt29o8//HHH2+6/ZGPfESRyMncU1BQoH/961+SpKSkJL3++uvdWmtrjPA6INnFCC8AAAAA9DQCrwOYwwsAAACgP2g+ItsfEXgd4GaEFwAAAEAXDeTtTM/0vRN4HdDU0swILwAAAIAOeDwelZWVDcjQa61VWVmZPB7PaV+DRasckJzIolUAAAAAOjd8+HAdOnRIx48fd7oUR3g8Hg0fPvy0n0/gdUDjHN4GRngBAAAAdCApKUmFhYVOl9FvEXgd0NjSfPvqzSr312toVoruuHicrpw6zOHKAAAAACB+EHgd8MbuUklSmb9eklRSGdDXntkqSYReAAAAAOgmLFrlgMfe3H/KsUBDWD96qe2NmwEAAAAAH1yPBV5jzMPGmGPGmH81O5ZjjPmrMWZn7Ht27LgxxtxnjNlljHnHGHNOs+csjZ2/0xiztKfq7U3Hq+vaPH64Mqj/eOJt/W79AR2qqO3lqgAAAAAgvvRkS/Mjkn4h6bFmx+6U9LK19m5jzJ2x+1+V9DFJY2JfMyX9StJMY0yOpG9LKpZkJW0yxvzJWlvRg3X3uIwUl6oCoVOOpyYnatP+Cr2w9X1JUlGeV3PH5Om8MbmaVeTTX989qnte2q7DlQHm/QIAAABAJ3os8FprXzXGFLQ6fIWkC2K3H5X0d0UD7xWSHrPRzaXeMsZkGWOGxM79q7W2XJKMMX+VdImk1T1Vd097bnOJ/HWnht2kRKPvf2KSrpgyVLuP1+jVHaV6bedxPbnhoB55c58Sogs7KxLbfot5vwAAAADQsd5etGqQtfb92O0jkgbFbg+TdLDZeYdix9o73m/d89J2tbX9rjfZ1RRcR+ena3R+upbNKVRdKKxN+yt0y2ObVNMqKAcawrrnpe0EXgAAAABog2OLVsVGc213Xc8Yc4sxZqMxZmNf3pT5cGWgzeNVgYY2j7tdifrIWbltjgp3dD0AAAAAGOh6O/AejbUqK/b9WOx4iaQRzc4bHjvW3vFTWGsftNYWW2uL8/Lyur3w7jI0K+UDHT/T5wEAAADAQNXbgfdPkhpXWl4q6Y/Nji+JrdY8S1JVrPX5JUkXGWOyYys6XxQ71m/dcfE4pSQltjiWkpSoOy4e1yPPAwAAAICBqsfm8BpjViu66FSuMeaQoqst3y3pKWPMckn7JS2Knf6ipI9L2iWpVtJNkmStLTfG/I+kDbHzvtu4gFV/1Tjf9oOuttz8eSWxNuavXMIqzQAAAADQHhOdShtfiouL7caNG50uo8e8e/iEPn7fa/rJorN11TnDnS4HAAAAAHqVMWaTtba4s/McW7QKp2/84HRlpSZp7e4yp0sBAAAAgD6LwNsPJSQYzSzM0Vt7CbwAAAAA0B4Cbz81q8ing+UBHaqodboUAAAAAOiTCLz91OyzfJKkt/b06zW8AAAAAKDHEHj7qbH56cpOTdJbe2hrBgAAAIC2EHj7qYQEo1lFPhauAgAAAIB2EHj7sVlFPpVUBnSwnHm8AAAAANAagbcfm1UUnce7lrZmAAAAADgFgbcfGzsoTTneZObxAgAAAEAbCLz9mDFGs4pytG5Puay1TpcDAAAAAH0KgbefOzmPN+B0KQAAAADQpxB4+7nZRY378dLWDAAAAADNEXj7udH5acpNYx4vAAAAALRG4O3njDGaWeTT2j1lzOMFAAAAgGYIvHFgVpFP71cFdYD9eAEAAACgCYE3DswuypEkrd1NWzMAAAAANCLwxoGz8tKUm+ZmHi8AAAAANEPgjQON+/G+xX68AAAAANCEwBsnZhX5dOREUPvKmMeLzj23uUTn3r1GhXe+oHPvXqPnNpc4XRIAAADQ7Qi8cWL2WezHi655bnOJvvbMVpVUBmQllVQG9LVnthJ6AQAAEHcIvHGiKNervHQ3C1ehUz/43/cUaAi3OBZoCOuel7Y7VBEAAADQM1xOF4DuYYzR7CKf3ortx2uMcbok9BEN4Yg27CvXK9uOac22Yzp6oq7N8w5XBnq5MgAAAKBnEXjjyKwin/70z8PaW+pXUV6a0+XAQcer6/T37cf0yvZjem1HqarrQkpOTNDMohyV1tSpKhA65TlDs1IcqBQAAADoOQTeODKrcT/ePWUE3gEmErH61+Eqrdl2TK9sO6Z/HqqSJA3KcOvSyUM0b3y+5ozOldftaprD27yt2Ui67YIih6oHAAAAegaBN44U5no1KMOtt/aU64aZo5wuBz2sOtig13eWRkPu9uMqramTMdKUEVn60kfHat74fH1oaMYp7e1XTh0mSbrnpe06XBmQLy1ZlbX1Wr3uoC4/e5gyU5KceDsAAABAtyPwxpHofrw+vbGLebzxyFqrPaX+prm4G/aVqyFsleFxae7YPM0fn6/zx+bJl+bu9FpXTh3WFHwl6R87jmvFoxu07JENenz5DKUm858GAAAA9H98qo0zs4t8+uOWw9p93K/R+bQ193d1obDW7SmPjeIe0/7YPstjB6Vp2ZxCzR+Xr2mjsuVKPLMF188fm6efXTtVn/3t27r18U16aGmx3K7E7ngLAAAAgGMIvHFmVtHJ/XgJvP3TkaqgXtkeHcV9Y1epauvDcrsS9JGzfFoxp1AXjMvXiJzUbn/dj08aors/OVlfefod3b56s355/TlnHKQBAAAAJxF448woX6oGZ3i0dk+ZFs9iHm9/EI5YbTlY2dSq/O77JyRJw7JSdNU5wzR/fL5mF+UqJbnnR1wXFY9QTTCk7/75XX3lD+/ox1efrYQEWuMBAADQPxF444wxRrPP8um1nceZx9uHVdU26B87j+uVbcf0jx3HVe6vV2KC0bSR2frqJeM1f3y+xg5Kc+TPb9mcQlUHQ7r3bzuU7nbpO5d/iL9HAAAA6JcIvHFoVlGOnt1cot3HazQ6P93pcqDoglM7jtY0bRu06UCFwhGr7NQkXTAuX/PG5+v8MXnKTO0bKyTfvmC0qoMNeuj1vUr3JOnLF49zuiQAAADgAyPwxqHGebxrd5cReB0UbAjrzd2xbYO2HVdJZUCSNHFIhj5z/lmaNz5fU0ZkKbEPtgwbY/SNSyeopi6kX7yyS+kel249/yynywIAAAA+EAJvHBqZk6qhmR69tadcn55d4HQ5A8qhitqmubhv7i5TXSii1OREnTs6V5+dP1rzxuVrcKbH6TK7xBijuz4xSTV1If3gf7cp3ZOk62eOdLosAAAAoMsIvHGocT/ef+xgHm9PC4Uj2rS/Qmu2R1uVdxytkRRdPOy6GSM1f3y+Zhbl9NstfhITjH6yaIr8dSF947mt8roTdcWUYZ0/EQAAAOgDCLxxatZZPj2zuUQ7j9Vo7CDamrtTub9e/9hxTC+/d0yv7jiuE8GQXAlGMwpztKh4hOaNz1dRrjduftGQ7ErQrxZP09KH1+tLT/1TaW6XFkwY5HRZAAAAQKcIvHFqdrP9eAm8Z8Zaq3ffP9HUqrz5YKWslXLT3Lr4Q4M1f3y+5ozJVbqnbyw41RM8SYl6aGmxFj+0Tp954m09ctN0feSsXKfLAgAAADpE4I1Tw7NTNCwrRWt3l2kJ83g/MH9dSG/sKtUr26MLTh05EZQknT08U59bMEbzx+frw0MzB9QetemeJD1y0wxd8+Ba3fzoRq1aMVNTR2Y7XRYAAADQLgJvnGqcx/vK9mOKROyACmana3+ZX2tio7jr9pSrPhxRmtul88bkav74fF0wLl956W6ny3RUtjdZq5bP1NX3r9WNv9mgJ2+dpfGDM5wuCwAAAGgTgTeOzSrK0R/ePqQdx6oJJW2oD0W0cV95NORuP6Y9x/2SpKI8r5bMHqX54/NVXJCjZFeCw5X2LfkZHj2xYqY+df9aLX5ovZ6+bbYKcr1OlwUAAACcgsAbxxr3431rdxmBN+ZYdVB/335cr2w7ptd2lqqmLqTkxATNLMrRp2dFQ+4oH+GtMyNyUrVqxQx96v61uuGhdfr9bbM1NCvF6bIAAACAFgi8cWxETqqGZ6forT3luvHcQqfLcUQkYrW1pEprth3TK9uP6Z1DVZKkwRkeXXb2EM0bl69zR+fK6+afwgc1Oj9djy2bqet//ZYWr1ynp26drdy0gd3yDQAAgL6FT/lxblaRT3977+iAmsd7Itig13eWas22Y/r79uMqramTMdLUEVn68kVjNW98viYOyYibbYOcNGl4plbeOF1LHl6nJSvXa/Uts5SZEr+rVQMAAKB/IfDGudlFPj296ZC2H63WhCHx2dZsrdXu4/6mbYM27CtXKGKV4XHp/HH5mj8+T+ePzVeON9npUuPSjMIc3b94mm5+bKOWPbJBjy+fodRk/tMCAAAA5/GpNM7NLMqRFN2PN54Cb7AhrHV7y5tC7oHyWknSuEHpWnFekeaPz9c5I7PkSmTBqd5wwbh8/ezaqfrsb9/WrY9v0kNLi+V2JTpdFgAAAAY4Am+cG56dqhE50f14b+rn83iPVAWbtg16Y1epAg1heZIS9JGzcnXz3CLNG5en4dmpTpc5YH180hDd/cnJ+srT7+j21Zv1y+vP4RcOAAAAcBSBdwCYXeTTS//uf/N4wxGrLQcrYiH3uN57/4QkaVhWiq6eNlzzx+dr9lk+eZIYSewrFhWPUE0wpO/++V199Q9bdc/Vk/vV3zkAAADEFwLvADCryKenNh7Se0dO6ENDM50up0OVtfX6x47otkH/2HFcFbUNSkwwmjYqW3d+bLzmj8/XmPw0Fpzqw5bNKVR1MKR7/7ZD6R6Xvn3ZRP68AAAA4AgC7wDQtB/vnvI+F3ittdp+tDq6bdC2Y9q0v0IRK+V4kzVvXL7mjc/X3DF5ykxl5d/+5PYFo1UdbNBDr+9VuselL100zumSAAAAMAAReAeAoVkpGuVL1Vt7yrR8jvPzeAP1Yb25++S2QSWVAUnSh4Zm6D/mjda88fk6e3iWEmmF7beMMfrGpRNUUxfSz9fsUrrHpVvmnuV0WQAAABhgCLwDxKxCn/73X+8rHLGOBMmD5bV6ZXt0wam1u8tUF4ooNTlRc0bn6j/nR0PuoAxPr9eFnmOM0V2fmKSaupC+/+I2pbmTdP3MkU6XBQAAgAGEwDtAzD7Lpyc3HtR775/Qh4f1fFtzQziiTfsr9Mr2aKvyjqM1kqQCX6qunzlS88fna0ZhDlvXxLnEBKOfLJoif11I33huq7zuRF0xZZjTZQEAAGCAIPAOEM334+2pwFtWU6d/7DiuNduO6dUdx3UiGFJSotGMwhwtKh6h+ePzVZSX1iOvjb4r2ZWgXy2epqUPr9eXnvqn0twuLZgwyOmyAAAAMAAQeAeIIZkpKojN411xXlG3XNNaq38fPqFXth3Tmu3HtOVgpayV8tLduuTDgzV/fL7OHZ2rdA8LTg10nqREPbS0WDc8tE6feeJtPXLTdH3krFynywIAAECcI/AOILPP8unP75zZPF5/XUiv7yrVK9uO6ZXtx3T0RJ2MkSYPz9LnF4zV/PH5+tDQDPZexSnSPUl69KYZWvTAWt386EatWjFTU0dmO10WAAAA4hiBdwCZVeTT6vUH9e7hE5o0vOttzftK/dFtg7Yf07o95aoPR5Tudmnu2DzNG5+v88fmKS/d3YOVI15ke5O1asVMfer+tbrxNxv05K2zNH5whtNlAQAAIE4ReAeQk/vxlnUYeOtDEW3YV960N+6eUr8kaXR+mm48t0DzxuWruCBbSYkJvVI34sugDI+eWDFTV9//phY/tF5P3zZbBblep8sCAABAHDLWWqdr6HbFxcV248aNTpfRJ03/3l91IhhSfSiioVkpuuPicbpy6jAdqw7q79uiC069vqtUNXUhJbsSNKvIpwXj8zVvXL5G+lKdLh9xZOfRai16YK1Sk136/W2zNTQrxemSAAAA0E8YYzZZa4s7PY/AO3A8t7lEX/r9PxWOnPwzdyUYDc7w6FBlQJI0JNOjC8blxxac8ik1mSYA9Jyth6p0/a/fUl6GW0/dOlu5abTGAwAAoHNdDbykmQHknpe2twi7khSKWB2rrtMdF4/TvHH5mjAkXcaw4BR6x6ThmVp543QteXidlqxcr9W3zFJmCqt6AwAAoHswCXMAORwbxW2tIRzRf8wbrYlDMwi76HUzCnN0/+Jp2nmsWssf2aDa+pDTJQEAACBOOBJ4jTFfMMb82xjzL2PMamOMxxhTaIxZZ4zZZYx50hiTHDvXHbu/K/Z4gRM1x4P25kgydxJOu2Bcvn527VS9faBCtz6+SXWhsNMlAQAAIA70euA1xgyTdLukYmvthyUlSrpW0g8l3WutHS2pQtLy2FOWS6qIHb83dh5Owx0Xj1NKUmKLYylJibrj4nEOVQSc9PFJQ3T3JyfrtZ2l+tzqLQqFI06XBAAAgH7OqZZml6QUY4xLUqqk9yXNl/R07PFHJV0Zu31F7L5ijy8w9N2eliunDtMPrpqkYVkpMpKGZaXoB1dN0pVThzldGiBJWlQ8Qt9aOFF/+fcRffUPWxWJxN+iegAAAOg9vb5olbW2xBjzY0kHJAUk/T9JmyRVWmsbJ+8dktSYwoZJOhh7bsgYUyXJJ6m0+XWNMbdIukWSRo4c2dNvo9+6cuowAi76tGVzClUdDOnev+1Quselb182kbnlAAAAOC1OtDRnKzpqWyhpqCSvpEvO9LrW2gettcXW2uK8vLwzvRwAB92+YLRWzCnUI2/u00/+usPpcgAAANBPObEt0YWS9lprj0uSMeYZSedKyjLGuGKjvMMllcTOL5E0QtKhWAt0pqSy3i8bQG8xxugbl05QdTCkn6/ZpXSPS7fMPcvpsgAAANDPODGH94CkWcaY1Nhc3AWS3pX0iqSrY+cslfTH2O0/xe4r9vgaay0T+4A4Z4zR96+apEsnD9H3X9ym36474HRJAAAA6GecmMNWCrM6AAAgAElEQVS7zhjztKS3JYUkbZb0oKQXJP3OGPO92LGVsaeslPS4MWaXpHJFV3QGMAAkJhjdu2iK/HUhfeO5rfK6E3XFFOagAwAAoGtMPA6WFhcX240bNzpdBoBuEqgPa+lv1uvt/RV64NPTtGDCIKdLAgAAgIOMMZustcWdnefUtkQA0GUpyYlaubRYE4dm6DNPvK03d5d2/iQAAAAMeAReAP1CuidJj940Q6NyUnXzoxu15WCl0yUBAACgjyPwAug3sr3JWrVipnxpbi19eL22H6l2uiQAAAD0YQReAP3KoAyPnlgxU56kBC1euU77Sv1OlwQAAIA+isALoN8ZkZOqVctnKhSO6IaH1un9qoDTJQEAAKAPIvAC6JfGDErXY8tm6kSgQYsfWqeymjqnSwIAAEAfQ+AF0G9NGp6plTdOV0llQEseXq+qQIPTJQEAAKAPIfAC6NdmFObo/sXTtONotZY/skG19SGnSwIAAEAfQeAF0O9dMC5fP7t2qt4+UKFbH9+kulDY6ZIAAADQBxB4AcSFj08aors/OVmv7SzV51ZvUSgccbokAAAAOIzACyBuLCoeoW8tnKi//PuIvvqHrYpErNMlAQAAwEEupwsAgO60bE6hTgQb9NO/7VS6x6VvXzZRxhinywIAAIADCLwA4s7nFoxRdTCkla/vVbrHpS9dNM7pkgAAAOAAAi+AuGOM0X9dOkE1wZB+vmaX0j0u3TL3LKfLAgAAQC8j8AKIS8YYff+qSaqpD+n7L25TuidJ180Y6XRZAAAA6EUEXgBxKzHB6N5FU+SvC+nrz26V1+3S5WcPdbosAAAA9BJWaQYQ15JdCfrVDdM0vSBHX3xyi9ZsO+p0SQAAAOglBF4AcS8lOVErlxZr4tAMfWbV21q7u8zpkgAAANALCLwABoR0T5IevWmGRuakasWjG7TlYKXTJQEAAKCHEXgBDBjZ3mStWjFTvjS3lj68XtuPVDtdEgAAAHoQgRfAgDIow6MnVsyUJylBi1eu075Sv9MlAQAAoIcQeAEMOCNyUrVq+UyFwhHd8NA6vV8VcLokAAAA9AACL4ABacygdD22bKZOBBq0+KF1Kqupc7okAAAAdDMCL4ABa9LwTK28cboOVQS05OH1qgo0OF0SAAAAuhGBF8CANqMwR/d/epp2HK3W8kc2qLY+5HRJAAAA6CYEXgAD3rxx+frpNVP19oEK3fr4JtWFwk6XBAAAgG5A4AUASZdOHqK7r5qs13aW6nOrtygUjjhdEgAAAM4QgRcAYhZNH6FvLpyov/z7iO58ZqsiEet0SQAAADgDLqcLAIC+ZPmcQlUHG/TTv+1Umtulb182UcYYp8sCAADAaSDwAkArn1swRtXBkFa+vlcZHpe+eNE4p0sCAADAaSDwAkArxhj916UTVBMM6b41u5TuSdLNc4ucLgsAAAAfEIEXANpgjNH3r5qkmvqQ7nrxPaV5XLpuxkinywIAAMAHQOAFgHYkJhjdu2iK/HUhff3ZrfK6Xbr87KFOlwUAAIAuYpVmAOhAsitBv7phmqYX5OiLT27Rmm1HnS4JAAAAXUTgBYBOpCQnauXSYk0cmqHPrHpba3eXOV0SAAAAuoDACwBdkO5J0qM3zdDInFSteHSDthysdLokAAAAdILACwBdlO1N1qoVM5WTlqylD6/X9iPVTpcEAACADhB4AeADGJTh0RPLZ8mTlKDFK9dpX6nf6ZIAAADQDgIvAHxAI32pWrV8pkLhiG54aJ3erwo4XRIAAADaQOAFgNMwZlC6Hls2U1WBBi1+aJ3KauqcLgkAAACtEHgB4DRNGp6plUuLdagioCUPr9eJYIPTJQEAAKAZAi8AnIGZRT7d/+lp2nG0Wssf2aBAfdjpkgAAABBD4AWAMzRvXL5+es1UbdpfoVtXbVJdiNALAADQFxB4AaAbXDp5iO6+arJe3XFcn//dFoXCEadLAgAAGPAIvADQTRZNH6FvLpyo//3XEd35zFZFItbpkgAAAAY0V1dOMsacJemQtbbOGHOBpMmSHrPWVvZkcQDQ3yyfU6jqYIN++redSnO79O3LJsoY43RZAAAAA1JXR3j/IClsjBkt6UFJIyT9tseqAoB+7HMLxmj5nEI98uY+3fvXHU6XAwAAMGB1aYRXUsRaGzLGfELSz621PzfGbO7JwgCgvzLG6L8unaCaYEj3rdmldE+Sbp5b5HRZAAAAA05XA2+DMeY6SUslXRY7ltQzJQFA/2eM0fevmqSa+pDuevE9pXlcum7GSKfLAgAAGFC6GnhvknSbpLustXuNMYWSHu+5sgCg/0tMMLp30RT560L6+rNb5XW7dPnZQ50uCwAAYMDocA6vMebBWBvzQWvt7dba1ZJkrd1rrf1hr1QIAP1YsitBv7phmqaPytEXn9yiNduOOl0SAADAgNHZolUrJZ0t6UVjzMvGmK8aY87uhboAIG6kJCdq5Y3FmjAkQ59Z9bbW7i5zuiQAAIABwVjbtX0ijTE+SRdJ+pii2xK9Lekv1tqneq6801NcXGw3btzodBkA0EK5v17XPLBWhysDeuLmWZoyIsvpkgAAAFp4bnOJ7nlpuw5XBjQ0K0V3XDxOV04d5nRZpzDGbLLWFnd2Xle3JZK1tsxau9pau8RaO0XSLyWNOZMiAWAgyfEma9WKmcpJS9aNv1mv7UeqnS4JAACgyXObS/S1Z7aqpDIgK6mkMqCvPbNVz20ucbq009Zp4DXGnG+MmRy7vcgY8wtjzOcl/ctae1ePVwgAcWRQhkdPLJ8ltytBi1eu0/4yv9MlAQAASJJ+9NI2BRrCLY4FGsK656XtDlV05jpcpdkY80tF25fdxpgdktIk/UXSuZIelnRDj1cIAHFmpC9Vq5bP1KIH1uqGh9bp97fN1pDMFKfLAgAAA0BDOKKSioD2lvm1v9SvfWW12lfm1/6yWh2uDLb5nMOVgV6usvt0ti3RPGvtRGOMR1KJpHxrbdgY84Ckd3q+PACIT2MGpeuxZTN13a/f0uKH1umpW2fLl+Z2uiwAABAHGsIRHSyv1f5YmN0XC7b7y/w6WBFQOHJyHSdvcqIKcr2aOCRDx6vrVFMXOuV6Q7P67y/mOwu8QUmy1gaNMfutteHYfWuMaejx6gAgjk0anqmVS4u15OH1WvLweq2+ZZYyPElOlwUAAPqB+lBEBytqW4TZvaXRkdqSypahNs3tUkFuqj48LFMLJw/VKF+qCnO9GuXzKjctWcYYSSfn8DZva05JStQdF4/r9ffXXToLvPnGmC9KMs1uK3Y/r0crA4ABYGaRT/d/eppueWyjlj+yQY8tm6mU5ESnywIAAH1AXSisg+W12lcaG6mNtR7vLfXrcGVAzTKt0j0uFeZ6dfaILF0xZagKfF4V5KZqlM8rn/dkqO1I42rM/WGV5q7qcFsiY8y3O3qytfa/u72ibsC2RAD6mxfeeV//ufptzRmTp18vmSa3i9ALAMBAEGyIhtrG0dnGYLuvtFaHqwJqHtcyYqG2IDY6W+BLVUGuVwU+r7JTk7oUauNFV7cl6nCEt6cCrTEmS9JDkj4syUpaJmm7pCclFUjaJ2mRtbbCRP/Ufibp45JqJd1orX27J+oCAKdcOnmI/HWT9ZU/vKPP/26Lfn7dVLkSu7xzHAAA6MOCDeE259PuK/Xr/RPBFqE2KzVJBT6vphdka5RveKz1ODUaar3Jzr2JfqqzVZq/Yq39kTHm54oG0xastbef5uv+TNJfrLVXG2OSJaVK+rqkl621dxtj7pR0p6SvSvqYovv9jpE0U9KvYt8BIK4smj5C1XUh/c+f39Wdz2zVjz45WQkJA+c3tQAA9GeB+rD2l/vbnFP7flXL1Y9zvMka5UvVrCJfdKQ2NxpoR/lSlZVKqO1Onc3hfS/2vdv6g40xmZLmSrpRkqy19ZLqjTFXSLogdtqjkv6uaOC9QtJjNtp7/ZYxJssYM8Ra+3531QQAfcXyOYWqDjbop3/bqTS3S9++bOKAak8CAKAv89eFtL8xzJb5tb/Z3NqjJ+panOvzJqsg16vZZ/lU6PNqVG60BXmUz6vMFBap7C2dtTQ/H/v+aDe+ZqGk45J+Y4w5W9ImSZ+TNKhZiD0iaVDs9jBJB5s9/1DsWIvAa4y5RdItkjRy5MhuLBcAetfnFozRiUBID7+xVxkel754Uf9dGREAgP6mpi4Uazc+2YLc2I58rLplqM1Nc6vAl6rzxuQ1hdnCXK9G+lLZeaGP6Kyl+U8dPW6tvfw0X/McSf9prV1njPmZou3Lza9rjTHtr6bVdi0PSnpQii5adRp1AUCfYIzRNxdOUE1dg+5bs0vpniTdPLfI6bIAAIgb1cGGNufU7i2tVWlNy1Cbl+5Woc+r88fmNS0QNSq2WFSau7OGWTitsz+h2YqOrq6WtE7R7YjO1CFJh6y162L3n1Y08B5tbFU2xgyRdCz2eImkEc2ePzx2DADiljFGP7hqsvx1Yd314ntK87h03Qy6VwAA6KoTwYaTYbY01oIcC7alNfUtzh2U4dYon1cLxudrVGw+bWOw9Q60UPvOU9LL35WqDkmZw6UF35ImL3K6qtPW2Z/eYEkflXSdpOslvSBptbX236f7gtbaI8aYg8aYcdba7ZIWSHo39rVU0t2x73+MPeVPkj5rjPmdootVVTF/F8BAkJhgdO81U+SvD+nrz25Vmtuly84e6nRZAAD0GVW1DS228dnfeLusVuX+lqF2SKZHo3ypunDCoNhIbbQFeZQvVanJAyzUtuedp6Tnb5caAtH7VQej96V+G3o73Ie3xYnGuBUNvvdI+m9r7S9O+0WNmaLotkTJkvZIuklSgqSnJI2UtF/RbYnKY9sS/ULSJYpuS3STtbbDRbTYhxdAPAnUh7X04fV6+0CFHlwyTfPHD+r8SQAAxInK2vqWe9Q2a0GuqG1oce7QTE9s1WNvyzm1OalKSWaP+zaFG6RApRQolx5ZKPmPnXpO5gjpC//q/do60NV9eDsNvLGge6miYbdA0RHXh621fbatmMALIN5UBxt0/a/XacfRaj26bIZmFfmcLgkAgG5hrVVFbKS2cR7t/tgo7b5Sv6oCJ0OtMdLQzBQV5MbCbLP5tCNzUuVJGsChNhySglXR4BqokGpj3xvvt3WstkKqr+7CxY30ncoefwsfRLcEXmPMY5I+LOlFSb+z1vatWN8OAi+AeFTur9c1D6zV4cqAfnvzLJ09IsvpkgAA6BJrrcr99S1aj/fGRmn3lfp1IhhqOjfBSEOzUqLzaJvNpy3ITdXw7AEQaiPhWHBtHVIr2g+ztRVSXVX71zQJkidLSsmWUnOi31NyWt3Plv5yp+Q/furz43WE1xgTkeSP3W1+olF0MeWMM6qyhxB4AcSroyeCuvr+N1UdDOnJW2Zr3OB0p0sCAEBSNNSW1tTHRmmbtSDH9qutrmsZaodnp0ZHZ1u1II/ISZHbFQehNhKR6k60DKWdBddARbS9WO1lNCN5MjsPrk3HYvfdmVJCQuc1t57DK0lJKdJl9/W5Obzd1tLcHxF4AcSzA2W1+tQDbypipadvm61RPq/TJQEABghrrY7X1DXtUdt8v9r9ZbWqaRZqExOMhmenxFqPT86nHeWLjtQmu7oQwPoCa6W66nZag9tpFW78spH2r+vOlFKyTg2uHYVZT6aU0MO/DOgnqzQTeAm8AOLYzqPVWvTAWnndLv3+ttkakpnidEkAgDhhrdWx6rrY4lAt96jdX+ZXbX246VxXgtGInGYjtb5UjYrtVTs8O0VJiX0o1For1fs7mNNa2Xb7cKBCioTav25yesvR1C4F1ywpkZWhzwSBl8ALIM5tPVSl6379lgZluPXUrbPlS3M7XRIAoJ+IRKKhNtp6fHKBqMaR2kBDy1A7Mie6MFTrFuShWQ6EWmujLbedLcbUVnAN17d/3STv6QVXV3LvvXc0IfASeAEMAOv2lGnJw+s1Oj9Nq2+ZpQxPktMlAQD6iEjE6siJ4Kl71JbWan+5X8GGk+22yYkJGpETXSgq2np8sgV5SKZHrp4KtQ3Bzue01paf3Dan8X64rv1rulJahdSsNua5thFmXfziuD8h8BJ4AQwQr2w/ppsf3aipI7P02LKZ7DMIAANIJGL1/ongyfbjZnvU7i+rVV2oZagd2ar1uHFbn6FZKUpMMKdfSKi+i8G11bzXUKD9ayYmR4Npi8WY2hlxbX4siWk+AwGBl8ALYAD58zuHdfvqzZozJk+/XjItPla3BABIksIRq8OVAe0vq9XeMr/2x0LtvjK/DpTXqr5ZqHW7EjQqtkBUQWx/2sYW5MEZns5Dbbjh1NHUDvdyjc17bfC3f80EVzsjrJ0s2JSUGt14F2hDVwMvM6UBIA4snDxU/rqQvvqHrfr877bo59dN7bn2MwBAtwuFI3q/Ktg0p3Zvsxbkg+UB1YdbhtoCn1dn5Xm1YHx+NNzG9qsdnOFRQoKRwqHYXq7lUuCodLRc2teFvVzrq9sv0iS2DKUZw6RBk1rNe21jnmtyGsEVjiHwAkCcuGb6SFUHQ/reC+/pzme26kefnBz90AMA6BNC4YhKKgOnLBC1r9SvgxW1agif7LxMSUrUKF+qxual6rKxKRqdVq9RKfUa5gkoS1VKCO4/GVQPlks7Wo3CBqvaL8QkRBdbagylaYOkvAlt7OXaasTVnUFwRb9D4AWAOLLivCJVB0P62cs7leZ26duXTZThwwkA9JqGcESHKgLRMNu89bi0RhUVZUq31cpSjbJMjQa5alXsrddVKUENHhFQXoJfmaZGqeFqJdVXygQqpD2Vktqbgmii+7I2htJUn+Qb00ZwzWk5CuvOlBLoAsLAQOAFgDjz+QvHqDoY0sNv7FVGSpK++NGxTpcEAPHDWtXXVunIkcM6evSIyo4fUXXFMdVWlSpUUyoTrFSmqpUlvyaZGp1vapST4Fe6rVFCcuTU6wViX+7Mk3NavdlSblEXtsTJlBJYswHoCIEXAOKMMUbfXDhBNXUNuu/lncrwuLTivCKnywKAvsVaqd7f7mJMYX+F/JXHVFddprC/XInBCiU3VMkbqVaywhopaWQbl61L9iqUnCml5ijJO0RJ6bkyXdnLNZGP5UBP4F8WAMQhY4x+cNVk+evC+t4L7ynN7dK1M9r6aAYA/Zy1UkOgC1viVJx6LFzf7mWD1q1qpanSRr/8iUMU9kxUYmqO3Om58mblKSs3X3l5g5WRM0gmNUfyZMntSha7uQJ9B4EXAOJUYoLRvddMkb8+pK89u1Vet0uXnT3U6bIAoH0NwS7u5dpq25xwXbuXtK4UhdyZCroyVW3SVW4H6ZgtVIk8OhTyqMKeDLVhd5bScwbJl5unYXk5KsyNbu8zwedVdmoSayIA/RCBFwDiWLIrQb+6YZqWPrxeX3hyi7zuRM0fP8jpsgDEu1B9F4NrRcv7oUD710xMbtkKnHNyjmuDO1tl4VS935CqgwG39vqTtf1Ekt6tSND+aitbc/Iy2alJ0W18RkT3qJ3g82qUL1WFuV5lpSb3/M8GQK8y1ra36lv/VVxcbDdu3Oh0GQDQZ1QHG3T9r9dpx9FqPbpshmYV+ZwuCUB/EG44dTQ1UNFBmK2M3q6vaf+aCa5T57Cm5JxcsKmdvVwD1q195Y17057c1mdfaa2OnAi2eIkcb7IKfNF9aZvvUVvg8yozNamHf2gAeoMxZpO1trjT8wi8wAD0zlPSy9+Vqg5JmcOlBd+SJi9yuir0sHJ/va55YK0OVwb025tn6ewRWU6XBKC3hEPRfVm7HFwrouG17kT71zSJHawinN1ucFVyWrt7ufrrQtF9actie9SW1mpvmV/7y/w6eqJl23JuWrJGNY7O+rwaletVoc+rkb5UZaYQaoF4R+Al8AJte+cp6fnbowt8NEpKkS67j9A7ABw9EdTV97+p6mBIT94yW+MGpztdEoAPIhKOBdeKNlqDO2gfDla1f02TEF0luK3gesp+rs3uuzPaDa4dqakLaV+p/2Swjd3eW+bX8erWodbdNI+2wBdtQS6IhdoMD6EWGMgIvAReoKVIODqi+9ACyX/81MfT8qXlf4t+T0rp/frQaw6U1erq+9+UlfT0bbM1yud1uiRg4IlEoqOnHa0i3NYobKBSUnuf3Ux0X9ZOg2urUVh3ppSQ0K1vrzrYEA2xpf5WLci1Kq1pGWrz092x1uOTgbYgFnLT3Cw3A6BtBF4CLwaicINUeUAq3yuV72n5VbFPijR07TrJaZI3L/qVli95cyVvfux+7Lg3djwl+7R+ww9n7TharWseWCuv26Xf3zZbQzL5JQdwWqyV6qo7X4zplOBaIdlI+9d1Z7aa09pRcG3cyzVTSkjstbd+ItjQFGIb59Puj90u87fc7mdQhrtpDu2o3FgLcizkegm1AE4DgZfAi3gVqpMq9p8aaMv3RMOuDZ88N8kbXcXSVxT9nlMUnbvb1ghvaq504XeijzV+1RyT/KWS/5hUW9b2h7ME18lw3JWAnEgLWl/xzqFKXf/rdRqU4dZTt86WL42dIzGAWSvV+ztejKm94BoJtX/d5PTO57S2DrOeLCmxb4TAqtqGpvm0+0qjC0btjQXb8lahdkimp2m14+YtyCNzUpWa3DfeD4D4QeAl8KI/awhER2Sbh9my3dGR26qDatHO5s44GWZbf6Xlnzr6erpzeCPh6Ic9//FoAPaXxgJx6/uxgBwKtn2dlOyWATgtv/3AnOxl9LiHrdtTpiUPr9fo/DStvmUWc+IQH+pru7AlThvBNVzf/jWTvKcXXF19f5ubCn990+hsixbkMr8qa1t2Bg3N9KigVaBtbEf2JPXe6DIAEHgJvOjr6mqkitatx7H7J0panpuS036oTc354KGwp1dptja6JUXzAOw/LtUcbzswByvbvo4r5dQR4tYBuTEkp2T3aitfPHll2zHd/NhGTR2ZpceWzVRKMj9H9BENwc7ntNY2boXT7LH2fuEmRf+70iKkZnUtuCZ5eu99dzNrrSoaR2qbtSA3BtuqwMlQa4w0NDOlxTY+jaO2I3IItQD6DgIvgRd9QbCq7UBbvkeqOdryXG9+qzBbePJ7SrYz9feWUL1UW9pGQG4nMDdv225kEqJt2W21UbcVkvvxh9ee8Od3Duv21Zt13pg8/XpJsZJd3buADQa4UH0Xg2tFy6+G2vavmZjcRlBta1XhVufE6aJ81lqV+eujIba0cVufWAtyqV/VwZNt1wlGGpqVEms9PhlsC3JTNTybUAugfyDwEnjRW2rLY0F296lzamvLWp6bPqRVmG325WZ7mC6JRKIjwk2BuJ05x40BucHf9nXcGc1GiNsLyLH7nswB0Vr95IYD+uofturjkwbrvmunypVI6EUr4YaWo6md7uVaGb1dX9P+NRNc7YywZnUSXFMHxL/L5qy1Kq2pbwqxjVv57I/tV1td1zLUDs9ObXNO7fDsFLldhFoA/VtXAy8rCACdsTYaotpaJKp8T6t2XBNtEc4plCZc1jLQZhdE56TizCQkRD/wpuZIeeM6P7/eHwvE7c05Pi6V7pT2vxn9oN7Wdh+JyV1fmCvV12cWm/mgrpk+UtXBkL73wnvyJm/VDz85WQkJAytQDBjhUGwv17ZWFm5v3mtldBud9pjEliOsGcOkQZOaHWsnuCanDbjg2hFrrY5X1zXNoW2+X+3+slrVNAu1iQlGI7JTNMrn1bSR2S3m0w7PTqVTAwBE4AWirJWqj7QTavdK9dUnzzUJUtbIaIiddHXLUJs1ilbZvibZG/3KLuj83HAoOirf5sJczUaRj70XfbzNBW5M9IN8hwG5+cJcqd39js/IivOKVB0M6Wcv71Sax6VvLZwoQxjpuyLhWHBtawucDkZhg1XtX9MkROesNobStEFS3oR22oWb3XdnEFy7yFqrY9V1be5Ru7/Mr9r6k9M2XAlGI3KiI7XTC3JU4EvVqFyvCn1eDctOURKdGADQIQIvBo5IJLoYVFuBtmJvy7liCa5oeM0pkkbObhVqR/aLVTdxGhJdUvqg6FdnrI2OdrVYiKuNhbne/2f0e107ASPJ2/lq1Y33PVnREe4e9vkLx6g6GNLDb+xVuidJX/zo2B5/zQEvEon+fWoRVDua49ps1LWtrgRJkom24zeG0tQcyTe6g71cY8fdmb3y9yzeRSJWR6uDLbfyKT05UhtoOBlqkxKjobbA59WsopwWLcjDslKYXgAAZ4DAi/gSDkknDrW9SFT5Xilcd/LcxGQpOzaXtuiClvNqM0f027ZU9BITCxOeTCl3dOfnNwTbWZirWUCuPCAd2hg9r709j09ZmKt5QG517DR/MWOM0TcXTlBNXYPue3mnMjwurTiv6LSuNeBYK9VVd74YU1vtwm0txtbIndlyTmv2qC6sLJzJyuU9LBKxOnIi2GJ0tnFu7f5yv4INJ/8dJycmaEROigp8Xp07Ojc6UuvzqjDXqyGZHkItAPQQPtGj/wk3RINBW+3HFfulSLM9A10p0QDrGy2NuajlSG3GUD4MovckeaLzuzOHd35uJBINQp0tzFW2KxqYQ4G2r+PJbHuecVsjyu70Fu2oxhj94KrJ8teF9b0X3lOa26VrZ4zsph9GP2BtdP53Z4sxtbW/ayTU/nWT01uOpmYO69qWOPwCzjGRiNXhqkDTPNoW2/qU16o+1CzUuhI0KicaZM8bk9tiTu3QrBQlMiceAHod/wdF39QQlCr3tx1qKw+2HAlJTouOzg76ULOFos6Kfk8fzJwy9D8JCbF25lwpf0LH5zYGs47mHPtLY/OOX40Gsra4PK1GjPOU6M3TzwryNKmiWn9+bqsGB2fpgnMmRhfm6o5fFvX0ftCN6ms7WYypjfbhQEU7c7RjkrwtF2PKn9C14Mp0iD4pHLE6XBk4uZVPqb/p9oFWodbtSmha+Xje+PzobZ9Xo3K9GpzhIdQCQB/DtkRwTn2tVLGvje189kY/ADefl+bOlG/TEfIAABzySURBVHxFp27lk1MU/XBOqAW6JlQfW5irnTnHrQNzm6OVJhp6O1qtuvn9tvY9fecp6fnbpYZmo9NJKdJl97Ufehv+f3t3Hl5XXe97/P1tkqZJOqRNS+mQDsxWKBQqFDgOCA44HFARceQcEfS5KA7ncg/nDlwfj8+j3nOuKPeqOHGFRz3AAVQUnC6i5yq2tUxtATlUoE1LoQOdm6ZJ+7t/rJV2J01a2qZ7Ze+8X8/TZ63922vv/U1YrOST32/9fjte5lquG3sG164d/X8/aht63dPa3M/aruN6Blcnp6s4Xbt28/zGHXmQ3dbj3tq2l7bTuWvvz5wRdcP29Mxm69Pu3T969AhnL5ekQcBliTQ4dGzpdR9tyX21W57veWxjSxZgp5+zb6htGGuolQZC7XAYPSn7dyApZYFx2zq2bVjNjfc8yM5NL3LFnJFMrdu6NxSveigLy6WzmZcaPmrfYdRL7+oZdiF7fO9nYPkf+r7vtXRiud5qhvcMpd3Xjf0F14axfYdxVayuXbtZtbF9z320pcv6tG3oGWob6mqY3tLIiRNH8cZZR+9Zo3ZGSxNHjao31EpSlbCHV4evfWM/k0Q9k/UalWo6Kr+n9tiek0SNnZn1rEgatF7atpNLv/lHVm9s54dXzuPU1l7/z+7cnk/MdYCZq7euyY7rT9NRfQwNbu57uPCe4NroH8WGiM5du1m5oX2fNWqfW7eNlRva6dq99/eaxuE1eQ9tPkFUd6/t+CzUuuSWJFWul9vDa+DVgaWU9bb0uUbtM9nQwVKjJudBdmavntqZ2cQ4kirWC5t28O5vPsiWHV3cftXZnHj0If4/fcMr81sXehnTCp9eenhFquLt7NrNyg3bew09zrYrN7SzqyTUjqyv3RNiS2c+nt7SyISRhlpJqlYGXgNv3/qbJCalrAemv1C7o3QN0ch+Kd0n0B4DY2fA8MaivjpJZbBi/XYuuelBAP71Y2czvaXp4N/kUO7hVVXp6NpF20vtPZby6b6/dtWGdkoyLaPqa3vcR9sdbmeMb6KlabihVpKGIAOvgXdfff2CGTUwahLs2Ag7t/Zsb57W9yRRY6dDbX3565c0aPz7i1t4zzf/SFN9LXd+7ByOHnMIkziVa5ZmFWZH5y5WbtjOs+u27xNsn9/YK9SOqGVmfg9td09td7AdZ6iVJPVi4DXw7uuGk2FT277ttSPgjL/pGWrHtLp8hqT9WrxyI+/79gKOHjOC26+aR8tI/xA2FO3o3MWKl7bn69Pmy/rkQ5Gf39RO6a8ZYxrq9vbO9rq3trmxzlArSXrZnKVZ++rrfjmArg648EvlrUVSxZs9tZnvXj6XD928kMv/z0J+eOU8Ro+oK7osHQHtO3ex/KW999OW3lu7evOOHqF2bGMd01uaOHPmuH2GIDc3+odUSVJ5GXiHkjFT++7hHTO1/LVIqgpnHdPCTR84gytvXcQV3/sTt374LBqG1xRdlg7B9p1dLF/fPfS45xDkFzb3XMt4XNNwZrQ0Mu+Ylp731rY0MabRP3pIkgYPA+9Qcv71cPdVQMmf4usasnZJOkTnnXQUX7nsNK75l0f42Pcf4tsfmsvw2mFFl6U+bOvo4rn1PZfy6R6C/OLmjh7Hjh85nOktTZx73Pjsntrx2dDjaS2NjGkw1EqSKoOBdyiZ+RogwYjmbNZlJ4mRNEDeNnsy2zq6+Pu7lvCp2x/hxsvmUFtj6C3C1o6ufdan7b63du2WnqF2wqh6ZrQ08urjJ+xZymdGvlbtKIenS5KqgIF3KFkxP9t+4C6YesD7uyXpoLznVdPYsqOLz9/7JE3Dl/Cld81m2DAnIToStuzo5Ll12/Pe2r1DkJ9bv511W3uG2qNG1TNjfBPnnTihxxq101uaGFnvrwGSpOrmT7qhpG1hNiPz0bOLrkRSlfrIq49hy44uvnr/04wcUcv1b5vlzLuHaFN7575r1Ob767ft7HHs0aNHML2lkfNPOmrPBFHT857aJkOtJGkI86fgUNI2Hyaf7nJDko6oT11wPFt2dHHzH55l1Ig6PvOGE4ouadDatL2TZ/Ne2u4e2+5gu2F7Z49jJ43JQu0bXzkxW6M2X9Zn2rhGGof741ySpL74E3Ko6GyH1Y/BOZ8ouhJJVS4i+G9vewVbOzq58f6nGT2ilo+8+piiyyrMhm07S4JsPvtxPlHUxpJQGwGTxzQwvaWRN588iZn5GrXd99SOqHP2a0mSDpaBd6hY9TDs7oLWs4quRNIQEBF84Z2z2dqR3dM7sr6Wy86cVnRZR0RKiQ3bO/Ohx9l9tM+V7G9q3zfUzhzfxFtPmdRjjdrWcYZaSZIGmoF3qGjLJ6wy8Eoqk5phwVfeM4dtHYv4hx8toam+lrefOrnosg5JSon123buM0FU9wzIW3Z07Tl2WMCUsQ3MaGni7adO2rM+7YzxTbSOa6C+1lArSVK5GHiHiraFMP4EaBxXdCWShpDhtcO46QNncPnNC/n07Y8ysr6W8046quiy+pRSYu3WjmyCqJKlfJav38byddvZ0tEz1E4d28iM8U3Mmdacz36cDUGeOtZQK0nSYGHgHQp274a2BXDSW4uuRNIQ1DC8hu/8zVze9+35fOz7D3HLh89k3jEthdSSUmLtlo4evbPL12/fMxx5285de46tGRa0jm1geksTc6eP27NG7YzxTUxpbmB4resMS5I02Bl4h4L1y6B9A7TOK7oSSUPU6BF13Prhs7j0m3/kI7cs4odXnsXsqc1H5LNSSry4uaPPNWqXr9/G9pJQWzssaB3XyIyWRs6cOY4ZLY35PbVNTBnbQF2NoVaSpEpm4B0KvH9X0iAwrmk437/iLN79zQf50M0LueOjZ3PCxFGH9F67dyde3LKjx1I+y/P95eu30965N9TW1XSH2ibOPqaFGfnQ45ktTUxuHkGtoVaSpKpl4B0KViyAhnEw/viiK5E0xB09ZgQ/uGIel9z0IO/6+h9orK9lzeYOJjc3cO2bTuTiOVP2HLt7d2L15h0sX5dPEJWvT7t8/XaWv7SNHZ279xw7vGYYreOy2Y/PPW58j57aSWMMtZIkDVUG3qGgbUHWuxtRdCWSxLSWRv723Bl86RdPsaUj64ldtbGda+98jJ88upKaYTXZRFEvbWdnV0morR3G9HFZ7+xrThi/Z43aGeMbmTSmgZphXuMkSVJPBt5qt209rH8aTntf0ZVI0h7fn79in7bOXYkHnlrHCRNHMnN8E+eddFS+pE8j08c3MWn0CIYZaiVJ0kEw8Fa7tgXZdpoTVkkaPJ7f2N5newC/+vRry1uMJEmqWt7UVO3aFsCwOpg8p+hKJGmPyc0NB9UuSZJ0KAy81a5tAUw6Fer8JVLS4HHtm06koa6mR1tDXQ3XvunEgiqSJEnVqLDAGxE1EfFIRPwsfzwzIhZExLKIuD0ihuft9fnjZfnzM4qqueJ07YRVDzucWdKgc/GcKXzhnacwpbmBAKY0N/CFd57SY5ZmSZKkw1XkPbyfBJ4ERuePvwTckFK6LSJuAq4AvpFvN6SUjouIy/Lj3lNEwRVn9WOwqwNazyy6Eknax8VzphhwJUnSEVVID29ETAXeCnwnfxzA64E780NuAS7O9y/KH5M/f35+vA6kbX62bbWHV5IkSdLQU9SQ5q8A/wnoXmCxBdiYUurKH68Euv/sPwVoA8if35QfrwNpWwBjZ8CoiUVXIkmSJEllV/bAGxFvA9aklB4a4Pe9KiIWRcSitWvXDuRbV6aUYMUCaD2r6EokSZIkqRBF9PCeC/x1RDwH3EY2lPmrQHNEdN9TPBVYle+vAloB8ufHAOt7v2lK6VsppbkppbkTJkw4sl9BJdjwLGxbY+CVJEmSNGSVPfCmlP4hpTQ1pTQDuAz4TUrp/cADwCX5YZcDP8n378kfkz//m5RSKmPJlaltYbZ1hmZJkiRJQ9RgWof374HPRMQysnt0v5u3fxdoyds/A1xXUH2VZcV8qB8NE04quhJJkiRJKkSRyxKRUvot8Nt8/xlgn/VzUko7gHeXtbBq0LYQpr4KhtUUXYkkSZIkFWIw9fBqoLRvhDVPeP+uJEmSpCHNwFuNVi4CEkwz8EqSJEkaugy81ahtAcQwmDK36EokSZIkqTAG3mrUNh8mngz1I4uuRJIkSZIKY+CtNru6YOVDLkckSZIkacgz8FabF5dC5zYnrJIkSZI05Bl4q03bgmxr4JUkSZI0xBl4q03bAhg9BZpbi65EkiRJkgpl4K02KxZA65lFVyFJkiRJhTPwVpNNK2HzSmh1wipJkiRJMvBWk+77d6d5/64kSZIkGXiryYoFUNeYrcErSZIkSUOcgbeatC2AKWdATV3RlUiSJElS4Qy81aJjK7ywxOWIJEmSJCln4K0Wqx6CtAumOWGVJEmSJIGBt3q0Lcy2U19VbB2SJEmSNEgYeKtF23yY8ApoaC66EkmSJEkaFAy81WD3bmj7k8sRSZIkSVIJA281WPtn6NgErd6/K0mSJEndDLzVoG1+tm09s9g6JEmSJGkQMfBWg7aF0DQBxh1TdCWSJEmSNGgYeKvBivnZ+rsRRVciSZIkSYOGgbfSbV0DG57NAq8kSZIkaQ8Db6VrW5BtpzlhlSRJkiSVMvBWuhXzoaYeJp1adCWSJEmSNKgYeCtd2wKYPAdq64uuRJIkSZIGFQNvJevcAc8/CtO8f1eSJEmSejPwVrLnH4HdnU5YJUmSJEl9MPBWsu4Jqwy8kiRJkrQPA28la1sA446FpvFFVyJJkiRJg46Bt1KllAVelyOSJEmSpD4ZeCvV+r/A9vUOZ5YkSZKkfhh4K1Xb/Gxr4JUkSZKkPhl4K9WK+TCiGcafUHQlkiRJkjQoGXgrVdvCrHd3mP8JJUmSJKkvpqVKtP0lWPcUtJ5ZdCWSJEmSNGgZeCvRyj9lW2doliRJkqR+GXgr0Yr5MKwWJp9edCWSJEmSNGgZeCvJ4jvghpPh91+GCPjzz4quSJIkSZIGrdqiC9DLtPgO+Ok10NmePd7VmT0GmH1pcXVJkiRJ0iBlD2+luP9ze8Nut872rF2SJEmStA8Db6XYtPLg2iVJkiRpiDPwVooxUw+uXZIkSZKGOANvpTj/eqgd0bOtriFrlyRJkiTtw8BbKWZfCsddkD8IGNMKb7/RCaskSZIkqR/O0lwpUoI1T8LM18DlPy26GkmSJEka9OzhrRSrH4OX/gInX1J0JZIkSZJUEQy8lWLpXTCsFl7x9qIrkSRJkqSKYOCtBLt3w9K74djzoXFc0dVIkiRJUkUw8FaClQth80o4xeHMkiRJkvRyGXgrwZI7syWJTryw6EokSZIkqWIYeAe7XV3wxI/hhDdD/aiiq5EkSZKkimHgHeye+zfYthZOflfRlUiSJElSRTHwDnZL74Lho+D4NxRdiSRJkiRVFAPvYNbVAU/+FF7xNqhrKLoaSZIkSaooBt7BbNn9sGOTw5klSZIk6RAYeAezpXdBwzg45nVFVyJJkiRJFcfAO1jt3AZP3QezLoKauqKrkSRJkqSKY+AdrP79F9C53eHMkiRJknSIDLyD1ZK7YNQkmH5O0ZVIkiRJUkUqe+CNiNaIeCAinoiIxyPik3n7uIj4dUQ8nW/H5u0RETdGxLKIWBwRp5e75rJr3wjLfg2vfAcMqym6GkmSJEmqSEX08HYBf5dSmgXMA66OiFnAdcD9KaXjgfvzxwAXAsfn/64CvlH+ksvsz/fCrp0OZ5YkSZKkw1D2wJtSWp1Sejjf3wI8CUwBLgJuyQ+7Bbg4378IuDVl5gPNETGpzGUPnMV3wA0nw2ebs+3iO/Y9Zumd0DwdppxR/vokSZIkqUoUeg9vRMwA5gALgIkppdX5Uy8AE/P9KUBbyctW5m293+uqiFgUEYvWrl17xGo+LIvvgJ9eA5vagJRtf3pNz9C7dS0887usdzeisFIlSZIkqdIVFngjYiRwF/CplNLm0udSSglIB/N+KaVvpZTmppTmTpgwYQArHUD3fw4623u2dbZn7d2e+DGkXXDKJeWtTZIkSZKqTCGBNyLqyMLuD1JKd+fNL3YPVc63a/L2VUBrycun5m2VZ9PKA7cvvRsmnARHzSpPTZIkSZJUpYqYpTmA7wJPppS+XPLUPcDl+f7lwE9K2j+Uz9Y8D9hUMvS5soyZuv/2TSthxYMOZ5YkSZKkAVBED++5wAeB10fEo/m/twBfBN4QEU8DF+SPAe4DngGWAd8G/kMBNQ+M86+HuoaebXUNWTvA4z/Kts7OLEmSJEmHrbbcH5hS+j3QX/fl+X0cn4Crj2hR5TL70mx7zzXQ1Q5jWrOw292+9C6YdBq0HFtcjZIkSZJUJcoeeIe82ZfCs7+DZb+BTy/d277+L/D8I/DGzxdXmyRJkiRVkUKXJRqy6kdDx5aebUvzubte+Y7y1yNJkiRJVcjAW4T6UbBzC+zelT1OCZbeCdPO7n9iK0mSJEnSQTHwFqF+dLbduTXbrnkC1v7ZyaokSZIkaQAZeItQPyrb7ticbZfcCVEDsy4uriZJkiRJqjIG3iKMyHt4O7bkw5nvgmNeCyMnFFuXJEmSJFURA28Runt4OzbDqodh43KHM0uSJEnSAHNZoiLUj8m2t70ftq/L9nd1FVePJEmSJFUhe3iLMP/r2bY77AL88jpYfEcx9UiSJElSFTLwltviO+Dxu/dt72yH+z9X/nokSZIkqUoZeMttf6F208ry1SFJkiRJVc7AW277C7VjppavDkmSJEmqcgbecus31Aacf31ZS5EkSZKkambgLbfzr4e6hl6NAXM/DLMvLaQkSZIkSapGLktUbt2h9v7PZcObx0zNQrBhV5IkSZIGlIG3CLMvNeBKkiRJ0hHmkGZJkiRJUlUy8EqSJEmSqpKBV5IkSZJUlQy8kiRJkqSqZOCVJEmSJFUlA68kSZIkqSoZeCVJkiRJVcnAK0mSJEmqSgZeSZIkSVJVMvBKkiRJkqqSgVeSJEmSVJUipVR0DQMuItYCy4uuYz/GA+uKLkJDgueaysVzTeXiuaZy8VxTuXiuHZrpKaUJBzqoKgPvYBcRi1JKc4uuQ9XPc03l4rmmcvFcU7l4rqlcPNeOLIc0S5IkSZKqkoFXkiRJklSVDLzF+FbRBWjI8FxTuXiuqVw811QunmsqF8+1I8h7eCVJkiRJVckeXkmSJElSVTLwSpIkSZKqkoG3zCLizRHxVEQsi4jriq5H1SUinouIJRHxaEQsytvGRcSvI+LpfDu26DpVeSLi5ohYExFLS9r6PLcic2N+nVscEacXV7kqTT/n2mcjYlV+bXs0It5S8tw/5OfaUxHxpmKqVqWJiNaIeCAinoiIxyPik3m71zUNqP2ca17XysTAW0YRUQN8DbgQmAW8NyJmFVuVqtB5KaXTStZzuw64P6V0PHB//lg6WN8D3tyrrb9z60Lg+PzfVcA3ylSjqsP32PdcA7ghv7adllK6DyD/GXoZ8Mr8NV/Pf9ZKB9IF/F1KaRYwD7g6P5+8rmmg9Xeugde1sjDwlteZwLKU0jMppZ3AbcBFBdek6ncRcEu+fwtwcYG1qEKllP4NeKlXc3/n1kXArSkzH2iOiEnlqVSVrp9zrT8XAbellDpSSs8Cy8h+1kr7lVJanVJ6ON/fAjwJTMHrmgbYfs61/nhdG2AG3vKaArSVPF7J/k946WAl4FcR8VBEXJW3TUwprc73XwAmFlOaqlB/55bXOh0JH8+Hkt5ccmuG55oOW0TMAOYAC/C6piOo17kGXtfKwsArVZe/SimdTjb06uqIeE3pkylbh8y1yDTgPLd0hH0DOBY4DVgN/M9iy1G1iIiRwF3Ap1JKm0uf87qmgdTHueZ1rUwMvOW1CmgteTw1b5MGREppVb5dA/yIbAjMi93DrvLtmuIqVJXp79zyWqcBlVJ6MaW0K6W0G/g2e4f3ea7pkEVEHVkA+UFK6e682euaBlxf55rXtfIx8JbXn4DjI2JmRAwnuyH9noJrUpWIiKaIGNW9D7wRWEp2jl2eH3Y58JNiKlQV6u/cugf4UD6r6TxgU8kQQemg9bpX8h1k1zbIzrXLIqI+ImaSTSi0sNz1qfJERADfBZ5MKX255CmvaxpQ/Z1rXtfKp7boAoaSlFJXRHwc+CVQA9ycUnq84LJUPSYCP8quq9QCP0wp/SIi/gTcERFXAMuBSwusURUqIv4FeB0wPiJWAv8d+CJ9n1v3AW8hm2hjO/C3ZS9YFaufc+11EXEa2fDS54CPAqSUHo+IO4AnyGZCvTqltKuIulVxzgU+CCyJiEfztv+M1zUNvP7Otfd6XSuPyG5PkCRJkiSpujikWZIkSZJUlQy8kiRJkqSqZOCVJEmSJFUlA68kSZIkqSoZeCVJkiRJVcnAK0kSEBEtEfFo/u+FiFhV8nj4EfrML0fE4xHxxUN8fU1EfC0ilkbEkohYGBHT8+d+2b0292HW+L8i4px8//cR8Wyv538WERvz/daIuD3fvyAiNuXfvyUR8auImJA/d3FEXH+4tUmSdCAuSyRJUi8R8Vlga0rpn3u1B9nPzt0D8BkBbADGvdz3i4jalFJXyeMPAm8F3pdS2h0R04DNKaWNh1tf/v4TgB+nlM7NH/8eGA1clVKaHxHjyNaWPz6l1NzrtRcAH08pXZw//qe8tn/Mv/ZHgHkppR0DUaskSX2xh1eSpP2IiOMi4omI+AHwODApIr4VEYvy3tnrS45dGRGfjYhHImJxRJyQt78+Ih7Lezsfjogm4F5gFPBwRFwSERMj4u78fRdGxLz8tZ+PiFsj4g/A93qVNwlY3R2YU0orusNuXktzRFxd0lP9XET8On/+woj4Y17P7XlNvb0b+HmvttuAy/L9S4A7e32vHu3jexjASLKAT8r+2v7/gLfs95svSdJhMvBKknRgJwE3pJRmpZRWAdellOYCpwJviIhZJce+mFKaA3wH+Ezedi1Zr+hpwGuAHcBfA1tSSqellO4EbgT+R/6+l+avL/3881NKH+hV123AO/OA/c8RcVrvwlNKX8s/90zgeeDLEXEUcF3+nqcDi4FP9vF1nws81Kvt18DrI2IY8B7g9r6/ZQCclwfgNuC19Azsi4BX7+e1kiQdNgOvJEkH9peU0qKSx++NiIeBh4FXAKWB9+58+xAwI9//A/DViPgEMDqltKuPz7gAuCkPiD8GxkZEQ/7cT/oa+ptSWgGcCPyXvOmBiHhdP1/D/wZ+nlL6OXBOXvOD+ee9v6TWUpOAtb3aOoH5ZL28NcDKfj4P4IE80E8FfgCU3qu8Bpi8n9dKknTYaosuQJKkCrCteycijifrDT0zpbQxIr4PjCg5tiPf7iL/OZtS+nxE3EN2v+38iDgf6DH5ExD5e+7s0RjR4/N7y4PwfcB9EbEOuAj4ba/3uBI4GvhoyWf9IqX0wf1/2bT3+tq63Qb8K/BfD/D6UveQhd5uI/L3lyTpiLGHV5KkgzMa2AJsjohJwJsO9IKIODaltDil9AWyXuET+zjs/wJXl7xmn+HJfbzvGXkN5EOMTwGW9zrmTOAa4INp70yVDwKvjYhj8mOa8iDf25PAcX20/5ast3Z/w5l7+yvgLyWPTwCWHsTrJUk6aAZeSZIOzsPAE8CfgVvJhisfyH/Mlw5aDGwFftXHMVcD5+aTXT0BXPky3vdo4N6IWAosIesx/UavYz4BjAN+l09cdVNK6UXgCuD2iHiMLACf0Mf73wu8rndjSml3SumfUkovHaC+8/LPfIxsCPS1pc/l7y9J0hHjskSSJKlP+ezKvwcuTCltHsD3nQx8L6X0xoF6T0mS+mLglSRJ/YqIs8lmkx6w4ccRcRbQnlJaPFDvKUlSXwy8kiRJkqSq5D28kiRJkqSqZOCVJEmSJFUlA68kSZIkqSoZeCVJkiRJVcnAK0mSJEmqSv8fa2ySvus6oR0AAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "import matplotlib.pyplot as plt\n", - "import matplotlib.dates as mdates\n", "%matplotlib inline\n", - "\n", - "plt.xlabel(\"Transfer Size (MiB)\")\n", - "plt.ylabel(\"MiB/s\")\n", - "plt.title(\"IOR Results\")\n", - "\n", - "fig = plt.gcf()\n", - "fig.set_size_inches(16, 6.5)\n", - "\n", - "plt.plot(df_disk[\"xsize\"] / 1.e6, df_disk[\"Max(MiB)\"],'-o',label=\"disk\")\n", - "plt.plot(df_pdwfs[\"xsize\"] / 1.e6, df_pdwfs[\"Max(MiB)\"],'-o',label=\"pdwfs\")\n", - "\n", - "plt.legend([\"disk\",\"pdwfs\"],loc=\"upper right\")" + "plot_results(df_disk, df_pdwfs, title=\"MPI-IO with collective operations\")" ] }, { @@ -281,68 +180,9 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "128:C 22 Mar 2019 14:24:06.966 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo\n", - "128:C 22 Mar 2019 14:24:06.967 # Redis version=5.0.3, bits=64, commit=00000000, modified=0, pid=128, just started\n", - "128:C 22 Mar 2019 14:24:06.967 # Configuration loaded\n", - "130:C 22 Mar 2019 14:24:06.969 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo\n", - "130:C 22 Mar 2019 14:24:06.969 # Redis version=5.0.3, bits=64, commit=00000000, modified=0, pid=130, just started\n", - "130:C 22 Mar 2019 14:24:06.969 # Configuration loaded\n", - "135:C 22 Mar 2019 14:24:06.972 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo\n", - "135:C 22 Mar 2019 14:24:06.972 # Redis version=5.0.3, bits=64, commit=00000000, modified=0, pid=135, just started\n", - "135:C 22 Mar 2019 14:24:06.972 # Configuration loaded\n", - "BlockSize: 262144000\n", - "BlockSize: 262144000\n", - "BlockSize: 262144000\n", - "IOR-3.2.0: MPI Coordinated Test of Parallel I/O\n", - "Began : Fri Mar 22 14:24:07 2019\n", - "Command line : ior -a MPIIO -t 250m -b 250m\n", - "Machine : Linux ff981a9089a8\n", - "TestID : 0\n", - "StartTime : Fri Mar 22 14:24:07 2019\n", - "Path : /home/luke/run\n", - "FS : 0.0 GiB Used FS: 0.0% Inodes: 0.0 Mi Used Inodes: 0.0%\n", - "\n", - "Options: \n", - "api : MPIIO\n", - "apiVersion : (3.1)\n", - "test filename : testFile\n", - "access : single-shared-file\n", - "type : independent\n", - "segments : 1\n", - "ordering in a file : sequential\n", - "ordering inter file : no tasks offsets\n", - "tasks : 2\n", - "clients per node : 2\n", - "repetitions : 1\n", - "xfersize : 250 MiB\n", - "blocksize : 250 MiB\n", - "aggregate filesize : 500 MiB\n", - "\n", - "Results: \n", - "\n", - "access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter\n", - "------ --------- ---------- --------- -------- -------- -------- -------- ----\n", - "write 353.08 256000 256000 0.019253 1.40 0.066392 1.42 0 \n", - "read 56.10 256000 256000 0.001259 8.88 1.11 8.91 0 \n", - "remove - - - - - - 0.128553 0 \n", - "Max Write: 353.08 MiB/sec (370.23 MB/sec)\n", - "Max Read: 56.10 MiB/sec (58.82 MB/sec)\n", - "\n", - "Summary of all tests:\n", - "Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum\n", - "write 353.08 353.08 353.08 0.00 1.41 1.41 1.41 0.00 1.41610 0 2 2 1 0 0 1 0 0 1 262144000 262144000 500.0 MPIIO 0\n", - "read 56.10 56.10 56.10 0.00 0.22 0.22 0.22 0.00 8.91322 0 2 2 1 0 0 1 0 0 1 262144000 262144000 500.0 MPIIO 0\n", - "Finished : Fri Mar 22 14:24:17 2019\n" - ] - } - ], + "outputs": [], "source": [ "%%bash\n", "cd run\n", @@ -353,8 +193,7 @@ "redis-server --daemonize yes --save \"\" --port 6380\n", "redis-server --daemonize yes --save \"\" --port 6381\n", "export PDWFS_REDIS=\":6379,:6380,:6381\"\n", - "export PDWFS_BLOCKSIZE=\"250\"\n", - "pdwfs -p . -- mpirun ior -a MPIIO -t 250m -b 250m" + "pdwfs -p . -- mpirun ior -a MPIIO -t 250m -b 250m -w" ] }, { diff --git a/examples/ior_benchmark/notebook/ior_script.jinja2 b/examples/ior_benchmark/notebook/ior_script.jinja2 deleted file mode 100644 index bc364eb..0000000 --- a/examples/ior_benchmark/notebook/ior_script.jinja2 +++ /dev/null @@ -1,16 +0,0 @@ -IOR START - api={{ api }} - verbose=0 - testFile=testFile - writeFile=1 - readFile={{ read }} -{% for case in cases %} -RUN - numTasks={{ case.numTasks }} - filePerProc={{ case.filePerProc }} - collective={{ case.collective }} - segmentCount={{ case.segmentCount }} - blockSize={{ case.transferSize }} - transferSize={{ case.transferSize }} -{% endfor %} -IOR STOP \ No newline at end of file diff --git a/examples/ior_benchmark/notebook/utils.py b/examples/ior_benchmark/notebook/utils.py deleted file mode 100644 index 25efcf1..0000000 --- a/examples/ior_benchmark/notebook/utils.py +++ /dev/null @@ -1,16 +0,0 @@ -import itertools as it -from collections import namedtuple -from jinja2 import Template - -IORCase = namedtuple("IORCase", - field_names=["numTasks", "filePerProc", "collective", "segmentCount", "transferSize"]) - -def make_ior_script(api, numTasks, filePerProc, collective, segmentCount, transferSize): - matrix = list(it.product(numTasks, filePerProc, collective, segmentCount, transferSize)) - cases = [IORCase(*case) for case in matrix] - - with open("ior_script.jinja2", "r") as f: - template = Template(f.read()) - - with open("ior_script_" + api, "w") as f: - f.write(template.render(api=api, cases=cases)) \ No newline at end of file diff --git a/examples/ior_benchmark/utils.py b/examples/ior_benchmark/utils.py new file mode 100644 index 0000000..6e4b53b --- /dev/null +++ b/examples/ior_benchmark/utils.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +import re +import itertools as it +from collections import namedtuple + +from jinja2 import Template +import matplotlib.pyplot as plt +import pandas + + +def build_ior_script(api, read, numTasks, filePerProc, collective, segmentCount, transferSize): + """ + Build IOR script from the jinja2 template ior_script.jinja2 and arguments + """ + with open("ior_script.jinja2", "r") as f: + template = Template(f.read()) + + with open("ior_script", "w") as f: + f.write(template.render(api=api, + read=read, + numTasks=numTasks, + filePerProc=filePerProc, + collective=collective, + segmentCount=segmentCount, + transferSize=transferSize)) + + +def parse_ior_results(filename): + """ + Parse IOR results in file given by filename and return a Pandas dataframe + """ + import re + + + start_line = None + end_line = None + with open(filename,'r') as f: + for i, line in enumerate(f.readlines()): + if re.search("Summary of all tests:", line): + start_line = i + 1 + if re.search("Finished", line): + end_line = i - 1 + + return pandas.read_csv(filename, sep='\s+', skiprows=start_line, nrows=end_line-start_line) + + +def plot_results(readOrWrite, df_disk, df_pdwfs, title=None, filename=None): + """ + Plot max write rate vs transfer size + """ + plt.style.use("ggplot") + + plt.xlabel("Transfer Size (MiB)") + plt.ylabel("Measured " + readOrWrite + " Rate (MiB/s)") + prefix = "IOR " + readOrWrite + " Rate Test Results" + title = prefix + " - " + title if title else prefix + plt.title(title) + + fig = plt.gcf() + fig.set_size_inches(16, 6.5) + + plt.plot(df_pdwfs["xsize"] / 1.e6, df_pdwfs["Max(MiB)"],'-o',label="pdwfs") + if readOrWrite == "write": + # plot only "write" results on disk, "read" results on disk are "polluted" by caching in RAM + plt.plot(df_disk["xsize"] / 1.e6, df_disk["Max(MiB)"],'-o',label="disk") + plt.legend(["pdwfs", "disk"],loc="upper right") + else: + plt.legend(["pdwfs"],loc="upper right") + + if filename: + plt.savefig(filename) + plt.clf() diff --git a/examples/ior_slurm/ior_pdwfs.sh b/examples/ior_slurm/ior_pdwfs.sh new file mode 100644 index 0000000..e685d43 --- /dev/null +++ b/examples/ior_slurm/ior_pdwfs.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# SBATCH --job-name=pdwfs +# SBATCH --nodes=12 +# SBATCH --exclusive + +# This script demonstrates how to run the I/O benchmarking tool ior +# with pdwfs using SLURM. +# pdwfs comes with the pdwfs-slurm CLI tool to manage Redis instances + +# pre-requisites: +# - pdwfs and Redis are installed and available in PATH +# - ior is installed with MPI support + +# Initialize the Redis instances: +# - 32 instances distributed on 4 nodes (8 per node) +# - bind Redis servers to ib0 network interface +pdwfs-slurm init -N 4 -n 8 -i ib0 + +# pdwfs-slurm produces a session file with some environment variables to source +source pdwfs.session + +# ior command will use MPI-IO in collective mode with data blocks of 100 MBytes +IOR_CMD="ior -a MPIIO -c -t 100m -b 100m -o $SCRATCHDIR/testFile" + +# pdwfs command will forward all I/O in $SCRATCHDIR in Redis instances +WITH_PDWFS="pdwfs -p $SCRATCHDIR" + +# Execute ior benchmark on 128 tasks +srun -N 8 --ntasks-per-node 16 $WITH_PDWFS $IOR_CMD + +# gracefully shuts down Redis instances +pdwfs-slurm finalize + +# pdwfs-slurm uses srun in background to execute Redis instances +# wait for background srun to complete +wait \ No newline at end of file diff --git a/pdwfs-spack.py b/pdwfs-spack.py new file mode 100644 index 0000000..40cc239 --- /dev/null +++ b/pdwfs-spack.py @@ -0,0 +1,39 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +from spack import * + + +class Pdwfs(MakefilePackage): + """ + pdwfs is an open source (Apache 2.0 licensed), preload library implementing + a distributed in-memory filesystem in user space suitable for intercepting + bulk I/O workloads typical of HPC simulations. It is using Redis as the + backend memory store. + + pdwfs (with Redis) provides a lightweight infrastructure to execute HPC simulation + workflows in transit, i.e. without writing/reading any intermediate data to/from + a (parallel) filesystem. + + pdwfs is written in Go and C and runs on Linux systems only. + """ + + homepage = "https://github.com/cea-hpc/pdwfs" + url = "https://github.com/cea-hpc/pdwfs/archive/v0.1.2.tar.gz" + git = "https://github.com/cea-hpc/pdwfs.git" + + version('develop', branch='develop') + version('0.1.2', sha256='78336ee06985d6ffa7a5e13ecb368cd0f39bcaeb84f99d54337823bce1eba371') + + depends_on('go@1.11:', type='build') + depends_on('redis', type='run') + + @property + def install_targets(self): + return [ + 'PREFIX={0}'.format(self.spec.prefix), + 'install' + ] \ No newline at end of file diff --git a/scripts/pdwfs-redis b/scripts/pdwfs-redis deleted file mode 100755 index 5cd868a..0000000 --- a/scripts/pdwfs-redis +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -usage="Usage: $(basename "$0") [-h] [command] - -Helper script to start a redis instance for testing purposes - -where [command] must be either: - start start a redis instance listening on localhost and default port (6379) - stop stop the running redis instance - restart restart the running redis instance - cli open redis CLI" - - -stop_redis() { - redis-cli shutdown -} - -start_redis() { - redis-server --daemonize yes --save "" -} - -while [ -n "$1" ]; do - case "$1" in - -h) - echo "$usage" - exit 0 - ;; - stop) - stop_redis - exit 0 - ;; - start) - start_redis - exit 0 - ;; - restart) - stop_redis - start_redis - exit 0 - ;; - cli) - redis-cli - exit 0 - ;; - *) - echo "Error: Command $1 not recognized" - echo "$usage" - exit 1 - ;; - esac -done - -echo "$usage" -exit 1 - - diff --git a/src/c/Makefile b/src/c/Makefile index e00cefa..eb31f00 100644 --- a/src/c/Makefile +++ b/src/c/Makefile @@ -1,40 +1,36 @@ -CFLAGS = -std=c99 -g -O2 -Wall -Werror +CFLAGS = -std=c99 -g -O2 -Wall -Werror $(shell pkg-config --cflags glib-2.0) LDFLAGS = +LIBS = $(shell pkg-config --libs glib-2.0) BUILDDIR=../../build -SCRIPTSDIR=../../scripts -C_TESTS = $(patsubst tests/%.c, $(BUILDDIR)/tests/%, $(wildcard tests/*.c)) +C_OBJ = $(BUILDDIR)/pdwfs.o $(BUILDDIR)/libc.o $(BUILDDIR)/utils.o -all: dirs $(BUILDDIR)/lib/pdwfs.so $(C_TESTS) +all: dirs $(BUILDDIR)/lib/pdwfs.so dirs: mkdir -p $(BUILDDIR)/tests -$(BUILDDIR)/pdwfs.o : pdwfs.c +$(BUILDDIR)/%.o : %.c gcc $(CFLAGS) -fPIC -I$(BUILDDIR)/include -c $< -o $@ -$(BUILDDIR)/lib/pdwfs.so: $(BUILDDIR)/lib/libpdwfs_go.so $(BUILDDIR)/pdwfs.o - gcc $(LDFLAGS) -fPIC -shared -Wl,-rpath=\$$ORIGIN/../lib -o $@ $(BUILDDIR)/pdwfs.o -ldl -lpdwfs_go -L$(BUILDDIR)/lib - -$(BUILDDIR)/tests/%: $(BUILDDIR)/tests/%.o - gcc $(LDFLAGS) -o $@ $^ - -$(BUILDDIR)/tests/%.o : tests/%.c - gcc $(CFLAGS) -c $< -o $@ +$(BUILDDIR)/lib/pdwfs.so: $(BUILDDIR)/lib/libpdwfs_go.so $(C_OBJ) + gcc $(LDFLAGS) -fPIC -shared -Wl,-rpath=\$$ORIGIN/../lib -o $@ $(C_OBJ) -ldl -lpdwfs_go -L$(BUILDDIR)/lib $(LIBS) clean: rm -f $(BUILDDIR)/pdwfs.o rm -f $(BUILDDIR)/lib/pdwfs.so - rm -rf $(BUILDDIR)/tests - -test-runs := $(addsuffix .test-disk,$(C_TESTS)) $(addsuffix .test-pdwfs,$(C_TESTS)) - -test: all $(test-runs) - -%.test-disk: # execute tests on disk - $(subst .test-disk,,$@) - -%.test-pdwfs: $(BUILDDIR)/lib/pdwfs.so # execute tests through pdwfs - $(BUILDDIR)/bin/pdwfs -p . -- $(subst .test-pdwfs,,$@) - @redis-cli flushall > /dev/null + make -C tests clean + +test: all + make -C tests + @echo "*****************************" + @echo "* Execute testsuite on disk *" + @echo "*****************************" + - $(BUILDDIR)/tests/testsuite + @echo "***********************************" + @echo "* Execute testsuite through pdwfs *" + @echo "***********************************" + redis-server --daemonize yes --save "" + - $(BUILDDIR)/bin/pdwfs -p . -- $(BUILDDIR)/tests/testsuite + redis-cli shutdown diff --git a/src/c/libc.c b/src/c/libc.c new file mode 100644 index 0000000..13bbd53 --- /dev/null +++ b/src/c/libc.c @@ -0,0 +1,507 @@ +/* +* Copyright 2019 CEA +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include "libc.h" + +static int (*ptr_open)(const char *pathname, int flags, ...) = NULL; +static int (*ptr_close)(int fd) = NULL; +static ssize_t (*ptr_write)(int fd, const void *buf, size_t count) = NULL; +static ssize_t (*ptr_read)(int fd, void *buf, size_t count) = NULL; +static int (*ptr_open64)(const char *pathname, int flags, ...) = NULL; +static int (*ptr_creat)(const char *pathname, mode_t mode) = NULL; +static int (*ptr_creat64)(const char *pathname, mode_t mode) = NULL; +static int (*ptr_fdatasync)(int fd) = NULL; +static int (*ptr_fsync)(int fd) = NULL; +static int (*ptr_ftruncate64)(int fd, off64_t length) = NULL; +static int (*ptr_ftruncate)(int fd, off_t length) = NULL; +static int (*ptr_truncate64)(const char *path, off64_t length) = NULL; +static int (*ptr_truncate)(const char *path, off_t length) = NULL; +static off64_t (*ptr_lseek64)(int fd, off64_t offset, int whence) = NULL; +static off_t (*ptr_lseek)(int fd, off_t offset, int whence) = NULL; +static ssize_t (*ptr_pread)(int fd, void *buf, size_t count, off_t offset) = NULL; +static ssize_t (*ptr_pread64)(int fd, void *buf, size_t count, off64_t offset) = NULL; +static ssize_t (*ptr_preadv)(int fd, const struct iovec *iov, int iovcnt, off_t offset) = NULL; +static ssize_t (*ptr_preadv64)(int fd, const struct iovec *iov, int iovcnt, off64_t offset) = NULL; +static ssize_t (*ptr_pwrite)(int fd, const void *buf, size_t count, off_t offset) = NULL; +static ssize_t (*ptr_pwrite64)(int fd, const void *buf, size_t count, off64_t offset) = NULL; +static ssize_t (*ptr_pwritev)(int fd, const struct iovec *iov, int iovcnt, off_t offset) = NULL; +static ssize_t (*ptr_pwritev64)(int fd, const struct iovec *iov, int iovcnt, off64_t offset) = NULL; +static ssize_t (*ptr_readv)(int fd, const struct iovec *iov, int iovcnt) = NULL; +static ssize_t (*ptr_writev)(int fd, const struct iovec *iov, int iovcnt) = NULL; +static int (*ptr_ioctl)(int fd, unsigned long request, void *argp) = NULL; +static int (*ptr_access)(const char *pathname, int mode) = NULL; +static int (*ptr_unlink)(const char *pathname) = NULL; +static int (*ptr___xstat)(int vers, const char *pathname, struct stat *buf) = NULL; +static int (*ptr___xstat64)(int vers, const char *pathname, struct stat64 *buf) = NULL; +static int (*ptr___lxstat)(int vers, const char *pathname, struct stat *buf) = NULL; +static int (*ptr___lxstat64)(int vers, const char *pathname, struct stat64 *buf) = NULL; +static int (*ptr___fxstat)(int vers, int fd, struct stat *buf) = NULL; +static int (*ptr___fxstat64)(int vers, int fd, struct stat64 *buf) = NULL; +static int (*ptr_statfs)(const char *path, struct statfs *buf) = NULL; +static int (*ptr_statfs64)(const char *path, struct statfs64 *buf) = NULL; +static int (*ptr_fstatfs)(int fd, struct statfs *buf) = NULL; +static int (*ptr_fstatfs64)(int fd, struct statfs64 *buf) = NULL; +static FILE* (*ptr_fdopen)(int fd, const char *mode) = NULL; +static FILE* (*ptr_fopen)(const char *path, const char *mode) = NULL; +static FILE* (*ptr_fopen64)(const char *path, const char *mode) = NULL; +static FILE* (*ptr_freopen)(const char *path, const char *mode, FILE *stream) = NULL; +static FILE* (*ptr_freopen64)(const char *path, const char *mode, FILE *stream) = NULL; +static int (*ptr_fclose)(FILE *stream) = NULL; +static int (*ptr_fflush)(FILE *stream) = NULL; +static int (*ptr_fputc)(int c, FILE *stream) = NULL; +static char* (*ptr_fgets)(char *s, int size, FILE *stream) = NULL; +static int (*ptr_fgetc)(FILE *stream) = NULL; +static int (*ptr_fgetpos)(FILE *stream, fpos_t *pos) = NULL; +static int (*ptr_fgetpos64)(FILE *stream, fpos64_t *pos) = NULL; +static int (*ptr_fseek)(FILE *stream, long offset, int whence) = NULL; +static int (*ptr_fseeko)(FILE *stream, off_t offset, int whence) = NULL; +static int (*ptr_fseeko64)(FILE *stream, off64_t offset, int whence) = NULL; +static int (*ptr_fsetpos)(FILE *stream, const fpos_t *pos) = NULL; +static int (*ptr_fsetpos64)(FILE *stream, const fpos64_t *pos) = NULL; +static int (*ptr_fputs)(const char *s, FILE *stream) = NULL; +static int (*ptr_putc)(int c, FILE *stream) = NULL; +static int (*ptr_getc)(FILE *stream) = NULL; +static int (*ptr_ungetc)(int c, FILE *stream) = NULL; +static long (*ptr_ftell)(FILE *stream) = NULL; +static off_t (*ptr_ftello)(FILE *stream) = NULL; +static off64_t (*ptr_ftello64)(FILE *stream) = NULL; +static size_t (*ptr_fread)(void *ptr, size_t size, size_t nmemb, FILE *stream) = NULL; +static size_t (*ptr_fwrite)(const void *ptr, size_t size, size_t nmemb, FILE *stream) = NULL; +static void (*ptr_rewind)(FILE *stream) = NULL; +static int (*ptr_dup2)(int oldfd, int newfd) = NULL; +static int (*ptr_unlinkat)(int dirfd, const char *pathname, int flags) = NULL; +static int (*ptr_openat)(int dirfd, const char *pathname, int flags, ...) = NULL; +static int (*ptr_faccessat)(int dirfd, const char *pathname, int mode, int flags) = NULL; +static int (*ptr___fxstatat)(int vers, int dirfd, const char *pathname, struct stat *buf, int flags) = NULL; +static int (*ptr___fxstatat64)(int vers, int dirfd, const char *pathname, struct stat64 *buf, int flags) = NULL; +static int (*ptr_mkdir)(const char *pathname, mode_t mode) = NULL; +static int (*ptr_mkdirat)(int dirfd, const char *pathname, mode_t mode) = NULL; +static int (*ptr_rmdir)(const char *pathname) = NULL; +static int (*ptr_rename)(const char *oldpath, const char *newpath) = NULL; +static int (*ptr_renameat)(int olddirfd, const char *oldpath, int newdirfd, const char *newpath) = NULL; +static int (*ptr_renameat2)(int olddirfd, const char *oldpath, int newdirfd, const char *newpath, unsigned int flags) = NULL; +static int (*ptr_posix_fadvise)(int fd, off_t offset, off_t len, int advice) = NULL; +static int (*ptr_posix_fadvise64)(int fd, off64_t offset, off64_t len, int advice) = NULL; +static int (*ptr_statvfs)(const char *pathname, struct statvfs *buf) = NULL; +static int (*ptr_statvfs64)(const char *pathname, struct statvfs64 *buf) = NULL; +static int (*ptr_fstatvfs)(int fd, struct statvfs *buf) = NULL; +static int (*ptr_fstatvfs64)(int fd, struct statvfs64 *buf) = NULL; +static ssize_t (*ptr_getdelim)(char **buf, size_t *bufsiz, int delimiter, FILE *fp) = NULL; +static ssize_t (*ptr_getline)(char **lineptr, size_t *n, FILE *stream) = NULL; +static DIR* (*ptr_opendir)(const char* path) = NULL; +static int (*ptr_feof)(FILE *stream) = NULL; +static int (*ptr_ferror)(FILE *stream) = NULL; + +static int g_do_trace = -1; + +#define RED "\033[31m" +#define BLUE "\033[34m" +#define DEFAULT "\033[39m" + +static int pdwfs_fprintf(FILE* stream, const char* color, const char *cat, const char* format, ...) { + va_list ap; + dprintf(fileno(stream), "%s[PDWFS][%d][%s]%s[C] ", color, getpid(), cat, DEFAULT);\ + va_start(ap, format); + int res = vfprintf(stream, format, ap); + va_end(ap); + return res; +} + +#define CALL_NEXT(symb, ...) {\ + if(g_do_trace < 0) {\ + g_do_trace = (getenv("PDWFS_CTRACES") != NULL);\ + }\ + if(g_do_trace) {\ + pdwfs_fprintf(stderr, BLUE, "TRACE", "calling libc %s\n", #symb);\ + }\ + if (! ptr_ ## symb) {\ + char *error;\ + dlerror();\ + ptr_ ## symb = dlsym(RTLD_NEXT, #symb);\ + if ((error = dlerror()) != NULL) {\ + pdwfs_fprintf(stderr, RED, "ERROR", "dlsym: %s\n", error);\ + exit(EXIT_FAILURE);\ + }\ + if (! ptr_ ## symb ) {\ + pdwfs_fprintf(stderr, RED, "ERROR", "symbol not found in dlsym: %s\n", #symb);\ + exit(EXIT_FAILURE);\ + }\ + }\ + return ptr_ ## symb(__VA_ARGS__);\ +} + +int libc_open(const char *pathname, int flags, int mode) { + CALL_NEXT(open, pathname, flags, mode) +} + +int libc_close(int fd) { + CALL_NEXT(close, fd) +} + +ssize_t libc_write(int fd, const void *buf, size_t count) { + CALL_NEXT(write, fd, buf, count) +} + +ssize_t libc_read(int fd, void *buf, size_t count) { + CALL_NEXT(read, fd, buf, count) +} + +int libc_open64(const char *pathname, int flags, int mode) { + CALL_NEXT(open64, pathname, flags, mode) +} + +int libc_creat(const char *pathname, mode_t mode) { + CALL_NEXT(creat, pathname, mode) +} + +int libc_creat64(const char *pathname, mode_t mode) { + CALL_NEXT(creat64, pathname, mode) +} + +int libc_fdatasync(int fd) { + CALL_NEXT(fdatasync, fd) +} + +int libc_fsync(int fd) { + CALL_NEXT(fsync, fd) +} + +int libc_ftruncate64(int fd, off64_t length) { + CALL_NEXT(ftruncate64, fd, length) +} + +int libc_ftruncate(int fd, off_t length) { + CALL_NEXT(ftruncate, fd, length) +} + +int libc_truncate64(const char *path, off64_t length) { + CALL_NEXT(truncate64, path, length) +} + +int libc_truncate(const char *path, off_t length) { + CALL_NEXT(truncate, path, length) +} + +off64_t libc_lseek64(int fd, off64_t offset, int whence) { + CALL_NEXT(lseek64, fd, offset, whence) +} + +off_t libc_lseek(int fd, off_t offset, int whence) { + CALL_NEXT(lseek, fd, offset, whence) +} + +ssize_t libc_pread(int fd, void *buf, size_t count, off_t offset) { + CALL_NEXT(pread, fd, buf, count, offset) +} + +ssize_t libc_pread64(int fd, void *buf, size_t count, off64_t offset) { + CALL_NEXT(pread64, fd, buf, count, offset) +} + +ssize_t libc_preadv(int fd, const struct iovec *iov, int iovcnt, off_t offset) { + CALL_NEXT(preadv, fd, iov, iovcnt, offset) +} + +ssize_t libc_preadv64(int fd, const struct iovec *iov, int iovcnt, off64_t offset) { + CALL_NEXT(preadv64, fd, iov, iovcnt, offset) +} + +ssize_t libc_pwrite(int fd, const void *buf, size_t count, off_t offset) { + CALL_NEXT(pwrite, fd, buf, count, offset) +} + +ssize_t libc_pwrite64(int fd, const void *buf, size_t count, off64_t offset) { + CALL_NEXT(pwrite64, fd, buf, count, offset) +} + +ssize_t libc_pwritev(int fd, const struct iovec *iov, int iovcnt, off_t offset) { + CALL_NEXT(pwritev, fd, iov, iovcnt, offset) +} + +ssize_t libc_pwritev64(int fd, const struct iovec *iov, int iovcnt, off64_t offset) { + CALL_NEXT(pwritev64, fd, iov, iovcnt, offset) +} + +ssize_t libc_readv(int fd, const struct iovec *iov, int iovcnt) { + CALL_NEXT(readv, fd, iov, iovcnt) +} + +ssize_t libc_writev(int fd, const struct iovec *iov, int iovcnt) { + CALL_NEXT(writev, fd, iov, iovcnt) +} + +int libc_ioctl(int fd, unsigned long request, void *argp) { + CALL_NEXT(ioctl, fd, request, argp) +} + +int libc_access(const char *pathname, int mode) { + CALL_NEXT(access, pathname, mode) +} + +int libc_unlink(const char *pathname) { + CALL_NEXT(unlink, pathname) +} + +int libc__xstat(int vers, const char *pathname, struct stat *buf) { + CALL_NEXT(__xstat, vers, pathname, buf) +} + +int libc__xstat64(int vers, const char *pathname, struct stat64 *buf) { + CALL_NEXT(__xstat64, vers, pathname, buf) +} + +int libc__lxstat(int vers, const char *pathname, struct stat *buf) { + CALL_NEXT(__lxstat, vers, pathname, buf) +} + +int libc__lxstat64(int vers, const char *pathname, struct stat64 *buf) { + CALL_NEXT(__lxstat64, vers, pathname, buf) +} + +int libc__fxstat(int vers, int fd, struct stat *buf) { + CALL_NEXT(__fxstat, vers, fd, buf) +} + +int libc__fxstat64(int vers, int fd, struct stat64 *buf) { + CALL_NEXT(__fxstat64, vers, fd, buf) +} + +int libc_statfs(const char *path, struct statfs *buf) { + CALL_NEXT(statfs, path, buf) +} + +int libc_statfs64(const char *path, struct statfs64 *buf) { + CALL_NEXT(statfs64, path, buf) +} + +int libc_fstatfs(int fd, struct statfs *buf) { + CALL_NEXT(fstatfs, fd, buf) +} + +int libc_fstatfs64(int fd, struct statfs64 *buf) { + CALL_NEXT(fstatfs64, fd, buf) +} + +FILE* libc_fdopen(int fd, const char *mode) { + CALL_NEXT(fdopen, fd, mode) +} + +FILE* libc_fopen(const char *path, const char *mode) { + CALL_NEXT(fopen, path, mode) +} + +FILE* libc_fopen64(const char *path, const char *mode) { + CALL_NEXT(fopen64, path, mode) +} + +FILE* libc_freopen(const char *path, const char *mode, FILE *stream) { + CALL_NEXT(freopen, path, mode, stream) +} + +FILE* libc_freopen64(const char *path, const char *mode, FILE *stream) { + CALL_NEXT(freopen64, path, mode, stream) +} + +int libc_fclose(FILE *stream) { + CALL_NEXT(fclose, stream) +} + +int libc_fflush(FILE *stream) { + CALL_NEXT(fflush, stream) +} + +int libc_fputc(int c, FILE *stream) { + CALL_NEXT(fputc, c, stream) +} + +char* libc_fgets(char *dst, int max, FILE *stream) { + CALL_NEXT(fgets, dst, max, stream) +} + +int libc_fgetc(FILE *stream) { + CALL_NEXT(fgetc, stream) +} + +int libc_fgetpos(FILE *stream, fpos_t *pos) { + CALL_NEXT(fgetpos, stream, pos) +} + +int libc_fgetpos64(FILE *stream, fpos64_t *pos) { + CALL_NEXT(fgetpos64, stream, pos) +} + +int libc_fseek(FILE *stream, long offset, int whence) { + CALL_NEXT(fseek, stream, offset, whence) +} + +int libc_fseeko(FILE *stream, off_t offset, int whence) { + CALL_NEXT(fseeko, stream, offset, whence) +} + +int libc_fseeko64(FILE *stream, off64_t offset, int whence) { + CALL_NEXT(fseeko64, stream, offset, whence) +} + +int libc_fsetpos(FILE *stream, const fpos_t *pos) { + CALL_NEXT(fsetpos, stream, pos) +} + +int libc_fsetpos64(FILE *stream, const fpos64_t *pos) { + CALL_NEXT(fsetpos64, stream, pos) +} + +int libc_fputs(const char *s, FILE *stream) { + CALL_NEXT(fputs, s, stream) +} + +int libc_putc(int c, FILE *stream) { + CALL_NEXT(putc, c, stream) +} + +int libc_getc(FILE *stream) { + CALL_NEXT(getc, stream) +} + +int libc_ungetc(int c, FILE *stream) { + CALL_NEXT(ungetc, c, stream) +} + +long libc_ftell(FILE *stream) { + CALL_NEXT(ftell, stream) +} + +off_t libc_ftello(FILE *stream) { + CALL_NEXT(ftello, stream) +} + +off64_t libc_ftello64(FILE *stream) { + CALL_NEXT(ftello64, stream) +} + +size_t libc_fread(void *ptr, size_t size, size_t nmemb, FILE *stream) { + CALL_NEXT(fread, ptr, size, nmemb, stream) +} + +size_t libc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream) { + CALL_NEXT(fwrite, ptr, size, nmemb, stream) +} + +void libc_rewind(FILE *stream) { + CALL_NEXT(rewind, stream) +} + +int libc_dup2(int oldfd, int newfd) { + CALL_NEXT(dup2, oldfd, newfd) +} + +int libc_unlinkat(int dirfd, const char *pathname, int flags) { + CALL_NEXT(unlinkat, dirfd, pathname, flags) +} + +int libc_faccessat(int dirfd, const char *pathname, int mode, int flags) { + CALL_NEXT(faccessat, dirfd, pathname, mode, flags) +} + +// __fxstatat is the glibc function corresponding to fstatat syscall +int libc__fxstatat(int vers, int dirfd, const char *pathname, struct stat *buf, int flags) { + CALL_NEXT(__fxstatat, vers, dirfd, pathname, buf, flags) +} + +// __fxstatat64 is the LARGEFILE64 version of glibc function corresponding to fstatat syscall +int libc__fxstatat64(int vers, int dirfd, const char *pathname, struct stat64 *buf, int flags) { + CALL_NEXT(__fxstatat64, vers, dirfd, pathname, buf, flags) +} + +int libc_openat(int dirfd, const char *pathname, int flags, int mode) { + CALL_NEXT(openat, dirfd, pathname, flags, mode) +} + +int libc_mkdir(const char *pathname, mode_t mode) { + CALL_NEXT(mkdir, pathname, mode) +} + +int libc_mkdirat(int dirfd, const char *pathname, mode_t mode) { + CALL_NEXT(mkdirat, dirfd, pathname, mode) +} + +int libc_rmdir(const char *pathname) { + CALL_NEXT(rmdir, pathname) +} + +int libc_rename(const char *oldpath, const char *newpath) { + CALL_NEXT(rename, oldpath, newpath) +} + +int libc_renameat(int olddirfd, const char *oldpath, int newdirfd, const char *newpath) { + CALL_NEXT(renameat, olddirfd, oldpath, newdirfd, newpath) +} + +int libc_renameat2(int olddirfd, const char *oldpath, int newdirfd, const char *newpath, unsigned int flags) { + CALL_NEXT(renameat2, olddirfd, oldpath, newdirfd, newpath, flags) +} + +int libc_posix_fadvise(int fd, off_t offset, off_t len, int advice) { + CALL_NEXT(posix_fadvise, fd, offset, len, advice) +} + +int libc_posix_fadvise64(int fd, off64_t offset, off64_t len, int advice) { + CALL_NEXT(posix_fadvise64, fd, offset, len, advice) +} + +int libc_statvfs(const char *pathname, struct statvfs *buf) { + CALL_NEXT(statvfs, pathname, buf) +} + +int libc_statvfs64(const char *pathname, struct statvfs64 *buf) { + CALL_NEXT(statvfs64, pathname, buf) +} + +int libc_fstatvfs(int fd, struct statvfs *buf) { + CALL_NEXT(fstatvfs, fd, buf) +} + +int libc_fstatvfs64(int fd, struct statvfs64 *buf) { + CALL_NEXT(fstatvfs64, fd, buf) +} + +ssize_t libc_getdelim(char **buf, size_t *bufsiz, int delimiter, FILE *fp) { + CALL_NEXT(getdelim, buf, bufsiz, delimiter, fp) +} + +ssize_t libc_getline(char **buf, size_t *bufsiz, FILE *stream) { + CALL_NEXT(getline, buf, bufsiz, stream) +} + +DIR* libc_opendir(const char* path) { + CALL_NEXT(opendir, path) +} + +int libc_feof(FILE *stream) { + CALL_NEXT(feof, stream) +} + +int libc_ferror(FILE *stream) { + CALL_NEXT(ferror, stream) +} \ No newline at end of file diff --git a/src/c/libc.h b/src/c/libc.h new file mode 100644 index 0000000..7832b4d --- /dev/null +++ b/src/c/libc.h @@ -0,0 +1,113 @@ +/* +* Copyright 2019 CEA +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#ifndef LIBC_H +#define LIBC_H + +#include +#include +#include + +int libc_open(const char *pathname, int flags, int mode); +int libc_close(int fd); +ssize_t libc_write(int fd, const void *buf, size_t count); +ssize_t libc_read(int fd, void *buf, size_t count); +int libc_open64(const char *pathname, int flags, int mode); +int libc_creat(const char *pathname, mode_t mode); +int libc_creat64(const char *pathname, mode_t mode); +int libc_fdatasync(int fd); +int libc_fsync(int fd); +int libc_ftruncate64(int fd, off64_t length); +int libc_ftruncate(int fd, off_t length); +int libc_truncate64(const char *path, off64_t length); +int libc_truncate(const char *path, off_t length); +off64_t libc_lseek64(int fd, off64_t offset, int whence); +off_t libc_lseek(int fd, off_t offset, int whence); +ssize_t libc_pread(int fd, void *buf, size_t count, off_t offset); +ssize_t libc_pread64(int fd, void *buf, size_t count, off64_t offset); +ssize_t libc_preadv(int fd, const struct iovec *iov, int iovcnt, off_t offset); +ssize_t libc_preadv64(int fd, const struct iovec *iov, int iovcnt, off64_t offset); +ssize_t libc_pwrite(int fd, const void *buf, size_t count, off_t offset); +ssize_t libc_pwrite64(int fd, const void *buf, size_t count, off64_t offset); +ssize_t libc_pwritev(int fd, const struct iovec *iov, int iovcnt, off_t offset); +ssize_t libc_pwritev64(int fd, const struct iovec *iov, int iovcnt, off64_t offset); +ssize_t libc_readv(int fd, const struct iovec *iov, int iovcnt); +ssize_t libc_writev(int fd, const struct iovec *iov, int iovcnt); +int libc_ioctl(int fd, unsigned long request, void *argp); +int libc_access(const char *pathname, int mode); +int libc_unlink(const char *pathname); +int libc__xstat(int vers, const char *pathname, struct stat *buf); +int libc__xstat64(int vers, const char *pathname, struct stat64 *buf); +int libc__lxstat(int vers, const char *pathname, struct stat *buf); +int libc__lxstat64(int vers, const char *pathname, struct stat64 *buf); +int libc__fxstat(int vers, int fd, struct stat *buf); +int libc__fxstat64(int vers, int fd, struct stat64 *buf); +int libc_statfs(const char *path, struct statfs *buf); +int libc_statfs64(const char *path, struct statfs64 *buf); +int libc_fstatfs(int fd, struct statfs *buf); +int libc_fstatfs64(int fd, struct statfs64 *buf); +FILE* libc_fdopen(int fd, const char *mode); +FILE* libc_fopen(const char *path, const char *mode); +FILE* libc_fopen64(const char *path, const char *mode); +FILE* libc_freopen(const char *path, const char *mode, FILE *stream); +FILE* libc_freopen64(const char *path, const char *mode, FILE *stream); +int libc_fclose(FILE *stream); +int libc_fflush(FILE *stream); +int libc_fputc(int c, FILE *stream); +char* libc_fgets(char *dst, int max, FILE *stream); +int libc_fgetc(FILE *stream); +int libc_fgetpos(FILE *stream, fpos_t *pos); +int libc_fgetpos64(FILE *stream, fpos64_t *pos); +int libc_fseek(FILE *stream, long offset, int whence); +int libc_fseeko(FILE *stream, off_t offset, int whence); +int libc_fseeko64(FILE *stream, off64_t offset, int whence); +int libc_fsetpos(FILE *stream, const fpos_t *pos); +int libc_fsetpos64(FILE *stream, const fpos64_t *pos); +int libc_fputs(const char *s, FILE *stream); +int libc_putc(int c, FILE *stream); +int libc_getc(FILE *stream); +int libc_ungetc(int c, FILE *stream); +long libc_ftell(FILE *stream); +off_t libc_ftello(FILE *stream); +off64_t libc_ftello64(FILE *stream); +size_t libc_fread(void *ptr, size_t size, size_t nmemb, FILE *stream); +size_t libc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); +void libc_rewind(FILE *stream); +int libc_dup2(int oldfd, int newfd); +int libc_unlinkat(int dirfd, const char *pathname, int flags); +int libc_faccessat(int dirfd, const char *pathname, int mode, int flags); +int libc__fxstatat(int vers, int dirfd, const char *pathname, struct stat *buf, int flags); +int libc__fxstatat64(int vers, int dirfd, const char *pathname, struct stat64 *buf, int flags); +int libc_openat(int dirfd, const char *pathname, int flags, int mode); +int libc_mkdir(const char *pathname, mode_t mode); +int libc_mkdirat(int dirfd, const char *pathname, mode_t mode); +int libc_rmdir(const char *pathname); +int libc_rename(const char *oldpath, const char *newpath); +int libc_renameat(int olddirfd, const char *oldpath, int newdirfd, const char *newpath); +int libc_renameat2(int olddirfd, const char *oldpath, int newdirfd, const char *newpath, unsigned int flags); +int libc_posix_fadvise(int fd, off_t offset, off_t len, int advice); +int libc_posix_fadvise64(int fd, off64_t offset, off64_t len, int advice); +int libc_statvfs(const char *pathname, struct statvfs *buf); +int libc_statvfs64(const char *pathname, struct statvfs64 *buf); +int libc_fstatvfs(int fd, struct statvfs *buf); +int libc_fstatvfs64(int fd, struct statvfs64 *buf); +ssize_t libc_getdelim(char **buf, size_t *bufsiz, int delimiter, FILE *fp); +ssize_t libc_getline(char **buf, size_t *bufsiz, FILE *stream); +DIR* libc_opendir(const char* path); +int libc_feof(FILE *stream); +int libc_ferror(FILE *stream); + +#endif \ No newline at end of file diff --git a/src/c/pdwfs.c b/src/c/pdwfs.c index 52b4e27..1f4c872 100644 --- a/src/c/pdwfs.c +++ b/src/c/pdwfs.c @@ -16,7 +16,6 @@ #define _GNU_SOURCE -#include #include #include #include @@ -28,111 +27,13 @@ #include #include #include -#include -#include "libpdwfs_go.h" +#include +#include -static int (*real_open)(const char *pathname, int flags, ...) = NULL; -static int (*real_close)(int fd) = NULL; -static ssize_t (*real_write)(int fd, const void *buf, size_t count) = NULL; -static ssize_t (*real_read)(int fd, void *buf, size_t count) = NULL; -static int (*real_open64)(const char *pathname, int flags, ...) = NULL; -static int (*real_creat)(const char *pathname, mode_t mode) = NULL; -static int (*real_creat64)(const char *pathname, mode_t mode) = NULL; -static int (*real_fdatasync)(int fd) = NULL; -static int (*real_fsync)(int fd) = NULL; -static int (*real_ftruncate64)(int fd, off64_t length) = NULL; -static int (*real_ftruncate)(int fd, off_t length) = NULL; -static int (*real_truncate64)(const char *path, off64_t length) = NULL; -static int (*real_truncate)(const char *path, off_t length) = NULL; -static off64_t (*real_lseek64)(int fd, off64_t offset, int whence) = NULL; -static off_t (*real_lseek)(int fd, off_t offset, int whence) = NULL; -static ssize_t (*real_pread)(int fd, void *buf, size_t count, off_t offset) = NULL; -static ssize_t (*real_pread64)(int fd, void *buf, size_t count, off64_t offset) = NULL; -static ssize_t (*real_preadv)(int fd, const struct iovec *iov, int iovcnt, off_t offset) = NULL; -static ssize_t (*real_preadv64)(int fd, const struct iovec *iov, int iovcnt, off64_t offset) = NULL; -static ssize_t (*real_pwrite)(int fd, const void *buf, size_t count, off_t offset) = NULL; -static ssize_t (*real_pwrite64)(int fd, const void *buf, size_t count, off64_t offset) = NULL; -static ssize_t (*real_pwritev)(int fd, const struct iovec *iov, int iovcnt, off_t offset) = NULL; -static ssize_t (*real_pwritev64)(int fd, const struct iovec *iov, int iovcnt, off64_t offset) = NULL; -static ssize_t (*real_readv)(int fd, const struct iovec *iov, int iovcnt) = NULL; -static ssize_t (*real_writev)(int fd, const struct iovec *iov, int iovcnt) = NULL; -//static int (*real_fcntl)(int fd, int cmd, long arg) = NULL; -static int (*real_ioctl)(int fd, unsigned long request, void *argp) = NULL; - -static int (*real_access)(const char *pathname, int mode) = NULL; - -static int (*real_unlink)(const char *pathname) = NULL; - -static int (*real___xstat)(int vers, const char *pathname, struct stat *buf) = NULL; -static int (*real___xstat64)(int vers, const char *pathname, struct stat64 *buf) = NULL; -static int (*real___lxstat)(int vers, const char *pathname, struct stat *buf) = NULL; -static int (*real___lxstat64)(int vers, const char *pathname, struct stat64 *buf) = NULL; -static int (*real___fxstat)(int vers, int fd, struct stat *buf) = NULL; -static int (*real___fxstat64)(int vers, int fd, struct stat64 *buf) = NULL; - - -static int (*real_statfs)(const char *path, struct statfs *buf) = NULL; -static int (*real_statfs64)(const char *path, struct statfs64 *buf) = NULL; -static int (*real_fstatfs)(int fd, struct statfs *buf) = NULL; -static int (*real_fstatfs64)(int fd, struct statfs64 *buf) = NULL; - -static FILE* (*real_fdopen)(int fd, const char *mode) = NULL; -static FILE* (*real_fopen)(const char *path, const char *mode) = NULL; -static FILE* (*real_fopen64)(const char *path, const char *mode) = NULL; -static FILE* (*real_freopen)(const char *path, const char *mode, FILE *stream) = NULL; -static FILE* (*real_freopen64)(const char *path, const char *mode, FILE *stream) = NULL; -static int (*real_fclose)(FILE *stream) = NULL; -static int (*real_fflush)(FILE *stream) = NULL; -static int (*real_fputc)(int c, FILE *stream) = NULL; -static char* (*real_fgets)(char *s, int size, FILE *stream) = NULL; -static int (*real_fgetc)(FILE *stream) = NULL; -static int (*real_fgetpos)(FILE *stream, fpos_t *pos) = NULL; -static int (*real_fgetpos64)(FILE *stream, fpos64_t *pos) = NULL; -static int (*real_fseek)(FILE *stream, long offset, int whence) = NULL; -static int (*real_fseeko)(FILE *stream, off_t offset, int whence) = NULL; -static int (*real_fseeko64)(FILE *stream, off64_t offset, int whence) = NULL; -static int (*real_fsetpos)(FILE *stream, const fpos_t *pos) = NULL; -static int (*real_fsetpos64)(FILE *stream, const fpos64_t *pos) = NULL; -static int (*real_fputs)(const char *s, FILE *stream) = NULL; -static int (*real_putc)(int c, FILE *stream) = NULL; -static int (*real_getc)(FILE *stream) = NULL; -static int (*real_ungetc)(int c, FILE *stream) = NULL; -static long (*real_ftell)(FILE *stream) = NULL; -static off_t (*real_ftello)(FILE *stream) = NULL; -static off64_t (*real_ftello64)(FILE *stream) = NULL; -static size_t (*real_fread)(void *ptr, size_t size, size_t nmemb, FILE *stream) = NULL; -static size_t (*real_fwrite)(const void *ptr, size_t size, size_t nmemb, FILE *stream) = NULL; -static int (*real_fprintf)(FILE *stream, const char *fmt, ...) = NULL; -static void (*real_rewind)(FILE *stream) = NULL; - -static int (*real_dup2)(int oldfd, int newfd) = NULL; - -static int (*real_unlinkat)(int dirfd, const char *pathname, int flags) = NULL; -static int (*real_openat)(int dirfd, const char *pathname, int flags, ...) = NULL; -static int (*real_faccessat)(int dirfd, const char *pathname, int mode, int flags) = NULL; -static int (*real___fxstatat)(int vers, int dirfd, const char *pathname, struct stat *buf, int flags) = NULL; -static int (*real___fxstatat64)(int vers, int dirfd, const char *pathname, struct stat64 *buf, int flags) = NULL; -static int (*real_mkdir)(const char *pathname, mode_t mode) = NULL; -static int (*real_mkdirat)(int dirfd, const char *pathname, mode_t mode) = NULL; -static int (*real_rmdir)(const char *pathname) = NULL; -static int (*real_rename)(const char *oldpath, const char *newpath) = NULL; -static int (*real_renameat)(int olddirfd, const char *oldpath, int newdirfd, const char *newpath) = NULL; -static int (*real_renameat2)(int olddirfd, const char *oldpath, int newdirfd, const char *newpath, unsigned int flags) = NULL; -static int (*real_posix_fadvise)(int fd, off_t offset, off_t len, int advice) = NULL; -static int (*real_posix_fadvise64)(int fd, off64_t offset, off64_t len, int advice) = NULL; - -static int (*real_statvfs)(const char *pathname, struct statvfs *buf) = NULL; -static int (*real_statvfs64)(const char *pathname, struct statvfs64 *buf) = NULL; -static int (*real_fstatvfs)(int fd, struct statvfs *buf) = NULL; -static int (*real_fstatvfs64)(int fd, struct statvfs64 *buf) = NULL; - -static ssize_t (*real_getdelim)(char **buf, size_t *bufsiz, int delimiter, FILE *fp) = NULL; -static ssize_t (*real_getline)(char **lineptr, size_t *n, FILE *stream) = NULL; -static DIR* (*real_opendir)(const char* path) = NULL; - -static int (*real_feof)(FILE *stream) = NULL; -static int (*real_ferror)(FILE *stream) = NULL; +#include "libc.h" +#include "libpdwfs_go.h" +#include "utils.h" static int g_do_trace = -1; @@ -163,55 +64,142 @@ static int pdwfs_fprintf(FILE* stream, const char* color, const char *cat, const #define DEBUG(fmt, ...) pdwfs_fprintf(stderr, YELLOW, "DEBUG", fmt, ##__VA_ARGS__); #define WARNING(fmt, ...) pdwfs_fprintf(stderr, MAGENTA, "WARNING", fmt, ##__VA_ARGS__); +#define NOT_IMPLEMENTED(symb) {\ + pdwfs_fprintf(stderr, RED, "ERROR", "%s not implemented by pdwfs\n", symb);\ + exit(EXIT_FAILURE);\ +} -static void raise(const char* format, ...) { - va_list ap; - dprintf(fileno(stderr), "%s[PDWFS][ERROR]%s[C] ", RED, DEFAULT);\ - va_start(ap, format); - vfprintf(stderr, format, ap); - va_end(ap); +#define IS_STD_FD(fd) (fd == STDIN_FILENO || fd == STDOUT_FILENO || fd == STDERR_FILENO) + +#define PATH_NOT_MANAGED(path) (!pdwfs_initialized || !contains_path(mount_register, path)) +#define FD_NOT_MANAGED(fd) (!pdwfs_initialized || IS_STD_FD(fd) || !contains_fd(fd_register, fd)) +#define STREAM_NOT_MANAGED(stream) FD_NOT_MANAGED(fileno(stream)) + + +//----------------------------------------------------------------------------------------- +// mount_register +// +// used to register pdwfs mount points and check if a filename belongs to one of the mount point +// if not, the call is passed to the system libc call + +// callback when a key is removed from the hash table +void mount_key_removed(gpointer key) { + g_free(key); } -#define NOT_IMPLEMENTED(symb) {\ - raise("%s not implemented by pdwfs\n", symb);\ - exit(EXIT_FAILURE);\ +GHashTable *new_mount_register() { + return g_hash_table_new_full(g_str_hash, g_str_equal, (GDestroyNotify)mount_key_removed, NULL); } +void free_mount_register(GHashTable *self) { + g_hash_table_destroy(self); +} -#define CALL_REAL_OP(symb, func, ...) {\ - if (!func) {\ - char *error;\ - dlerror();\ - func = dlsym(RTLD_NEXT, symb);\ - if ((error = dlerror()) != NULL) {\ - raise("dlsym: %s\n", error);\ - exit(EXIT_FAILURE);\ - }\ - if (! func ) {\ - raise("symbol not found in dlsym: %s\n", symb);\ - exit(EXIT_FAILURE);\ - }\ - }\ - return func(__VA_ARGS__);\ +// register pdwfs mount points into the hash table +void register_mounts(GHashTable *self, gchar **mounts) { + int i = 0; + gchar* mount = mounts[0]; + while (mount != NULL) { + if (g_strcmp0(mount, "") !=0) { + g_hash_table_insert(self, g_strdup(mount), ""); + } + i++; + mount = mounts[i]; + } } -#define IS_STD_FD(fd) (fd == STDIN_FILENO || fd == STDOUT_FILENO || fd == STDERR_FILENO) -#define IS_STD_STREAM(s) (s == stdin || s == stdout || s == stderr) +// lookup function used in g_hash_table_find +gboolean finder(gpointer mount, gpointer unused, gpointer abspath) { + return g_str_has_prefix(abspath, mount); +} + +// returns 1 if the path in argument belongs to one of the mount points registered +int contains_path(GHashTable *self, const char *path) { + + char *apath = abspath(path); + if (!apath) { + return 0; + } + gpointer item_ptr = g_hash_table_find(self, (GHRFunc)finder, apath); + free(apath); + return (item_ptr) ? 1 : 0; +} + +// end of mount_register +//----------------------------------------------------------------------------------------- + + +//----------------------------------------------------------------------------------------- +// fd_register +// +// when a newly created file is managed by pdwfs, the fd_register creates a "twin" local temporary file +// to provide a valid system file descriptor or valid FILE stream object + +// callback when a key is removed +void fd_key_removed(gpointer fd_ptr) { + close(GPOINTER_TO_INT(fd_ptr)); +} + +GHashTable *new_fd_register() { + return g_hash_table_new_full(g_direct_hash, g_direct_equal, (GDestroyNotify)fd_key_removed, NULL); +} + +void free_fd_register(GHashTable *self) { + g_hash_table_destroy(self); +} + +// returns a new FILE stream object and registers its file descriptor +FILE* get_new_stream(GHashTable *self) { + FILE *fp = tmpfile(); + g_hash_table_insert(self, GINT_TO_POINTER(fileno(fp)), GINT_TO_POINTER(0)); + return fp; +} + +// returns a new file descriptor and registers it +int get_new_fd(GHashTable *self) { + return fileno(get_new_stream(self)); +} + +void remove_fd(GHashTable *self, int fd) { + g_hash_table_remove(self, GINT_TO_POINTER(fd)); +} + +int contains_fd(GHashTable *self, int fd) { + return g_hash_table_contains(self, GINT_TO_POINTER(fd)); +} + +// end of fd_register +//----------------------------------------------------------------------------------------- + static int pdwfs_initialized = 0; // there are cases where pdwfs is not yet initialized and a another library constructor // (called before pdwfs.so constructor) does some IO (e.g libselinux, libnuma) -// in such case we cannot cross the cgo layer to check if the file/fd is managed by pdwfs, +// in such case we do not know yet if the file/fd is managed by pdwfs, // so we defer the call to the real system calls (there's hardly any chance that these IOs // calls are the one we intend to intercept anyway) +static GHashTable *fd_register = NULL; +static GHashTable *mount_register = NULL; + static __attribute__((constructor)) void init_pdwfs(void) { - InitPdwfs(); + char buf[1024]; + GoSlice mounts_buf = {&buf, 0, 1024}; + InitPdwfs(mounts_buf); + + mount_register = new_mount_register(); + // parse the list of mount point paths obtained from the Go layer + gchar **mounts = g_strsplit((char*)mounts_buf.data, "@", -1); + register_mounts(mount_register, mounts); + g_strfreev(mounts); + fd_register = new_fd_register(); pdwfs_initialized = 1; } static __attribute__((destructor)) void finalize_pdwfs(void) { FinalizePdwfs(); + free_fd_register(fd_register); + free_mount_register(mount_register); } int open(const char *pathname, int flags, ...) { @@ -227,35 +215,40 @@ int open(const char *pathname, int flags, ...) { TRACE("intercepting open(pathname=%s, flags=%d, mode=%d)\n", pathname, flags, mode) - GoString filename = {pathname, strlen(pathname)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc open\n"); - CALL_REAL_OP("open", real_open, pathname, flags, mode) + if PATH_NOT_MANAGED(pathname) { + return libc_open(pathname, flags, mode); } - int ret = Open(filename, flags, mode); + GoString filename = {pathname, strlen(pathname)}; + + int fd = get_new_fd(fd_register); + + int ret = Open(filename, flags, mode, fd); if (ret < 0) { errno = GetErrno(); + remove_fd(fd_register, fd); } return ret; } +int open64(const char *pathname, int flags, ...) __attribute__((alias("open"))); + int close(int fd) { TRACE("intercepting close(fd=%d)\n", fd) - if (!pdwfs_initialized || IS_STD_FD(fd) || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc close\n"); - CALL_REAL_OP("close", real_close, fd) + if FD_NOT_MANAGED(fd) { + return libc_close(fd); } - return Close(fd); + int ret = Close(fd); + remove_fd(fd_register, fd); + return ret; } ssize_t write(int fd, const void *buf, size_t count) { TRACE("intercepting write(fd=%d, buf=%p, count=%lu)\n", fd, buf, count) - if (!pdwfs_initialized || IS_STD_FD(fd) || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc write\n"); - CALL_REAL_OP("write", real_write, fd, buf, count) + if FD_NOT_MANAGED(fd) { + return libc_write(fd, buf, count); } GoSlice buffer = {(void*)buf, count, count}; return Write(fd, buffer); @@ -264,67 +257,29 @@ ssize_t write(int fd, const void *buf, size_t count) { ssize_t read(int fd, void *buf, size_t count) { TRACE("intercepting read(fd=%d, buf=%p, count=%lu)\n", fd, buf, count) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc read\n"); - CALL_REAL_OP("read", real_read, fd, buf, count) + if FD_NOT_MANAGED(fd) { + return libc_read(fd, buf, count); } GoSlice buffer = {buf, count, count}; return Read(fd, buffer); } -int open64(const char *pathname, int flags, ...) { - - int mode = 0; - - if (__OPEN_NEEDS_MODE (flags)) { - va_list arg; - va_start(arg, flags); - mode = va_arg(arg, int); - va_end(arg); - } - - TRACE("intercepting open64(pathname=%s, flags=%d, mode=%d)\n", pathname, flags, mode) - - GoString filename = {pathname, strlen(pathname)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc open64\n"); - CALL_REAL_OP("open64", real_open64, pathname, flags, mode) - } - int ret = Open(filename, flags, mode); - if (ret < 0) { - errno = GetErrno(); - } - return ret; -} - int creat(const char *pathname, mode_t mode) { TRACE("intercepting creat(pathname=%s, mode=%d)\n", pathname, mode) - GoString filename = {pathname, strlen(pathname)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc creat\n"); - CALL_REAL_OP("creat", real_creat, pathname, mode) + if PATH_NOT_MANAGED(pathname) { + return libc_creat(pathname, mode); } NOT_IMPLEMENTED("creat") } -int creat64(const char *pathname, mode_t mode) { - TRACE("intercepting creat64(pathname=%s, mode=%d)\n", pathname, mode) - - GoString filename = {pathname, strlen(pathname)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc creat64\n"); - CALL_REAL_OP("creat64", real_creat64, pathname, mode) - } - NOT_IMPLEMENTED("creat64") -} +int creat64(const char *pathname, mode_t mode) __attribute__((alias("creat"))); int fdatasync(int fd) { TRACE("intercepting fdatasync(fd=%d)\n", fd) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc fdatasync\n"); - CALL_REAL_OP("fdatasync", real_fdatasync, fd) + if FD_NOT_MANAGED(fd) { + return libc_fdatasync(fd); } NOT_IMPLEMENTED("fdatasync") } @@ -332,9 +287,8 @@ int fdatasync(int fd) { int fsync(int fd) { TRACE("intercepting fsync(fd=%d)\n", fd) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc fsync\n"); - CALL_REAL_OP("fsync", real_fsync, fd) + if FD_NOT_MANAGED(fd) { + return libc_fsync(fd); } NOT_IMPLEMENTED("fsync") } @@ -342,9 +296,8 @@ int fsync(int fd) { int ftruncate64(int fd, off64_t length) { TRACE("intercepting ftrunctate64(fd=%d, length=%ld)\n", fd, length) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc ftruncate64\n"); - CALL_REAL_OP("ftuncate64", real_ftruncate64, fd, length) + if FD_NOT_MANAGED(fd) { + return libc_ftruncate64(fd, length); } return Ftruncate(fd, length); } @@ -352,9 +305,8 @@ int ftruncate64(int fd, off64_t length) { int ftruncate(int fd, off_t length) { TRACE("callled ftruncate(fd=%d; length=%ld)\n", fd, length) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc ftruncate\n"); - CALL_REAL_OP("ftruncate", real_ftruncate, fd, length) + if FD_NOT_MANAGED(fd) { + return libc_ftruncate(fd, length); } return Ftruncate(fd, length); } @@ -362,10 +314,8 @@ int ftruncate(int fd, off_t length) { int truncate64(const char *path, off64_t length) { TRACE("intercepting truncate64(path=%s, length=%ld)\n", path, length) - GoString filename = {path, strlen(path)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc truncate64\n"); - CALL_REAL_OP("truncate64", real_truncate64, path, length) + if PATH_NOT_MANAGED(path) { + return libc_truncate64(path, length); } NOT_IMPLEMENTED("truncate64") } @@ -373,10 +323,8 @@ int truncate64(const char *path, off64_t length) { int truncate(const char *path, off_t length) { TRACE("intercepting truncate(path=%s, length=%ld)\n", path, length) - GoString filename = {path, strlen(path)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc truncate\n"); - CALL_REAL_OP("truncate", real_truncate, path, length) + if PATH_NOT_MANAGED(path) { + return libc_truncate(path, length); } NOT_IMPLEMENTED("truncate") } @@ -384,9 +332,8 @@ int truncate(const char *path, off_t length) { off64_t lseek64(int fd, off64_t offset, int whence) { TRACE("intercepting lseek64(fd=%d, offset=%ld; whence=%d)\n", fd, offset, whence) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc lseek64\n"); - CALL_REAL_OP("lseek64", real_lseek64, fd, offset, whence) + if FD_NOT_MANAGED(fd) { + return libc_lseek64(fd, offset, whence); } return Lseek(fd, offset, whence); } @@ -394,9 +341,8 @@ off64_t lseek64(int fd, off64_t offset, int whence) { off_t lseek(int fd, off_t offset, int whence) { TRACE("intercepting lseek(fd=%d; offset=%ld, whence=%d)\n", fd, offset, whence) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc lseek\n"); - CALL_REAL_OP("lseek", real_lseek, fd, offset, whence) + if FD_NOT_MANAGED(fd) { + return libc_lseek(fd, offset, whence); } return Lseek(fd, offset, whence); } @@ -404,9 +350,8 @@ off_t lseek(int fd, off_t offset, int whence) { ssize_t pread(int fd, void *buf, size_t count, off_t offset) { TRACE("intercepting pread(fd=%d, buf=%p, count=%lu, offset=%ld)\n", fd, buf, count, offset) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc pread\n"); - CALL_REAL_OP("pread", real_pread, fd, buf, count, offset) + if FD_NOT_MANAGED(fd) { + return libc_pread(fd, buf, count, offset); } GoSlice buffer = {buf, count, count}; return Pread(fd, buffer, offset); @@ -415,9 +360,8 @@ ssize_t pread(int fd, void *buf, size_t count, off_t offset) { ssize_t pread64(int fd, void *buf, size_t count, off64_t offset) { TRACE("intercepting pread64(fd=%d, buf=%p, count=%lu, offset=%ld)\n", fd, buf, count, offset) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc pread64\n"); - CALL_REAL_OP("pread64", real_pread64, fd, buf, count, offset) + if FD_NOT_MANAGED(fd) { + return libc_pread64(fd, buf, count, offset); } GoSlice buffer = {buf, count, count}; return Pread(fd, buffer, offset); @@ -426,9 +370,8 @@ ssize_t pread64(int fd, void *buf, size_t count, off64_t offset) { ssize_t preadv(int fd, const struct iovec *iov, int iovcnt, off_t offset) { TRACE("intercepting preadv(fd=%d, iov=%p, iovcnt=%d, offset=%ld)\n", fd, iov, iovcnt, offset) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc preadv\n"); - CALL_REAL_OP("preadv", real_preadv, fd, iov, iovcnt, offset) + if FD_NOT_MANAGED(fd) { + return libc_preadv(fd, iov, iovcnt, offset); } GoSlice vec[iovcnt]; @@ -445,9 +388,8 @@ ssize_t preadv(int fd, const struct iovec *iov, int iovcnt, off_t offset) { ssize_t preadv64(int fd, const struct iovec *iov, int iovcnt, off64_t offset) { TRACE("intercepting preadv64(fd=%d, iov=%p, iovcnt=%d, offset=%ld)\n", fd, iov, iovcnt, offset) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc preadv64\n"); - CALL_REAL_OP("preadv64", real_preadv64, fd, iov, iovcnt, offset) + if FD_NOT_MANAGED(fd) { + return libc_preadv64(fd, iov, iovcnt, offset); } GoSlice vec[iovcnt]; @@ -464,9 +406,8 @@ ssize_t preadv64(int fd, const struct iovec *iov, int iovcnt, off64_t offset) { ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset) { TRACE("intercepting pwrite(fd=%d, buf=%p, count=%lu, offset=%ld)\n", fd, buf, count, offset) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc pwrite\n"); - CALL_REAL_OP("pwrite", real_pwrite, fd, buf, count, offset) + if FD_NOT_MANAGED(fd) { + return libc_pwrite(fd, buf, count, offset); } GoSlice buffer = {(void*)buf, count, count}; return Pwrite(fd, buffer, offset); @@ -475,9 +416,8 @@ ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset) { ssize_t pwrite64(int fd, const void *buf, size_t count, off64_t offset) { TRACE("intercepting pwrite64(fd=%d, buf=%p, count=%lu, offset=%ld)\n", fd, buf, count, offset) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc pwrite64\n"); - CALL_REAL_OP("pwrite64", real_pwrite64, fd, buf, count, offset) + if FD_NOT_MANAGED(fd) { + return libc_pwrite64(fd, buf, count, offset); } GoSlice buffer = {(void*)buf, count, count}; return Pwrite(fd, buffer, offset); @@ -486,9 +426,8 @@ ssize_t pwrite64(int fd, const void *buf, size_t count, off64_t offset) { ssize_t pwritev(int fd, const struct iovec *iov, int iovcnt, off_t offset) { TRACE("intercepting pwritev(fd=%d, iov=%p, iovcnt=%d, offset=%ld)\n", fd, iov, iovcnt, offset) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc pwritev\n"); - CALL_REAL_OP("pwritev", real_pwritev, fd, iov, iovcnt, offset) + if FD_NOT_MANAGED(fd) { + return libc_pwritev(fd, iov, iovcnt, offset); } GoSlice vec[iovcnt]; @@ -505,9 +444,8 @@ ssize_t pwritev(int fd, const struct iovec *iov, int iovcnt, off_t offset) { ssize_t pwritev64(int fd, const struct iovec *iov, int iovcnt, off64_t offset) { TRACE("intercepting pwritev64(fd=%d, iov=%p, iovcnt=%d, offset=%ld)\n", fd, iov, iovcnt, offset) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc pwritev64\n"); - CALL_REAL_OP("pwritev64", real_pwritev64, fd, iov, iovcnt, offset) + if FD_NOT_MANAGED(fd) { + return libc_pwritev64(fd, iov, iovcnt, offset); } GoSlice vec[iovcnt]; @@ -524,9 +462,8 @@ ssize_t pwritev64(int fd, const struct iovec *iov, int iovcnt, off64_t offset) { ssize_t readv(int fd, const struct iovec *iov, int iovcnt) { TRACE("intercepting readv(fd=%d, iov=%p, iovcnt=%d)\n", fd, iov, iovcnt) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc readv\n"); - CALL_REAL_OP("readv", real_readv, fd, iov, iovcnt) + if FD_NOT_MANAGED(fd) { + return libc_readv(fd, iov, iovcnt); } GoSlice vec[iovcnt]; @@ -543,9 +480,8 @@ ssize_t readv(int fd, const struct iovec *iov, int iovcnt) { ssize_t writev(int fd, const struct iovec *iov, int iovcnt) { TRACE("intercepting writev(fd=%d, iov=%p, iovcnt=%d)\n", fd, iov, iovcnt) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc writev\n"); - CALL_REAL_OP("writev", real_writev, fd, iov, iovcnt) + if FD_NOT_MANAGED(fd) { + return libc_writev(fd, iov, iovcnt); } GoSlice vec[iovcnt]; @@ -559,52 +495,32 @@ ssize_t writev(int fd, const struct iovec *iov, int iovcnt) { return Writev(fd, iovSlice); } - -// we don't intercept fcntl because its signature has variadic args with a too complex -// behaviour to be reproduced and passed down to the real fcntl -/*** -int fcntl(int fd, int cmd, long arg) { - TRACE("intercepting fcntl(fd=%d, cmd=%d, arg=%ld)\n", fd, cmd, arg) - - if (IsFdManaged(fd)) { - WARNING("fcntl called on a managed fd, fcntl is not intercepted") - } - TRACE("calling libc fcntl\n"); - CALL_REAL_OP("fcntl", real_fcntl, fd, cmd, arg) -***/ - - int ioctl(int fd, unsigned long request, void *argp) { TRACE("intercepting ioctl(fd=%d, request=%lu, argp=%p)\n", fd, request, argp) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc ioctl\n"); - CALL_REAL_OP("ioctl", real_ioctl, fd, request, argp) + if FD_NOT_MANAGED(fd) { + return libc_ioctl(fd, request, argp); } NOT_IMPLEMENTED("ioctl") } - int access(const char *pathname, int mode) { TRACE("intercepting access(pathname=%s, mode=%d)\n", pathname, mode) - GoString filename = {pathname, strlen(pathname)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc access\n"); - CALL_REAL_OP("access", real_access, pathname, mode) + if PATH_NOT_MANAGED(pathname) { + return libc_access(pathname, mode); } + GoString filename = {pathname, strlen(pathname)}; return Access(filename, mode); } - int unlink(const char *pathname) { TRACE("intercepting unlink(pathname=%s)\n", pathname) - GoString filename = {pathname, strlen(pathname)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc unlink\n"); - CALL_REAL_OP("unlink", real_unlink, pathname) + if PATH_NOT_MANAGED(pathname) { + return libc_unlink(pathname); } + GoString filename = {pathname, strlen(pathname)}; int ret = Unlink(filename); if (ret < 0) { errno = GetErrno(); @@ -612,57 +528,51 @@ int unlink(const char *pathname) { return ret; } - int __xstat(int vers, const char *pathname, struct stat *buf) { TRACE("intercepting __xstat(vers=%d, pathname=%s, buf=%p)\n", vers, pathname, buf) - GoString filename = {pathname, strlen(pathname)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc __xstat\n"); - CALL_REAL_OP("__xstat", real___xstat, vers, pathname, buf) + if PATH_NOT_MANAGED(pathname) { + return libc__xstat(vers, pathname, buf); } + GoString filename = {pathname, strlen(pathname)}; return Stat(filename, buf); } int __xstat64(int vers, const char *pathname, struct stat64 *buf) { TRACE("intercepting __xstat64(vers=%d, pathname=%s, buf=%p)\n", vers, pathname, buf) - GoString filename = {pathname, strlen(pathname)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc __xstat64\n"); - CALL_REAL_OP("__xstat64", real___xstat64, vers, pathname, buf) + if PATH_NOT_MANAGED(pathname) { + return libc__xstat64(vers, pathname, buf); } + GoString filename = {pathname, strlen(pathname)}; return Stat64(filename, buf); } int __lxstat(int vers, const char *pathname, struct stat *buf) { TRACE("intercepting __lxstat(vers=%d, pathname=%s, buf=%p)\n", vers, pathname, buf) - GoString filename = {pathname, strlen(pathname)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc __lxstat\n"); - CALL_REAL_OP("__lxstat", real___lxstat, vers, pathname, buf) + if PATH_NOT_MANAGED(pathname) { + return libc__lxstat(vers, pathname, buf); } + GoString filename = {pathname, strlen(pathname)}; return Lstat(filename, buf); } int __lxstat64(int vers, const char *pathname, struct stat64 *buf) { TRACE("intercepting __lxstat64(vers=%d, pathname=%s, buf=%p)\n", vers, pathname, buf) - GoString filename = {pathname, strlen(pathname)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc __lxstat64\n"); - CALL_REAL_OP("__lxstat64", real___lxstat64, vers, pathname, buf) + if PATH_NOT_MANAGED(pathname) { + return libc__lxstat64(vers, pathname, buf); } + GoString filename = {pathname, strlen(pathname)}; return Lstat64(filename, buf); } int __fxstat(int vers, int fd, struct stat *buf) { TRACE("intercepting __fxstat(vers=%d, fd=%d, buf=%p)\n", vers, fd, buf) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc __fxstat\n"); - CALL_REAL_OP("__fxstat", real___fxstat, vers, fd, buf) + if FD_NOT_MANAGED(fd) { + return libc__fxstat(vers, fd, buf); } return Fstat(fd, buf); } @@ -670,41 +580,37 @@ int __fxstat(int vers, int fd, struct stat *buf) { int __fxstat64(int vers, int fd, struct stat64 *buf) { TRACE("intercepting __fxstat64(vers=%d, fd=%d, buf=%p)\n", vers, fd, buf) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc __fxstat64\n"); - CALL_REAL_OP("__fxstat64", real___fxstat64, vers, fd, buf) + if FD_NOT_MANAGED(fd) { + return libc__fxstat64(vers, fd, buf); } return Fstat64(fd, buf); } - int statfs(const char *path, struct statfs *buf) { TRACE("intercepting statfs(path=%s, buf=%p)\n", path, buf) - GoString filename = {path, strlen(path)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc statfs\n"); - CALL_REAL_OP("statfs", real_statfs, path, buf) + if PATH_NOT_MANAGED(path) { + return libc_statfs(path, buf); } + GoString filename = {path, strlen(path)}; return Statfs(filename, buf); } int statfs64(const char *path, struct statfs64 *buf) { TRACE("intercepting statfs64(path=%s, buf=%p)\n", path, buf) - GoString filename = {path, strlen(path)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc statfs64\n"); - CALL_REAL_OP("statfs64", real_statfs64, path, buf) + + if PATH_NOT_MANAGED(path) { + return libc_statfs64(path, buf); } + GoString filename = {path, strlen(path)}; return Statfs64(filename, buf); } int fstatfs(int fd, struct statfs *buf) { TRACE("intercepting fstatfs(fd=%d, buf=%p)\n", fd, buf) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc fstatfs\n"); - CALL_REAL_OP("fstatfs", real_fstatfs, fd, buf) + if FD_NOT_MANAGED(fd) { + return libc_fstatfs(fd, buf); } NOT_IMPLEMENTED("fstatfs") } @@ -712,9 +618,8 @@ int fstatfs(int fd, struct statfs *buf) { int fstatfs64(int fd, struct statfs64 *buf) { TRACE("intercepting fstatfs64(fd=%d, buf=%p)\n", fd, buf) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc fstatfs64\n"); - CALL_REAL_OP("fstatfs64", real_fstatfs64, fd, buf) + if FD_NOT_MANAGED(fd) { + return libc_fstatfs64(fd, buf); } NOT_IMPLEMENTED("fstatfs64") } @@ -722,9 +627,8 @@ int fstatfs64(int fd, struct statfs64 *buf) { FILE* fdopen(int fd, const char *mode) { TRACE("intercepting fdopen(fd=%d, mode=%s)\n", fd, mode) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc fdopen\n"); - CALL_REAL_OP("fdopen", real_fdopen, fd, mode) + if FD_NOT_MANAGED(fd) { + return libc_fdopen(fd, mode); } NOT_IMPLEMENTED("fdopen") } @@ -732,55 +636,39 @@ FILE* fdopen(int fd, const char *mode) { FILE* fopen(const char *path, const char *mode) { TRACE("intercepting fopen(path=%s, mode=%s)\n", path, mode) - GoString filename = {path, strlen(path)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc fopen\n"); - CALL_REAL_OP("fopen", real_fopen, path, mode) + if PATH_NOT_MANAGED(path) { + return libc_fopen(path, mode); } - GoString gomode = {mode, strlen(mode)}; - return Fopen(filename, gomode); -} - -FILE* fopen64(const char *path, const char *mode) { - TRACE("intercepting fopen64(path=%s, mode=%s)\n", path, mode) - GoString filename = {path, strlen(path)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc fopen64\n"); - CALL_REAL_OP("fopen64", real_fopen64, path, mode) - } + FILE *stream = get_new_stream(fd_register); + GoString gopath = {path, strlen(path)}; GoString gomode = {mode, strlen(mode)}; - return Fopen(filename, gomode); + int ret = Fopen(gopath, gomode, fileno(stream)); + if (ret < 0) { + remove_fd(fd_register, fileno(stream)); + return (FILE*)(NULL); + } + return stream; } +FILE* fopen64(const char *path, const char *mode) __attribute__((alias("fopen"))); + FILE* freopen(const char *path, const char *mode, FILE *stream) { TRACE("intercepting fropen(path=%s, mode=%s, stream=%p)\n", path, mode, stream) - GoString filename = {path, strlen(path)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc freopen\n"); - CALL_REAL_OP("freopen", real_freopen, path, mode, stream) + if PATH_NOT_MANAGED(path) { + return libc_freopen(path, mode, stream); } NOT_IMPLEMENTED("freopen") } -FILE* freopen64(const char *path, const char *mode, FILE *stream) { - TRACE("intercepting fropen64(path=%s, mode=%s, stream=%p)\n", path, mode, stream) - - GoString filename = {path, strlen(path)}; - if (!pdwfs_initialized || !IsFileManaged(filename)) { - TRACE("calling libc freopen64\n"); - CALL_REAL_OP("freopen64", real_freopen64, path, mode, stream) - } - NOT_IMPLEMENTED("freopen64") -} +FILE* freopen64(const char *path, const char *mode, FILE *stream) __attribute__((alias("fopen"))); int fclose(FILE *stream) { TRACE("intercepting fclose(stream=%p)\n", stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc fclose\n"); - CALL_REAL_OP("fclose", real_fclose, stream) + if STREAM_NOT_MANAGED(stream) { + return libc_fclose(stream); } Fflush(stream); return close(fileno(stream)); @@ -789,9 +677,8 @@ int fclose(FILE *stream) { int fflush(FILE *stream) { TRACE("intercepting fflush(stream=%p)\n", stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc fflush\n"); - CALL_REAL_OP("fflush", real_fflush, stream) + if STREAM_NOT_MANAGED(stream) { + return libc_fflush(stream); } return Fflush(stream); } @@ -799,9 +686,8 @@ int fflush(FILE *stream) { int fputc(int c, FILE *stream) { TRACE("intercepting fputc(c=%d, stream=%p)\n", c, stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc fputc\n"); - CALL_REAL_OP("fputc", real_fputc, c, stream) + if STREAM_NOT_MANAGED(stream) { + return libc_fputc(c, stream); } GoSlice cBuf = {(char*)&c, 1, 1}; int n = Write(fileno(stream), cBuf); @@ -813,9 +699,8 @@ int fputc(int c, FILE *stream) { char* fgets(char *dst, int max, FILE *stream) { TRACE("intercepting fgets(dst=%s, max=%d, stream=%p)\n", dst, max, stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc fgets\n"); - CALL_REAL_OP("fgets", real_fgets, dst, max, stream) + if STREAM_NOT_MANAGED(stream) { + return libc_fgets(dst, max, stream); } size_t n = max; @@ -841,13 +726,11 @@ char* fgets(char *dst, int max, FILE *stream) { return result; } - int fgetc(FILE *stream) { TRACE("intercepting fgetc(stream=%p)\n", stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc fgetc\n"); - CALL_REAL_OP("fgetc", real_fgetc, stream) + if STREAM_NOT_MANAGED(stream) { + return libc_fgetc(stream); } char c; @@ -863,9 +746,8 @@ int fgetc(FILE *stream) { int fgetpos(FILE *stream, fpos_t *pos) { TRACE("intercepting fgetpos(stream=%p, pos=%p)\n", stream, pos) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc fgetpos\n"); - CALL_REAL_OP("fgetpos", real_fgetpos, stream, pos) + if STREAM_NOT_MANAGED(stream) { + return libc_fgetpos(stream, pos); } NOT_IMPLEMENTED("fgetpos") } @@ -873,9 +755,8 @@ int fgetpos(FILE *stream, fpos_t *pos) { int fgetpos64(FILE *stream, fpos64_t *pos) { TRACE("intercepting fgetpos64(stream=%p, pos=%p)\n", stream, pos) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc fgetpos64\n"); - CALL_REAL_OP("fgetpos64", real_fgetpos64, stream, pos) + if STREAM_NOT_MANAGED(stream) { + return libc_fgetpos64(stream, pos); } NOT_IMPLEMENTED("fgetpos64") } @@ -883,9 +764,8 @@ int fgetpos64(FILE *stream, fpos64_t *pos) { int fseek(FILE *stream, long offset, int whence) { TRACE("intercepting fseek(stream=%p, offset=%ld, whence=%d)\n", stream, offset, whence) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc fseek\n"); - CALL_REAL_OP("fseek", real_fseek, stream, offset, whence) + if STREAM_NOT_MANAGED(stream) { + return libc_fseek(stream, offset, whence); } NOT_IMPLEMENTED("fseek") } @@ -893,9 +773,8 @@ int fseek(FILE *stream, long offset, int whence) { int fseeko(FILE *stream, off_t offset, int whence) { TRACE("intercepting fseeko(stream=%p, offset=%ld, whence=%d)\n", stream, offset, whence) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc fseeko\n"); - CALL_REAL_OP("fseeko", real_fseeko, stream, offset, whence) + if STREAM_NOT_MANAGED(stream) { + return libc_fseeko(stream, offset, whence); } NOT_IMPLEMENTED("fseeko") } @@ -903,9 +782,8 @@ int fseeko(FILE *stream, off_t offset, int whence) { int fseeko64(FILE *stream, off64_t offset, int whence) { TRACE("intercepting fseeko64(stream=%p, offset=%ld, whence=%d)\n", stream, offset, whence) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc fseeko64\n"); - CALL_REAL_OP("fseeko64", real_fseeko64, stream, offset, whence) + if STREAM_NOT_MANAGED(stream) { + return libc_fseeko64(stream, offset, whence); } NOT_IMPLEMENTED("fseeko") } @@ -913,9 +791,8 @@ int fseeko64(FILE *stream, off64_t offset, int whence) { int fsetpos(FILE *stream, const fpos_t *pos) { TRACE("intercepting fsetpos(stream=%p, pos=%p)\n", stream, pos) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc fsetpos\n"); - CALL_REAL_OP("fsetpos", real_fsetpos, stream, pos) + if STREAM_NOT_MANAGED(stream) { + return libc_fsetpos(stream, pos); } NOT_IMPLEMENTED("fsetpos") } @@ -923,9 +800,8 @@ int fsetpos(FILE *stream, const fpos_t *pos) { int fsetpos64(FILE *stream, const fpos64_t *pos) { TRACE("intercepting fsetpos64(stream=%p, pos=%p)\n", stream, pos) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc fsetpos64\n"); - CALL_REAL_OP("fsetpos64", real_fsetpos64, stream, pos) + if STREAM_NOT_MANAGED(stream) { + return libc_fsetpos64(stream, pos); } NOT_IMPLEMENTED("fsetpos64") } @@ -933,19 +809,18 @@ int fsetpos64(FILE *stream, const fpos64_t *pos) { int fputs(const char *s, FILE *stream) { TRACE("intercepting fputs(s=%s, stream=%p)\n", s, stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc fputs\n"); - CALL_REAL_OP("fputs", real_fputs, s, stream) + if STREAM_NOT_MANAGED(stream) { + return libc_fputs(s, stream); } - NOT_IMPLEMENTED("fputs") + size_t len = strlen(s); + return (fwrite(s, 1, len, stream)==len) - 1; } int putc(int c, FILE *stream) { TRACE("intercepting putc(c=%d, stream=%p)\n", c, stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc putc\n"); - CALL_REAL_OP("putc", real_putc, c, stream) + if STREAM_NOT_MANAGED(stream) { + return libc_putc(c, stream); } NOT_IMPLEMENTED("putc") } @@ -953,9 +828,8 @@ int putc(int c, FILE *stream) { int getc(FILE *stream) { TRACE("intercepting getc(stream=%p)\n", stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc getc\n"); - CALL_REAL_OP("getc", real_getc, stream) + if STREAM_NOT_MANAGED(stream) { + return libc_getc(stream); } NOT_IMPLEMENTED("getc") } @@ -963,9 +837,8 @@ int getc(FILE *stream) { int ungetc(int c, FILE *stream) { TRACE("intercepting fgetc(c=%d, stream=%p)\n", c, stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc ungetc\n"); - CALL_REAL_OP("ungetc", real_ungetc, c, stream) + if STREAM_NOT_MANAGED(stream) { + return libc_ungetc(c, stream); } NOT_IMPLEMENTED("ungetc") } @@ -973,9 +846,8 @@ int ungetc(int c, FILE *stream) { long ftell(FILE *stream) { TRACE("intercepting ftell(stream=%p)\n", stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc ftell\n"); - CALL_REAL_OP("ftell", real_ftell, stream) + if STREAM_NOT_MANAGED(stream) { + return libc_ftell(stream); } NOT_IMPLEMENTED("ftell") } @@ -983,9 +855,8 @@ long ftell(FILE *stream) { off_t ftello(FILE *stream) { TRACE("intercepting ftello(stream=%p)\n", stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc ftello\n"); - CALL_REAL_OP("ftello", real_ftello, stream) + if STREAM_NOT_MANAGED(stream) { + return libc_ftello(stream); } NOT_IMPLEMENTED("ftello") } @@ -993,9 +864,8 @@ off_t ftello(FILE *stream) { off64_t ftello64(FILE *stream) { TRACE("intercepting ftello64(stream=%p)\n", stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc ftello64\n"); - CALL_REAL_OP("ftello64", real_ftello64, stream) + if STREAM_NOT_MANAGED(stream) { + return libc_ftello64(stream); } NOT_IMPLEMENTED("ftello64") } @@ -1003,9 +873,8 @@ off64_t ftello64(FILE *stream) { size_t fread(void *ptr, size_t size, size_t nmemb, FILE *stream) { TRACE("intercepting fread(ptr=%p, size=%lu, nmemb=%lu, stream=%p)\n", ptr, size, nmemb, stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc fread\n"); - CALL_REAL_OP("fread", real_fread, ptr, size, nmemb, stream) + if STREAM_NOT_MANAGED(stream) { + return libc_fread(ptr, size, nmemb, stream); } GoSlice buffer = {ptr, size * nmemb, size * nmemb}; return Read(fileno(stream), buffer); @@ -1014,27 +883,13 @@ size_t fread(void *ptr, size_t size, size_t nmemb, FILE *stream) { size_t fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream) { TRACE("intercepting fwrite(ptr=%p, size=%lu, nmemb=%lu, stream=%p)\n", ptr, size, nmemb, stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc fwrite\n"); - CALL_REAL_OP("fwrite", real_fwrite, ptr, size, nmemb, stream) + if STREAM_NOT_MANAGED(stream) { + return libc_fwrite(ptr, size, nmemb, stream); } GoSlice buffer = {(void*)ptr, size * nmemb, size * nmemb}; return Write(fileno(stream), buffer); } -int _fprint(FILE *stream, const char *msg, int size) { - - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc fprintf\n"); - if (!real_fprintf) { - real_fprintf = dlsym(RTLD_NEXT, "fprintf"); - } - return real_fprintf(stream, "%s", msg); - } - GoSlice buffer = {(char*)msg, size, size}; - return Write(fileno(stream), buffer); -} - int __fprintf_chk(FILE *stream, int flag, const char *fmt, ...) { TRACE("intercepting __fprintf_chk(stream=%p, ...)\n", stream) @@ -1062,7 +917,7 @@ int __fprintf_chk(FILE *stream, int flag, const char *fmt, ...) { } va_end(ap); - int ret = _fprint(stream, msg, size); + int ret = fputs(msg, stream); free(msg); return ret; } @@ -1094,18 +949,17 @@ int fprintf(FILE *stream, const char *fmt, ...) { } va_end(ap); - int ret = _fprint(stream, msg, size); + int ret = fputs(msg, stream); free(msg); return ret; } - void rewind(FILE *stream) { TRACE("intercepting rewind(stream=%p)\n", stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc rewind\n"); - CALL_REAL_OP("rewind", real_rewind, stream) + if STREAM_NOT_MANAGED(stream) { + libc_rewind(stream); + return; } NOT_IMPLEMENTED("rewind") } @@ -1113,37 +967,32 @@ void rewind(FILE *stream) { int dup2(int oldfd, int newfd) { TRACE("intercepting dup2(oldfd=%d, newfd=%d)\n", oldfd, newfd) - if (!pdwfs_initialized || (!IsFdManaged(oldfd) && !IsFdManaged(newfd))) { - TRACE("calling libc dup2\n"); - CALL_REAL_OP("dup2", real_dup2, oldfd, newfd) -} + if (FD_NOT_MANAGED(oldfd) && FD_NOT_MANAGED(newfd)) { + return libc_dup2(oldfd, newfd); + } NOT_IMPLEMENTED("dup2") } int unlinkat(int dirfd, const char *pathname, int flags) { - TRACE("intercepting unlinkat(dirfd=%d, pathname=%s, flags=%d)\n", dirfd, pathname, flags) - TRACE("calling libc unlinkat (INTERCEPTION NOT IMPLEMENTED)\n") - CALL_REAL_OP("unlinkat", real_unlinkat, dirfd, pathname, flags) + TRACE("intercepting unlinkat(dirfd=%d, pathname=%s, flags=%d) (PASS THOUGH)\n", dirfd, pathname, flags) + return libc_unlinkat(dirfd, pathname, flags); } int faccessat(int dirfd, const char *pathname, int mode, int flags) { - TRACE("intercepting faccessat(dirfd=%d, pathname=%s, mode=%d, flags=%d)\n", dirfd, pathname, mode, flags) - TRACE("calling libc faccessat (INTERCEPTION NOT IMPLEMENTED)\n") - CALL_REAL_OP("faccessat", real_faccessat, dirfd, pathname, mode, flags) + TRACE("intercepting faccessat(dirfd=%d, pathname=%s, mode=%d, flags=%d) (PASS THOUGH)\n", dirfd, pathname, mode, flags) + return libc_faccessat(dirfd, pathname, mode, flags); } // __fxstatat is the glibc function corresponding to fstatat syscall int __fxstatat(int vers, int dirfd, const char *pathname, struct stat *buf, int flags) { - TRACE("intercepting __fxstatat(vers=%d, dirfd=%d, pathname=%s, buf=%p, flags=%d)\n", vers, dirfd, pathname, buf, flags) - TRACE("calling libc __fxstatat (INTERCEPTION NOT IMPLEMENTED)\n") - CALL_REAL_OP("__fxstatat", real___fxstatat, vers, dirfd, pathname, buf, flags) + TRACE("intercepting __fxstatat(vers=%d, dirfd=%d, pathname=%s, buf=%p, flags=%d) (PASS THOUGH)\n", vers, dirfd, pathname, buf, flags) + return libc__fxstatat(vers, dirfd, pathname, buf, flags); } // __fxstatat64 is the LARGEFILE64 version of glibc function corresponding to fstatat syscall int __fxstatat64(int vers, int dirfd, const char *pathname, struct stat64 *buf, int flags) { - TRACE("intercepting __fxstatat64(vers=%d, dirfd=%d, pathname=%s, buf=%p, flags=%d)\n", dirfd, pathname, buf, flags) - TRACE("calling libc __fxstatat64 (INTERCEPTION NOT IMPLEMENTED)\n") - CALL_REAL_OP("__fxstatat64", real___fxstatat64, vers, dirfd, pathname, buf, flags) + TRACE("intercepting __fxstatat64(vers=%d, dirfd=%d, pathname=%s, buf=%p, flags=%d) (PASS THOUGH)\n", dirfd, pathname, buf, flags) + return libc__fxstatat64(vers, dirfd, pathname, buf, flags); } int openat(int dirfd, const char *pathname, int flags, ...) { @@ -1157,70 +1006,60 @@ int openat(int dirfd, const char *pathname, int flags, ...) { va_end(arg); } - TRACE("intercepting openat(dirfd=%d, pathname=%s, flags=%d, mode=%d)\n", dirfd, pathname, flags, mode) + TRACE("intercepting openat(dirfd=%d, pathname=%s, flags=%d, mode=%d) (PASS THOUGH)\n", dirfd, pathname, flags, mode) // NOTE: see the comment of func (fs *PdwFS) registerFile in pdwfs.go before implementing openat interception - TRACE("calling libc openat (INTERCEPTION NOT IMPLEMENTED)\n") - CALL_REAL_OP("openat", real_openat, dirfd, pathname, flags, mode) + return libc_openat(dirfd, pathname, flags, mode); } int mkdir(const char *pathname, mode_t mode) { TRACE("intercepting mkdir(pathname=%s, mode=%d)\n", pathname, mode) - GoString gopath = {pathname, strlen(pathname)}; - if (!pdwfs_initialized || !IsFileManaged(gopath)) { - TRACE("calling libc mkdir\n"); - CALL_REAL_OP("mkdir", real_mkdir, pathname, mode) + if PATH_NOT_MANAGED(pathname) { + return libc_mkdir(pathname, mode); } + GoString gopath = {pathname, strlen(pathname)}; return Mkdir(gopath, mode); } int mkdirat(int dirfd, const char *pathname, mode_t mode) { - TRACE("intercepting mkdirat(dirfd=%d, pathname=%s, mode=%d)\n", dirfd, pathname, mode) - TRACE("calling libc mkdirat (INTERCEPTION NOT IMPLEMENTED)\n") - CALL_REAL_OP("mkdirat", real_mkdirat, dirfd, pathname, mode) + TRACE("intercepting mkdirat(dirfd=%d, pathname=%s, mode=%d) (PASS THROUGH)\n", dirfd, pathname, mode) + return libc_mkdirat(dirfd, pathname, mode); } int rmdir(const char *pathname) { TRACE("intercepting rmdir(pathname=%s)\n", pathname) - GoString gopath = {pathname, strlen(pathname)}; - if (!pdwfs_initialized || !IsFileManaged(gopath)) { - TRACE("calling libc rmdir\n"); - CALL_REAL_OP("rmdir", real_rmdir, pathname) + if PATH_NOT_MANAGED(pathname) { + return libc_rmdir(pathname); } + GoString gopath = {pathname, strlen(pathname)}; return Rmdir(gopath); } int rename(const char *oldpath, const char *newpath) { TRACE("intercepting rename(oldname=%s, newpath=%s)\n", oldpath, newpath) - GoString old = {oldpath, strlen(oldpath)}; - GoString new = {newpath, strlen(newpath)}; - if (!pdwfs_initialized || (!IsFileManaged(old) && !IsFileManaged(new))) { - TRACE("calling libc rename\n"); - CALL_REAL_OP("rename", real_rename, oldpath, newpath) + if (PATH_NOT_MANAGED(oldpath) && PATH_NOT_MANAGED(newpath)) { + return libc_rename(oldpath, newpath); } NOT_IMPLEMENTED("rename") } int renameat(int olddirfd, const char *oldpath, int newdirfd, const char *newpath) { - TRACE("intercepting renameat(olddirfd=%d, oldpath=%s, newdirfd=%d, newpath=%s)\n", olddirfd, oldpath, newdirfd, newpath) - TRACE("calling libc renameat (INTERCEPTION NOT IMPLEMENTED)\n") - CALL_REAL_OP("renameat", real_renameat, olddirfd, oldpath, newdirfd, newpath) + TRACE("intercepting renameat(olddirfd=%d, oldpath=%s, newdirfd=%d, newpath=%s) (PASS THROUGH)\n", olddirfd, oldpath, newdirfd, newpath) + return libc_renameat(olddirfd, oldpath, newdirfd, newpath); } int renameat2(int olddirfd, const char *oldpath, int newdirfd, const char *newpath, unsigned int flags) { - TRACE("intercepting renameat2(olddirfd=%d, oldpath=%s, newdirfd=%d, newpath=%s, flags=%d)\n", olddirfd, oldpath, newdirfd, newpath, flags) - TRACE("calling libc renameat2 (INTERCEPTION NOT IMPLEMENTED)\n") - CALL_REAL_OP("renameat2", real_renameat2, olddirfd, oldpath, newdirfd, newpath, flags) + TRACE("intercepting renameat2(olddirfd=%d, oldpath=%s, newdirfd=%d, newpath=%s, flags=%d) (PASS THROUGH)\n", olddirfd, oldpath, newdirfd, newpath, flags) + return libc_renameat2(olddirfd, oldpath, newdirfd, newpath, flags); } int posix_fadvise(int fd, off_t offset, off_t len, int advice) { TRACE("intercepting posix_fadvise(fd=%d, offset=%d, len=%d, advice=%d)\n", fd, offset, len, advice) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc posix_fadvise\n"); - CALL_REAL_OP("posix_fadvise", real_posix_fadvise, fd, offset, len, advice) + if FD_NOT_MANAGED(fd) { + return libc_posix_fadvise(fd, offset, len, advice); } return Fadvise(fd, offset, len, advice); } @@ -1228,9 +1067,8 @@ int posix_fadvise(int fd, off_t offset, off_t len, int advice) { int posix_fadvise64(int fd, off64_t offset, off64_t len, int advice) { TRACE("intercepting posix_fadvise64(fd=%d, offset=%d, len=%d, advice=%d)\n", fd, offset, len, advice) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc posix_fadvise64\n"); - CALL_REAL_OP("posix_fadvise64", real_posix_fadvise64, fd, offset, len, advice) + if FD_NOT_MANAGED(fd) { + return libc_posix_fadvise64(fd, offset, len, advice); } return Fadvise(fd, offset, len, advice); } @@ -1238,31 +1076,28 @@ int posix_fadvise64(int fd, off64_t offset, off64_t len, int advice) { int statvfs(const char *pathname, struct statvfs *buf) { TRACE("intercepting statvfs(path=%s, buf=%p)\n", pathname, buf) - GoString gopath = {pathname, strlen(pathname)}; - if (!pdwfs_initialized || !IsFileManaged(gopath)) { - TRACE("calling libc statvfs\n"); - CALL_REAL_OP("statvfs", real_statvfs, pathname, buf) + if PATH_NOT_MANAGED(pathname) { + return libc_statvfs(pathname, buf); } + GoString gopath = {pathname, strlen(pathname)}; return Statvfs(gopath, buf); } int statvfs64(const char *pathname, struct statvfs64 *buf) { TRACE("intercepting statvfs64(path=%s, buf=%p)\n", pathname, buf) - GoString gopath = {pathname, strlen(pathname)}; - if (!pdwfs_initialized || !IsFileManaged(gopath)) { - TRACE("calling libc statvfs64\n"); - CALL_REAL_OP("statvfs64", real_statvfs64, pathname, buf) + if PATH_NOT_MANAGED(pathname) { + return libc_statvfs64(pathname, buf); } + GoString gopath = {pathname, strlen(pathname)}; return Statvfs64(gopath, buf); } int fstatvfs(int fd, struct statvfs *buf) { TRACE("intercepting fstatvfs(fd=%d, buf=%p)\n", fd, buf) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc fstatvfs\n"); - CALL_REAL_OP("fstatvfs", real_fstatvfs, fd, buf) + if FD_NOT_MANAGED(fd) { + return libc_fstatvfs(fd, buf); } NOT_IMPLEMENTED("fstatvfs") } @@ -1270,20 +1105,17 @@ int fstatvfs(int fd, struct statvfs *buf) { int fstatvfs64(int fd, struct statvfs64 *buf) { TRACE("intercepting fstatvfs64(fd=%d, buf=%p)\n", fd, buf) - if (!pdwfs_initialized || IS_STD_FD(fd) || !IsFdManaged(fd)) { - TRACE("calling libc fstatvfs64\n"); - CALL_REAL_OP("fstatvfs64", real_fstatvfs64, fd, buf) + if FD_NOT_MANAGED(fd) { + return libc_fstatvfs64(fd, buf); } NOT_IMPLEMENTED("fstatvfs64") } - -ssize_t getdelim(char **buf, size_t *bufsiz, int delimiter, FILE *fp) { - TRACE("intercepting getdelim(buf=%p, bufsiz=%p, delimiter=%d, stream=%p)\n", buf, bufsiz, delimiter, fp) +ssize_t getdelim(char **buf, size_t *bufsiz, int delimiter, FILE *stream) { + TRACE("intercepting getdelim(buf=%p, bufsiz=%p, delimiter=%d, stream=%p)\n", buf, bufsiz, delimiter, stream) - if (!pdwfs_initialized || IS_STD_STREAM(fp) || !IsFdManaged(fileno(fp))) { - TRACE("calling libc getdelim\n"); - CALL_REAL_OP("getdelim", real_getdelim, buf, bufsiz, delimiter, fp) + if STREAM_NOT_MANAGED(stream) { + return libc_getdelim(buf, bufsiz, delimiter, stream); } char *ptr, *eptr; @@ -1296,9 +1128,9 @@ ssize_t getdelim(char **buf, size_t *bufsiz, int delimiter, FILE *fp) { } for (ptr = *buf, eptr = *buf + *bufsiz;;) { - int c = fgetc(fp); + int c = fgetc(stream); if (c == -1) { - if (feof(fp)) + if (feof(stream)) return ptr == *buf ? -1 : ptr - *buf; else return -1; @@ -1322,13 +1154,11 @@ ssize_t getdelim(char **buf, size_t *bufsiz, int delimiter, FILE *fp) { } } - ssize_t getline(char **buf, size_t *bufsiz, FILE *stream) { TRACE("intercepting getline(buf=%p, bufsiz=%p, stream=%p)\n", buf, bufsiz, stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc getline\n"); - CALL_REAL_OP("getline", real_getline, buf, bufsiz, stream) + if STREAM_NOT_MANAGED(stream) { + return libc_getline(buf, bufsiz, stream); } return getdelim(buf, bufsiz, '\n', stream); } @@ -1336,10 +1166,8 @@ ssize_t getline(char **buf, size_t *bufsiz, FILE *stream) { DIR* opendir(const char* path) { TRACE("intercepting opendir(path=%s)\n", path) - GoString gopath = {path, strlen(path)}; - if (!pdwfs_initialized || !IsFileManaged(gopath)) { - TRACE("calling libc opendir\n"); - CALL_REAL_OP("opendir", real_opendir, path) + if PATH_NOT_MANAGED(path) { + return libc_opendir(path); } NOT_IMPLEMENTED("opendir") } @@ -1347,9 +1175,8 @@ DIR* opendir(const char* path) { int feof(FILE *stream) { TRACE("intercepting feof(stream=%p)\n", stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc feof\n"); - CALL_REAL_OP("feof", real_feof, stream) + if STREAM_NOT_MANAGED(stream) { + return libc_feof(stream); } int fd = fileno(stream); off_t cur_off = lseek(fd, 0, SEEK_CUR); @@ -1362,9 +1189,8 @@ int feof(FILE *stream) { int ferror(FILE *stream) { TRACE("intercepting ferror(stream=%p)\n", stream) - if (!pdwfs_initialized || IS_STD_STREAM(stream) || !IsFdManaged(fileno(stream))) { - TRACE("calling libc ferror\n"); - CALL_REAL_OP("ferror", real_ferror, stream) + if STREAM_NOT_MANAGED(stream) { + return libc_ferror(stream); } NOT_IMPLEMENTED("ferror") } \ No newline at end of file diff --git a/src/c/tests/Makefile b/src/c/tests/Makefile new file mode 100644 index 0000000..1569225 --- /dev/null +++ b/src/c/tests/Makefile @@ -0,0 +1,28 @@ + +BUILDDIR=../../../build/tests +CFLAGS = -std=c99 -g -O2 -Wall -Werror + +SRCS = $(wildcard *.c) +OBJS = $(patsubst %.c, $(BUILDDIR)/%.o, $(SRCS)) +EXE = $(BUILDDIR)/testsuite + +all: $(EXE) + +$(EXE): $(OBJS) + $(CC) $(LDFLAGS) -o $@ $(OBJS) $(LIBS) + +$(BUILDDIR)/%.o: %.c + $(CC) $(CFLAGS) -o $@ -c $< + +clean: + rm -f $(OBJS) $(EXE) + +relink: + rm -f $(EXE) + make all + +test: $(EXE) + - $(EXE) + rm -rf test_file + +retest: relink test \ No newline at end of file diff --git a/src/c/tests/test_access.c b/src/c/tests/test_access.c index 0468abf..5c4c668 100644 --- a/src/c/tests/test_access.c +++ b/src/c/tests/test_access.c @@ -19,7 +19,7 @@ #include #include "tests.h" -void test_access() { +int test_access() { int ret = access(TESTFILE, F_OK); assert(ret == -1); @@ -35,8 +35,6 @@ void test_access() { ret = access(TESTFILE, F_OK); assert(ret == -1); -} -int main() { - test_access(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_feof.c b/src/c/tests/test_feof.c index e8ecb6f..1c38563 100644 --- a/src/c/tests/test_feof.c +++ b/src/c/tests/test_feof.c @@ -19,7 +19,7 @@ #include #include "tests.h" -void test_feof() { +int test_feof() { FILE* f = fopen(TESTFILE, "w"); CHECK_NULL(f, "fopen") @@ -45,8 +45,6 @@ void test_feof() { fclose(f); unlink(TESTFILE); -} -int main() { - test_feof(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_fgets.c b/src/c/tests/test_fgets.c index 31dd164..e6251f7 100644 --- a/src/c/tests/test_fgets.c +++ b/src/c/tests/test_fgets.c @@ -19,7 +19,7 @@ #include #include "tests.h" -void test_fgets() { +int test_fgets() { FILE* f = fopen(TESTFILE, "w"); CHECK_NULL(f, "fopen") @@ -45,8 +45,6 @@ void test_fgets() { fclose(f); unlink(TESTFILE); -} -int main() { - test_fgets(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_fopen_fclose.c b/src/c/tests/test_fopen_fclose.c index 004df71..5a75137 100644 --- a/src/c/tests/test_fopen_fclose.c +++ b/src/c/tests/test_fopen_fclose.c @@ -18,7 +18,7 @@ #include #include "tests.h" -void test_fopen() { +int test_fopen_fclose() { FILE *f = NULL; @@ -33,8 +33,6 @@ void test_fopen() { ret = unlink(TESTFILE); CHECK_ERROR(ret, "unlink") -} -int main() { - test_fopen(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_fprintf.c b/src/c/tests/test_fprintf.c index 115a06c..9da1071 100644 --- a/src/c/tests/test_fprintf.c +++ b/src/c/tests/test_fprintf.c @@ -19,7 +19,7 @@ #include #include "tests.h" -void test_fprintf() { +int test_fprintf() { FILE* f = fopen(TESTFILE, "w"); CHECK_NULL(f, "fopen") @@ -45,8 +45,6 @@ void test_fprintf() { fclose(f); unlink(TESTFILE); -} -int main() { - test_fprintf(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_fputc_fgetc.c b/src/c/tests/test_fputc_fgetc.c index db4cb67..368ad21 100644 --- a/src/c/tests/test_fputc_fgetc.c +++ b/src/c/tests/test_fputc_fgetc.c @@ -19,7 +19,7 @@ #include #include "tests.h" -void test_fputc_fgetc() { +int test_fputc_fgetc() { FILE* f = fopen(TESTFILE, "w"); CHECK_NULL(f, "fopen") @@ -41,8 +41,6 @@ void test_fputc_fgetc() { fclose(f); unlink(TESTFILE); -} -int main() { - test_fputc_fgetc(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_ftruncate.c b/src/c/tests/test_ftruncate.c index ce43e40..b31054a 100644 --- a/src/c/tests/test_ftruncate.c +++ b/src/c/tests/test_ftruncate.c @@ -21,7 +21,7 @@ #include #include "tests.h" -void test_ftruncate() { +int test_ftruncate() { int fd = open(TESTFILE, O_CREAT|O_RDWR, 0777); CHECK_ERROR(fd, "open") @@ -46,8 +46,6 @@ void test_ftruncate() { close(fd); unlink(TESTFILE); -} -int main() { - test_ftruncate(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_fwrite_fread.c b/src/c/tests/test_fwrite_fread.c index 66f110c..629db86 100644 --- a/src/c/tests/test_fwrite_fread.c +++ b/src/c/tests/test_fwrite_fread.c @@ -19,7 +19,7 @@ #include #include "tests.h" -void test_fwrite_fread() { +int test_fwrite_fread() { FILE* f = fopen(TESTFILE, "w"); CHECK_NULL(f, "fopen") @@ -48,8 +48,6 @@ void test_fwrite_fread() { fclose(f); unlink(TESTFILE); -} -int main() { - test_fwrite_fread(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_lseek.c b/src/c/tests/test_lseek.c index 4e0731a..f83a304 100644 --- a/src/c/tests/test_lseek.c +++ b/src/c/tests/test_lseek.c @@ -20,7 +20,7 @@ #include #include "tests.h" -void test_lseek() { +int test_lseek() { int fd = open(TESTFILE, O_CREAT|O_RDWR|O_TRUNC, 0777); CHECK_ERROR(fd, "open") @@ -55,9 +55,6 @@ void test_lseek() { close(fd); unlink(TESTFILE); -} - -int main() { - test_lseek(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_mkdir_rmdir.c b/src/c/tests/test_mkdir_rmdir.c index a14c421..084d5f9 100644 --- a/src/c/tests/test_mkdir_rmdir.c +++ b/src/c/tests/test_mkdir_rmdir.c @@ -19,7 +19,7 @@ #include #include "tests.h" -void test_mkdir_rmdir() { +int test_mkdir_rmdir() { int ret = mkdir(TESTDIR, 0777); assert(ret == 0); @@ -38,8 +38,6 @@ void test_mkdir_rmdir() { err = stat(TESTDIR, &dirstats); assert(err != 0); -} -int main() { - test_mkdir_rmdir(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_open_close.c b/src/c/tests/test_open_close.c index a399eb0..3599b9c 100644 --- a/src/c/tests/test_open_close.c +++ b/src/c/tests/test_open_close.c @@ -19,7 +19,7 @@ #include #include "tests.h" -void test_open_close() { +int test_open_close() { int fd = open(TESTFILE, O_CREAT|O_RDWR, 0777); CHECK_ERROR(fd, "open") @@ -28,8 +28,6 @@ void test_open_close() { ret = unlink(TESTFILE); CHECK_ERROR(ret, "unlink") -} -int main() { - test_open_close(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_pread.c b/src/c/tests/test_pread.c index f459fb1..7d9f6b4 100644 --- a/src/c/tests/test_pread.c +++ b/src/c/tests/test_pread.c @@ -21,7 +21,7 @@ #include #include "tests.h" -void test_pread() { +int test_pread() { int fd = open(TESTFILE, O_CREAT|O_RDWR, 0777); CHECK_ERROR(fd, "open") @@ -46,8 +46,6 @@ void test_pread() { close(fd); unlink(TESTFILE); -} -int main() { - test_pread(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_preadv.c b/src/c/tests/test_preadv.c index d32f970..96ded00 100644 --- a/src/c/tests/test_preadv.c +++ b/src/c/tests/test_preadv.c @@ -22,7 +22,7 @@ #include #include "tests.h" -void test_preadv() { +int test_preadv() { int fd = open(TESTFILE, O_CREAT|O_RDWR, 0777); CHECK_ERROR(fd, "open") @@ -50,8 +50,6 @@ void test_preadv() { close(fd); unlink(TESTFILE); -} -int main() { - test_preadv(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_pwrite.c b/src/c/tests/test_pwrite.c index 096c7ff..b1dd6c5 100644 --- a/src/c/tests/test_pwrite.c +++ b/src/c/tests/test_pwrite.c @@ -21,7 +21,7 @@ #include #include "tests.h" -void test_pwrite() { +int test_pwrite() { int fd = open(TESTFILE, O_CREAT|O_RDWR, 0777); CHECK_ERROR(fd, "open") @@ -49,8 +49,6 @@ void test_pwrite() { close(fd); unlink(TESTFILE); -} -int main() { - test_pwrite(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_pwritev.c b/src/c/tests/test_pwritev.c index d0d0f75..7523e06 100644 --- a/src/c/tests/test_pwritev.c +++ b/src/c/tests/test_pwritev.c @@ -22,7 +22,7 @@ #include #include "tests.h" -void test_pwritev() { +int test_pwritev() { int fd = open(TESTFILE, O_CREAT|O_RDWR, 0777); CHECK_ERROR(fd, "open") @@ -55,8 +55,6 @@ void test_pwritev() { close(fd); unlink(TESTFILE); -} -int main() { - test_pwritev(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_readv.c b/src/c/tests/test_readv.c index 0df7b22..345079d 100644 --- a/src/c/tests/test_readv.c +++ b/src/c/tests/test_readv.c @@ -20,7 +20,7 @@ #include #include "tests.h" -void test_readv() { +int test_readv() { int fd = open(TESTFILE, O_CREAT|O_RDWR, 0777); CHECK_ERROR(fd, "open") @@ -48,8 +48,6 @@ void test_readv() { close(fd); unlink(TESTFILE); -} -int main() { - test_readv(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_stat.c b/src/c/tests/test_stat.c index d37a3e0..b46b8af 100644 --- a/src/c/tests/test_stat.c +++ b/src/c/tests/test_stat.c @@ -20,7 +20,7 @@ #include #include "tests.h" -void test_stat() { +int test_stat() { int fd = open(TESTFILE, O_CREAT|O_RDWR, 0777); CHECK_ERROR(fd, "open") @@ -46,9 +46,11 @@ void test_stat() { close(fd); unlink(TESTFILE); + + return 0; } -void test_stat_size() { +int test_stat_size() { int fd = open(TESTFILE, O_CREAT|O_RDWR, 0777); CHECK_ERROR(fd, "open") @@ -64,9 +66,6 @@ void test_stat_size() { close(fd); unlink(TESTFILE); -} -int main() { - test_stat(); - test_stat_size(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_statfs.c b/src/c/tests/test_statfs.c index 567dc9c..fee39e6 100644 --- a/src/c/tests/test_statfs.c +++ b/src/c/tests/test_statfs.c @@ -20,7 +20,7 @@ #include #include "tests.h" -void test_statfs() { +int test_statfs() { int fd = open(TESTFILE, O_CREAT|O_RDWR, 0777); CHECK_ERROR(fd, "open") @@ -31,13 +31,11 @@ void test_statfs() { CHECK_ERROR(err, "statfs") if (getenv("PDWFS")) { - // if running on pdwfs, we check it "fakes" a Lustre filesystem (see lustre_user.h) - assert(fsstats.f_type == 0x0BD00BD0); + // if running on pdwfs, we check it "fakes" an ext2 filesystem + assert(fsstats.f_type == 0xEF53); } close(fd); unlink(TESTFILE); -} -int main() { - test_statfs(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_unlink.c b/src/c/tests/test_unlink.c index 8a1bc80..ee2747e 100644 --- a/src/c/tests/test_unlink.c +++ b/src/c/tests/test_unlink.c @@ -19,7 +19,7 @@ #include #include "tests.h" -void test_unlink() { +int test_unlink() { int fd = open(TESTFILE, O_CREAT|O_RDWR, 0777); CHECK_ERROR(fd, "open") @@ -46,8 +46,6 @@ void test_unlink() { close(fd); unlink(TESTFILE); -} -int main() { - test_unlink(); - } \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_write_read.c b/src/c/tests/test_write_read.c index 7cf76a0..fad1e44 100644 --- a/src/c/tests/test_write_read.c +++ b/src/c/tests/test_write_read.c @@ -19,7 +19,7 @@ #include #include "tests.h" -void test_write_read() { +int test_write_read() { int fd = open(TESTFILE, O_CREAT|O_RDWR, 0777); CHECK_ERROR(fd, "open") @@ -45,8 +45,6 @@ void test_write_read() { close(fd); unlink(TESTFILE); -} -int main() { - test_write_read(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/test_writev.c b/src/c/tests/test_writev.c index f191c98..65046c1 100644 --- a/src/c/tests/test_writev.c +++ b/src/c/tests/test_writev.c @@ -20,7 +20,7 @@ #include #include "tests.h" -void test_writev() { +int test_writev() { int fd = open(TESTFILE, O_CREAT|O_RDWR, 0777); CHECK_ERROR(fd, "open") @@ -50,8 +50,6 @@ void test_writev() { close(fd); unlink(TESTFILE); -} -int main() { - test_writev(); -} \ No newline at end of file + return 0; +} diff --git a/src/c/tests/testsuite.c b/src/c/tests/testsuite.c new file mode 100644 index 0000000..b6f084d --- /dev/null +++ b/src/c/tests/testsuite.c @@ -0,0 +1,58 @@ +/* +* Copyright 2019 CEA +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +// from libc-testsuite + +#include + +#define RUN_TEST(a) { \ +extern int test_ ##a (void); \ +int e = test_ ##a (); \ +if (e) printf("%s test failed, %d error(s)\n", #a, e); \ +else printf("%s test passed\n", #a); \ +err += e; \ +} + +int main() +{ + int err=0; + + RUN_TEST(access); + RUN_TEST(feof); + RUN_TEST(fgets); + RUN_TEST(fopen_fclose); + RUN_TEST(fprintf); + RUN_TEST(fputc_fgetc); + RUN_TEST(ftruncate); + RUN_TEST(fwrite_fread); + RUN_TEST(lseek); + RUN_TEST(mkdir_rmdir); + RUN_TEST(open_close); + RUN_TEST(pread); + RUN_TEST(preadv); + RUN_TEST(pwrite); + RUN_TEST(pwritev); + RUN_TEST(readv); + RUN_TEST(stat); + RUN_TEST(stat_size); + RUN_TEST(statfs); + RUN_TEST(unlink); + RUN_TEST(write_read); + RUN_TEST(writev); + + printf("\ntotal errors: %d\n", err); + return !!err; +} diff --git a/src/c/utils.c b/src/c/utils.c new file mode 100644 index 0000000..8aaa94c --- /dev/null +++ b/src/c/utils.c @@ -0,0 +1,103 @@ +/* +* Copyright 2019 CEA +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#include "utils.h" + +char* abspath (const char *name) { + // implementation of realpath with resolved=NULL, without links, and no need for file to exist on fs (from glibc) + + char *rpath, *dest = NULL; + const char *start, *end, *rpath_limit; + + if (name == NULL) { + errno = EINVAL; + return NULL; + } + + if (name[0] == '\0') { + errno = ENOENT; + return NULL; + } + + rpath = malloc(PATH_MAX); + if (rpath == NULL) + return NULL; + rpath_limit = rpath + PATH_MAX; + + if (name[0] != '/') { + if (!getcwd (rpath, PATH_MAX)) { + free(rpath); + return NULL; + } + dest = memchr(rpath, '\0', (size_t)-1); + } + else { + rpath[0] = '/'; + dest = rpath + 1; + } + for (start = end = name; *start; start = end) { + + /* Skip sequence of multiple path-separators. */ + while (*start == '/') + ++start; + + /* Find end of path component. */ + for (end = start; *end && *end != '/'; ++end) + /* Nothing. */; + + if (end - start == 0) + break; + else if (end - start == 1 && start[0] == '.') + /* nothing */; + else if (end - start == 2 && start[0] == '.' && start[1] == '.') { + /* Back up to previous component, ignore if at root already. */ + if (dest > rpath + 1) + while ((--dest)[-1] != '/'); + } else { + size_t new_size; + + if (dest[-1] != '/') + *dest++ = '/'; + + if (dest + (end - start) >= rpath_limit) { + ptrdiff_t dest_offset = dest - rpath; + char *new_rpath; + + new_size = rpath_limit - rpath; + if (end - start + 1 > PATH_MAX) + new_size += end - start + 1; + else + new_size += PATH_MAX; + new_rpath = (char *) realloc (rpath, new_size); + if (new_rpath == NULL) { + free(rpath); + return NULL; + } + rpath = new_rpath; + rpath_limit = rpath + new_size; + + dest = rpath + dest_offset; + } + + dest = mempcpy(dest, start, end - start); + *dest = '\0'; + } + } + if (dest > rpath + 1 && dest[-1] == '/') + --dest; + *dest = '\0'; + return rpath; +} diff --git a/src/c/utils.h b/src/c/utils.h new file mode 100644 index 0000000..4694e53 --- /dev/null +++ b/src/c/utils.h @@ -0,0 +1,30 @@ +/* +* Copyright 2019 CEA +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#define _GNU_SOURCE + +#ifndef UTILS_H +#define UTILS_H + +#include +#include +#include +#include +#include +#include + +char* abspath (const char *name); + +#endif \ No newline at end of file diff --git a/src/go/Makefile b/src/go/Makefile index 786b5d4..722c673 100644 --- a/src/go/Makefile +++ b/src/go/Makefile @@ -1,7 +1,6 @@ BUILDDIR=../../build -SCRIPTSDIR=../../scripts -GO_FILES = $(wildcard config/*.go) $(wildcard redisfs/*.go) +GO_FILES = $(shell find . -type f -name '*.go') $(BUILDDIR)/lib/libpdwfs_go.so: $(GO_FILES) go build -mod=vendor -o $@ -buildmode=c-shared @@ -13,4 +12,9 @@ clean: test: go vet -mod=vendor ./... - go test -mod=vendor ./... + # count=1 deactivate test results caching + go test -mod=vendor -failfast -race -count=1 -timeout 15s ./... + +tidy: + go mod tidy + go mod vendor diff --git a/src/go/config/config.go b/src/go/config/config.go index b0a9e2f..a008f68 100644 --- a/src/go/config/config.go +++ b/src/go/config/config.go @@ -25,37 +25,53 @@ import ( "strings" ) +// DefaultStripeSize is the default maximum size of file stripe +const DefaultStripeSize = 50 * 1024 * 1024 // 50MB +const maxRedisString = 512 * 1024 * 1024 // 512MB + +func try(err error) { + if err != nil { + panic(err) + } +} + +var check = try + //Mount point configuration type Mount struct { - Path string - BlockSize int - WriteParallel bool - ReadParallel bool + Path string + StripeSize int } //Redis connection configuration type Redis struct { - RedisAddrs []string - RedisCluster bool - RedisClusterAddrs []string + Addrs []string + Cluster bool + ClusterAddrs []string +} + +// NewRedisConf generates a default configuration +func NewRedisConf() *Redis { + return &Redis{ + Addrs: []string{":6379"}, + Cluster: false, + ClusterAddrs: []string{":7001", ":7002", ":7003", ":7004", ":7005", ":7006"}, + } + } //Pdwfs configuration type Pdwfs struct { - Mounts map[string]*Mount - RedisConf *Redis + Mounts map[string]*Mount + Redis *Redis } func validateMountPath(path string) string { path, err := filepath.Abs(path) - if err != nil { - panic(err) - } + check(err) if _, err = os.Stat(path); os.IsExist(err) { entries, err := ioutil.ReadDir(path) - if err != nil { - panic(err) - } + check(err) if len(entries) != 0 { log.Printf("WARNING mountPath '%s' is not empty, files will not be available for reading through pdwfs", path) } @@ -66,56 +82,50 @@ func validateMountPath(path string) string { //New returns a new config object func New() *Pdwfs { - defaultRedis := Redis{ - RedisAddrs: []string{":6379"}, - RedisCluster: false, - RedisClusterAddrs: []string{":7001", ":7002", ":7003", ":7004", ":7005", ":7006"}, - } + defaultRedis := NewRedisConf() conf := Pdwfs{ - Mounts: map[string]*Mount{}, - RedisConf: &defaultRedis, + Mounts: map[string]*Mount{}, + Redis: defaultRedis, + } + + if confFile := os.Getenv("PDWFS_CONF"); confFile != "" { + jsonFile, err := os.Open(confFile) + check(err) + defer jsonFile.Close() + content, _ := ioutil.ReadAll(jsonFile) + try(json.Unmarshal([]byte(content), &conf)) } if addrs := os.Getenv("PDWFS_REDIS"); addrs != "" { - conf.RedisConf.RedisAddrs = strings.Split(addrs, ",") + s := strings.Split(addrs, ",") + var a []string + for _, i := range s { + if i != "" { + a = append(a, i) + } + } + conf.Redis.Addrs = a } if path := os.Getenv("PDWFS_MOUNTPATH"); path != "" { - mount := Mount{ - Path: path, - BlockSize: 1 * 1024 * 1024, // 1MB - WriteParallel: true, - ReadParallel: true, + conf.Mounts[path] = &Mount{ + Path: path, + StripeSize: DefaultStripeSize, } - conf.Mounts[path] = &mount } - if blockSize := os.Getenv("PDWFS_BLOCKSIZE"); blockSize != "" { + if stripeSize := os.Getenv("PDWFS_STRIPESIZE"); stripeSize != "" { for _, mount := range conf.Mounts { - size, err := strconv.Atoi(blockSize) + size, err := strconv.Atoi(stripeSize) if err != nil { - log.Fatalln("Can't convert BlockSize in PDWFS_BLOCKSIZE to int") + log.Fatalln("Can't convert StripeSize in PDWFS_STRIPESIZE to int") } - mount.BlockSize = size * 1024 * 1024 - fmt.Println("BlockSize: ", mount.BlockSize) - } - } - - if confFile := os.Getenv("PDWFS_CONF"); confFile != "" { - jsonFile, err := os.Open(confFile) - if err != nil { - panic(err) - } - defer jsonFile.Close() - content, _ := ioutil.ReadAll(jsonFile) - err = json.Unmarshal([]byte(content), &conf) - if err != nil { - panic(err) + mount.StripeSize = size * 1024 * 1024 } } - // Options normalization + // Options verifications and normalization if val := os.Getenv("PDWFS_LOGS"); val == "" { log.SetOutput(ioutil.Discard) @@ -127,8 +137,10 @@ func New() *Pdwfs { for path, conf := range conf.Mounts { conf.Path = validateMountPath(path) - // NOTE: we may add a different Redis database index per mount point for isolation - // (but not suitable with Redis Cluster) + if conf.StripeSize > maxRedisString { + err := fmt.Sprintf("Mount point '%s' block size (%dMB) is above what Redis can sustain, set block size <= 512MB", path, conf.StripeSize/(1024*1024)) + panic(err) + } normalized[conf.Path] = conf } conf.Mounts = normalized @@ -137,14 +149,8 @@ func New() *Pdwfs { } // Dump writes the configuration in a JSON file -func (c *Pdwfs) Dump() error { +func (c *Pdwfs) Dump() { content, err := json.MarshalIndent(c, "", " ") - if err != nil { - return err - } - err = ioutil.WriteFile("pdwfs.json", content, 0644) - if err != nil { - return err - } - return nil + check(err) + try(ioutil.WriteFile("pdwfs.json", content, 0644)) } diff --git a/src/go/go.mod b/src/go/go.mod index 582611e..d60441a 100644 --- a/src/go/go.mod +++ b/src/go/go.mod @@ -1,8 +1 @@ module github.com/cea-hpc/pdwfs - -require ( - github.com/go-redis/redis v6.15.1+incompatible - github.com/onsi/ginkgo v1.8.0 - github.com/onsi/gomega v1.4.3 - golang.org/x/sys v0.0.0-20190204203706-41f3e6584952 // indirect -) diff --git a/src/go/go.sum b/src/go/go.sum index 47ddd30..e69de29 100644 --- a/src/go/go.sum +++ b/src/go/go.sum @@ -1,32 +0,0 @@ -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/go-redis/redis v6.15.1+incompatible h1:BZ9s4/vHrIqwOb0OPtTQ5uABxETJ3NRuUNoSUurnkew= -github.com/go-redis/redis v6.15.1+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= -github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190204203706-41f3e6584952 h1:FDfvYgoVsA7TTZSbgiqjAbfPbK47CNHdWl3h/PJtii0= -golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/src/go/pdwfs.go b/src/go/pdwfs.go index d11bcea..aa94b4d 100644 --- a/src/go/pdwfs.go +++ b/src/go/pdwfs.go @@ -11,6 +11,9 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +// +// Main entry points into Go code of pdwfs. +// This file contains exported functions to the C layer of pdwfs package main @@ -27,16 +30,15 @@ package main import "C" import ( + "bytes" "errors" "fmt" "io" - "io/ioutil" "os" "path/filepath" "strings" "sync" "syscall" - "unsafe" "github.com/cea-hpc/pdwfs/config" "github.com/cea-hpc/pdwfs/redisfs" @@ -47,46 +49,48 @@ import ( var ( errFdPoolTempCreate = errors.New("Failed to create a tempfile to get a valid fd") errInvalidFd = errors.New("Invalid file descriptor") + errFdInUse = errors.New("file descriptor already used") ) -func check(err error) { +// helpers for error checking + +func try(err error) { if err != nil { panic(err) } } -type fileTuple struct { - tempFileName string - cFile *C.FILE - redisFile *redisfs.File -} +var check = try -//PdwFS is a virtual filesystem object built on top of github.com/cea-hpc/pdwfs/redisfs +// PdwFS manages multiple redisfs mount points and keeps a map of opened fd <-> opened redisfs.File. +// This map is used to translate I/O calls coming from the C layer and addressed by a system file descriptor +// to pdwfs implementation of Files (redisfs.File). type PdwFS struct { mounts map[string]*redisfs.RedisFS conf *config.Pdwfs prefix string - fdFileMap map[int]*fileTuple + fdFileMap map[int]*redisfs.File lock sync.RWMutex } -//NewPdwFS returns a pdwfs virtual filesystem +//NewPdwFS returns a new PdwFS instance with newly created redisfs mount points based on configuration info func NewPdwFS(conf *config.Pdwfs) *PdwFS { if len(conf.Mounts) == 0 { panic("No mount path specified...") } mounts := map[string]*redisfs.RedisFS{} for path, mountConf := range conf.Mounts { - mounts[path] = redisfs.NewRedisFS(conf.RedisConf, mountConf) + mounts[path] = redisfs.NewRedisFS(conf.Redis, mountConf) } return &PdwFS{ mounts: mounts, conf: conf, - fdFileMap: make(map[int]*fileTuple), + fdFileMap: make(map[int]*redisfs.File), lock: sync.RWMutex{}, } } +// parse a filename to return the correponding mount point if found func (fs *PdwFS) getMount(filename string) (*redisfs.RedisFS, error) { if filename == "" { //short-circuit filepath.Abs as Abs behaviour is to return working directory on empty string @@ -105,78 +109,61 @@ func (fs *PdwFS) getMount(filename string) (*redisfs.RedisFS, error) { return nil, nil } -func (fs *PdwFS) registerFile(redisFile *redisfs.File) (*C.FILE, error) { - // create a temporary file to get a valid system file descritor - // NOTE: TempFile uses os.OpenFile which uses openat syscall which is currently not intercepted - // so if a user set a mount point in the same temp folder used by TempFile, we're not - // entering in a recursive loop (intercepting a file in temp, creating a twin temp file, etc) - tempFile, err := ioutil.TempFile("", "pdwfs") - check(err) - tempFileName := tempFile.Name() - tempFile.Close() - path := C.CString(tempFileName) - mode := C.CString("r") - cFile, err := C.fopen(path, mode) - C.free(unsafe.Pointer(path)) - C.free(unsafe.Pointer(mode)) - check(err) - fd := int(C.fileno(cFile)) - fs.fdFileMap[fd] = &fileTuple{tempFileName, cFile, redisFile} - return cFile, nil +// register a new redisfs.File and its associated system file descriptor +func (fs *PdwFS) registerFile(fd int, redisFile *redisfs.File) error { + if _, ok := fs.fdFileMap[fd]; ok { + return errFdInUse + } + fs.fdFileMap[fd] = redisFile + return nil } -func (fs *PdwFS) closeFd(fd int) error { - //FIXME: not thread-safe if multiple thread handle the same fd... +// remove a file descriptor and its associated redisfs.File +func (fs *PdwFS) removeFd(fd int) error { if _, ok := fs.fdFileMap[fd]; !ok { return errInvalidFd } delete(fs.fdFileMap, fd) - // deleting fd from the managed fd map first ensures the subsequent call to close(fd), - // (which will be intercepted by pdwfs), will be passed to the real libc close call - C.close(C.int(fd)) return nil } func (fs *PdwFS) getFileFromFd(fd int) (*redisfs.File, error) { - if fileTuple, ok := fs.fdFileMap[fd]; ok { - return fileTuple.redisFile, nil + if f, ok := fs.fdFileMap[fd]; ok { + return f, nil } return nil, errInvalidFd } -func (fs *PdwFS) isFdManaged(fd int) bool { - if _, ok := fs.fdFileMap[fd]; ok { - return true - } - return false -} - func (fs *PdwFS) finalize() { for _, mount := range fs.mounts { - err := mount.Finalize() - check(err) - } - // clean up all temp files created - for fd, fileTuple := range fs.fdFileMap { - fs.closeFd(fd) - err := os.Remove(fileTuple.tempFileName) - check(err) + mount.Finalize() } } // ----------------Exported to C ---------------- +// function below are exported to the C layer using cgo system var pdwfs *PdwFS -// InitPdwfs is called once when pdwfs.so library is loaded (gcc constructor attribute) +// InitPdwfs is called only once when pdwfs.so library is loaded (gcc constructor attribute). +// The mountBuf argument is used to communicate the list of mount points back to the C layer. +// The C layer uses the mount points information for its own triage of filename (pdwfs I/O calls vs libc I/O calls). +// This is necessary as the configuration mechanism is in the Go layer. //export InitPdwfs -func InitPdwfs() { +func InitPdwfs(mountBuf []byte) { conf := config.New() if dump := os.Getenv("PDWFS_DUMPCONF"); dump != "" { - err := conf.Dump() - check(err) + conf.Dump() } pdwfs = NewPdwFS(conf) + + // writes in mountBuf the mount point paths + b := bytes.NewBuffer(mountBuf) + for path := range pdwfs.mounts { + b.WriteString(path) + b.WriteString("@") // separator + } + b.WriteString("\000") // end sentinel } // FinalizePdwfs is called once when pdwfs.so library is unloaded (gcc destructor attribute) @@ -185,26 +172,6 @@ func FinalizePdwfs() { pdwfs.finalize() } -//IsFileManaged returns 1 if the directory in argument is managed by pdwfs from config -//export IsFileManaged -func IsFileManaged(filename string) int { - mount, err := pdwfs.getMount(filename) - check(err) - if mount != nil { - return 1 - } - return 0 -} - -//IsFdManaged checks whether a file descriptor is managed -//export IsFdManaged -func IsFdManaged(fd int) int { - if pdwfs.isFdManaged(fd) { - return 1 - } - return 0 -} - var errno C.int //GetErrno is used by C functions to retrieve the error number set by Go function @@ -220,7 +187,7 @@ func setErrno(err C.int) { //Open implements open libc call //export Open -func Open(filename string, flags int, mode int) int { +func Open(filename string, flags, mode, fd int) int { pdwfs.lock.Lock() defer pdwfs.lock.Unlock() mount, err := pdwfs.getMount(filename) @@ -239,14 +206,13 @@ func Open(filename string, flags int, mode int) int { } return -1 } - cFile, err := pdwfs.registerFile(&file) - check(err) - return int(C.fileno(cFile)) + try(pdwfs.registerFile(fd, &file)) + return fd } //Fopen implements fopen libc call //export Fopen -func Fopen(filename string, mode string) *C.FILE { +func Fopen(filename string, mode string, fd int) int { pdwfs.lock.Lock() defer pdwfs.lock.Unlock() mount, err := pdwfs.getMount(filename) @@ -272,11 +238,10 @@ func Fopen(filename string, mode string) *C.FILE { } else { panic(fmt.Sprintf("unhandled %T in Fopen: %s", err, err)) } - return (*C.FILE)(C.NULL) + return -1 } - cFile, err := pdwfs.registerFile(&file) - check(err) - return cFile + try(pdwfs.registerFile(fd, &file)) + return fd } //Close implements close libc call @@ -287,11 +252,8 @@ func Close(fd int) int { file, err := pdwfs.getFileFromFd(fd) check(err) - err = (*file).Close() - check(err) // no known conversion to errno, just panic if err != nil - - err = pdwfs.closeFd(fd) - check(err) + try((*file).Close()) // no known conversion to errno, just panic if err != nil + try(pdwfs.removeFd(fd)) return 0 } @@ -304,6 +266,7 @@ func Write(fd int, buf []byte) int { check(err) n, err := (*file).Write(buf) + check(err) // no known conversion to errno, just panic if err != nil return n } @@ -658,13 +621,13 @@ func Lstat64(filename string, stats *C.struct_stat64) int { func statfs() syscall.Statfs_t { return syscall.Statfs_t{ - Type: 0x0BD00BD0, // we fake a Lustre file system, see lustre_user.h (ext2 is 0xEF53, see man statfs) - Bsize: 1, // block size - Blocks: 1, // number of blocks - Bfree: 1, // total free blocks - Bavail: 1, // free blocks available to user (unpriviledged) - Files: 1, // total file nodes in fs - Ffree: 1, // free file nodes in fs + Type: 0xEF53, // ext2 filesystem + Bsize: 1, // block size + Blocks: 1, // number of blocks + Bfree: 1, // total free blocks + Bavail: 1, // free blocks available to user (unpriviledged) + Files: 1, // total file nodes in fs + Ffree: 1, // free file nodes in fs } } diff --git a/src/go/pdwfs_test.go b/src/go/pdwfs_test.go index a331d44..ea42c88 100644 --- a/src/go/pdwfs_test.go +++ b/src/go/pdwfs_test.go @@ -15,35 +15,14 @@ package main import ( - "fmt" "io/ioutil" "os" - "path/filepath" - "reflect" - "runtime" "testing" "github.com/cea-hpc/pdwfs/config" + "github.com/cea-hpc/pdwfs/util" ) -// Ok fails the test if an err is not nil. -func Ok(tb testing.TB, err error) { - if err != nil { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error()) - tb.FailNow() - } -} - -// Equals fails the test if exp is not equal to act. -func Equals(tb testing.TB, exp, act interface{}, msg string) { - if !reflect.DeepEqual(exp, act) { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("\033[31m%s:%d: %s\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, msg, exp, act) - tb.FailNow() - } -} - func writeFile(pdwfs *PdwFS, filename string, data []byte, perm os.FileMode) (int, error) { mount, err := pdwfs.getMount(filename) if err != nil { @@ -70,29 +49,34 @@ func readFile(pdwfs *PdwFS, filename string) ([]byte, error) { func TestMultiMount(t *testing.T) { + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() + conf := config.New() + conf.Redis = redisConf // create two fake mount paths conf.Mounts["/rebels/luke"] = &config.Mount{ - Path: "/rebels/luke", - BlockSize: 2 * 1024, // 2KB + Path: "/rebels/luke", + StripeSize: 2 * 1024, // 2KB } conf.Mounts["/empire/vader"] = &config.Mount{ - Path: "/empire/vader", - BlockSize: 1024, // 1KB + Path: "/empire/vader", + StripeSize: 1024, // 1KB } pdwfs := NewPdwFS(conf) + defer pdwfs.finalize() _, err := writeFile(pdwfs, "/rebels/luke/quotes", []byte("Vader's on that ship.\n"), os.FileMode(0777)) - Ok(t, err) + util.Ok(t, err) _, err = writeFile(pdwfs, "/empire/vader/quotes", []byte("The Force is strong with this one.\n"), os.FileMode(0777)) - Ok(t, err) + util.Ok(t, err) data, err := readFile(pdwfs, "/rebels/luke/quotes") - Equals(t, "Vader's on that ship.\n", string(data), "Bad quote !") + util.Equals(t, "Vader's on that ship.\n", string(data), "Bad quote !") data, err = readFile(pdwfs, "/empire/vader/quotes") - Equals(t, "The Force is strong with this one.\n", string(data), "Bad quote !") + util.Equals(t, "The Force is strong with this one.\n", string(data), "Bad quote !") } diff --git a/src/go/vendor/gopkg.in/yaml.v2/LICENSE b/src/go/redigo/LICENSE similarity index 89% rename from src/go/vendor/gopkg.in/yaml.v2/LICENSE rename to src/go/redigo/LICENSE index 8dada3e..67db858 100644 --- a/src/go/vendor/gopkg.in/yaml.v2/LICENSE +++ b/src/go/redigo/LICENSE @@ -1,3 +1,4 @@ + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -172,30 +173,3 @@ defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/src/go/redigo/README.markdown b/src/go/redigo/README.markdown new file mode 100644 index 0000000..87bee8e --- /dev/null +++ b/src/go/redigo/README.markdown @@ -0,0 +1,61 @@ +# Note by JCapul - 05/13/2019 + +I am integrating redigo into pdwfs main code base. + +For performance reason we need to have a Redis client that allows receiving large strings (GET and GETRANGE) into a user provided buffer to avoid the cost of additional data copies. + +Redigo seems to be a fairly simple Redis Go client that I can easily modify to suit this requirement. + +I could have done a fork on Github but since it is not supporting yet Go Modules and there seem to be complicated stuff going-on ([issue336](https://github.com/gomodule/redigo/issues/366)), I am "hard-vendoring" it for the moment. + +Redigo +====== + +[![Build Status](https://travis-ci.org/gomodule/redigo.svg?branch=master)](https://travis-ci.org/gomodule/redigo) +[![GoDoc](https://godoc.org/github.com/gomodule/redigo/redis?status.svg)](https://godoc.org/github.com/gomodule/redigo/redis) + +Redigo is a [Go](http://golang.org/) client for the [Redis](http://redis.io/) database. + +Features +------- + +* A [Print-like](http://godoc.org/github.com/gomodule/redigo/redis#hdr-Executing_Commands) API with support for all Redis commands. +* [Pipelining](http://godoc.org/github.com/gomodule/redigo/redis#hdr-Pipelining), including pipelined transactions. +* [Publish/Subscribe](http://godoc.org/github.com/gomodule/redigo/redis#hdr-Publish_and_Subscribe). +* [Connection pooling](http://godoc.org/github.com/gomodule/redigo/redis#Pool). +* [Script helper type](http://godoc.org/github.com/gomodule/redigo/redis#Script) with optimistic use of EVALSHA. +* [Helper functions](http://godoc.org/github.com/gomodule/redigo/redis#hdr-Reply_Helpers) for working with command replies. + +Documentation +------------- + +- [API Reference](http://godoc.org/github.com/gomodule/redigo/redis) +- [FAQ](https://github.com/gomodule/redigo/wiki/FAQ) +- [Examples](https://godoc.org/github.com/gomodule/redigo/redis#pkg-examples) + +Installation +------------ + +Install Redigo using the "go get" command: + + go get github.com/gomodule/redigo/redis + +The Go distribution is Redigo's only dependency. + +Related Projects +---------------- + +- [rafaeljusto/redigomock](https://godoc.org/github.com/rafaeljusto/redigomock) - A mock library for Redigo. +- [chasex/redis-go-cluster](https://github.com/chasex/redis-go-cluster) - A Redis cluster client implementation. +- [FZambia/sentinel](https://github.com/FZambia/sentinel) - Redis Sentinel support for Redigo +- [mna/redisc](https://github.com/mna/redisc) - Redis Cluster client built on top of Redigo + +Contributing +------------ + +See [CONTRIBUTING.md](https://github.com/gomodule/redigo/blob/master/.github/CONTRIBUTING.md). + +License +------- + +Redigo is available under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html). diff --git a/src/go/redigo/redis/commandinfo.go b/src/go/redigo/redis/commandinfo.go new file mode 100644 index 0000000..b6df6a2 --- /dev/null +++ b/src/go/redigo/redis/commandinfo.go @@ -0,0 +1,55 @@ +// Copyright 2014 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "strings" +) + +const ( + connectionWatchState = 1 << iota + connectionMultiState + connectionSubscribeState + connectionMonitorState +) + +type commandInfo struct { + // Set or Clear these states on connection. + Set, Clear int +} + +var commandInfos = map[string]commandInfo{ + "WATCH": {Set: connectionWatchState}, + "UNWATCH": {Clear: connectionWatchState}, + "MULTI": {Set: connectionMultiState}, + "EXEC": {Clear: connectionWatchState | connectionMultiState}, + "DISCARD": {Clear: connectionWatchState | connectionMultiState}, + "PSUBSCRIBE": {Set: connectionSubscribeState}, + "SUBSCRIBE": {Set: connectionSubscribeState}, + "MONITOR": {Set: connectionMonitorState}, +} + +func init() { + for n, ci := range commandInfos { + commandInfos[strings.ToLower(n)] = ci + } +} + +func lookupCommandInfo(commandName string) commandInfo { + if ci, ok := commandInfos[commandName]; ok { + return ci + } + return commandInfos[strings.ToUpper(commandName)] +} diff --git a/src/go/redigo/redis/commandinfo_test.go b/src/go/redigo/redis/commandinfo_test.go new file mode 100644 index 0000000..799b929 --- /dev/null +++ b/src/go/redigo/redis/commandinfo_test.go @@ -0,0 +1,27 @@ +package redis + +import "testing" + +func TestLookupCommandInfo(t *testing.T) { + for _, n := range []string{"watch", "WATCH", "wAtch"} { + if lookupCommandInfo(n) == (commandInfo{}) { + t.Errorf("LookupCommandInfo(%q) = CommandInfo{}, expected non-zero value", n) + } + } +} + +func benchmarkLookupCommandInfo(b *testing.B, names ...string) { + for i := 0; i < b.N; i++ { + for _, c := range names { + lookupCommandInfo(c) + } + } +} + +func BenchmarkLookupCommandInfoCorrectCase(b *testing.B) { + benchmarkLookupCommandInfo(b, "watch", "WATCH", "monitor", "MONITOR") +} + +func BenchmarkLookupCommandInfoMixedCase(b *testing.B) { + benchmarkLookupCommandInfo(b, "wAtch", "WeTCH", "monItor", "MONiTOR") +} diff --git a/src/go/redigo/redis/conn.go b/src/go/redigo/redis/conn.go new file mode 100644 index 0000000..8278ade --- /dev/null +++ b/src/go/redigo/redis/conn.go @@ -0,0 +1,721 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "bufio" + "bytes" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "net/url" + "regexp" + "strconv" + "sync" + "time" +) + +var ( + _ ConnWithTimeout = (*conn)(nil) +) + +// conn is the low-level implementation of Conn +type conn struct { + // Shared + mu sync.Mutex + pending int + err error + conn net.Conn + + // Read + readTimeout time.Duration + br *bufio.Reader + dst []byte + + // Write + writeTimeout time.Duration + bw *bufio.Writer + + // Scratch space for formatting argument length. + // '*' or '$', length, "\r\n" + lenScratch [32]byte + + // Scratch space for formatting integers and floats. + numScratch [40]byte +} + +// DialTimeout acts like Dial but takes timeouts for establishing the +// connection to the server, writing a command and reading a reply. +// +// Deprecated: Use Dial with options instead. +func DialTimeout(network, address string, connectTimeout, readTimeout, writeTimeout time.Duration) (Conn, error) { + return Dial(network, address, + DialConnectTimeout(connectTimeout), + DialReadTimeout(readTimeout), + DialWriteTimeout(writeTimeout)) +} + +// DialOption specifies an option for dialing a Redis server. +type DialOption struct { + f func(*dialOptions) +} + +type dialOptions struct { + readTimeout time.Duration + writeTimeout time.Duration + dialer *net.Dialer + dial func(network, addr string) (net.Conn, error) + db int + password string + clientName string + useTLS bool + skipVerify bool + tlsConfig *tls.Config +} + +// DialReadTimeout specifies the timeout for reading a single command reply. +func DialReadTimeout(d time.Duration) DialOption { + return DialOption{func(do *dialOptions) { + do.readTimeout = d + }} +} + +// DialWriteTimeout specifies the timeout for writing a single command. +func DialWriteTimeout(d time.Duration) DialOption { + return DialOption{func(do *dialOptions) { + do.writeTimeout = d + }} +} + +// DialConnectTimeout specifies the timeout for connecting to the Redis server when +// no DialNetDial option is specified. +func DialConnectTimeout(d time.Duration) DialOption { + return DialOption{func(do *dialOptions) { + do.dialer.Timeout = d + }} +} + +// DialKeepAlive specifies the keep-alive period for TCP connections to the Redis server +// when no DialNetDial option is specified. +// If zero, keep-alives are not enabled. If no DialKeepAlive option is specified then +// the default of 5 minutes is used to ensure that half-closed TCP sessions are detected. +func DialKeepAlive(d time.Duration) DialOption { + return DialOption{func(do *dialOptions) { + do.dialer.KeepAlive = d + }} +} + +// DialNetDial specifies a custom dial function for creating TCP +// connections, otherwise a net.Dialer customized via the other options is used. +// DialNetDial overrides DialConnectTimeout and DialKeepAlive. +func DialNetDial(dial func(network, addr string) (net.Conn, error)) DialOption { + return DialOption{func(do *dialOptions) { + do.dial = dial + }} +} + +// DialDatabase specifies the database to select when dialing a connection. +func DialDatabase(db int) DialOption { + return DialOption{func(do *dialOptions) { + do.db = db + }} +} + +// DialPassword specifies the password to use when connecting to +// the Redis server. +func DialPassword(password string) DialOption { + return DialOption{func(do *dialOptions) { + do.password = password + }} +} + +// DialClientName specifies a client name to be used +// by the Redis server connection. +func DialClientName(name string) DialOption { + return DialOption{func(do *dialOptions) { + do.clientName = name + }} +} + +// DialTLSConfig specifies the config to use when a TLS connection is dialed. +// Has no effect when not dialing a TLS connection. +func DialTLSConfig(c *tls.Config) DialOption { + return DialOption{func(do *dialOptions) { + do.tlsConfig = c + }} +} + +// DialTLSSkipVerify disables server name verification when connecting over +// TLS. Has no effect when not dialing a TLS connection. +func DialTLSSkipVerify(skip bool) DialOption { + return DialOption{func(do *dialOptions) { + do.skipVerify = skip + }} +} + +// DialUseTLS specifies whether TLS should be used when connecting to the +// server. This option is ignore by DialURL. +func DialUseTLS(useTLS bool) DialOption { + return DialOption{func(do *dialOptions) { + do.useTLS = useTLS + }} +} + +// Dial connects to the Redis server at the given network and +// address using the specified options. +func Dial(network, address string, options ...DialOption) (Conn, error) { + do := dialOptions{ + dialer: &net.Dialer{ + KeepAlive: time.Minute * 5, + }, + } + for _, option := range options { + option.f(&do) + } + if do.dial == nil { + do.dial = do.dialer.Dial + } + + netConn, err := do.dial(network, address) + if err != nil { + return nil, err + } + + if do.useTLS { + var tlsConfig *tls.Config + if do.tlsConfig == nil { + tlsConfig = &tls.Config{InsecureSkipVerify: do.skipVerify} + } else { + tlsConfig = cloneTLSConfig(do.tlsConfig) + } + if tlsConfig.ServerName == "" { + host, _, err := net.SplitHostPort(address) + if err != nil { + netConn.Close() + return nil, err + } + tlsConfig.ServerName = host + } + + tlsConn := tls.Client(netConn, tlsConfig) + if err := tlsConn.Handshake(); err != nil { + netConn.Close() + return nil, err + } + netConn = tlsConn + } + + c := &conn{ + conn: netConn, + bw: bufio.NewWriter(netConn), + br: bufio.NewReader(netConn), + readTimeout: do.readTimeout, + writeTimeout: do.writeTimeout, + } + + if do.password != "" { + if _, err := c.Do("AUTH", do.password); err != nil { + netConn.Close() + return nil, err + } + } + + if do.clientName != "" { + if _, err := c.Do("CLIENT", "SETNAME", do.clientName); err != nil { + netConn.Close() + return nil, err + } + } + + if do.db != 0 { + if _, err := c.Do("SELECT", do.db); err != nil { + netConn.Close() + return nil, err + } + } + + return c, nil +} + +var pathDBRegexp = regexp.MustCompile(`/(\d*)\z`) + +// DialURL connects to a Redis server at the given URL using the Redis +// URI scheme. URLs should follow the draft IANA specification for the +// scheme (https://www.iana.org/assignments/uri-schemes/prov/redis). +func DialURL(rawurl string, options ...DialOption) (Conn, error) { + u, err := url.Parse(rawurl) + if err != nil { + return nil, err + } + + if u.Scheme != "redis" && u.Scheme != "rediss" { + return nil, fmt.Errorf("invalid redis URL scheme: %s", u.Scheme) + } + + // As per the IANA draft spec, the host defaults to localhost and + // the port defaults to 6379. + host, port, err := net.SplitHostPort(u.Host) + if err != nil { + // assume port is missing + host = u.Host + port = "6379" + } + if host == "" { + host = "localhost" + } + address := net.JoinHostPort(host, port) + + if u.User != nil { + password, isSet := u.User.Password() + if isSet { + options = append(options, DialPassword(password)) + } + } + + match := pathDBRegexp.FindStringSubmatch(u.Path) + if len(match) == 2 { + db := 0 + if len(match[1]) > 0 { + db, err = strconv.Atoi(match[1]) + if err != nil { + return nil, fmt.Errorf("invalid database: %s", u.Path[1:]) + } + } + if db != 0 { + options = append(options, DialDatabase(db)) + } + } else if u.Path != "" { + return nil, fmt.Errorf("invalid database: %s", u.Path[1:]) + } + + options = append(options, DialUseTLS(u.Scheme == "rediss")) + + return Dial("tcp", address, options...) +} + +// NewConn returns a new Redigo connection for the given net connection. +func NewConn(netConn net.Conn, readTimeout, writeTimeout time.Duration) Conn { + return &conn{ + conn: netConn, + bw: bufio.NewWriter(netConn), + br: bufio.NewReader(netConn), + readTimeout: readTimeout, + writeTimeout: writeTimeout, + } +} + +func (c *conn) Close() error { + c.mu.Lock() + err := c.err + if c.err == nil { + c.err = errors.New("redigo: closed") + err = c.conn.Close() + } + c.mu.Unlock() + return err +} + +func (c *conn) fatal(err error) error { + c.mu.Lock() + if c.err == nil { + c.err = err + // Close connection to force errors on subsequent calls and to unblock + // other reader or writer. + c.conn.Close() + } + c.mu.Unlock() + return err +} + +func (c *conn) Err() error { + c.mu.Lock() + err := c.err + c.mu.Unlock() + return err +} + +func (c *conn) writeLen(prefix byte, n int) error { + c.lenScratch[len(c.lenScratch)-1] = '\n' + c.lenScratch[len(c.lenScratch)-2] = '\r' + i := len(c.lenScratch) - 3 + for { + c.lenScratch[i] = byte('0' + n%10) + i -= 1 + n = n / 10 + if n == 0 { + break + } + } + c.lenScratch[i] = prefix + _, err := c.bw.Write(c.lenScratch[i:]) + return err +} + +func (c *conn) writeString(s string) error { + c.writeLen('$', len(s)) + c.bw.WriteString(s) + _, err := c.bw.WriteString("\r\n") + return err +} + +func (c *conn) writeBytes(p []byte) error { + c.writeLen('$', len(p)) + c.bw.Write(p) + _, err := c.bw.WriteString("\r\n") + return err +} + +func (c *conn) writeInt64(n int64) error { + return c.writeBytes(strconv.AppendInt(c.numScratch[:0], n, 10)) +} + +func (c *conn) writeFloat64(n float64) error { + return c.writeBytes(strconv.AppendFloat(c.numScratch[:0], n, 'g', -1, 64)) +} + +func (c *conn) writeCommand(cmd string, args []interface{}) error { + c.writeLen('*', 1+len(args)) + if err := c.writeString(cmd); err != nil { + return err + } + for _, arg := range args { + if err := c.writeArg(arg, true); err != nil { + return err + } + } + return nil +} + +func (c *conn) writeArg(arg interface{}, argumentTypeOK bool) (err error) { + switch arg := arg.(type) { + case string: + return c.writeString(arg) + case []byte: + return c.writeBytes(arg) + case int: + return c.writeInt64(int64(arg)) + case int64: + return c.writeInt64(arg) + case float64: + return c.writeFloat64(arg) + case bool: + if arg { + return c.writeString("1") + } else { + return c.writeString("0") + } + case nil: + return c.writeString("") + case Argument: + if argumentTypeOK { + return c.writeArg(arg.RedisArg(), false) + } + // See comment in default clause below. + var buf bytes.Buffer + fmt.Fprint(&buf, arg) + return c.writeBytes(buf.Bytes()) + default: + // This default clause is intended to handle builtin numeric types. + // The function should return an error for other types, but this is not + // done for compatibility with previous versions of the package. + var buf bytes.Buffer + fmt.Fprint(&buf, arg) + return c.writeBytes(buf.Bytes()) + } +} + +type protocolError string + +func (pe protocolError) Error() string { + return fmt.Sprintf("redigo: %s (possible server error or unsupported concurrent read by application)", string(pe)) +} + +// readLine reads a line of input from the RESP stream. +func (c *conn) readLine() ([]byte, error) { + // To avoid allocations, attempt to read the line using ReadSlice. This + // call typically succeeds. The known case where the call fails is when + // reading the output from the MONITOR command. + p, err := c.br.ReadSlice('\n') + if err == bufio.ErrBufferFull { + // The line does not fit in the bufio.Reader's buffer. Fall back to + // allocating a buffer for the line. + buf := append([]byte{}, p...) + for err == bufio.ErrBufferFull { + p, err = c.br.ReadSlice('\n') + buf = append(buf, p...) + } + p = buf + } + if err != nil { + return nil, err + } + i := len(p) - 2 + if i < 0 || p[i] != '\r' { + return nil, protocolError("bad response line terminator") + } + return p[:i], nil +} + +// parseLen parses bulk string and array lengths. +func parseLen(p []byte) (int, error) { + if len(p) == 0 { + return -1, protocolError("malformed length") + } + + if p[0] == '-' && len(p) == 2 && p[1] == '1' { + // handle $-1 and $-1 null replies. + return -1, nil + } + + var n int + for _, b := range p { + n *= 10 + if b < '0' || b > '9' { + return -1, protocolError("illegal bytes in length") + } + n += int(b - '0') + } + + return n, nil +} + +// parseInt parses an integer reply. +func parseInt(p []byte) (interface{}, error) { + if len(p) == 0 { + return 0, protocolError("malformed integer") + } + + var negate bool + if p[0] == '-' { + negate = true + p = p[1:] + if len(p) == 0 { + return 0, protocolError("malformed integer") + } + } + + var n int64 + for _, b := range p { + n *= 10 + if b < '0' || b > '9' { + return 0, protocolError("illegal bytes in length") + } + n += int64(b - '0') + } + + if negate { + n = -n + } + return n, nil +} + +var ( + okReply interface{} = "OK" + pongReply interface{} = "PONG" +) + +func (c *conn) SetReadBuffer(buf []byte) { + c.dst = buf +} + +func (c *conn) UnsetReadBuffer() { + c.dst = nil +} + +func (c *conn) readBulkString(dst []byte) (int, error) { + read, err := io.ReadFull(c.br, dst) + if err != nil { + return read, err + } + if line, err := c.readLine(); err != nil { + return read, err + } else if len(line) != 0 { + return read, protocolError("bad bulk string format") + } + return read, nil +} + +func (c *conn) readReply() (interface{}, error) { + line, err := c.readLine() + if err != nil { + return nil, err + } + if len(line) == 0 { + return nil, protocolError("short response line") + } + switch line[0] { + case '+': + switch { + case len(line) == 3 && line[1] == 'O' && line[2] == 'K': + // Avoid allocation for frequent "+OK" response. + return okReply, nil + case len(line) == 5 && line[1] == 'P' && line[2] == 'O' && line[3] == 'N' && line[4] == 'G': + // Avoid allocation in PING command benchmarks :) + return pongReply, nil + default: + return string(line[1:]), nil + } + case '-': + return Error(string(line[1:])), nil + case ':': + return parseInt(line[1:]) + case '$': + n, err := parseLen(line[1:]) + if n < 0 || err != nil { + return nil, err + } + if c.dst == nil { + p := make([]byte, n) + _, err := c.readBulkString(p) + return p, err + } + // use the destination buffer + if len(c.dst) < n { + return nil, protocolError("destination buffer is too small") + } + return c.readBulkString(c.dst[:n]) + case '*': + n, err := parseLen(line[1:]) + if n < 0 || err != nil { + return nil, err + } + r := make([]interface{}, n) + for i := range r { + r[i], err = c.readReply() + if err != nil { + return nil, err + } + } + return r, nil + } + return nil, protocolError("unexpected response line") +} + +func (c *conn) Send(cmd string, args ...interface{}) error { + c.mu.Lock() + c.pending += 1 + c.mu.Unlock() + if c.writeTimeout != 0 { + c.conn.SetWriteDeadline(time.Now().Add(c.writeTimeout)) + } + if err := c.writeCommand(cmd, args); err != nil { + return c.fatal(err) + } + return nil +} + +func (c *conn) Flush() error { + if c.writeTimeout != 0 { + c.conn.SetWriteDeadline(time.Now().Add(c.writeTimeout)) + } + if err := c.bw.Flush(); err != nil { + return c.fatal(err) + } + return nil +} + +func (c *conn) Receive() (interface{}, error) { + return c.ReceiveWithTimeout(c.readTimeout) +} + +func (c *conn) ReceiveWithTimeout(timeout time.Duration) (reply interface{}, err error) { + var deadline time.Time + if timeout != 0 { + deadline = time.Now().Add(timeout) + } + c.conn.SetReadDeadline(deadline) + + if reply, err = c.readReply(); err != nil { + return nil, c.fatal(err) + } + // When using pub/sub, the number of receives can be greater than the + // number of sends. To enable normal use of the connection after + // unsubscribing from all channels, we do not decrement pending to a + // negative value. + // + // The pending field is decremented after the reply is read to handle the + // case where Receive is called before Send. + c.mu.Lock() + if c.pending > 0 { + c.pending -= 1 + } + c.mu.Unlock() + if err, ok := reply.(Error); ok { + return nil, err + } + return +} + +func (c *conn) Do(cmd string, args ...interface{}) (interface{}, error) { + return c.DoWithTimeout(c.readTimeout, cmd, args...) +} + +func (c *conn) DoWithTimeout(readTimeout time.Duration, cmd string, args ...interface{}) (interface{}, error) { + c.mu.Lock() + pending := c.pending + c.pending = 0 + c.mu.Unlock() + + if cmd == "" && pending == 0 { + return nil, nil + } + + if c.writeTimeout != 0 { + c.conn.SetWriteDeadline(time.Now().Add(c.writeTimeout)) + } + + if cmd != "" { + if err := c.writeCommand(cmd, args); err != nil { + return nil, c.fatal(err) + } + } + + if err := c.bw.Flush(); err != nil { + return nil, c.fatal(err) + } + + var deadline time.Time + if readTimeout != 0 { + deadline = time.Now().Add(readTimeout) + } + c.conn.SetReadDeadline(deadline) + + if cmd == "" { + reply := make([]interface{}, pending) + for i := range reply { + r, e := c.readReply() + if e != nil { + return nil, c.fatal(e) + } + reply[i] = r + } + return reply, nil + } + + var err error + var reply interface{} + for i := 0; i <= pending; i++ { + var e error + if reply, e = c.readReply(); e != nil { + return nil, c.fatal(e) + } + if e, ok := reply.(Error); ok && err == nil { + err = e + } + } + return reply, err +} diff --git a/src/go/redigo/redis/conn_test.go b/src/go/redigo/redis/conn_test.go new file mode 100644 index 0000000..ff40de1 --- /dev/null +++ b/src/go/redigo/redis/conn_test.go @@ -0,0 +1,942 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis_test + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "math" + "net" + "os" + "reflect" + "strings" + "testing" + "time" + + "github.com/cea-hpc/pdwfs/redigo/redis" +) + +type testConn struct { + io.Reader + io.Writer + readDeadline time.Time + writeDeadline time.Time +} + +func (*testConn) Close() error { return nil } +func (*testConn) LocalAddr() net.Addr { return nil } +func (*testConn) RemoteAddr() net.Addr { return nil } +func (c *testConn) SetDeadline(t time.Time) error { c.readDeadline = t; c.writeDeadline = t; return nil } +func (c *testConn) SetReadDeadline(t time.Time) error { c.readDeadline = t; return nil } +func (c *testConn) SetWriteDeadline(t time.Time) error { c.writeDeadline = t; return nil } + +func dialTestConn(r string, w io.Writer) redis.DialOption { + return redis.DialNetDial(func(network, addr string) (net.Conn, error) { + return &testConn{Reader: strings.NewReader(r), Writer: w}, nil + }) +} + +type tlsTestConn struct { + net.Conn + done chan struct{} +} + +func (c *tlsTestConn) Close() error { + c.Conn.Close() + <-c.done + return nil +} + +func dialTestConnTLS(r string, w io.Writer) redis.DialOption { + return redis.DialNetDial(func(network, addr string) (net.Conn, error) { + client, server := net.Pipe() + tlsServer := tls.Server(server, &serverTLSConfig) + go io.Copy(tlsServer, strings.NewReader(r)) + done := make(chan struct{}) + go func() { + io.Copy(w, tlsServer) + close(done) + }() + return &tlsTestConn{Conn: client, done: done}, nil + }) +} + +type durationArg struct { + time.Duration +} + +func (t durationArg) RedisArg() interface{} { + return t.Seconds() +} + +type recursiveArg int + +func (v recursiveArg) RedisArg() interface{} { return v } + +var writeTests = []struct { + args []interface{} + expected string +}{ + { + []interface{}{"SET", "key", "value"}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$5\r\nvalue\r\n", + }, + { + []interface{}{"SET", "key", "value"}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$5\r\nvalue\r\n", + }, + { + []interface{}{"SET", "key", byte(100)}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$3\r\n100\r\n", + }, + { + []interface{}{"SET", "key", 100}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$3\r\n100\r\n", + }, + { + []interface{}{"SET", "key", int64(math.MinInt64)}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$20\r\n-9223372036854775808\r\n", + }, + { + []interface{}{"SET", "key", float64(1349673917.939762)}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$21\r\n1.349673917939762e+09\r\n", + }, + { + []interface{}{"SET", "key", ""}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$0\r\n\r\n", + }, + { + []interface{}{"SET", "key", nil}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$0\r\n\r\n", + }, + { + []interface{}{"SET", "key", durationArg{time.Minute}}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$2\r\n60\r\n", + }, + { + []interface{}{"SET", "key", recursiveArg(123)}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$3\r\n123\r\n", + }, + { + []interface{}{"ECHO", true, false}, + "*3\r\n$4\r\nECHO\r\n$1\r\n1\r\n$1\r\n0\r\n", + }, +} + +func TestWrite(t *testing.T) { + for _, tt := range writeTests { + var buf bytes.Buffer + c, _ := redis.Dial("", "", dialTestConn("", &buf)) + err := c.Send(tt.args[0].(string), tt.args[1:]...) + if err != nil { + t.Errorf("Send(%v) returned error %v", tt.args, err) + continue + } + c.Flush() + actual := buf.String() + if actual != tt.expected { + t.Errorf("Send(%v) = %q, want %q", tt.args, actual, tt.expected) + } + } +} + +var errorSentinel = &struct{}{} + +var readTests = []struct { + reply string + expected interface{} +}{ + { + "+OK\r\n", + "OK", + }, + { + "+PONG\r\n", + "PONG", + }, + { + "+OK\n\n", // no \r + errorSentinel, + }, + { + "@OK\r\n", + errorSentinel, + }, + { + "$6\r\nfoobar\r\n", + []byte("foobar"), + }, + { + "$-1\r\n", + nil, + }, + { + ":1\r\n", + int64(1), + }, + { + ":-2\r\n", + int64(-2), + }, + { + "*0\r\n", + []interface{}{}, + }, + { + "*-1\r\n", + nil, + }, + { + "*4\r\n$3\r\nfoo\r\n$3\r\nbar\r\n$5\r\nHello\r\n$5\r\nWorld\r\n", + []interface{}{[]byte("foo"), []byte("bar"), []byte("Hello"), []byte("World")}, + }, + { + "*3\r\n$3\r\nfoo\r\n$-1\r\n$3\r\nbar\r\n", + []interface{}{[]byte("foo"), nil, []byte("bar")}, + }, + + { + // "" is not a valid length + "$\r\nfoobar\r\n", + errorSentinel, + }, + { + // "x" is not a valid length + "$x\r\nfoobar\r\n", + errorSentinel, + }, + { + // -2 is not a valid length + "$-2\r\n", + errorSentinel, + }, + { + // "" is not a valid integer + ":\r\n", + errorSentinel, + }, + { + // "x" is not a valid integer + ":x\r\n", + errorSentinel, + }, + { + // missing \r\n following value + "$6\r\nfoobar", + errorSentinel, + }, + { + // short value + "$6\r\nxx", + errorSentinel, + }, + { + // long value + "$6\r\nfoobarx\r\n", + errorSentinel, + }, +} + +func TestRead(t *testing.T) { + for _, tt := range readTests { + c, _ := redis.Dial("", "", dialTestConn(tt.reply, nil)) + actual, err := c.Receive() + if tt.expected == errorSentinel { + if err == nil { + t.Errorf("Receive(%q) did not return expected error", tt.reply) + } + } else { + if err != nil { + t.Errorf("Receive(%q) returned error %v", tt.reply, err) + continue + } + if !reflect.DeepEqual(actual, tt.expected) { + t.Errorf("Receive(%q) = %v, want %v", tt.reply, actual, tt.expected) + } + } + } +} + +func TestReadString(t *testing.T) { + // n is value of bufio.defaultBufSize + const n = 4096 + + // Test read string lengths near bufio.Reader buffer boundaries. + testRanges := [][2]int{{0, 64}, {n - 64, n + 64}, {2*n - 64, 2*n + 64}} + + p := make([]byte, 2*n+64) + for i := range p { + p[i] = byte('a' + i%26) + } + s := string(p) + + for _, r := range testRanges { + for i := r[0]; i < r[1]; i++ { + c, _ := redis.Dial("", "", dialTestConn("+"+s[:i]+"\r\n", nil)) + actual, err := c.Receive() + if err != nil || actual != s[:i] { + t.Fatalf("Receive(string len %d) -> err=%v, equal=%v", i, err, actual != s[:i]) + } + } + } +} + +var testCommands = []struct { + args []interface{} + expected interface{} +}{ + { + []interface{}{"PING"}, + "PONG", + }, + { + []interface{}{"SET", "foo", "bar"}, + "OK", + }, + { + []interface{}{"GET", "foo"}, + []byte("bar"), + }, + { + []interface{}{"GET", "nokey"}, + nil, + }, + { + []interface{}{"MGET", "nokey", "foo"}, + []interface{}{nil, []byte("bar")}, + }, + { + []interface{}{"INCR", "mycounter"}, + int64(1), + }, + { + []interface{}{"LPUSH", "mylist", "foo"}, + int64(1), + }, + { + []interface{}{"LPUSH", "mylist", "bar"}, + int64(2), + }, + { + []interface{}{"LRANGE", "mylist", 0, -1}, + []interface{}{[]byte("bar"), []byte("foo")}, + }, + { + []interface{}{"MULTI"}, + "OK", + }, + { + []interface{}{"LRANGE", "mylist", 0, -1}, + "QUEUED", + }, + { + []interface{}{"PING"}, + "QUEUED", + }, + { + []interface{}{"EXEC"}, + []interface{}{ + []interface{}{[]byte("bar"), []byte("foo")}, + "PONG", + }, + }, +} + +func TestDoCommands(t *testing.T) { + c, err := redis.DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer c.Close() + + for _, cmd := range testCommands { + actual, err := c.Do(cmd.args[0].(string), cmd.args[1:]...) + if err != nil { + t.Errorf("Do(%v) returned error %v", cmd.args, err) + continue + } + if !reflect.DeepEqual(actual, cmd.expected) { + t.Errorf("Do(%v) = %v, want %v", cmd.args, actual, cmd.expected) + } + } +} + +func TestPipelineCommands(t *testing.T) { + c, err := redis.DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer c.Close() + + for _, cmd := range testCommands { + if err := c.Send(cmd.args[0].(string), cmd.args[1:]...); err != nil { + t.Fatalf("Send(%v) returned error %v", cmd.args, err) + } + } + if err := c.Flush(); err != nil { + t.Errorf("Flush() returned error %v", err) + } + for _, cmd := range testCommands { + actual, err := c.Receive() + if err != nil { + t.Fatalf("Receive(%v) returned error %v", cmd.args, err) + } + if !reflect.DeepEqual(actual, cmd.expected) { + t.Errorf("Receive(%v) = %v, want %v", cmd.args, actual, cmd.expected) + } + } +} + +func TestBlankCommand(t *testing.T) { + c, err := redis.DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer c.Close() + + for _, cmd := range testCommands { + if err := c.Send(cmd.args[0].(string), cmd.args[1:]...); err != nil { + t.Fatalf("Send(%v) returned error %v", cmd.args, err) + } + } + reply, err := redis.Values(c.Do("")) + if err != nil { + t.Fatalf("Do() returned error %v", err) + } + if len(reply) != len(testCommands) { + t.Fatalf("len(reply)=%d, want %d", len(reply), len(testCommands)) + } + for i, cmd := range testCommands { + actual := reply[i] + if !reflect.DeepEqual(actual, cmd.expected) { + t.Errorf("Receive(%v) = %v, want %v", cmd.args, actual, cmd.expected) + } + } +} + +func TestRecvBeforeSend(t *testing.T) { + c, err := redis.DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer c.Close() + done := make(chan struct{}) + go func() { + c.Receive() + close(done) + }() + time.Sleep(time.Millisecond) + c.Send("PING") + c.Flush() + <-done + _, err = c.Do("") + if err != nil { + t.Fatalf("error=%v", err) + } +} + +func TestError(t *testing.T) { + c, err := redis.DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer c.Close() + + c.Do("SET", "key", "val") + _, err = c.Do("HSET", "key", "fld", "val") + if err == nil { + t.Errorf("Expected err for HSET on string key.") + } + if c.Err() != nil { + t.Errorf("Conn has Err()=%v, expect nil", c.Err()) + } + _, err = c.Do("SET", "key", "val") + if err != nil { + t.Errorf("Do(SET, key, val) returned error %v, expected nil.", err) + } +} + +func TestReadTimeout(t *testing.T) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("net.Listen returned %v", err) + } + defer l.Close() + + go func() { + for { + c, err := l.Accept() + if err != nil { + return + } + go func() { + time.Sleep(time.Second) + c.Write([]byte("+OK\r\n")) + c.Close() + }() + } + }() + + // Do + + c1, err := redis.Dial(l.Addr().Network(), l.Addr().String(), redis.DialReadTimeout(time.Millisecond)) + if err != nil { + t.Fatalf("redis.Dial returned %v", err) + } + defer c1.Close() + + _, err = c1.Do("PING") + if err == nil { + t.Fatalf("c1.Do() returned nil, expect error") + } + if c1.Err() == nil { + t.Fatalf("c1.Err() = nil, expect error") + } + + // Send/Flush/Receive + + c2, err := redis.Dial(l.Addr().Network(), l.Addr().String(), redis.DialReadTimeout(time.Millisecond)) + if err != nil { + t.Fatalf("redis.Dial returned %v", err) + } + defer c2.Close() + + c2.Send("PING") + c2.Flush() + _, err = c2.Receive() + if err == nil { + t.Fatalf("c2.Receive() returned nil, expect error") + } + if c2.Err() == nil { + t.Fatalf("c2.Err() = nil, expect error") + } +} + +var dialErrors = []struct { + rawurl string + expectedError string +}{ + { + "localhost", + "invalid redis URL scheme", + }, + // The error message for invalid hosts is different in different + // versions of Go, so just check that there is an error message. + { + "redis://weird url", + "", + }, + { + "redis://foo:bar:baz", + "", + }, + { + "http://www.google.com", + "invalid redis URL scheme: http", + }, + { + "redis://localhost:6379/abc123", + "invalid database: abc123", + }, +} + +func TestDialURLErrors(t *testing.T) { + for _, d := range dialErrors { + _, err := redis.DialURL(d.rawurl) + if err == nil || !strings.Contains(err.Error(), d.expectedError) { + t.Errorf("DialURL did not return expected error (expected %v to contain %s)", err, d.expectedError) + } + } +} + +func TestDialURLPort(t *testing.T) { + checkPort := func(network, address string) (net.Conn, error) { + if address != "localhost:6379" { + t.Errorf("DialURL did not set port to 6379 by default (got %v)", address) + } + return nil, nil + } + _, err := redis.DialURL("redis://localhost", redis.DialNetDial(checkPort)) + if err != nil { + t.Error("dial error:", err) + } +} + +func TestDialURLHost(t *testing.T) { + checkHost := func(network, address string) (net.Conn, error) { + if address != "localhost:6379" { + t.Errorf("DialURL did not set host to localhost by default (got %v)", address) + } + return nil, nil + } + _, err := redis.DialURL("redis://:6379", redis.DialNetDial(checkHost)) + if err != nil { + t.Error("dial error:", err) + } +} + +var dialURLTests = []struct { + description string + url string + r string + w string +}{ + {"password", "redis://x:abc123@localhost", "+OK\r\n", "*2\r\n$4\r\nAUTH\r\n$6\r\nabc123\r\n"}, + {"database 3", "redis://localhost/3", "+OK\r\n", "*2\r\n$6\r\nSELECT\r\n$1\r\n3\r\n"}, + {"database 99", "redis://localhost/99", "+OK\r\n", "*2\r\n$6\r\nSELECT\r\n$2\r\n99\r\n"}, + {"no database", "redis://localhost/", "+OK\r\n", ""}, +} + +func TestDialURL(t *testing.T) { + for _, tt := range dialURLTests { + var buf bytes.Buffer + // UseTLS should be ignored in all of these tests. + _, err := redis.DialURL(tt.url, dialTestConn(tt.r, &buf), redis.DialUseTLS(true)) + if err != nil { + t.Errorf("%s dial error: %v", tt.description, err) + continue + } + if w := buf.String(); w != tt.w { + t.Errorf("%s commands = %q, want %q", tt.description, w, tt.w) + } + } +} + +func checkPingPong(t *testing.T, buf *bytes.Buffer, c redis.Conn) { + resp, err := c.Do("PING") + if err != nil { + t.Fatal("ping error:", err) + } + // Close connection to ensure that writes to buf are complete. + c.Close() + expected := "*1\r\n$4\r\nPING\r\n" + actual := buf.String() + if actual != expected { + t.Errorf("commands = %q, want %q", actual, expected) + } + if resp != "PONG" { + t.Errorf("resp = %v, want %v", resp, "PONG") + } +} + +const pingResponse = "+PONG\r\n" + +func TestDialURLTLS(t *testing.T) { + var buf bytes.Buffer + c, err := redis.DialURL("rediss://example.com/", + redis.DialTLSConfig(&clientTLSConfig), + dialTestConnTLS(pingResponse, &buf)) + if err != nil { + t.Fatal("dial error:", err) + } + checkPingPong(t, &buf, c) +} + +func TestDialUseTLS(t *testing.T) { + var buf bytes.Buffer + c, err := redis.Dial("tcp", "example.com:6379", + redis.DialTLSConfig(&clientTLSConfig), + dialTestConnTLS(pingResponse, &buf), + redis.DialUseTLS(true)) + if err != nil { + t.Fatal("dial error:", err) + } + checkPingPong(t, &buf, c) +} + +func TestDialTLSSKipVerify(t *testing.T) { + var buf bytes.Buffer + c, err := redis.Dial("tcp", "example.com:6379", + dialTestConnTLS(pingResponse, &buf), + redis.DialTLSSkipVerify(true), + redis.DialUseTLS(true)) + if err != nil { + t.Fatal("dial error:", err) + } + checkPingPong(t, &buf, c) +} + +func TestDialClientName(t *testing.T) { + var buf bytes.Buffer + _, err := redis.Dial("tcp", ":6379", + dialTestConn(pingResponse, &buf), + redis.DialClientName("redis-connection"), + ) + if err != nil { + t.Fatal("dial error:", err) + } + expected := "*3\r\n$6\r\nCLIENT\r\n$7\r\nSETNAME\r\n$16\r\nredis-connection\r\n" + if w := buf.String(); w != expected { + t.Errorf("got %q, want %q", w, expected) + } + + // testing against a real server + connectionName := "test-connection" + c, err := redis.DialDefaultServer(redis.DialClientName(connectionName)) + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer c.Close() + + v, err := c.Do("CLIENT", "GETNAME") + if err != nil { + t.Fatalf("CLIENT GETNAME returned error %v", err) + } + + vs, err := redis.String(v, nil) + if err != nil { + t.Fatalf("String(v) returned error %v", err) + } + + if vs != connectionName { + t.Fatalf("wrong connection name. Got '%s', expected '%s'", vs, connectionName) + } +} + +// Connect to local instance of Redis running on the default port. +func ExampleDial() { + c, err := redis.Dial("tcp", ":6379") + if err != nil { + // handle error + } + defer c.Close() +} + +// Connect to remote instance of Redis using a URL. +func ExampleDialURL() { + c, err := redis.DialURL(os.Getenv("REDIS_URL")) + if err != nil { + // handle connection error + } + defer c.Close() +} + +// TextExecError tests handling of errors in a transaction. See +// http://redis.io/topics/transactions for information on how Redis handles +// errors in a transaction. +func TestExecError(t *testing.T) { + c, err := redis.DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer c.Close() + + // Execute commands that fail before EXEC is called. + + c.Do("DEL", "k0") + c.Do("ZADD", "k0", 0, 0) + c.Send("MULTI") + c.Send("NOTACOMMAND", "k0", 0, 0) + c.Send("ZINCRBY", "k0", 0, 0) + v, err := c.Do("EXEC") + if err == nil { + t.Fatalf("EXEC returned values %v, expected error", v) + } + + // Execute commands that fail after EXEC is called. The first command + // returns an error. + + c.Do("DEL", "k1") + c.Do("ZADD", "k1", 0, 0) + c.Send("MULTI") + c.Send("HSET", "k1", 0, 0) + c.Send("ZINCRBY", "k1", 0, 0) + v, err = c.Do("EXEC") + if err != nil { + t.Fatalf("EXEC returned error %v", err) + } + + vs, err := redis.Values(v, nil) + if err != nil { + t.Fatalf("Values(v) returned error %v", err) + } + + if len(vs) != 2 { + t.Fatalf("len(vs) == %d, want 2", len(vs)) + } + + if _, ok := vs[0].(error); !ok { + t.Fatalf("first result is type %T, expected error", vs[0]) + } + + if _, ok := vs[1].([]byte); !ok { + t.Fatalf("second result is type %T, expected []byte", vs[1]) + } + + // Execute commands that fail after EXEC is called. The second command + // returns an error. + + c.Do("ZADD", "k2", 0, 0) + c.Send("MULTI") + c.Send("ZINCRBY", "k2", 0, 0) + c.Send("HSET", "k2", 0, 0) + v, err = c.Do("EXEC") + if err != nil { + t.Fatalf("EXEC returned error %v", err) + } + + vs, err = redis.Values(v, nil) + if err != nil { + t.Fatalf("Values(v) returned error %v", err) + } + + if len(vs) != 2 { + t.Fatalf("len(vs) == %d, want 2", len(vs)) + } + + if _, ok := vs[0].([]byte); !ok { + t.Fatalf("first result is type %T, expected []byte", vs[0]) + } + + if _, ok := vs[1].(error); !ok { + t.Fatalf("second result is type %T, expected error", vs[2]) + } +} + +func BenchmarkDoEmpty(b *testing.B) { + b.StopTimer() + c, err := redis.DialDefaultServer() + if err != nil { + b.Fatal(err) + } + defer c.Close() + b.StartTimer() + for i := 0; i < b.N; i++ { + if _, err := c.Do(""); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkDoPing(b *testing.B) { + b.StopTimer() + c, err := redis.DialDefaultServer() + if err != nil { + b.Fatal(err) + } + defer c.Close() + b.StartTimer() + for i := 0; i < b.N; i++ { + if _, err := c.Do("PING"); err != nil { + b.Fatal(err) + } + } +} + +var clientTLSConfig, serverTLSConfig tls.Config + +func init() { + // The certificate and key for testing TLS dial options was created + // using the command + // + // go run GOROOT/src/crypto/tls/generate_cert.go \ + // --rsa-bits 1024 \ + // --host 127.0.0.1,::1,example.com --ca \ + // --start-date "Jan 1 00:00:00 1970" \ + // --duration=1000000h + // + // where GOROOT is the value of GOROOT reported by go env. + localhostCert := []byte(` +-----BEGIN CERTIFICATE----- +MIICFDCCAX2gAwIBAgIRAJfBL4CUxkXcdlFurb3K+iowDQYJKoZIhvcNAQELBQAw +EjEQMA4GA1UEChMHQWNtZSBDbzAgFw03MDAxMDEwMDAwMDBaGA8yMDg0MDEyOTE2 +MDAwMFowEjEQMA4GA1UEChMHQWNtZSBDbzCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEArizw8WxMUQ3bGHLeuJ4fDrEpy+L2pqrbYRlKk1DasJ/VkB8bImzIpe6+ +LGjiYIxvnDCOJ3f3QplcQuiuMyl6f2irJlJsbFT8Lo/3obnuTKAIaqUdJUqBg6y+ +JaL8Auk97FvunfKFv8U1AIhgiLzAfQ/3Eaq1yi87Ra6pMjGbTtcCAwEAAaNoMGYw +DgYDVR0PAQH/BAQDAgKkMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQF +MAMBAf8wLgYDVR0RBCcwJYILZXhhbXBsZS5jb22HBH8AAAGHEAAAAAAAAAAAAAAA +AAAAAAEwDQYJKoZIhvcNAQELBQADgYEAdZ8daIVkyhVwflt5I19m0oq1TycbGO1+ +ach7T6cZiBQeNR/SJtxr/wKPEpmvUgbv2BfFrKJ8QoIHYsbNSURTWSEa02pfw4k9 +6RQhij3ZkG79Ituj5OYRORV6Z0HUW32r670BtcuHuAhq7YA6Nxy4FtSt7bAlVdRt +rrKgNsltzMk= +-----END CERTIFICATE-----`) + + localhostKey := []byte(` +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQCuLPDxbExRDdsYct64nh8OsSnL4vamqtthGUqTUNqwn9WQHxsi +bMil7r4saOJgjG+cMI4nd/dCmVxC6K4zKXp/aKsmUmxsVPwuj/ehue5MoAhqpR0l +SoGDrL4lovwC6T3sW+6d8oW/xTUAiGCIvMB9D/cRqrXKLztFrqkyMZtO1wIDAQAB +AoGACrc5G6FOEK6JjDeE/Fa+EmlT6PdNtXNNi+vCas3Opo8u1G8VfEi1D4BgstrB +Eq+RLkrOdB8tVyuYQYWPMhabMqF+hhKJN72j0OwfuPlVvTInwb/cKjo/zbH1IA+Y +HenHNK4ywv7/p/9/MvQPJ3I32cQBCgGUW5chVSH5M1sj5gECQQDabQAI1X0uDqCm +KbX9gXVkAgxkFddrt6LBHt57xujFcqEKFE7nwKhDh7DweVs/VEJ+kpid4z+UnLOw +KjtP9JolAkEAzCNBphQ//IsbH5rNs10wIUw3Ks/Oepicvr6kUFbIv+neRzi1iJHa +m6H7EayK3PWgax6BAsR/t0Jc9XV7r2muSwJAVzN09BHnK+ADGtNEKLTqXMbEk6B0 +pDhn7ZmZUOkUPN+Kky+QYM11X6Bob1jDqQDGmymDbGUxGO+GfSofC8inUQJAGfci +Eo3g1a6b9JksMPRZeuLG4ZstGErxJRH6tH1Va5PDwitka8qhk8o2tTjNMO3NSdLH +diKoXBcE2/Pll5pJoQJBAIMiiMIzXJhnN4mX8may44J/HvMlMf2xuVH2gNMwmZuc +Bjqn3yoLHaoZVvbWOi0C2TCN4FjXjaLNZGifQPbIcaA= +-----END RSA PRIVATE KEY-----`) + + cert, err := tls.X509KeyPair(localhostCert, localhostKey) + if err != nil { + panic(fmt.Sprintf("error creating key pair: %v", err)) + } + serverTLSConfig.Certificates = []tls.Certificate{cert} + + certificate, err := x509.ParseCertificate(serverTLSConfig.Certificates[0].Certificate[0]) + if err != nil { + panic(fmt.Sprintf("error parsing x509 certificate: %v", err)) + } + + clientTLSConfig.RootCAs = x509.NewCertPool() + clientTLSConfig.RootCAs.AddCert(certificate) +} + +func TestWithTimeout(t *testing.T) { + for _, recv := range []bool{true, false} { + for _, defaultTimout := range []time.Duration{0, time.Minute} { + var buf bytes.Buffer + nc := &testConn{Reader: strings.NewReader("+OK\r\n+OK\r\n+OK\r\n+OK\r\n+OK\r\n+OK\r\n+OK\r\n+OK\r\n+OK\r\n+OK\r\n"), Writer: &buf} + c, _ := redis.Dial("", "", redis.DialReadTimeout(defaultTimout), redis.DialNetDial(func(network, addr string) (net.Conn, error) { return nc, nil })) + for i := 0; i < 4; i++ { + var minDeadline, maxDeadline time.Time + + // Alternate between default and specified timeout. + if i%2 == 0 { + if defaultTimout != 0 { + minDeadline = time.Now().Add(defaultTimout) + } + if recv { + c.Receive() + } else { + c.Do("PING") + } + if defaultTimout != 0 { + maxDeadline = time.Now().Add(defaultTimout) + } + } else { + timeout := 10 * time.Minute + minDeadline = time.Now().Add(timeout) + if recv { + redis.ReceiveWithTimeout(c, timeout) + } else { + redis.DoWithTimeout(c, timeout, "PING") + } + maxDeadline = time.Now().Add(timeout) + } + + // Expect set deadline in expected range. + if nc.readDeadline.Before(minDeadline) || nc.readDeadline.After(maxDeadline) { + t.Errorf("recv %v, %d: do deadline error: %v, %v, %v", recv, i, minDeadline, nc.readDeadline, maxDeadline) + } + } + } + } +} diff --git a/src/go/redigo/redis/doc.go b/src/go/redigo/redis/doc.go new file mode 100644 index 0000000..69ad506 --- /dev/null +++ b/src/go/redigo/redis/doc.go @@ -0,0 +1,177 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Package redis is a client for the Redis database. +// +// The Redigo FAQ (https://github.com/gomodule/redigo/wiki/FAQ) contains more +// documentation about this package. +// +// Connections +// +// The Conn interface is the primary interface for working with Redis. +// Applications create connections by calling the Dial, DialWithTimeout or +// NewConn functions. In the future, functions will be added for creating +// sharded and other types of connections. +// +// The application must call the connection Close method when the application +// is done with the connection. +// +// Executing Commands +// +// The Conn interface has a generic method for executing Redis commands: +// +// Do(commandName string, args ...interface{}) (reply interface{}, err error) +// +// The Redis command reference (http://redis.io/commands) lists the available +// commands. An example of using the Redis APPEND command is: +// +// n, err := conn.Do("APPEND", "key", "value") +// +// The Do method converts command arguments to bulk strings for transmission +// to the server as follows: +// +// Go Type Conversion +// []byte Sent as is +// string Sent as is +// int, int64 strconv.FormatInt(v) +// float64 strconv.FormatFloat(v, 'g', -1, 64) +// bool true -> "1", false -> "0" +// nil "" +// all other types fmt.Fprint(w, v) +// +// Redis command reply types are represented using the following Go types: +// +// Redis type Go type +// error redis.Error +// integer int64 +// simple string string +// bulk string []byte or nil if value not present. +// array []interface{} or nil if value not present. +// +// Use type assertions or the reply helper functions to convert from +// interface{} to the specific Go type for the command result. +// +// Pipelining +// +// Connections support pipelining using the Send, Flush and Receive methods. +// +// Send(commandName string, args ...interface{}) error +// Flush() error +// Receive() (reply interface{}, err error) +// +// Send writes the command to the connection's output buffer. Flush flushes the +// connection's output buffer to the server. Receive reads a single reply from +// the server. The following example shows a simple pipeline. +// +// c.Send("SET", "foo", "bar") +// c.Send("GET", "foo") +// c.Flush() +// c.Receive() // reply from SET +// v, err = c.Receive() // reply from GET +// +// The Do method combines the functionality of the Send, Flush and Receive +// methods. The Do method starts by writing the command and flushing the output +// buffer. Next, the Do method receives all pending replies including the reply +// for the command just sent by Do. If any of the received replies is an error, +// then Do returns the error. If there are no errors, then Do returns the last +// reply. If the command argument to the Do method is "", then the Do method +// will flush the output buffer and receive pending replies without sending a +// command. +// +// Use the Send and Do methods to implement pipelined transactions. +// +// c.Send("MULTI") +// c.Send("INCR", "foo") +// c.Send("INCR", "bar") +// r, err := c.Do("EXEC") +// fmt.Println(r) // prints [1, 1] +// +// Concurrency +// +// Connections support one concurrent caller to the Receive method and one +// concurrent caller to the Send and Flush methods. No other concurrency is +// supported including concurrent calls to the Do and Close methods. +// +// For full concurrent access to Redis, use the thread-safe Pool to get, use +// and release a connection from within a goroutine. Connections returned from +// a Pool have the concurrency restrictions described in the previous +// paragraph. +// +// Publish and Subscribe +// +// Use the Send, Flush and Receive methods to implement Pub/Sub subscribers. +// +// c.Send("SUBSCRIBE", "example") +// c.Flush() +// for { +// reply, err := c.Receive() +// if err != nil { +// return err +// } +// // process pushed message +// } +// +// The PubSubConn type wraps a Conn with convenience methods for implementing +// subscribers. The Subscribe, PSubscribe, Unsubscribe and PUnsubscribe methods +// send and flush a subscription management command. The receive method +// converts a pushed message to convenient types for use in a type switch. +// +// psc := redis.PubSubConn{Conn: c} +// psc.Subscribe("example") +// for { +// switch v := psc.Receive().(type) { +// case redis.Message: +// fmt.Printf("%s: message: %s\n", v.Channel, v.Data) +// case redis.Subscription: +// fmt.Printf("%s: %s %d\n", v.Channel, v.Kind, v.Count) +// case error: +// return v +// } +// } +// +// Reply Helpers +// +// The Bool, Int, Bytes, String, Strings and Values functions convert a reply +// to a value of a specific type. To allow convenient wrapping of calls to the +// connection Do and Receive methods, the functions take a second argument of +// type error. If the error is non-nil, then the helper function returns the +// error. If the error is nil, the function converts the reply to the specified +// type: +// +// exists, err := redis.Bool(c.Do("EXISTS", "foo")) +// if err != nil { +// // handle error return from c.Do or type conversion error. +// } +// +// The Scan function converts elements of a array reply to Go types: +// +// var value1 int +// var value2 string +// reply, err := redis.Values(c.Do("MGET", "key1", "key2")) +// if err != nil { +// // handle error +// } +// if _, err := redis.Scan(reply, &value1, &value2); err != nil { +// // handle error +// } +// +// Errors +// +// Connection methods return error replies from the server as type redis.Error. +// +// Call the connection Err() method to determine if the connection encountered +// non-recoverable error such as a network error or protocol parsing error. If +// Err() returns a non-nil value, then the connection is not usable and should +// be closed. +package redis diff --git a/src/go/redigo/redis/go17.go b/src/go/redigo/redis/go17.go new file mode 100644 index 0000000..5f36379 --- /dev/null +++ b/src/go/redigo/redis/go17.go @@ -0,0 +1,29 @@ +// +build go1.7,!go1.8 + +package redis + +import "crypto/tls" + +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + return &tls.Config{ + Rand: cfg.Rand, + Time: cfg.Time, + Certificates: cfg.Certificates, + NameToCertificate: cfg.NameToCertificate, + GetCertificate: cfg.GetCertificate, + RootCAs: cfg.RootCAs, + NextProtos: cfg.NextProtos, + ServerName: cfg.ServerName, + ClientAuth: cfg.ClientAuth, + ClientCAs: cfg.ClientCAs, + InsecureSkipVerify: cfg.InsecureSkipVerify, + CipherSuites: cfg.CipherSuites, + PreferServerCipherSuites: cfg.PreferServerCipherSuites, + ClientSessionCache: cfg.ClientSessionCache, + MinVersion: cfg.MinVersion, + MaxVersion: cfg.MaxVersion, + CurvePreferences: cfg.CurvePreferences, + DynamicRecordSizingDisabled: cfg.DynamicRecordSizingDisabled, + Renegotiation: cfg.Renegotiation, + } +} diff --git a/src/go/redigo/redis/go18.go b/src/go/redigo/redis/go18.go new file mode 100644 index 0000000..558363b --- /dev/null +++ b/src/go/redigo/redis/go18.go @@ -0,0 +1,9 @@ +// +build go1.8 + +package redis + +import "crypto/tls" + +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + return cfg.Clone() +} diff --git a/src/go/redigo/redis/list_test.go b/src/go/redigo/redis/list_test.go new file mode 100644 index 0000000..9c34bed --- /dev/null +++ b/src/go/redigo/redis/list_test.go @@ -0,0 +1,85 @@ +// Copyright 2018 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// +build go1.9 + +package redis + +import "testing" + +func TestPoolList(t *testing.T) { + var idle idleList + var a, b, c poolConn + + check := func(pcs ...*poolConn) { + if idle.count != len(pcs) { + t.Fatal("idle.count != len(pcs)") + } + if len(pcs) == 0 { + if idle.front != nil { + t.Fatalf("front not nil") + } + if idle.back != nil { + t.Fatalf("back not nil") + } + return + } + if idle.front != pcs[0] { + t.Fatal("front != pcs[0]") + } + if idle.back != pcs[len(pcs)-1] { + t.Fatal("back != pcs[len(pcs)-1]") + } + if idle.front.prev != nil { + t.Fatal("front.prev != nil") + } + if idle.back.next != nil { + t.Fatal("back.next != nil") + } + for i := 1; i < len(pcs)-1; i++ { + if pcs[i-1].next != pcs[i] { + t.Fatal("pcs[i-1].next != pcs[i]") + } + if pcs[i+1].prev != pcs[i] { + t.Fatal("pcs[i+1].prev != pcs[i]") + } + } + } + + idle.pushFront(&c) + check(&c) + idle.pushFront(&b) + check(&b, &c) + idle.pushFront(&a) + check(&a, &b, &c) + idle.popFront() + check(&b, &c) + idle.popFront() + check(&c) + idle.popFront() + check() + + idle.pushFront(&c) + check(&c) + idle.pushFront(&b) + check(&b, &c) + idle.pushFront(&a) + check(&a, &b, &c) + idle.popBack() + check(&a, &b) + idle.popBack() + check(&a) + idle.popBack() + check() +} diff --git a/src/go/redigo/redis/log.go b/src/go/redigo/redis/log.go new file mode 100644 index 0000000..a06db9d --- /dev/null +++ b/src/go/redigo/redis/log.go @@ -0,0 +1,146 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "bytes" + "fmt" + "log" + "time" +) + +var ( + _ ConnWithTimeout = (*loggingConn)(nil) +) + +// NewLoggingConn returns a logging wrapper around a connection. +func NewLoggingConn(conn Conn, logger *log.Logger, prefix string) Conn { + if prefix != "" { + prefix = prefix + "." + } + return &loggingConn{conn, logger, prefix, nil} +} + +//NewLoggingConnFilter returns a logging wrapper around a connection and a filter function. +func NewLoggingConnFilter(conn Conn, logger *log.Logger, prefix string, skip func(cmdName string) bool) Conn { + if prefix != "" { + prefix = prefix + "." + } + return &loggingConn{conn, logger, prefix, skip} +} + +type loggingConn struct { + Conn + logger *log.Logger + prefix string + skip func(cmdName string) bool +} + +func (c *loggingConn) Close() error { + err := c.Conn.Close() + var buf bytes.Buffer + fmt.Fprintf(&buf, "%sClose() -> (%v)", c.prefix, err) + c.logger.Output(2, buf.String()) + return err +} + +func (c *loggingConn) printValue(buf *bytes.Buffer, v interface{}) { + const chop = 32 + switch v := v.(type) { + case []byte: + if len(v) > chop { + fmt.Fprintf(buf, "%q...", v[:chop]) + } else { + fmt.Fprintf(buf, "%q", v) + } + case string: + if len(v) > chop { + fmt.Fprintf(buf, "%q...", v[:chop]) + } else { + fmt.Fprintf(buf, "%q", v) + } + case []interface{}: + if len(v) == 0 { + buf.WriteString("[]") + } else { + sep := "[" + fin := "]" + if len(v) > chop { + v = v[:chop] + fin = "...]" + } + for _, vv := range v { + buf.WriteString(sep) + c.printValue(buf, vv) + sep = ", " + } + buf.WriteString(fin) + } + default: + fmt.Fprint(buf, v) + } +} + +func (c *loggingConn) print(method, commandName string, args []interface{}, reply interface{}, err error) { + if c.skip != nil && c.skip(commandName) { + return + } + var buf bytes.Buffer + fmt.Fprintf(&buf, "%s%s(", c.prefix, method) + if method != "Receive" { + buf.WriteString(commandName) + for _, arg := range args { + buf.WriteString(", ") + c.printValue(&buf, arg) + } + } + buf.WriteString(") -> (") + if method != "Send" { + c.printValue(&buf, reply) + buf.WriteString(", ") + } + fmt.Fprintf(&buf, "%v)", err) + c.logger.Output(3, buf.String()) +} + +func (c *loggingConn) Do(commandName string, args ...interface{}) (interface{}, error) { + reply, err := c.Conn.Do(commandName, args...) + c.print("Do", commandName, args, reply, err) + return reply, err +} + +func (c *loggingConn) DoWithTimeout(timeout time.Duration, commandName string, args ...interface{}) (interface{}, error) { + reply, err := DoWithTimeout(c.Conn, timeout, commandName, args...) + c.print("DoWithTimeout", commandName, args, reply, err) + return reply, err +} + +func (c *loggingConn) Send(commandName string, args ...interface{}) error { + err := c.Conn.Send(commandName, args...) + c.print("Send", commandName, args, nil, err) + return err +} + +func (c *loggingConn) Receive() (interface{}, error) { + reply, err := c.Conn.Receive() + c.print("Receive", "", nil, reply, err) + return reply, err +} + +func (c *loggingConn) ReceiveWithTimeout(timeout time.Duration) (interface{}, error) { + reply, err := ReceiveWithTimeout(c.Conn, timeout) + c.print("ReceiveWithTimeout", "", nil, reply, err) + return reply, err +} diff --git a/src/go/redigo/redis/pool.go b/src/go/redigo/redis/pool.go new file mode 100644 index 0000000..98fe4a0 --- /dev/null +++ b/src/go/redigo/redis/pool.go @@ -0,0 +1,633 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha1" + "errors" + "io" + "strconv" + "sync" + "sync/atomic" + "time" +) + +var ( + _ ConnWithTimeout = (*activeConn)(nil) + _ ConnWithTimeout = (*errorConn)(nil) +) + +var nowFunc = time.Now // for testing + +// ErrPoolExhausted is returned from a pool connection method (Do, Send, +// Receive, Flush, Err) when the maximum number of database connections in the +// pool has been reached. +var ErrPoolExhausted = errors.New("redigo: connection pool exhausted") + +var ( + errPoolClosed = errors.New("redigo: connection pool closed") + errConnClosed = errors.New("redigo: connection closed") +) + +// Pool maintains a pool of connections. The application calls the Get method +// to get a connection from the pool and the connection's Close method to +// return the connection's resources to the pool. +// +// The following example shows how to use a pool in a web application. The +// application creates a pool at application startup and makes it available to +// request handlers using a package level variable. The pool configuration used +// here is an example, not a recommendation. +// +// func newPool(addr string) *redis.Pool { +// return &redis.Pool{ +// MaxIdle: 3, +// IdleTimeout: 240 * time.Second, +// // Dial or DialContext must be set. When both are set, DialContext takes precedence over Dial. +// Dial: func () (redis.Conn, error) { return redis.Dial("tcp", addr) }, +// } +// } +// +// var ( +// pool *redis.Pool +// redisServer = flag.String("redisServer", ":6379", "") +// ) +// +// func main() { +// flag.Parse() +// pool = newPool(*redisServer) +// ... +// } +// +// A request handler gets a connection from the pool and closes the connection +// when the handler is done: +// +// func serveHome(w http.ResponseWriter, r *http.Request) { +// conn := pool.Get() +// defer conn.Close() +// ... +// } +// +// Use the Dial function to authenticate connections with the AUTH command or +// select a database with the SELECT command: +// +// pool := &redis.Pool{ +// // Other pool configuration not shown in this example. +// Dial: func () (redis.Conn, error) { +// c, err := redis.Dial("tcp", server) +// if err != nil { +// return nil, err +// } +// if _, err := c.Do("AUTH", password); err != nil { +// c.Close() +// return nil, err +// } +// if _, err := c.Do("SELECT", db); err != nil { +// c.Close() +// return nil, err +// } +// return c, nil +// }, +// } +// +// Use the TestOnBorrow function to check the health of an idle connection +// before the connection is returned to the application. This example PINGs +// connections that have been idle more than a minute: +// +// pool := &redis.Pool{ +// // Other pool configuration not shown in this example. +// TestOnBorrow: func(c redis.Conn, t time.Time) error { +// if time.Since(t) < time.Minute { +// return nil +// } +// _, err := c.Do("PING") +// return err +// }, +// } +// +type Pool struct { + // Dial is an application supplied function for creating and configuring a + // connection. + // + // The connection returned from Dial must not be in a special state + // (subscribed to pubsub channel, transaction started, ...). + Dial func() (Conn, error) + + // DialContext is an application supplied function for creating and configuring a + // connection with the given context. + // + // The connection returned from Dial must not be in a special state + // (subscribed to pubsub channel, transaction started, ...). + DialContext func(ctx context.Context) (Conn, error) + + // TestOnBorrow is an optional application supplied function for checking + // the health of an idle connection before the connection is used again by + // the application. Argument t is the time that the connection was returned + // to the pool. If the function returns an error, then the connection is + // closed. + TestOnBorrow func(c Conn, t time.Time) error + + // Maximum number of idle connections in the pool. + MaxIdle int + + // Maximum number of connections allocated by the pool at a given time. + // When zero, there is no limit on the number of connections in the pool. + MaxActive int + + // Close connections after remaining idle for this duration. If the value + // is zero, then idle connections are not closed. Applications should set + // the timeout to a value less than the server's timeout. + IdleTimeout time.Duration + + // If Wait is true and the pool is at the MaxActive limit, then Get() waits + // for a connection to be returned to the pool before returning. + Wait bool + + // Close connections older than this duration. If the value is zero, then + // the pool does not close connections based on age. + MaxConnLifetime time.Duration + + chInitialized uint32 // set to 1 when field ch is initialized + + mu sync.Mutex // mu protects the following fields + closed bool // set to true when the pool is closed. + active int // the number of open connections in the pool + ch chan struct{} // limits open connections when p.Wait is true + idle idleList // idle connections + waitCount int64 // total number of connections waited for. + waitDuration time.Duration // total time waited for new connections. +} + +// NewPool creates a new pool. +// +// Deprecated: Initialize the Pool directory as shown in the example. +func NewPool(newFn func() (Conn, error), maxIdle int) *Pool { + return &Pool{Dial: newFn, MaxIdle: maxIdle} +} + +// Get gets a connection. The application must close the returned connection. +// This method always returns a valid connection so that applications can defer +// error handling to the first use of the connection. If there is an error +// getting an underlying connection, then the connection Err, Do, Send, Flush +// and Receive methods return that error. +func (p *Pool) Get() Conn { + pc, err := p.get(nil) + if err != nil { + return errorConn{err} + } + return &activeConn{p: p, pc: pc} +} + +// GetContext gets a connection using the provided context. +// +// The provided Context must be non-nil. If the context expires before the +// connection is complete, an error is returned. Any expiration on the context +// will not affect the returned connection. +// +// If the function completes without error, then the application must close the +// returned connection. +func (p *Pool) GetContext(ctx context.Context) (Conn, error) { + pc, err := p.get(ctx) + if err != nil { + return errorConn{err}, err + } + return &activeConn{p: p, pc: pc}, nil +} + +// PoolStats contains pool statistics. +type PoolStats struct { + // ActiveCount is the number of connections in the pool. The count includes + // idle connections and connections in use. + ActiveCount int + // IdleCount is the number of idle connections in the pool. + IdleCount int + + // WaitCount is the total number of connections waited for. + // This value is currently not guaranteed to be 100% accurate. + WaitCount int64 + + // WaitDuration is the total time blocked waiting for a new connection. + // This value is currently not guaranteed to be 100% accurate. + WaitDuration time.Duration +} + +// Stats returns pool's statistics. +func (p *Pool) Stats() PoolStats { + p.mu.Lock() + stats := PoolStats{ + ActiveCount: p.active, + IdleCount: p.idle.count, + WaitCount: p.waitCount, + WaitDuration: p.waitDuration, + } + p.mu.Unlock() + + return stats +} + +// ActiveCount returns the number of connections in the pool. The count +// includes idle connections and connections in use. +func (p *Pool) ActiveCount() int { + p.mu.Lock() + active := p.active + p.mu.Unlock() + return active +} + +// IdleCount returns the number of idle connections in the pool. +func (p *Pool) IdleCount() int { + p.mu.Lock() + idle := p.idle.count + p.mu.Unlock() + return idle +} + +// Close releases the resources used by the pool. +func (p *Pool) Close() error { + p.mu.Lock() + if p.closed { + p.mu.Unlock() + return nil + } + p.closed = true + p.active -= p.idle.count + pc := p.idle.front + p.idle.count = 0 + p.idle.front, p.idle.back = nil, nil + if p.ch != nil { + close(p.ch) + } + p.mu.Unlock() + for ; pc != nil; pc = pc.next { + pc.c.Close() + } + return nil +} + +func (p *Pool) lazyInit() { + // Fast path. + if atomic.LoadUint32(&p.chInitialized) == 1 { + return + } + // Slow path. + p.mu.Lock() + if p.chInitialized == 0 { + p.ch = make(chan struct{}, p.MaxActive) + if p.closed { + close(p.ch) + } else { + for i := 0; i < p.MaxActive; i++ { + p.ch <- struct{}{} + } + } + atomic.StoreUint32(&p.chInitialized, 1) + } + p.mu.Unlock() +} + +// get prunes stale connections and returns a connection from the idle list or +// creates a new connection. +func (p *Pool) get(ctx context.Context) (*poolConn, error) { + + // Handle limit for p.Wait == true. + var waited time.Duration + if p.Wait && p.MaxActive > 0 { + p.lazyInit() + + // wait indicates if we believe it will block so its not 100% accurate + // however for stats it should be good enough. + wait := len(p.ch) == 0 + var start time.Time + if wait { + start = time.Now() + } + if ctx == nil { + <-p.ch + } else { + select { + case <-p.ch: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + if wait { + waited = time.Since(start) + } + } + + p.mu.Lock() + + if waited > 0 { + p.waitCount++ + p.waitDuration += waited + } + + // Prune stale connections at the back of the idle list. + if p.IdleTimeout > 0 { + n := p.idle.count + for i := 0; i < n && p.idle.back != nil && p.idle.back.t.Add(p.IdleTimeout).Before(nowFunc()); i++ { + pc := p.idle.back + p.idle.popBack() + p.mu.Unlock() + pc.c.Close() + p.mu.Lock() + p.active-- + } + } + + // Get idle connection from the front of idle list. + for p.idle.front != nil { + pc := p.idle.front + p.idle.popFront() + p.mu.Unlock() + if (p.TestOnBorrow == nil || p.TestOnBorrow(pc.c, pc.t) == nil) && + (p.MaxConnLifetime == 0 || nowFunc().Sub(pc.created) < p.MaxConnLifetime) { + return pc, nil + } + pc.c.Close() + p.mu.Lock() + p.active-- + } + + // Check for pool closed before dialing a new connection. + if p.closed { + p.mu.Unlock() + return nil, errors.New("redigo: get on closed pool") + } + + // Handle limit for p.Wait == false. + if !p.Wait && p.MaxActive > 0 && p.active >= p.MaxActive { + p.mu.Unlock() + return nil, ErrPoolExhausted + } + + p.active++ + p.mu.Unlock() + c, err := p.dial(ctx) + if err != nil { + c = nil + p.mu.Lock() + p.active-- + if p.ch != nil && !p.closed { + p.ch <- struct{}{} + } + p.mu.Unlock() + } + return &poolConn{c: c, created: nowFunc()}, err +} + +func (p *Pool) dial(ctx context.Context) (Conn, error) { + if p.DialContext != nil { + return p.DialContext(ctx) + } + if p.Dial != nil { + return p.Dial() + } + return nil, errors.New("redigo: must pass Dial or DialContext to pool") +} + +func (p *Pool) put(pc *poolConn, forceClose bool) error { + p.mu.Lock() + if !p.closed && !forceClose { + pc.t = nowFunc() + p.idle.pushFront(pc) + if p.idle.count > p.MaxIdle { + pc = p.idle.back + p.idle.popBack() + } else { + pc = nil + } + } + + if pc != nil { + p.mu.Unlock() + pc.c.Close() + p.mu.Lock() + p.active-- + } + + if p.ch != nil && !p.closed { + p.ch <- struct{}{} + } + p.mu.Unlock() + return nil +} + +type activeConn struct { + p *Pool + pc *poolConn + state int +} + +var ( + sentinel []byte + sentinelOnce sync.Once +) + +func initSentinel() { + p := make([]byte, 64) + if _, err := rand.Read(p); err == nil { + sentinel = p + } else { + h := sha1.New() + io.WriteString(h, "Oops, rand failed. Use time instead.") + io.WriteString(h, strconv.FormatInt(time.Now().UnixNano(), 10)) + sentinel = h.Sum(nil) + } +} + +func (ac *activeConn) Close() error { + pc := ac.pc + if pc == nil { + return nil + } + ac.pc = nil + + if ac.state&connectionMultiState != 0 { + pc.c.Send("DISCARD") + ac.state &^= (connectionMultiState | connectionWatchState) + } else if ac.state&connectionWatchState != 0 { + pc.c.Send("UNWATCH") + ac.state &^= connectionWatchState + } + if ac.state&connectionSubscribeState != 0 { + pc.c.Send("UNSUBSCRIBE") + pc.c.Send("PUNSUBSCRIBE") + // To detect the end of the message stream, ask the server to echo + // a sentinel value and read until we see that value. + sentinelOnce.Do(initSentinel) + pc.c.Send("ECHO", sentinel) + pc.c.Flush() + for { + p, err := pc.c.Receive() + if err != nil { + break + } + if p, ok := p.([]byte); ok && bytes.Equal(p, sentinel) { + ac.state &^= connectionSubscribeState + break + } + } + } + pc.c.Do("") + ac.p.put(pc, ac.state != 0 || pc.c.Err() != nil) + return nil +} + +func (ac *activeConn) Err() error { + pc := ac.pc + if pc == nil { + return errConnClosed + } + return pc.c.Err() +} + +func (ac *activeConn) Do(commandName string, args ...interface{}) (reply interface{}, err error) { + pc := ac.pc + if pc == nil { + return nil, errConnClosed + } + ci := lookupCommandInfo(commandName) + ac.state = (ac.state | ci.Set) &^ ci.Clear + return pc.c.Do(commandName, args...) +} + +func (ac *activeConn) DoWithTimeout(timeout time.Duration, commandName string, args ...interface{}) (reply interface{}, err error) { + pc := ac.pc + if pc == nil { + return nil, errConnClosed + } + cwt, ok := pc.c.(ConnWithTimeout) + if !ok { + return nil, errTimeoutNotSupported + } + ci := lookupCommandInfo(commandName) + ac.state = (ac.state | ci.Set) &^ ci.Clear + return cwt.DoWithTimeout(timeout, commandName, args...) +} + +func (ac *activeConn) Send(commandName string, args ...interface{}) error { + pc := ac.pc + if pc == nil { + return errConnClosed + } + ci := lookupCommandInfo(commandName) + ac.state = (ac.state | ci.Set) &^ ci.Clear + return pc.c.Send(commandName, args...) +} + +func (ac *activeConn) Flush() error { + pc := ac.pc + if pc == nil { + return errConnClosed + } + return pc.c.Flush() +} + +func (ac *activeConn) Receive() (reply interface{}, err error) { + pc := ac.pc + if pc == nil { + return nil, errConnClosed + } + return pc.c.Receive() +} + +func (ac *activeConn) ReceiveWithTimeout(timeout time.Duration) (reply interface{}, err error) { + pc := ac.pc + if pc == nil { + return nil, errConnClosed + } + cwt, ok := pc.c.(ConnWithTimeout) + if !ok { + return nil, errTimeoutNotSupported + } + return cwt.ReceiveWithTimeout(timeout) +} + +func (ac *activeConn) SetReadBuffer(dst []byte) { + pc := ac.pc + pc.c.SetReadBuffer(dst) +} + +func (ac *activeConn) UnsetReadBuffer() { + pc := ac.pc + pc.c.UnsetReadBuffer() +} + +type errorConn struct{ err error } + +func (ec errorConn) Do(string, ...interface{}) (interface{}, error) { return nil, ec.err } +func (ec errorConn) DoWithTimeout(time.Duration, string, ...interface{}) (interface{}, error) { + return nil, ec.err +} +func (ec errorConn) Send(string, ...interface{}) error { return ec.err } +func (ec errorConn) Err() error { return ec.err } +func (ec errorConn) Close() error { return nil } +func (ec errorConn) Flush() error { return ec.err } +func (ec errorConn) Receive() (interface{}, error) { return nil, ec.err } +func (ec errorConn) ReceiveWithTimeout(time.Duration) (interface{}, error) { return nil, ec.err } +func (ec errorConn) SetReadBuffer([]byte) { return } +func (ec errorConn) UnsetReadBuffer() { return } + +type idleList struct { + count int + front, back *poolConn +} + +type poolConn struct { + c Conn + t time.Time + created time.Time + next, prev *poolConn +} + +func (l *idleList) pushFront(pc *poolConn) { + pc.next = l.front + pc.prev = nil + if l.count == 0 { + l.back = pc + } else { + l.front.prev = pc + } + l.front = pc + l.count++ + return +} + +func (l *idleList) popFront() { + pc := l.front + l.count-- + if l.count == 0 { + l.front, l.back = nil, nil + } else { + pc.next.prev = nil + l.front = pc.next + } + pc.next, pc.prev = nil, nil +} + +func (l *idleList) popBack() { + pc := l.back + l.count-- + if l.count == 0 { + l.front, l.back = nil, nil + } else { + pc.prev.next = nil + l.back = pc.prev + } + pc.next, pc.prev = nil, nil +} diff --git a/src/go/redigo/redis/pool_test.go b/src/go/redigo/redis/pool_test.go new file mode 100644 index 0000000..bdb0050 --- /dev/null +++ b/src/go/redigo/redis/pool_test.go @@ -0,0 +1,875 @@ +// Copyright 2011 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis_test + +import ( + "context" + "errors" + "io" + "reflect" + "sync" + "testing" + "time" + + "github.com/cea-hpc/pdwfs/redigo/redis" +) + +const ( + testGoRoutines = 10 +) + +type poolTestConn struct { + d *poolDialer + err error + redis.Conn +} + +func (c *poolTestConn) Close() error { + c.d.mu.Lock() + c.d.open -= 1 + c.d.mu.Unlock() + return c.Conn.Close() +} + +func (c *poolTestConn) Err() error { return c.err } + +func (c *poolTestConn) Do(commandName string, args ...interface{}) (interface{}, error) { + if commandName == "ERR" { + c.err = args[0].(error) + commandName = "PING" + } + if commandName != "" { + c.d.commands = append(c.d.commands, commandName) + } + return c.Conn.Do(commandName, args...) +} + +func (c *poolTestConn) Send(commandName string, args ...interface{}) error { + c.d.commands = append(c.d.commands, commandName) + return c.Conn.Send(commandName, args...) +} + +type poolDialer struct { + mu sync.Mutex + t *testing.T + dialed int + open int + commands []string + dialErr error +} + +func (d *poolDialer) dial() (redis.Conn, error) { + d.mu.Lock() + d.dialed += 1 + dialErr := d.dialErr + d.mu.Unlock() + if dialErr != nil { + return nil, d.dialErr + } + c, err := redis.DialDefaultServer() + if err != nil { + return nil, err + } + d.mu.Lock() + d.open += 1 + d.mu.Unlock() + return &poolTestConn{d: d, Conn: c}, nil +} + +func (d *poolDialer) dialContext(ctx context.Context) (redis.Conn, error) { + return d.dial() +} + +func (d *poolDialer) check(message string, p *redis.Pool, dialed, open, inuse int) { + d.checkAll(message, p, dialed, open, inuse, 0, 0) +} + +func (d *poolDialer) checkAll(message string, p *redis.Pool, dialed, open, inuse int, waitCountMax int64, waitDurationMax time.Duration) { + d.mu.Lock() + defer d.mu.Unlock() + + if d.dialed != dialed { + d.t.Errorf("%s: dialed=%d, want %d", message, d.dialed, dialed) + } + if d.open != open { + d.t.Errorf("%s: open=%d, want %d", message, d.open, open) + } + + stats := p.Stats() + + if stats.ActiveCount != open { + d.t.Errorf("%s: active=%d, want %d", message, stats.ActiveCount, open) + } + if stats.IdleCount != open-inuse { + d.t.Errorf("%s: idle=%d, want %d", message, stats.IdleCount, open-inuse) + } + + if stats.WaitCount > waitCountMax { + d.t.Errorf("%s: unexpected wait=%d want at most %d", message, stats.WaitCount, waitCountMax) + } + + if waitCountMax == 0 { + if stats.WaitDuration != 0 { + d.t.Errorf("%s: unexpected waitDuration=%v want %v", message, stats.WaitDuration, 0) + } + return + } + + if stats.WaitDuration > waitDurationMax { + d.t.Errorf("%s: unexpected waitDuration=%v want < %v", message, stats.WaitDuration, waitDurationMax) + } +} + +func TestPoolReuse(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 2, + Dial: d.dial, + } + + for i := 0; i < 10; i++ { + c1 := p.Get() + c1.Do("PING") + c2 := p.Get() + c2.Do("PING") + c1.Close() + c2.Close() + } + + d.check("before close", p, 2, 2, 0) + p.Close() + d.check("after close", p, 2, 0, 0) +} + +func TestPoolMaxIdle(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 2, + Dial: d.dial, + } + defer p.Close() + + for i := 0; i < 10; i++ { + c1 := p.Get() + c1.Do("PING") + c2 := p.Get() + c2.Do("PING") + c3 := p.Get() + c3.Do("PING") + c1.Close() + c2.Close() + c3.Close() + } + d.check("before close", p, 12, 2, 0) + p.Close() + d.check("after close", p, 12, 0, 0) +} + +func TestPoolError(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 2, + Dial: d.dial, + } + defer p.Close() + + c := p.Get() + c.Do("ERR", io.EOF) + if c.Err() == nil { + t.Errorf("expected c.Err() != nil") + } + c.Close() + + c = p.Get() + c.Do("ERR", io.EOF) + c.Close() + + d.check(".", p, 2, 0, 0) +} + +func TestPoolClose(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 2, + Dial: d.dial, + } + defer p.Close() + + c1 := p.Get() + c1.Do("PING") + c2 := p.Get() + c2.Do("PING") + c3 := p.Get() + c3.Do("PING") + + c1.Close() + if _, err := c1.Do("PING"); err == nil { + t.Errorf("expected error after connection closed") + } + + c2.Close() + c2.Close() + + p.Close() + + d.check("after pool close", p, 3, 1, 1) + + if _, err := c1.Do("PING"); err == nil { + t.Errorf("expected error after connection and pool closed") + } + + c3.Close() + + d.check("after conn close", p, 3, 0, 0) + + c1 = p.Get() + if _, err := c1.Do("PING"); err == nil { + t.Errorf("expected error after pool closed") + } +} + +func TestPoolClosedConn(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 2, + IdleTimeout: 300 * time.Second, + Dial: d.dial, + } + defer p.Close() + c := p.Get() + if c.Err() != nil { + t.Fatal("get failed") + } + c.Close() + if err := c.Err(); err == nil { + t.Fatal("Err on closed connection did not return error") + } + if _, err := c.Do("PING"); err == nil { + t.Fatal("Do on closed connection did not return error") + } + if err := c.Send("PING"); err == nil { + t.Fatal("Send on closed connection did not return error") + } + if err := c.Flush(); err == nil { + t.Fatal("Flush on closed connection did not return error") + } + if _, err := c.Receive(); err == nil { + t.Fatal("Receive on closed connection did not return error") + } +} + +func TestPoolIdleTimeout(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 2, + IdleTimeout: 300 * time.Second, + Dial: d.dial, + } + defer p.Close() + + now := time.Now() + redis.SetNowFunc(func() time.Time { return now }) + defer redis.SetNowFunc(time.Now) + + c := p.Get() + c.Do("PING") + c.Close() + + d.check("1", p, 1, 1, 0) + + now = now.Add(p.IdleTimeout + 1) + + c = p.Get() + c.Do("PING") + c.Close() + + d.check("2", p, 2, 1, 0) +} + +func TestPoolMaxLifetime(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 2, + MaxConnLifetime: 300 * time.Second, + Dial: d.dial, + } + defer p.Close() + + now := time.Now() + redis.SetNowFunc(func() time.Time { return now }) + defer redis.SetNowFunc(time.Now) + + c := p.Get() + c.Do("PING") + c.Close() + + d.check("1", p, 1, 1, 0) + + now = now.Add(p.MaxConnLifetime + 1) + + c = p.Get() + c.Do("PING") + c.Close() + + d.check("2", p, 2, 1, 0) +} + +func TestPoolConcurrenSendReceive(t *testing.T) { + p := &redis.Pool{ + Dial: func() (redis.Conn, error) { return redis.DialDefaultServer() }, + } + defer p.Close() + + c := p.Get() + done := make(chan error, 1) + go func() { + _, err := c.Receive() + done <- err + }() + c.Send("PING") + c.Flush() + err := <-done + if err != nil { + t.Fatalf("Receive() returned error %v", err) + } + _, err = c.Do("") + if err != nil { + t.Fatalf("Do() returned error %v", err) + } + c.Close() +} + +func TestPoolBorrowCheck(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 2, + Dial: d.dial, + TestOnBorrow: func(redis.Conn, time.Time) error { return redis.Error("BLAH") }, + } + defer p.Close() + + for i := 0; i < 10; i++ { + c := p.Get() + c.Do("PING") + c.Close() + } + d.check("1", p, 10, 1, 0) +} + +func TestPoolMaxActive(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 2, + MaxActive: 2, + Dial: d.dial, + } + defer p.Close() + + c1 := p.Get() + c1.Do("PING") + c2 := p.Get() + c2.Do("PING") + + d.check("1", p, 2, 2, 2) + + c3 := p.Get() + if _, err := c3.Do("PING"); err != redis.ErrPoolExhausted { + t.Errorf("expected pool exhausted") + } + + c3.Close() + d.check("2", p, 2, 2, 2) + c2.Close() + d.check("3", p, 2, 2, 1) + + c3 = p.Get() + if _, err := c3.Do("PING"); err != nil { + t.Errorf("expected good channel, err=%v", err) + } + c3.Close() + + d.check("4", p, 2, 2, 1) +} + +func TestPoolWaitStats(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + Wait: true, + MaxIdle: 2, + MaxActive: 2, + Dial: d.dial, + } + defer p.Close() + + c1 := p.Get() + c1.Do("PING") + c2 := p.Get() + c2.Do("PING") + + d.checkAll("1", p, 2, 2, 2, 0, 0) + + start := time.Now() + go func() { + time.Sleep(time.Millisecond * 100) + c1.Close() + }() + + c3 := p.Get() + d.checkAll("2", p, 2, 2, 2, 1, time.Since(start)) + + if _, err := c3.Do("PING"); err != nil { + t.Errorf("expected good channel, err=%v", err) + } +} + +func TestPoolMonitorCleanup(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 2, + MaxActive: 2, + Dial: d.dial, + } + defer p.Close() + + c := p.Get() + c.Send("MONITOR") + c.Close() + + d.check("", p, 1, 0, 0) +} + +func TestPoolPubSubCleanup(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 2, + MaxActive: 2, + Dial: d.dial, + } + defer p.Close() + + c := p.Get() + c.Send("SUBSCRIBE", "x") + c.Close() + + want := []string{"SUBSCRIBE", "UNSUBSCRIBE", "PUNSUBSCRIBE", "ECHO"} + if !reflect.DeepEqual(d.commands, want) { + t.Errorf("got commands %v, want %v", d.commands, want) + } + d.commands = nil + + c = p.Get() + c.Send("PSUBSCRIBE", "x*") + c.Close() + + want = []string{"PSUBSCRIBE", "UNSUBSCRIBE", "PUNSUBSCRIBE", "ECHO"} + if !reflect.DeepEqual(d.commands, want) { + t.Errorf("got commands %v, want %v", d.commands, want) + } + d.commands = nil +} + +func TestPoolTransactionCleanup(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 2, + MaxActive: 2, + Dial: d.dial, + } + defer p.Close() + + c := p.Get() + c.Do("WATCH", "key") + c.Do("PING") + c.Close() + + want := []string{"WATCH", "PING", "UNWATCH"} + if !reflect.DeepEqual(d.commands, want) { + t.Errorf("got commands %v, want %v", d.commands, want) + } + d.commands = nil + + c = p.Get() + c.Do("WATCH", "key") + c.Do("UNWATCH") + c.Do("PING") + c.Close() + + want = []string{"WATCH", "UNWATCH", "PING"} + if !reflect.DeepEqual(d.commands, want) { + t.Errorf("got commands %v, want %v", d.commands, want) + } + d.commands = nil + + c = p.Get() + c.Do("WATCH", "key") + c.Do("MULTI") + c.Do("PING") + c.Close() + + want = []string{"WATCH", "MULTI", "PING", "DISCARD"} + if !reflect.DeepEqual(d.commands, want) { + t.Errorf("got commands %v, want %v", d.commands, want) + } + d.commands = nil + + c = p.Get() + c.Do("WATCH", "key") + c.Do("MULTI") + c.Do("DISCARD") + c.Do("PING") + c.Close() + + want = []string{"WATCH", "MULTI", "DISCARD", "PING"} + if !reflect.DeepEqual(d.commands, want) { + t.Errorf("got commands %v, want %v", d.commands, want) + } + d.commands = nil + + c = p.Get() + c.Do("WATCH", "key") + c.Do("MULTI") + c.Do("EXEC") + c.Do("PING") + c.Close() + + want = []string{"WATCH", "MULTI", "EXEC", "PING"} + if !reflect.DeepEqual(d.commands, want) { + t.Errorf("got commands %v, want %v", d.commands, want) + } + d.commands = nil +} + +func startGoroutines(p *redis.Pool, cmd string, args ...interface{}) chan error { + errs := make(chan error, testGoRoutines) + for i := 0; i < cap(errs); i++ { + go func() { + c := p.Get() + _, err := c.Do(cmd, args...) + c.Close() + errs <- err + }() + } + + return errs +} + +func TestWaitPool(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 1, + MaxActive: 1, + Dial: d.dial, + Wait: true, + } + defer p.Close() + + c := p.Get() + start := time.Now() + errs := startGoroutines(p, "PING") + d.check("before close", p, 1, 1, 1) + c.Close() + timeout := time.After(2 * time.Second) + for i := 0; i < cap(errs); i++ { + select { + case err := <-errs: + if err != nil { + t.Fatal(err) + } + case <-timeout: + t.Fatalf("timeout waiting for blocked goroutine %d", i) + } + } + d.checkAll("done", p, 1, 1, 0, testGoRoutines, time.Since(start)*testGoRoutines) +} + +func TestWaitPoolClose(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 1, + MaxActive: 1, + Dial: d.dial, + Wait: true, + } + defer p.Close() + + c := p.Get() + if _, err := c.Do("PING"); err != nil { + t.Fatal(err) + } + start := time.Now() + errs := startGoroutines(p, "PING") + d.check("before close", p, 1, 1, 1) + p.Close() + timeout := time.After(2 * time.Second) + for i := 0; i < cap(errs); i++ { + select { + case err := <-errs: + switch err { + case nil: + t.Fatal("blocked goroutine did not get error") + case redis.ErrPoolExhausted: + t.Fatal("blocked goroutine got pool exhausted error") + } + case <-timeout: + t.Fatal("timeout waiting for blocked goroutine") + } + } + c.Close() + d.checkAll("done", p, 1, 0, 0, testGoRoutines, time.Since(start)*testGoRoutines) +} + +func TestWaitPoolCommandError(t *testing.T) { + testErr := errors.New("test") + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 1, + MaxActive: 1, + Dial: d.dial, + Wait: true, + } + defer p.Close() + + c := p.Get() + start := time.Now() + errs := startGoroutines(p, "ERR", testErr) + d.check("before close", p, 1, 1, 1) + c.Close() + timeout := time.After(2 * time.Second) + for i := 0; i < cap(errs); i++ { + select { + case err := <-errs: + if err != nil { + t.Fatal(err) + } + case <-timeout: + t.Fatalf("timeout waiting for blocked goroutine %d", i) + } + } + d.checkAll("done", p, cap(errs), 0, 0, testGoRoutines, time.Since(start)*testGoRoutines) +} + +func TestWaitPoolDialError(t *testing.T) { + testErr := errors.New("test") + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 1, + MaxActive: 1, + Dial: d.dial, + Wait: true, + } + defer p.Close() + + c := p.Get() + start := time.Now() + errs := startGoroutines(p, "ERR", testErr) + d.check("before close", p, 1, 1, 1) + + d.dialErr = errors.New("dial") + c.Close() + + nilCount := 0 + errCount := 0 + timeout := time.After(2 * time.Second) + for i := 0; i < cap(errs); i++ { + select { + case err := <-errs: + switch err { + case nil: + nilCount++ + case d.dialErr: + errCount++ + default: + t.Fatalf("expected dial error or nil, got %v", err) + } + case <-timeout: + t.Fatalf("timeout waiting for blocked goroutine %d", i) + } + } + if nilCount != 1 { + t.Errorf("expected one nil error, got %d", nilCount) + } + if errCount != cap(errs)-1 { + t.Errorf("expected %d dial errors, got %d", cap(errs)-1, errCount) + } + d.checkAll("done", p, cap(errs), 0, 0, testGoRoutines, time.Since(start)*testGoRoutines) +} + +// Borrowing requires us to iterate over the idle connections, unlock the pool, +// and perform a blocking operation to check the connection still works. If +// TestOnBorrow fails, we must reacquire the lock and continue iteration. This +// test ensures that iteration will work correctly if multiple threads are +// iterating simultaneously. +func TestLocking_TestOnBorrowFails_PoolDoesntCrash(t *testing.T) { + const count = 100 + + // First we'll Create a pool where the pilfering of idle connections fails. + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: count, + MaxActive: count, + Dial: d.dial, + TestOnBorrow: func(c redis.Conn, t time.Time) error { + return errors.New("No way back into the real world.") + }, + } + defer p.Close() + + // Fill the pool with idle connections. + conns := make([]redis.Conn, count) + for i := range conns { + conns[i] = p.Get() + } + for i := range conns { + conns[i].Close() + } + + // Spawn a bunch of goroutines to thrash the pool. + var wg sync.WaitGroup + wg.Add(count) + for i := 0; i < count; i++ { + go func() { + c := p.Get() + if c.Err() != nil { + t.Errorf("pool get failed: %v", c.Err()) + } + c.Close() + wg.Done() + }() + } + wg.Wait() + if d.dialed != count*2 { + t.Errorf("Expected %d dials, got %d", count*2, d.dialed) + } +} + +func BenchmarkPoolGet(b *testing.B) { + b.StopTimer() + p := redis.Pool{Dial: func() (redis.Conn, error) { return redis.DialDefaultServer() }, MaxIdle: 2} + c := p.Get() + if err := c.Err(); err != nil { + b.Fatal(err) + } + c.Close() + defer p.Close() + b.StartTimer() + for i := 0; i < b.N; i++ { + c = p.Get() + c.Close() + } +} + +func BenchmarkPoolGetErr(b *testing.B) { + b.StopTimer() + p := redis.Pool{Dial: func() (redis.Conn, error) { return redis.DialDefaultServer() }, MaxIdle: 2} + c := p.Get() + if err := c.Err(); err != nil { + b.Fatal(err) + } + c.Close() + defer p.Close() + b.StartTimer() + for i := 0; i < b.N; i++ { + c = p.Get() + if err := c.Err(); err != nil { + b.Fatal(err) + } + c.Close() + } +} + +func BenchmarkPoolGetPing(b *testing.B) { + b.StopTimer() + p := redis.Pool{Dial: func() (redis.Conn, error) { return redis.DialDefaultServer() }, MaxIdle: 2} + c := p.Get() + if err := c.Err(); err != nil { + b.Fatal(err) + } + c.Close() + defer p.Close() + b.StartTimer() + for i := 0; i < b.N; i++ { + c = p.Get() + if _, err := c.Do("PING"); err != nil { + b.Fatal(err) + } + c.Close() + } +} + +func TestWaitPoolGetContext(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 1, + MaxActive: 1, + Dial: d.dial, + Wait: true, + } + defer p.Close() + c, err := p.GetContext(context.Background()) + if err != nil { + t.Fatalf("GetContext returned %v", err) + } + defer c.Close() +} + +func TestWaitPoolGetContextWithDialContext(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 1, + MaxActive: 1, + DialContext: d.dialContext, + Wait: true, + } + defer p.Close() + c, err := p.GetContext(context.Background()) + if err != nil { + t.Fatalf("GetContext returned %v", err) + } + defer c.Close() +} + +func TestWaitPoolGetAfterClose(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 1, + MaxActive: 1, + Dial: d.dial, + Wait: true, + } + p.Close() + _, err := p.GetContext(context.Background()) + if err == nil { + t.Fatal("expected error") + } +} + +func TestWaitPoolGetCanceledContext(t *testing.T) { + d := poolDialer{t: t} + p := &redis.Pool{ + MaxIdle: 1, + MaxActive: 1, + Dial: d.dial, + Wait: true, + } + defer p.Close() + ctx, f := context.WithCancel(context.Background()) + f() + c := p.Get() + defer c.Close() + _, err := p.GetContext(ctx) + if err != context.Canceled { + t.Fatalf("got error %v, want %v", err, context.Canceled) + } +} diff --git a/src/go/redigo/redis/pubsub.go b/src/go/redigo/redis/pubsub.go new file mode 100644 index 0000000..2da6021 --- /dev/null +++ b/src/go/redigo/redis/pubsub.go @@ -0,0 +1,148 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "errors" + "time" +) + +// Subscription represents a subscribe or unsubscribe notification. +type Subscription struct { + // Kind is "subscribe", "unsubscribe", "psubscribe" or "punsubscribe" + Kind string + + // The channel that was changed. + Channel string + + // The current number of subscriptions for connection. + Count int +} + +// Message represents a message notification. +type Message struct { + // The originating channel. + Channel string + + // The matched pattern, if any + Pattern string + + // The message data. + Data []byte +} + +// Pong represents a pubsub pong notification. +type Pong struct { + Data string +} + +// PubSubConn wraps a Conn with convenience methods for subscribers. +type PubSubConn struct { + Conn Conn +} + +// Close closes the connection. +func (c PubSubConn) Close() error { + return c.Conn.Close() +} + +// Subscribe subscribes the connection to the specified channels. +func (c PubSubConn) Subscribe(channel ...interface{}) error { + c.Conn.Send("SUBSCRIBE", channel...) + return c.Conn.Flush() +} + +// PSubscribe subscribes the connection to the given patterns. +func (c PubSubConn) PSubscribe(channel ...interface{}) error { + c.Conn.Send("PSUBSCRIBE", channel...) + return c.Conn.Flush() +} + +// Unsubscribe unsubscribes the connection from the given channels, or from all +// of them if none is given. +func (c PubSubConn) Unsubscribe(channel ...interface{}) error { + c.Conn.Send("UNSUBSCRIBE", channel...) + return c.Conn.Flush() +} + +// PUnsubscribe unsubscribes the connection from the given patterns, or from all +// of them if none is given. +func (c PubSubConn) PUnsubscribe(channel ...interface{}) error { + c.Conn.Send("PUNSUBSCRIBE", channel...) + return c.Conn.Flush() +} + +// Ping sends a PING to the server with the specified data. +// +// The connection must be subscribed to at least one channel or pattern when +// calling this method. +func (c PubSubConn) Ping(data string) error { + c.Conn.Send("PING", data) + return c.Conn.Flush() +} + +// Receive returns a pushed message as a Subscription, Message, Pong or error. +// The return value is intended to be used directly in a type switch as +// illustrated in the PubSubConn example. +func (c PubSubConn) Receive() interface{} { + return c.receiveInternal(c.Conn.Receive()) +} + +// ReceiveWithTimeout is like Receive, but it allows the application to +// override the connection's default timeout. +func (c PubSubConn) ReceiveWithTimeout(timeout time.Duration) interface{} { + return c.receiveInternal(ReceiveWithTimeout(c.Conn, timeout)) +} + +func (c PubSubConn) receiveInternal(replyArg interface{}, errArg error) interface{} { + reply, err := Values(replyArg, errArg) + if err != nil { + return err + } + + var kind string + reply, err = Scan(reply, &kind) + if err != nil { + return err + } + + switch kind { + case "message": + var m Message + if _, err := Scan(reply, &m.Channel, &m.Data); err != nil { + return err + } + return m + case "pmessage": + var m Message + if _, err := Scan(reply, &m.Pattern, &m.Channel, &m.Data); err != nil { + return err + } + return m + case "subscribe", "psubscribe", "unsubscribe", "punsubscribe": + s := Subscription{Kind: kind} + if _, err := Scan(reply, &s.Channel, &s.Count); err != nil { + return err + } + return s + case "pong": + var p Pong + if _, err := Scan(reply, &p.Data); err != nil { + return err + } + return p + } + return errors.New("redigo: unknown pubsub notification") +} diff --git a/src/go/redigo/redis/pubsub_example_test.go b/src/go/redigo/redis/pubsub_example_test.go new file mode 100644 index 0000000..a6b3d77 --- /dev/null +++ b/src/go/redigo/redis/pubsub_example_test.go @@ -0,0 +1,165 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// +build go1.7 + +package redis_test + +import ( + "context" + "fmt" + "time" + + "github.com/cea-hpc/pdwfs/redigo/redis" +) + +// listenPubSubChannels listens for messages on Redis pubsub channels. The +// onStart function is called after the channels are subscribed. The onMessage +// function is called for each message. +func listenPubSubChannels(ctx context.Context, redisServerAddr string, + onStart func() error, + onMessage func(channel string, data []byte) error, + channels ...string) error { + // A ping is set to the server with this period to test for the health of + // the connection and server. + const healthCheckPeriod = time.Minute + + c, err := redis.Dial("tcp", redisServerAddr, + // Read timeout on server should be greater than ping period. + redis.DialReadTimeout(healthCheckPeriod+10*time.Second), + redis.DialWriteTimeout(10*time.Second)) + if err != nil { + return err + } + defer c.Close() + + psc := redis.PubSubConn{Conn: c} + + if err := psc.Subscribe(redis.Args{}.AddFlat(channels)...); err != nil { + return err + } + + done := make(chan error, 1) + + // Start a goroutine to receive notifications from the server. + go func() { + for { + switch n := psc.Receive().(type) { + case error: + done <- n + return + case redis.Message: + if err := onMessage(n.Channel, n.Data); err != nil { + done <- err + return + } + case redis.Subscription: + switch n.Count { + case len(channels): + // Notify application when all channels are subscribed. + if err := onStart(); err != nil { + done <- err + return + } + case 0: + // Return from the goroutine when all channels are unsubscribed. + done <- nil + return + } + } + } + }() + + ticker := time.NewTicker(healthCheckPeriod) + defer ticker.Stop() +loop: + for err == nil { + select { + case <-ticker.C: + // Send ping to test health of connection and server. If + // corresponding pong is not received, then receive on the + // connection will timeout and the receive goroutine will exit. + if err = psc.Ping(""); err != nil { + break loop + } + case <-ctx.Done(): + break loop + case err := <-done: + // Return error from the receive goroutine. + return err + } + } + + // Signal the receiving goroutine to exit by unsubscribing from all channels. + psc.Unsubscribe() + + // Wait for goroutine to complete. + return <-done +} + +func publish() { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + + c.Do("PUBLISH", "c1", "hello") + c.Do("PUBLISH", "c2", "world") + c.Do("PUBLISH", "c1", "goodbye") +} + +// This example shows how receive pubsub notifications with cancelation and +// health checks. +func ExamplePubSubConn() { + redisServerAddr, err := serverAddr() + if err != nil { + fmt.Println(err) + return + } + + ctx, cancel := context.WithCancel(context.Background()) + + err = listenPubSubChannels(ctx, + redisServerAddr, + func() error { + // The start callback is a good place to backfill missed + // notifications. For the purpose of this example, a goroutine is + // started to send notifications. + go publish() + return nil + }, + func(channel string, message []byte) error { + fmt.Printf("channel: %s, message: %s\n", channel, message) + + // For the purpose of this example, cancel the listener's context + // after receiving last message sent by publish(). + if string(message) == "goodbye" { + cancel() + } + return nil + }, + "c1", "c2") + + if err != nil { + fmt.Println(err) + return + } + + // Output: + // channel: c1, message: hello + // channel: c2, message: world + // channel: c1, message: goodbye +} diff --git a/src/go/redigo/redis/pubsub_test.go b/src/go/redigo/redis/pubsub_test.go new file mode 100644 index 0000000..d716f64 --- /dev/null +++ b/src/go/redigo/redis/pubsub_test.go @@ -0,0 +1,74 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis_test + +import ( + "reflect" + "testing" + "time" + + "github.com/cea-hpc/pdwfs/redigo/redis" +) + +func expectPushed(t *testing.T, c redis.PubSubConn, message string, expected interface{}) { + actual := c.Receive() + if !reflect.DeepEqual(actual, expected) { + t.Errorf("%s = %v, want %v", message, actual, expected) + } +} + +func TestPushed(t *testing.T) { + pc, err := redis.DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer pc.Close() + + sc, err := redis.DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer sc.Close() + + c := redis.PubSubConn{Conn: sc} + + c.Subscribe("c1") + expectPushed(t, c, "Subscribe(c1)", redis.Subscription{Kind: "subscribe", Channel: "c1", Count: 1}) + c.Subscribe("c2") + expectPushed(t, c, "Subscribe(c2)", redis.Subscription{Kind: "subscribe", Channel: "c2", Count: 2}) + c.PSubscribe("p1") + expectPushed(t, c, "PSubscribe(p1)", redis.Subscription{Kind: "psubscribe", Channel: "p1", Count: 3}) + c.PSubscribe("p2") + expectPushed(t, c, "PSubscribe(p2)", redis.Subscription{Kind: "psubscribe", Channel: "p2", Count: 4}) + c.PUnsubscribe() + expectPushed(t, c, "Punsubscribe(p1)", redis.Subscription{Kind: "punsubscribe", Channel: "p1", Count: 3}) + expectPushed(t, c, "Punsubscribe()", redis.Subscription{Kind: "punsubscribe", Channel: "p2", Count: 2}) + + pc.Do("PUBLISH", "c1", "hello") + expectPushed(t, c, "PUBLISH c1 hello", redis.Message{Channel: "c1", Data: []byte("hello")}) + + c.Ping("hello") + expectPushed(t, c, `Ping("hello")`, redis.Pong{Data: "hello"}) + + c.Conn.Send("PING") + c.Conn.Flush() + expectPushed(t, c, `Send("PING")`, redis.Pong{}) + + c.Ping("timeout") + got := c.ReceiveWithTimeout(time.Minute) + if want := (redis.Pong{Data: "timeout"}); want != got { + t.Errorf("recv /w timeout got %v, want %v", got, want) + } +} diff --git a/src/go/redigo/redis/redis.go b/src/go/redigo/redis/redis.go new file mode 100644 index 0000000..d7fae14 --- /dev/null +++ b/src/go/redigo/redis/redis.go @@ -0,0 +1,120 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "errors" + "time" +) + +// Error represents an error returned in a command reply. +type Error string + +func (err Error) Error() string { return string(err) } + +// Conn represents a connection to a Redis server. +type Conn interface { + // Close closes the connection. + Close() error + + // Err returns a non-nil value when the connection is not usable. + Err() error + + // Do sends a command to the server and returns the received reply. + Do(commandName string, args ...interface{}) (reply interface{}, err error) + + // Send writes the command to the client's output buffer. + Send(commandName string, args ...interface{}) error + + // Flush flushes the output buffer to the Redis server. + Flush() error + + // Receive receives a single reply from the Redis server + Receive() (reply interface{}, err error) + + SetReadBuffer(dst []byte) + UnsetReadBuffer() +} + +// Argument is the interface implemented by an object which wants to control how +// the object is converted to Redis bulk strings. +type Argument interface { + // RedisArg returns a value to be encoded as a bulk string per the + // conversions listed in the section 'Executing Commands'. + // Implementations should typically return a []byte or string. + RedisArg() interface{} +} + +// Scanner is implemented by an object which wants to control its value is +// interpreted when read from Redis. +type Scanner interface { + // RedisScan assigns a value from a Redis value. The argument src is one of + // the reply types listed in the section `Executing Commands`. + // + // An error should be returned if the value cannot be stored without + // loss of information. + RedisScan(src interface{}) error +} + +// ConnWithTimeout is an optional interface that allows the caller to override +// a connection's default read timeout. This interface is useful for executing +// the BLPOP, BRPOP, BRPOPLPUSH, XREAD and other commands that block at the +// server. +// +// A connection's default read timeout is set with the DialReadTimeout dial +// option. Applications should rely on the default timeout for commands that do +// not block at the server. +// +// All of the Conn implementations in this package satisfy the ConnWithTimeout +// interface. +// +// Use the DoWithTimeout and ReceiveWithTimeout helper functions to simplify +// use of this interface. +type ConnWithTimeout interface { + Conn + + // Do sends a command to the server and returns the received reply. + // The timeout overrides the read timeout set when dialing the + // connection. + DoWithTimeout(timeout time.Duration, commandName string, args ...interface{}) (reply interface{}, err error) + + // Receive receives a single reply from the Redis server. The timeout + // overrides the read timeout set when dialing the connection. + ReceiveWithTimeout(timeout time.Duration) (reply interface{}, err error) +} + +var errTimeoutNotSupported = errors.New("redis: connection does not support ConnWithTimeout") + +// DoWithTimeout executes a Redis command with the specified read timeout. If +// the connection does not satisfy the ConnWithTimeout interface, then an error +// is returned. +func DoWithTimeout(c Conn, timeout time.Duration, cmd string, args ...interface{}) (interface{}, error) { + cwt, ok := c.(ConnWithTimeout) + if !ok { + return nil, errTimeoutNotSupported + } + return cwt.DoWithTimeout(timeout, cmd, args...) +} + +// ReceiveWithTimeout receives a reply with the specified read timeout. If the +// connection does not satisfy the ConnWithTimeout interface, then an error is +// returned. +func ReceiveWithTimeout(c Conn, timeout time.Duration) (interface{}, error) { + cwt, ok := c.(ConnWithTimeout) + if !ok { + return nil, errTimeoutNotSupported + } + return cwt.ReceiveWithTimeout(timeout) +} diff --git a/src/go/redigo/redis/redis_test.go b/src/go/redigo/redis/redis_test.go new file mode 100644 index 0000000..ae76aef --- /dev/null +++ b/src/go/redigo/redis/redis_test.go @@ -0,0 +1,73 @@ +// Copyright 2017 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis_test + +import ( + "testing" + "time" + + "github.com/cea-hpc/pdwfs/redigo/redis" +) + +type timeoutTestConn int + +func (tc timeoutTestConn) Do(string, ...interface{}) (interface{}, error) { + return time.Duration(-1), nil +} +func (tc timeoutTestConn) DoWithTimeout(timeout time.Duration, cmd string, args ...interface{}) (interface{}, error) { + return timeout, nil +} + +func (tc timeoutTestConn) Receive() (interface{}, error) { + return time.Duration(-1), nil +} +func (tc timeoutTestConn) ReceiveWithTimeout(timeout time.Duration) (interface{}, error) { + return timeout, nil +} + +func (tc timeoutTestConn) Send(string, ...interface{}) error { return nil } +func (tc timeoutTestConn) Err() error { return nil } +func (tc timeoutTestConn) Close() error { return nil } +func (tc timeoutTestConn) Flush() error { return nil } +func (tc timeoutTestConn) SetReadBuffer([]byte) { return } +func (tc timeoutTestConn) UnsetReadBuffer() { return } + +func testTimeout(t *testing.T, c redis.Conn) { + r, err := c.Do("PING") + if r != time.Duration(-1) || err != nil { + t.Errorf("Do() = %v, %v, want %v, %v", r, err, time.Duration(-1), nil) + } + r, err = redis.DoWithTimeout(c, time.Minute, "PING") + if r != time.Minute || err != nil { + t.Errorf("DoWithTimeout() = %v, %v, want %v, %v", r, err, time.Minute, nil) + } + r, err = c.Receive() + if r != time.Duration(-1) || err != nil { + t.Errorf("Receive() = %v, %v, want %v, %v", r, err, time.Duration(-1), nil) + } + r, err = redis.ReceiveWithTimeout(c, time.Minute) + if r != time.Minute || err != nil { + t.Errorf("ReceiveWithTimeout() = %v, %v, want %v, %v", r, err, time.Minute, nil) + } +} + +func TestConnTimeout(t *testing.T) { + testTimeout(t, timeoutTestConn(0)) +} + +func TestPoolConnTimeout(t *testing.T) { + p := &redis.Pool{Dial: func() (redis.Conn, error) { return timeoutTestConn(0), nil }} + testTimeout(t, p.Get()) +} diff --git a/src/go/redigo/redis/reply.go b/src/go/redigo/redis/reply.go new file mode 100644 index 0000000..50971e8 --- /dev/null +++ b/src/go/redigo/redis/reply.go @@ -0,0 +1,499 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "errors" + "fmt" + "strconv" +) + +// ErrNil indicates that a reply value is nil. +var ErrNil = errors.New("redigo: nil returned") + +// Int is a helper that converts a command reply to an integer. If err is not +// equal to nil, then Int returns 0, err. Otherwise, Int converts the +// reply to an int as follows: +// +// Reply type Result +// integer int(reply), nil +// bulk string parsed reply, nil +// nil 0, ErrNil +// other 0, error +func Int(reply interface{}, err error) (int, error) { + if err != nil { + return 0, err + } + switch reply := reply.(type) { + case int64: + x := int(reply) + if int64(x) != reply { + return 0, strconv.ErrRange + } + return x, nil + case []byte: + n, err := strconv.ParseInt(string(reply), 10, 0) + return int(n), err + case nil: + return 0, ErrNil + case Error: + return 0, reply + } + return 0, fmt.Errorf("redigo: unexpected type for Int, got type %T", reply) +} + +// Int64 is a helper that converts a command reply to 64 bit integer. If err is +// not equal to nil, then Int returns 0, err. Otherwise, Int64 converts the +// reply to an int64 as follows: +// +// Reply type Result +// integer reply, nil +// bulk string parsed reply, nil +// nil 0, ErrNil +// other 0, error +func Int64(reply interface{}, err error) (int64, error) { + if err != nil { + return 0, err + } + switch reply := reply.(type) { + case int64: + return reply, nil + case []byte: + n, err := strconv.ParseInt(string(reply), 10, 64) + return n, err + case nil: + return 0, ErrNil + case Error: + return 0, reply + } + return 0, fmt.Errorf("redigo: unexpected type for Int64, got type %T", reply) +} + +var errNegativeInt = errors.New("redigo: unexpected value for Uint64") + +// Uint64 is a helper that converts a command reply to 64 bit integer. If err is +// not equal to nil, then Int returns 0, err. Otherwise, Int64 converts the +// reply to an int64 as follows: +// +// Reply type Result +// integer reply, nil +// bulk string parsed reply, nil +// nil 0, ErrNil +// other 0, error +func Uint64(reply interface{}, err error) (uint64, error) { + if err != nil { + return 0, err + } + switch reply := reply.(type) { + case int64: + if reply < 0 { + return 0, errNegativeInt + } + return uint64(reply), nil + case []byte: + n, err := strconv.ParseUint(string(reply), 10, 64) + return n, err + case nil: + return 0, ErrNil + case Error: + return 0, reply + } + return 0, fmt.Errorf("redigo: unexpected type for Uint64, got type %T", reply) +} + +// Float64 is a helper that converts a command reply to 64 bit float. If err is +// not equal to nil, then Float64 returns 0, err. Otherwise, Float64 converts +// the reply to an int as follows: +// +// Reply type Result +// bulk string parsed reply, nil +// nil 0, ErrNil +// other 0, error +func Float64(reply interface{}, err error) (float64, error) { + if err != nil { + return 0, err + } + switch reply := reply.(type) { + case []byte: + n, err := strconv.ParseFloat(string(reply), 64) + return n, err + case nil: + return 0, ErrNil + case Error: + return 0, reply + } + return 0, fmt.Errorf("redigo: unexpected type for Float64, got type %T", reply) +} + +// String is a helper that converts a command reply to a string. If err is not +// equal to nil, then String returns "", err. Otherwise String converts the +// reply to a string as follows: +// +// Reply type Result +// bulk string string(reply), nil +// simple string reply, nil +// nil "", ErrNil +// other "", error +func String(reply interface{}, err error) (string, error) { + if err != nil { + return "", err + } + switch reply := reply.(type) { + case []byte: + return string(reply), nil + case string: + return reply, nil + case nil: + return "", ErrNil + case Error: + return "", reply + } + return "", fmt.Errorf("redigo: unexpected type for String, got type %T", reply) +} + +// Bytes is a helper that converts a command reply to a slice of bytes. If err +// is not equal to nil, then Bytes returns nil, err. Otherwise Bytes converts +// the reply to a slice of bytes as follows: +// +// Reply type Result +// bulk string reply, nil +// simple string []byte(reply), nil +// nil nil, ErrNil +// other nil, error +func Bytes(reply interface{}, err error) ([]byte, error) { + if err != nil { + return nil, err + } + switch reply := reply.(type) { + case []byte: + return reply, nil + case string: + return []byte(reply), nil + case nil: + return nil, ErrNil + case Error: + return nil, reply + } + return nil, fmt.Errorf("redigo: unexpected type for Bytes, got type %T", reply) +} + +// ReadBytes is a helper that converts a command reply to the number of bytes read. +func ReadBytes(reply interface{}, err error) (int, error) { + if err != nil { + return -1, err + } + switch reply := reply.(type) { + case []byte: + return len(reply), nil + case string: + return len(reply), nil + case int: + return reply, nil + case nil: + return -1, ErrNil + case Error: + return -1, reply + } + return -1, fmt.Errorf("redigo: unexpected type for Bytes, got type %T", reply) +} + +// Bool is a helper that converts a command reply to a boolean. If err is not +// equal to nil, then Bool returns false, err. Otherwise Bool converts the +// reply to boolean as follows: +// +// Reply type Result +// integer value != 0, nil +// bulk string strconv.ParseBool(reply) +// nil false, ErrNil +// other false, error +func Bool(reply interface{}, err error) (bool, error) { + if err != nil { + return false, err + } + switch reply := reply.(type) { + case int64: + return reply != 0, nil + case []byte: + return strconv.ParseBool(string(reply)) + case nil: + return false, ErrNil + case Error: + return false, reply + } + return false, fmt.Errorf("redigo: unexpected type for Bool, got type %T", reply) +} + +// MultiBulk is a helper that converts an array command reply to a []interface{}. +// +// Deprecated: Use Values instead. +func MultiBulk(reply interface{}, err error) ([]interface{}, error) { return Values(reply, err) } + +// Values is a helper that converts an array command reply to a []interface{}. +// If err is not equal to nil, then Values returns nil, err. Otherwise, Values +// converts the reply as follows: +// +// Reply type Result +// array reply, nil +// nil nil, ErrNil +// other nil, error +func Values(reply interface{}, err error) ([]interface{}, error) { + if err != nil { + return nil, err + } + switch reply := reply.(type) { + case []interface{}: + return reply, nil + case nil: + return nil, ErrNil + case Error: + return nil, reply + } + return nil, fmt.Errorf("redigo: unexpected type for Values, got type %T", reply) +} + +func sliceHelper(reply interface{}, err error, name string, makeSlice func(int), assign func(int, interface{}) error) error { + if err != nil { + return err + } + switch reply := reply.(type) { + case []interface{}: + makeSlice(len(reply)) + for i := range reply { + if reply[i] == nil { + continue + } + if err := assign(i, reply[i]); err != nil { + return err + } + } + return nil + case nil: + return ErrNil + case Error: + return reply + } + return fmt.Errorf("redigo: unexpected type for %s, got type %T", name, reply) +} + +// Float64s is a helper that converts an array command reply to a []float64. If +// err is not equal to nil, then Float64s returns nil, err. Nil array items are +// converted to 0 in the output slice. Floats64 returns an error if an array +// item is not a bulk string or nil. +func Float64s(reply interface{}, err error) ([]float64, error) { + var result []float64 + err = sliceHelper(reply, err, "Float64s", func(n int) { result = make([]float64, n) }, func(i int, v interface{}) error { + p, ok := v.([]byte) + if !ok { + return fmt.Errorf("redigo: unexpected element type for Floats64, got type %T", v) + } + f, err := strconv.ParseFloat(string(p), 64) + result[i] = f + return err + }) + return result, err +} + +// Strings is a helper that converts an array command reply to a []string. If +// err is not equal to nil, then Strings returns nil, err. Nil array items are +// converted to "" in the output slice. Strings returns an error if an array +// item is not a bulk string or nil. +func Strings(reply interface{}, err error) ([]string, error) { + var result []string + err = sliceHelper(reply, err, "Strings", func(n int) { result = make([]string, n) }, func(i int, v interface{}) error { + switch v := v.(type) { + case string: + result[i] = v + return nil + case []byte: + result[i] = string(v) + return nil + default: + return fmt.Errorf("redigo: unexpected element type for Strings, got type %T", v) + } + }) + return result, err +} + +// ByteSlices is a helper that converts an array command reply to a [][]byte. +// If err is not equal to nil, then ByteSlices returns nil, err. Nil array +// items are stay nil. ByteSlices returns an error if an array item is not a +// bulk string or nil. +func ByteSlices(reply interface{}, err error) ([][]byte, error) { + var result [][]byte + err = sliceHelper(reply, err, "ByteSlices", func(n int) { result = make([][]byte, n) }, func(i int, v interface{}) error { + p, ok := v.([]byte) + if !ok { + return fmt.Errorf("redigo: unexpected element type for ByteSlices, got type %T", v) + } + result[i] = p + return nil + }) + return result, err +} + +// Int64s is a helper that converts an array command reply to a []int64. +// If err is not equal to nil, then Int64s returns nil, err. Nil array +// items are stay nil. Int64s returns an error if an array item is not a +// bulk string or nil. +func Int64s(reply interface{}, err error) ([]int64, error) { + var result []int64 + err = sliceHelper(reply, err, "Int64s", func(n int) { result = make([]int64, n) }, func(i int, v interface{}) error { + switch v := v.(type) { + case int64: + result[i] = v + return nil + case []byte: + n, err := strconv.ParseInt(string(v), 10, 64) + result[i] = n + return err + default: + return fmt.Errorf("redigo: unexpected element type for Int64s, got type %T", v) + } + }) + return result, err +} + +// Ints is a helper that converts an array command reply to a []in. +// If err is not equal to nil, then Ints returns nil, err. Nil array +// items are stay nil. Ints returns an error if an array item is not a +// bulk string or nil. +func Ints(reply interface{}, err error) ([]int, error) { + var result []int + err = sliceHelper(reply, err, "Ints", func(n int) { result = make([]int, n) }, func(i int, v interface{}) error { + switch v := v.(type) { + case int64: + n := int(v) + if int64(n) != v { + return strconv.ErrRange + } + result[i] = n + return nil + case []byte: + n, err := strconv.Atoi(string(v)) + result[i] = n + return err + default: + return fmt.Errorf("redigo: unexpected element type for Ints, got type %T", v) + } + }) + return result, err +} + +// StringMap is a helper that converts an array of strings (alternating key, value) +// into a map[string]string. The HGETALL and CONFIG GET commands return replies in this format. +// Requires an even number of values in result. +func StringMap(result interface{}, err error) (map[string]string, error) { + values, err := Values(result, err) + if err != nil { + return nil, err + } + if len(values)%2 != 0 { + return nil, errors.New("redigo: StringMap expects even number of values result") + } + m := make(map[string]string, len(values)/2) + for i := 0; i < len(values); i += 2 { + key, okKey := values[i].([]byte) + value, okValue := values[i+1].([]byte) + if !okKey || !okValue { + return nil, errors.New("redigo: StringMap key not a bulk string value") + } + m[string(key)] = string(value) + } + return m, nil +} + +// IntMap is a helper that converts an array of strings (alternating key, value) +// into a map[string]int. The HGETALL commands return replies in this format. +// Requires an even number of values in result. +func IntMap(result interface{}, err error) (map[string]int, error) { + values, err := Values(result, err) + if err != nil { + return nil, err + } + if len(values)%2 != 0 { + return nil, errors.New("redigo: IntMap expects even number of values result") + } + m := make(map[string]int, len(values)/2) + for i := 0; i < len(values); i += 2 { + key, ok := values[i].([]byte) + if !ok { + return nil, errors.New("redigo: IntMap key not a bulk string value") + } + value, err := Int(values[i+1], nil) + if err != nil { + return nil, err + } + m[string(key)] = value + } + return m, nil +} + +// Int64Map is a helper that converts an array of strings (alternating key, value) +// into a map[string]int64. The HGETALL commands return replies in this format. +// Requires an even number of values in result. +func Int64Map(result interface{}, err error) (map[string]int64, error) { + values, err := Values(result, err) + if err != nil { + return nil, err + } + if len(values)%2 != 0 { + return nil, errors.New("redigo: Int64Map expects even number of values result") + } + m := make(map[string]int64, len(values)/2) + for i := 0; i < len(values); i += 2 { + key, ok := values[i].([]byte) + if !ok { + return nil, errors.New("redigo: Int64Map key not a bulk string value") + } + value, err := Int64(values[i+1], nil) + if err != nil { + return nil, err + } + m[string(key)] = value + } + return m, nil +} + +// Positions is a helper that converts an array of positions (lat, long) +// into a [][2]float64. The GEOPOS command returns replies in this format. +func Positions(result interface{}, err error) ([]*[2]float64, error) { + values, err := Values(result, err) + if err != nil { + return nil, err + } + positions := make([]*[2]float64, len(values)) + for i := range values { + if values[i] == nil { + continue + } + p, ok := values[i].([]interface{}) + if !ok { + return nil, fmt.Errorf("redigo: unexpected element type for interface slice, got type %T", values[i]) + } + if len(p) != 2 { + return nil, fmt.Errorf("redigo: unexpected number of values for a member position, got %d", len(p)) + } + lat, err := Float64(p[0], nil) + if err != nil { + return nil, err + } + long, err := Float64(p[1], nil) + if err != nil { + return nil, err + } + positions[i] = &[2]float64{lat, long} + } + return positions, nil +} diff --git a/src/go/redigo/redis/reply_test.go b/src/go/redigo/redis/reply_test.go new file mode 100644 index 0000000..593600d --- /dev/null +++ b/src/go/redigo/redis/reply_test.go @@ -0,0 +1,209 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis_test + +import ( + "fmt" + "reflect" + "testing" + + "github.com/cea-hpc/pdwfs/redigo/redis" +) + +type valueError struct { + v interface{} + err error +} + +func ve(v interface{}, err error) valueError { + return valueError{v, err} +} + +var replyTests = []struct { + name interface{} + actual valueError + expected valueError +}{ + { + "ints([[]byte, []byte])", + ve(redis.Ints([]interface{}{[]byte("4"), []byte("5")}, nil)), + ve([]int{4, 5}, nil), + }, + { + "ints([nt64, int64])", + ve(redis.Ints([]interface{}{int64(4), int64(5)}, nil)), + ve([]int{4, 5}, nil), + }, + { + "ints([[]byte, nil, []byte])", + ve(redis.Ints([]interface{}{[]byte("4"), nil, []byte("5")}, nil)), + ve([]int{4, 0, 5}, nil), + }, + { + "ints(nil)", + ve(redis.Ints(nil, nil)), + ve([]int(nil), redis.ErrNil), + }, + { + "int64s([[]byte, []byte])", + ve(redis.Int64s([]interface{}{[]byte("4"), []byte("5")}, nil)), + ve([]int64{4, 5}, nil), + }, + { + "int64s([int64, int64])", + ve(redis.Int64s([]interface{}{int64(4), int64(5)}, nil)), + ve([]int64{4, 5}, nil), + }, + { + "strings([[]byte, []bytev2])", + ve(redis.Strings([]interface{}{[]byte("v1"), []byte("v2")}, nil)), + ve([]string{"v1", "v2"}, nil), + }, + { + "strings([string, string])", + ve(redis.Strings([]interface{}{"v1", "v2"}, nil)), + ve([]string{"v1", "v2"}, nil), + }, + { + "byteslices([v1, v2])", + ve(redis.ByteSlices([]interface{}{[]byte("v1"), []byte("v2")}, nil)), + ve([][]byte{[]byte("v1"), []byte("v2")}, nil), + }, + { + "float64s([v1, v2])", + ve(redis.Float64s([]interface{}{[]byte("1.234"), []byte("5.678")}, nil)), + ve([]float64{1.234, 5.678}, nil), + }, + { + "values([v1, v2])", + ve(redis.Values([]interface{}{[]byte("v1"), []byte("v2")}, nil)), + ve([]interface{}{[]byte("v1"), []byte("v2")}, nil), + }, + { + "values(nil)", + ve(redis.Values(nil, nil)), + ve([]interface{}(nil), redis.ErrNil), + }, + { + "float64(1.0)", + ve(redis.Float64([]byte("1.0"), nil)), + ve(float64(1.0), nil), + }, + { + "float64(nil)", + ve(redis.Float64(nil, nil)), + ve(float64(0.0), redis.ErrNil), + }, + { + "uint64(1)", + ve(redis.Uint64(int64(1), nil)), + ve(uint64(1), nil), + }, + { + "uint64(-1)", + ve(redis.Uint64(int64(-1), nil)), + ve(uint64(0), redis.ErrNegativeInt), + }, + { + "positions([[1, 2], nil, [3, 4]])", + ve(redis.Positions([]interface{}{[]interface{}{[]byte("1"), []byte("2")}, nil, []interface{}{[]byte("3"), []byte("4")}}, nil)), + ve([]*[2]float64{{1.0, 2.0}, nil, {3.0, 4.0}}, nil), + }, +} + +func TestReply(t *testing.T) { + for _, rt := range replyTests { + if rt.actual.err != rt.expected.err { + t.Errorf("%s returned err %v, want %v", rt.name, rt.actual.err, rt.expected.err) + continue + } + if !reflect.DeepEqual(rt.actual.v, rt.expected.v) { + t.Errorf("%s=%+v, want %+v", rt.name, rt.actual.v, rt.expected.v) + } + } +} + +// dial wraps DialDefaultServer() with a more suitable function name for examples. +func dial() (redis.Conn, error) { + return redis.DialDefaultServer() +} + +// serverAddr wraps DefaultServerAddr() with a more suitable function name for examples. +func serverAddr() (string, error) { + return redis.DefaultServerAddr() +} + +func ExampleBool() { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + + c.Do("SET", "foo", 1) + exists, _ := redis.Bool(c.Do("EXISTS", "foo")) + fmt.Printf("%#v\n", exists) + // Output: + // true +} + +func ExampleInt() { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + + c.Do("SET", "k1", 1) + n, _ := redis.Int(c.Do("GET", "k1")) + fmt.Printf("%#v\n", n) + n, _ = redis.Int(c.Do("INCR", "k1")) + fmt.Printf("%#v\n", n) + // Output: + // 1 + // 2 +} + +func ExampleInts() { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + + c.Do("SADD", "set_with_integers", 4, 5, 6) + ints, _ := redis.Ints(c.Do("SMEMBERS", "set_with_integers")) + fmt.Printf("%#v\n", ints) + // Output: + // []int{4, 5, 6} +} + +func ExampleString() { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + + c.Do("SET", "hello", "world") + s, err := redis.String(c.Do("GET", "hello")) + fmt.Printf("%#v\n", s) + // Output: + // "world" +} diff --git a/src/go/redigo/redis/scan.go b/src/go/redigo/redis/scan.go new file mode 100644 index 0000000..5227aac --- /dev/null +++ b/src/go/redigo/redis/scan.go @@ -0,0 +1,630 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "errors" + "fmt" + "reflect" + "strconv" + "strings" + "sync" +) + +func ensureLen(d reflect.Value, n int) { + if n > d.Cap() { + d.Set(reflect.MakeSlice(d.Type(), n, n)) + } else { + d.SetLen(n) + } +} + +func cannotConvert(d reflect.Value, s interface{}) error { + var sname string + switch s.(type) { + case string: + sname = "Redis simple string" + case Error: + sname = "Redis error" + case int64: + sname = "Redis integer" + case []byte: + sname = "Redis bulk string" + case []interface{}: + sname = "Redis array" + case nil: + sname = "Redis nil" + default: + sname = reflect.TypeOf(s).String() + } + return fmt.Errorf("cannot convert from %s to %s", sname, d.Type()) +} + +func convertAssignNil(d reflect.Value) (err error) { + switch d.Type().Kind() { + case reflect.Slice, reflect.Interface: + d.Set(reflect.Zero(d.Type())) + default: + err = cannotConvert(d, nil) + } + return err +} + +func convertAssignError(d reflect.Value, s Error) (err error) { + if d.Kind() == reflect.String { + d.SetString(string(s)) + } else if d.Kind() == reflect.Slice && d.Type().Elem().Kind() == reflect.Uint8 { + d.SetBytes([]byte(s)) + } else { + err = cannotConvert(d, s) + } + return +} + +func convertAssignString(d reflect.Value, s string) (err error) { + switch d.Type().Kind() { + case reflect.Float32, reflect.Float64: + var x float64 + x, err = strconv.ParseFloat(s, d.Type().Bits()) + d.SetFloat(x) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + var x int64 + x, err = strconv.ParseInt(s, 10, d.Type().Bits()) + d.SetInt(x) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + var x uint64 + x, err = strconv.ParseUint(s, 10, d.Type().Bits()) + d.SetUint(x) + case reflect.Bool: + var x bool + x, err = strconv.ParseBool(s) + d.SetBool(x) + case reflect.String: + d.SetString(s) + case reflect.Slice: + if d.Type().Elem().Kind() == reflect.Uint8 { + d.SetBytes([]byte(s)) + } else { + err = cannotConvert(d, s) + } + default: + err = cannotConvert(d, s) + } + return +} + +func convertAssignBulkString(d reflect.Value, s []byte) (err error) { + switch d.Type().Kind() { + case reflect.Slice: + // Handle []byte destination here to avoid unnecessary + // []byte -> string -> []byte converion. + if d.Type().Elem().Kind() == reflect.Uint8 { + d.SetBytes(s) + } else { + err = cannotConvert(d, s) + } + default: + err = convertAssignString(d, string(s)) + } + return err +} + +func convertAssignInt(d reflect.Value, s int64) (err error) { + switch d.Type().Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + d.SetInt(s) + if d.Int() != s { + err = strconv.ErrRange + d.SetInt(0) + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + if s < 0 { + err = strconv.ErrRange + } else { + x := uint64(s) + d.SetUint(x) + if d.Uint() != x { + err = strconv.ErrRange + d.SetUint(0) + } + } + case reflect.Bool: + d.SetBool(s != 0) + default: + err = cannotConvert(d, s) + } + return +} + +func convertAssignValue(d reflect.Value, s interface{}) (err error) { + if d.Kind() != reflect.Ptr { + if d.CanAddr() { + d2 := d.Addr() + if d2.CanInterface() { + if scanner, ok := d2.Interface().(Scanner); ok { + return scanner.RedisScan(s) + } + } + } + } else if d.CanInterface() { + // Already a reflect.Ptr + if d.IsNil() { + d.Set(reflect.New(d.Type().Elem())) + } + if scanner, ok := d.Interface().(Scanner); ok { + return scanner.RedisScan(s) + } + } + + switch s := s.(type) { + case nil: + err = convertAssignNil(d) + case []byte: + err = convertAssignBulkString(d, s) + case int64: + err = convertAssignInt(d, s) + case string: + err = convertAssignString(d, s) + case Error: + err = convertAssignError(d, s) + default: + err = cannotConvert(d, s) + } + return err +} + +func convertAssignArray(d reflect.Value, s []interface{}) error { + if d.Type().Kind() != reflect.Slice { + return cannotConvert(d, s) + } + ensureLen(d, len(s)) + for i := 0; i < len(s); i++ { + if err := convertAssignValue(d.Index(i), s[i]); err != nil { + return err + } + } + return nil +} + +func convertAssign(d interface{}, s interface{}) (err error) { + if scanner, ok := d.(Scanner); ok { + return scanner.RedisScan(s) + } + + // Handle the most common destination types using type switches and + // fall back to reflection for all other types. + switch s := s.(type) { + case nil: + // ignore + case []byte: + switch d := d.(type) { + case *string: + *d = string(s) + case *int: + *d, err = strconv.Atoi(string(s)) + case *bool: + *d, err = strconv.ParseBool(string(s)) + case *[]byte: + *d = s + case *interface{}: + *d = s + case nil: + // skip value + default: + if d := reflect.ValueOf(d); d.Type().Kind() != reflect.Ptr { + err = cannotConvert(d, s) + } else { + err = convertAssignBulkString(d.Elem(), s) + } + } + case int64: + switch d := d.(type) { + case *int: + x := int(s) + if int64(x) != s { + err = strconv.ErrRange + x = 0 + } + *d = x + case *bool: + *d = s != 0 + case *interface{}: + *d = s + case nil: + // skip value + default: + if d := reflect.ValueOf(d); d.Type().Kind() != reflect.Ptr { + err = cannotConvert(d, s) + } else { + err = convertAssignInt(d.Elem(), s) + } + } + case string: + switch d := d.(type) { + case *string: + *d = s + case *interface{}: + *d = s + case nil: + // skip value + default: + err = cannotConvert(reflect.ValueOf(d), s) + } + case []interface{}: + switch d := d.(type) { + case *[]interface{}: + *d = s + case *interface{}: + *d = s + case nil: + // skip value + default: + if d := reflect.ValueOf(d); d.Type().Kind() != reflect.Ptr { + err = cannotConvert(d, s) + } else { + err = convertAssignArray(d.Elem(), s) + } + } + case Error: + err = s + default: + err = cannotConvert(reflect.ValueOf(d), s) + } + return +} + +// Scan copies from src to the values pointed at by dest. +// +// Scan uses RedisScan if available otherwise: +// +// The values pointed at by dest must be an integer, float, boolean, string, +// []byte, interface{} or slices of these types. Scan uses the standard strconv +// package to convert bulk strings to numeric and boolean types. +// +// If a dest value is nil, then the corresponding src value is skipped. +// +// If a src element is nil, then the corresponding dest value is not modified. +// +// To enable easy use of Scan in a loop, Scan returns the slice of src +// following the copied values. +func Scan(src []interface{}, dest ...interface{}) ([]interface{}, error) { + if len(src) < len(dest) { + return nil, errors.New("redigo.Scan: array short") + } + var err error + for i, d := range dest { + err = convertAssign(d, src[i]) + if err != nil { + err = fmt.Errorf("redigo.Scan: cannot assign to dest %d: %v", i, err) + break + } + } + return src[len(dest):], err +} + +type fieldSpec struct { + name string + index []int + omitEmpty bool +} + +type structSpec struct { + m map[string]*fieldSpec + l []*fieldSpec +} + +func (ss *structSpec) fieldSpec(name []byte) *fieldSpec { + return ss.m[string(name)] +} + +func compileStructSpec(t reflect.Type, depth map[string]int, index []int, ss *structSpec) { + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + switch { + case f.PkgPath != "" && !f.Anonymous: + // Ignore unexported fields. + case f.Anonymous: + // TODO: Handle pointers. Requires change to decoder and + // protection against infinite recursion. + if f.Type.Kind() == reflect.Struct { + compileStructSpec(f.Type, depth, append(index, i), ss) + } + default: + fs := &fieldSpec{name: f.Name} + tag := f.Tag.Get("redis") + p := strings.Split(tag, ",") + if len(p) > 0 { + if p[0] == "-" { + continue + } + if len(p[0]) > 0 { + fs.name = p[0] + } + for _, s := range p[1:] { + switch s { + case "omitempty": + fs.omitEmpty = true + default: + panic(fmt.Errorf("redigo: unknown field tag %s for type %s", s, t.Name())) + } + } + } + d, found := depth[fs.name] + if !found { + d = 1 << 30 + } + switch { + case len(index) == d: + // At same depth, remove from result. + delete(ss.m, fs.name) + j := 0 + for i := 0; i < len(ss.l); i++ { + if fs.name != ss.l[i].name { + ss.l[j] = ss.l[i] + j += 1 + } + } + ss.l = ss.l[:j] + case len(index) < d: + fs.index = make([]int, len(index)+1) + copy(fs.index, index) + fs.index[len(index)] = i + depth[fs.name] = len(index) + ss.m[fs.name] = fs + ss.l = append(ss.l, fs) + } + } + } +} + +var ( + structSpecMutex sync.RWMutex + structSpecCache = make(map[reflect.Type]*structSpec) + defaultFieldSpec = &fieldSpec{} +) + +func structSpecForType(t reflect.Type) *structSpec { + + structSpecMutex.RLock() + ss, found := structSpecCache[t] + structSpecMutex.RUnlock() + if found { + return ss + } + + structSpecMutex.Lock() + defer structSpecMutex.Unlock() + ss, found = structSpecCache[t] + if found { + return ss + } + + ss = &structSpec{m: make(map[string]*fieldSpec)} + compileStructSpec(t, make(map[string]int), nil, ss) + structSpecCache[t] = ss + return ss +} + +var errScanStructValue = errors.New("redigo.ScanStruct: value must be non-nil pointer to a struct") + +// ScanStruct scans alternating names and values from src to a struct. The +// HGETALL and CONFIG GET commands return replies in this format. +// +// ScanStruct uses exported field names to match values in the response. Use +// 'redis' field tag to override the name: +// +// Field int `redis:"myName"` +// +// Fields with the tag redis:"-" are ignored. +// +// Each field uses RedisScan if available otherwise: +// Integer, float, boolean, string and []byte fields are supported. Scan uses the +// standard strconv package to convert bulk string values to numeric and +// boolean types. +// +// If a src element is nil, then the corresponding field is not modified. +func ScanStruct(src []interface{}, dest interface{}) error { + d := reflect.ValueOf(dest) + if d.Kind() != reflect.Ptr || d.IsNil() { + return errScanStructValue + } + d = d.Elem() + if d.Kind() != reflect.Struct { + return errScanStructValue + } + ss := structSpecForType(d.Type()) + + if len(src)%2 != 0 { + return errors.New("redigo.ScanStruct: number of values not a multiple of 2") + } + + for i := 0; i < len(src); i += 2 { + s := src[i+1] + if s == nil { + continue + } + name, ok := src[i].([]byte) + if !ok { + return fmt.Errorf("redigo.ScanStruct: key %d not a bulk string value", i) + } + fs := ss.fieldSpec(name) + if fs == nil { + continue + } + if err := convertAssignValue(d.FieldByIndex(fs.index), s); err != nil { + return fmt.Errorf("redigo.ScanStruct: cannot assign field %s: %v", fs.name, err) + } + } + return nil +} + +var ( + errScanSliceValue = errors.New("redigo.ScanSlice: dest must be non-nil pointer to a struct") +) + +// ScanSlice scans src to the slice pointed to by dest. The elements the dest +// slice must be integer, float, boolean, string, struct or pointer to struct +// values. +// +// Struct fields must be integer, float, boolean or string values. All struct +// fields are used unless a subset is specified using fieldNames. +func ScanSlice(src []interface{}, dest interface{}, fieldNames ...string) error { + d := reflect.ValueOf(dest) + if d.Kind() != reflect.Ptr || d.IsNil() { + return errScanSliceValue + } + d = d.Elem() + if d.Kind() != reflect.Slice { + return errScanSliceValue + } + + isPtr := false + t := d.Type().Elem() + if t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct { + isPtr = true + t = t.Elem() + } + + if t.Kind() != reflect.Struct { + ensureLen(d, len(src)) + for i, s := range src { + if s == nil { + continue + } + if err := convertAssignValue(d.Index(i), s); err != nil { + return fmt.Errorf("redigo.ScanSlice: cannot assign element %d: %v", i, err) + } + } + return nil + } + + ss := structSpecForType(t) + fss := ss.l + if len(fieldNames) > 0 { + fss = make([]*fieldSpec, len(fieldNames)) + for i, name := range fieldNames { + fss[i] = ss.m[name] + if fss[i] == nil { + return fmt.Errorf("redigo.ScanSlice: ScanSlice bad field name %s", name) + } + } + } + + if len(fss) == 0 { + return errors.New("redigo.ScanSlice: no struct fields") + } + + n := len(src) / len(fss) + if n*len(fss) != len(src) { + return errors.New("redigo.ScanSlice: length not a multiple of struct field count") + } + + ensureLen(d, n) + for i := 0; i < n; i++ { + d := d.Index(i) + if isPtr { + if d.IsNil() { + d.Set(reflect.New(t)) + } + d = d.Elem() + } + for j, fs := range fss { + s := src[i*len(fss)+j] + if s == nil { + continue + } + if err := convertAssignValue(d.FieldByIndex(fs.index), s); err != nil { + return fmt.Errorf("redigo.ScanSlice: cannot assign element %d to field %s: %v", i*len(fss)+j, fs.name, err) + } + } + } + return nil +} + +// Args is a helper for constructing command arguments from structured values. +type Args []interface{} + +// Add returns the result of appending value to args. +func (args Args) Add(value ...interface{}) Args { + return append(args, value...) +} + +// AddFlat returns the result of appending the flattened value of v to args. +// +// Maps are flattened by appending the alternating keys and map values to args. +// +// Slices are flattened by appending the slice elements to args. +// +// Structs are flattened by appending the alternating names and values of +// exported fields to args. If v is a nil struct pointer, then nothing is +// appended. The 'redis' field tag overrides struct field names. See ScanStruct +// for more information on the use of the 'redis' field tag. +// +// Other types are appended to args as is. +func (args Args) AddFlat(v interface{}) Args { + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.Struct: + args = flattenStruct(args, rv) + case reflect.Slice: + for i := 0; i < rv.Len(); i++ { + args = append(args, rv.Index(i).Interface()) + } + case reflect.Map: + for _, k := range rv.MapKeys() { + args = append(args, k.Interface(), rv.MapIndex(k).Interface()) + } + case reflect.Ptr: + if rv.Type().Elem().Kind() == reflect.Struct { + if !rv.IsNil() { + args = flattenStruct(args, rv.Elem()) + } + } else { + args = append(args, v) + } + default: + args = append(args, v) + } + return args +} + +func flattenStruct(args Args, v reflect.Value) Args { + ss := structSpecForType(v.Type()) + for _, fs := range ss.l { + fv := v.FieldByIndex(fs.index) + if fs.omitEmpty { + var empty = false + switch fv.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + empty = fv.Len() == 0 + case reflect.Bool: + empty = !fv.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + empty = fv.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + empty = fv.Uint() == 0 + case reflect.Float32, reflect.Float64: + empty = fv.Float() == 0 + case reflect.Interface, reflect.Ptr: + empty = fv.IsNil() + } + if empty { + continue + } + } + args = append(args, fs.name, fv.Interface()) + } + return args +} diff --git a/src/go/redigo/redis/scan_test.go b/src/go/redigo/redis/scan_test.go new file mode 100644 index 0000000..a54d432 --- /dev/null +++ b/src/go/redigo/redis/scan_test.go @@ -0,0 +1,513 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis_test + +import ( + "fmt" + "math" + "reflect" + "testing" + "time" + + "github.com/cea-hpc/pdwfs/redigo/redis" +) + +type durationScan struct { + time.Duration `redis:"sd"` +} + +func (t *durationScan) RedisScan(src interface{}) (err error) { + if t == nil { + return fmt.Errorf("nil pointer") + } + switch src := src.(type) { + case string: + t.Duration, err = time.ParseDuration(src) + case []byte: + t.Duration, err = time.ParseDuration(string(src)) + case int64: + t.Duration = time.Duration(src) + default: + err = fmt.Errorf("cannot convert from %T to %T", src, t) + } + return err +} + +var scanConversionTests = []struct { + src interface{} + dest interface{} +}{ + {[]byte("-inf"), math.Inf(-1)}, + {[]byte("+inf"), math.Inf(1)}, + {[]byte("0"), float64(0)}, + {[]byte("3.14159"), float64(3.14159)}, + {[]byte("3.14"), float32(3.14)}, + {[]byte("-100"), int(-100)}, + {[]byte("101"), int(101)}, + {int64(102), int(102)}, + {[]byte("103"), uint(103)}, + {int64(104), uint(104)}, + {[]byte("105"), int8(105)}, + {int64(106), int8(106)}, + {[]byte("107"), uint8(107)}, + {int64(108), uint8(108)}, + {[]byte("0"), false}, + {int64(0), false}, + {[]byte("f"), false}, + {[]byte("1"), true}, + {int64(1), true}, + {[]byte("t"), true}, + {"hello", "hello"}, + {[]byte("hello"), "hello"}, + {[]byte("world"), []byte("world")}, + {nil, ""}, + {nil, []byte(nil)}, + + {[]interface{}{[]byte("b1")}, []interface{}{[]byte("b1")}}, + {[]interface{}{[]byte("b2")}, []string{"b2"}}, + {[]interface{}{[]byte("b3"), []byte("b4")}, []string{"b3", "b4"}}, + {[]interface{}{[]byte("b5")}, [][]byte{[]byte("b5")}}, + {[]interface{}{[]byte("1")}, []int{1}}, + {[]interface{}{[]byte("1"), []byte("2")}, []int{1, 2}}, + {[]interface{}{[]byte("1"), []byte("2")}, []float64{1, 2}}, + {[]interface{}{[]byte("1")}, []byte{1}}, + {[]interface{}{[]byte("1")}, []bool{true}}, + + {[]interface{}{"s1"}, []interface{}{"s1"}}, + {[]interface{}{"s2"}, [][]byte{[]byte("s2")}}, + {[]interface{}{"s3", "s4"}, []string{"s3", "s4"}}, + {[]interface{}{"s5"}, [][]byte{[]byte("s5")}}, + {[]interface{}{"1"}, []int{1}}, + {[]interface{}{"1", "2"}, []int{1, 2}}, + {[]interface{}{"1", "2"}, []float64{1, 2}}, + {[]interface{}{"1"}, []byte{1}}, + {[]interface{}{"1"}, []bool{true}}, + + {[]interface{}{nil, "2"}, []interface{}{nil, "2"}}, + {[]interface{}{nil, []byte("2")}, [][]byte{nil, []byte("2")}}, + + {[]interface{}{redis.Error("e1")}, []interface{}{redis.Error("e1")}}, + {[]interface{}{redis.Error("e2")}, [][]byte{[]byte("e2")}}, + {[]interface{}{redis.Error("e3")}, []string{"e3"}}, + + {"1m", durationScan{Duration: time.Minute}}, + {[]byte("1m"), durationScan{Duration: time.Minute}}, + {time.Minute.Nanoseconds(), durationScan{Duration: time.Minute}}, + {[]interface{}{[]byte("1m")}, []durationScan{{Duration: time.Minute}}}, + {[]interface{}{[]byte("1m")}, []*durationScan{{Duration: time.Minute}}}, +} + +func TestScanConversion(t *testing.T) { + for _, tt := range scanConversionTests { + values := []interface{}{tt.src} + dest := reflect.New(reflect.TypeOf(tt.dest)) + values, err := redis.Scan(values, dest.Interface()) + if err != nil { + t.Errorf("Scan(%v) returned error %v", tt, err) + continue + } + if !reflect.DeepEqual(tt.dest, dest.Elem().Interface()) { + t.Errorf("Scan(%v) returned %v, want %v", tt, dest.Elem().Interface(), tt.dest) + } + } +} + +var scanConversionErrorTests = []struct { + src interface{} + dest interface{} +}{ + {[]byte("1234"), byte(0)}, + {int64(1234), byte(0)}, + {[]byte("-1"), byte(0)}, + {int64(-1), byte(0)}, + {[]byte("junk"), false}, + {redis.Error("blah"), false}, + {redis.Error("blah"), durationScan{Duration: time.Minute}}, + {"invalid", durationScan{Duration: time.Minute}}, +} + +func TestScanConversionError(t *testing.T) { + for _, tt := range scanConversionErrorTests { + values := []interface{}{tt.src} + dest := reflect.New(reflect.TypeOf(tt.dest)) + values, err := redis.Scan(values, dest.Interface()) + if err == nil { + t.Errorf("Scan(%v) did not return error", tt) + } + } +} + +func ExampleScan() { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + + c.Send("HMSET", "album:1", "title", "Red", "rating", 5) + c.Send("HMSET", "album:2", "title", "Earthbound", "rating", 1) + c.Send("HMSET", "album:3", "title", "Beat") + c.Send("LPUSH", "albums", "1") + c.Send("LPUSH", "albums", "2") + c.Send("LPUSH", "albums", "3") + values, err := redis.Values(c.Do("SORT", "albums", + "BY", "album:*->rating", + "GET", "album:*->title", + "GET", "album:*->rating")) + if err != nil { + fmt.Println(err) + return + } + + for len(values) > 0 { + var title string + rating := -1 // initialize to illegal value to detect nil. + values, err = redis.Scan(values, &title, &rating) + if err != nil { + fmt.Println(err) + return + } + if rating == -1 { + fmt.Println(title, "not-rated") + } else { + fmt.Println(title, rating) + } + } + // Output: + // Beat not-rated + // Earthbound 1 + // Red 5 +} + +type s0 struct { + X int + Y int `redis:"y"` + Bt bool +} + +type s1 struct { + X int `redis:"-"` + I int `redis:"i"` + U uint `redis:"u"` + S string `redis:"s"` + P []byte `redis:"p"` + B bool `redis:"b"` + Bt bool + Bf bool + s0 + Sd durationScan `redis:"sd"` + Sdp *durationScan `redis:"sdp"` +} + +var scanStructTests = []struct { + title string + reply []string + value interface{} +}{ + {"basic", + []string{ + "i", "-1234", + "u", "5678", + "s", "hello", + "p", "world", + "b", "t", + "Bt", "1", + "Bf", "0", + "X", "123", + "y", "456", + "sd", "1m", + "sdp", "1m", + }, + &s1{ + I: -1234, + U: 5678, + S: "hello", + P: []byte("world"), + B: true, + Bt: true, + Bf: false, + s0: s0{X: 123, Y: 456}, + Sd: durationScan{Duration: time.Minute}, + Sdp: &durationScan{Duration: time.Minute}, + }, + }, + {"absent values", + []string{}, + &s1{}, + }, +} + +func TestScanStruct(t *testing.T) { + for _, tt := range scanStructTests { + + var reply []interface{} + for _, v := range tt.reply { + reply = append(reply, []byte(v)) + } + + value := reflect.New(reflect.ValueOf(tt.value).Type().Elem()) + + if err := redis.ScanStruct(reply, value.Interface()); err != nil { + t.Fatalf("ScanStruct(%s) returned error %v", tt.title, err) + } + + if !reflect.DeepEqual(value.Interface(), tt.value) { + t.Fatalf("ScanStruct(%s) returned %v, want %v", tt.title, value.Interface(), tt.value) + } + } +} + +func TestBadScanStructArgs(t *testing.T) { + x := []interface{}{"A", "b"} + test := func(v interface{}) { + if err := redis.ScanStruct(x, v); err == nil { + t.Errorf("Expect error for ScanStruct(%T, %T)", x, v) + } + } + + test(nil) + + var v0 *struct{} + test(v0) + + var v1 int + test(&v1) + + x = x[:1] + v2 := struct{ A string }{} + test(&v2) +} + +var scanSliceTests = []struct { + src []interface{} + fieldNames []string + ok bool + dest interface{} +}{ + { + []interface{}{[]byte("1"), nil, []byte("-1")}, + nil, + true, + []int{1, 0, -1}, + }, + { + []interface{}{[]byte("1"), nil, []byte("2")}, + nil, + true, + []uint{1, 0, 2}, + }, + { + []interface{}{[]byte("-1")}, + nil, + false, + []uint{1}, + }, + { + []interface{}{[]byte("hello"), nil, []byte("world")}, + nil, + true, + [][]byte{[]byte("hello"), nil, []byte("world")}, + }, + { + []interface{}{[]byte("hello"), nil, []byte("world")}, + nil, + true, + []string{"hello", "", "world"}, + }, + { + []interface{}{[]byte("a1"), []byte("b1"), []byte("a2"), []byte("b2")}, + nil, + true, + []struct{ A, B string }{{"a1", "b1"}, {"a2", "b2"}}, + }, + { + []interface{}{[]byte("a1"), []byte("b1")}, + nil, + false, + []struct{ A, B, C string }{{"a1", "b1", ""}}, + }, + { + []interface{}{[]byte("a1"), []byte("b1"), []byte("a2"), []byte("b2")}, + nil, + true, + []*struct{ A, B string }{{A: "a1", B: "b1"}, {A: "a2", B: "b2"}}, + }, + { + []interface{}{[]byte("a1"), []byte("b1"), []byte("a2"), []byte("b2")}, + []string{"A", "B"}, + true, + []struct{ A, C, B string }{{"a1", "", "b1"}, {"a2", "", "b2"}}, + }, + { + []interface{}{[]byte("a1"), []byte("b1"), []byte("a2"), []byte("b2")}, + nil, + false, + []struct{}{}, + }, +} + +func TestScanSlice(t *testing.T) { + for _, tt := range scanSliceTests { + + typ := reflect.ValueOf(tt.dest).Type() + dest := reflect.New(typ) + + err := redis.ScanSlice(tt.src, dest.Interface(), tt.fieldNames...) + if tt.ok != (err == nil) { + t.Errorf("ScanSlice(%v, []%s, %v) returned error %v", tt.src, typ, tt.fieldNames, err) + continue + } + if tt.ok && !reflect.DeepEqual(dest.Elem().Interface(), tt.dest) { + t.Errorf("ScanSlice(src, []%s) returned %#v, want %#v", typ, dest.Elem().Interface(), tt.dest) + } + } +} + +func ExampleScanSlice() { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + + c.Send("HMSET", "album:1", "title", "Red", "rating", 5) + c.Send("HMSET", "album:2", "title", "Earthbound", "rating", 1) + c.Send("HMSET", "album:3", "title", "Beat", "rating", 4) + c.Send("LPUSH", "albums", "1") + c.Send("LPUSH", "albums", "2") + c.Send("LPUSH", "albums", "3") + values, err := redis.Values(c.Do("SORT", "albums", + "BY", "album:*->rating", + "GET", "album:*->title", + "GET", "album:*->rating")) + if err != nil { + fmt.Println(err) + return + } + + var albums []struct { + Title string + Rating int + } + if err := redis.ScanSlice(values, &albums); err != nil { + fmt.Println(err) + return + } + fmt.Printf("%v\n", albums) + // Output: + // [{Earthbound 1} {Beat 4} {Red 5}] +} + +var argsTests = []struct { + title string + actual redis.Args + expected redis.Args +}{ + {"struct ptr", + redis.Args{}.AddFlat(&struct { + I int `redis:"i"` + U uint `redis:"u"` + S string `redis:"s"` + P []byte `redis:"p"` + M map[string]string `redis:"m"` + Bt bool + Bf bool + }{ + -1234, 5678, "hello", []byte("world"), map[string]string{"hello": "world"}, true, false, + }), + redis.Args{"i", int(-1234), "u", uint(5678), "s", "hello", "p", []byte("world"), "m", map[string]string{"hello": "world"}, "Bt", true, "Bf", false}, + }, + {"struct", + redis.Args{}.AddFlat(struct{ I int }{123}), + redis.Args{"I", 123}, + }, + {"slice", + redis.Args{}.Add(1).AddFlat([]string{"a", "b", "c"}).Add(2), + redis.Args{1, "a", "b", "c", 2}, + }, + {"struct omitempty", + redis.Args{}.AddFlat(&struct { + Sdp *durationArg `redis:"Sdp,omitempty"` + }{ + nil, + }), + redis.Args{}, + }, +} + +func TestArgs(t *testing.T) { + for _, tt := range argsTests { + if !reflect.DeepEqual(tt.actual, tt.expected) { + t.Fatalf("%s is %v, want %v", tt.title, tt.actual, tt.expected) + } + } +} + +func ExampleArgs() { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + + var p1, p2 struct { + Title string `redis:"title"` + Author string `redis:"author"` + Body string `redis:"body"` + } + + p1.Title = "Example" + p1.Author = "Gary" + p1.Body = "Hello" + + if _, err := c.Do("HMSET", redis.Args{}.Add("id1").AddFlat(&p1)...); err != nil { + fmt.Println(err) + return + } + + m := map[string]string{ + "title": "Example2", + "author": "Steve", + "body": "Map", + } + + if _, err := c.Do("HMSET", redis.Args{}.Add("id2").AddFlat(m)...); err != nil { + fmt.Println(err) + return + } + + for _, id := range []string{"id1", "id2"} { + + v, err := redis.Values(c.Do("HGETALL", id)) + if err != nil { + fmt.Println(err) + return + } + + if err := redis.ScanStruct(v, &p2); err != nil { + fmt.Println(err) + return + } + + fmt.Printf("%+v\n", p2) + } + + // Output: + // {Title:Example Author:Gary Body:Hello} + // {Title:Example2 Author:Steve Body:Map} +} diff --git a/src/go/redigo/redis/script.go b/src/go/redigo/redis/script.go new file mode 100644 index 0000000..0ef1c82 --- /dev/null +++ b/src/go/redigo/redis/script.go @@ -0,0 +1,91 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "crypto/sha1" + "encoding/hex" + "io" + "strings" +) + +// Script encapsulates the source, hash and key count for a Lua script. See +// http://redis.io/commands/eval for information on scripts in Redis. +type Script struct { + keyCount int + src string + hash string +} + +// NewScript returns a new script object. If keyCount is greater than or equal +// to zero, then the count is automatically inserted in the EVAL command +// argument list. If keyCount is less than zero, then the application supplies +// the count as the first value in the keysAndArgs argument to the Do, Send and +// SendHash methods. +func NewScript(keyCount int, src string) *Script { + h := sha1.New() + io.WriteString(h, src) + return &Script{keyCount, src, hex.EncodeToString(h.Sum(nil))} +} + +func (s *Script) args(spec string, keysAndArgs []interface{}) []interface{} { + var args []interface{} + if s.keyCount < 0 { + args = make([]interface{}, 1+len(keysAndArgs)) + args[0] = spec + copy(args[1:], keysAndArgs) + } else { + args = make([]interface{}, 2+len(keysAndArgs)) + args[0] = spec + args[1] = s.keyCount + copy(args[2:], keysAndArgs) + } + return args +} + +// Hash returns the script hash. +func (s *Script) Hash() string { + return s.hash +} + +// Do evaluates the script. Under the covers, Do optimistically evaluates the +// script using the EVALSHA command. If the command fails because the script is +// not loaded, then Do evaluates the script using the EVAL command (thus +// causing the script to load). +func (s *Script) Do(c Conn, keysAndArgs ...interface{}) (interface{}, error) { + v, err := c.Do("EVALSHA", s.args(s.hash, keysAndArgs)...) + if e, ok := err.(Error); ok && strings.HasPrefix(string(e), "NOSCRIPT ") { + v, err = c.Do("EVAL", s.args(s.src, keysAndArgs)...) + } + return v, err +} + +// SendHash evaluates the script without waiting for the reply. The script is +// evaluated with the EVALSHA command. The application must ensure that the +// script is loaded by a previous call to Send, Do or Load methods. +func (s *Script) SendHash(c Conn, keysAndArgs ...interface{}) error { + return c.Send("EVALSHA", s.args(s.hash, keysAndArgs)...) +} + +// Send evaluates the script without waiting for the reply. +func (s *Script) Send(c Conn, keysAndArgs ...interface{}) error { + return c.Send("EVAL", s.args(s.src, keysAndArgs)...) +} + +// Load loads the script without evaluating it. +func (s *Script) Load(c Conn) error { + _, err := c.Do("SCRIPT", "LOAD", s.src) + return err +} diff --git a/src/go/redigo/redis/script_test.go b/src/go/redigo/redis/script_test.go new file mode 100644 index 0000000..756f7d1 --- /dev/null +++ b/src/go/redigo/redis/script_test.go @@ -0,0 +1,100 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis_test + +import ( + "fmt" + "reflect" + "testing" + "time" + + "github.com/cea-hpc/pdwfs/redigo/redis" +) + +var ( + // These variables are declared at package level to remove distracting + // details from the examples. + c redis.Conn + reply interface{} + err error +) + +func ExampleScript() { + // Initialize a package-level variable with a script. + var getScript = redis.NewScript(1, `return redis.call('get', KEYS[1])`) + + // In a function, use the script Do method to evaluate the script. The Do + // method optimistically uses the EVALSHA command. If the script is not + // loaded, then the Do method falls back to the EVAL command. + reply, err = getScript.Do(c, "foo") +} + +func TestScript(t *testing.T) { + c, err := redis.DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer c.Close() + + // To test fall back in Do, we make script unique by adding comment with current time. + script := fmt.Sprintf("--%d\nreturn {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}", time.Now().UnixNano()) + s := redis.NewScript(2, script) + reply := []interface{}{[]byte("key1"), []byte("key2"), []byte("arg1"), []byte("arg2")} + + v, err := s.Do(c, "key1", "key2", "arg1", "arg2") + if err != nil { + t.Errorf("s.Do(c, ...) returned %v", err) + } + + if !reflect.DeepEqual(v, reply) { + t.Errorf("s.Do(c, ..); = %v, want %v", v, reply) + } + + err = s.Load(c) + if err != nil { + t.Errorf("s.Load(c) returned %v", err) + } + + err = s.SendHash(c, "key1", "key2", "arg1", "arg2") + if err != nil { + t.Errorf("s.SendHash(c, ...) returned %v", err) + } + + err = c.Flush() + if err != nil { + t.Errorf("c.Flush() returned %v", err) + } + + v, err = c.Receive() + if !reflect.DeepEqual(v, reply) { + t.Errorf("s.SendHash(c, ..); c.Receive() = %v, want %v", v, reply) + } + + err = s.Send(c, "key1", "key2", "arg1", "arg2") + if err != nil { + t.Errorf("s.Send(c, ...) returned %v", err) + } + + err = c.Flush() + if err != nil { + t.Errorf("c.Flush() returned %v", err) + } + + v, err = c.Receive() + if !reflect.DeepEqual(v, reply) { + t.Errorf("s.Send(c, ..); c.Receive() = %v, want %v", v, reply) + } + +} diff --git a/src/go/redigo/redis/test_test.go b/src/go/redigo/redis/test_test.go new file mode 100644 index 0000000..dfa48bf --- /dev/null +++ b/src/go/redigo/redis/test_test.go @@ -0,0 +1,183 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "bufio" + "errors" + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +func SetNowFunc(f func() time.Time) { + nowFunc = f +} + +var ( + ErrNegativeInt = errNegativeInt + + serverPath = flag.String("redis-server", "redis-server", "Path to redis server binary") + serverAddress = flag.String("redis-address", "127.0.0.1", "The address of the server") + serverBasePort = flag.Int("redis-port", 16379, "Beginning of port range for test servers") + serverLogName = flag.String("redis-log", "", "Write Redis server logs to `filename`") + serverLog = ioutil.Discard + + defaultServerMu sync.Mutex + defaultServer *Server + defaultServerErr error +) + +type Server struct { + name string + cmd *exec.Cmd + done chan struct{} +} + +func NewServer(name string, args ...string) (*Server, error) { + s := &Server{ + name: name, + cmd: exec.Command(*serverPath, args...), + done: make(chan struct{}), + } + + r, err := s.cmd.StdoutPipe() + if err != nil { + return nil, err + } + + err = s.cmd.Start() + if err != nil { + return nil, err + } + + ready := make(chan error, 1) + go s.watch(r, ready) + + select { + case err = <-ready: + case <-time.After(time.Second * 10): + err = errors.New("timeout waiting for server to start") + } + + if err != nil { + s.Stop() + return nil, err + } + + return s, nil +} + +func (s *Server) watch(r io.Reader, ready chan error) { + fmt.Fprintf(serverLog, "%d START %s \n", s.cmd.Process.Pid, s.name) + var listening bool + var text string + scn := bufio.NewScanner(r) + for scn.Scan() { + text = scn.Text() + fmt.Fprintf(serverLog, "%s\n", text) + if !listening { + if strings.Contains(text, " * Ready to accept connections") || + strings.Contains(text, " * The server is now ready to accept connections on port") { + listening = true + ready <- nil + } + } + } + if !listening { + ready <- fmt.Errorf("server exited: %s", text) + } + s.cmd.Wait() + fmt.Fprintf(serverLog, "%d STOP %s \n", s.cmd.Process.Pid, s.name) + close(s.done) +} + +func (s *Server) Stop() { + s.cmd.Process.Signal(os.Interrupt) + <-s.done +} + +// stopDefaultServer stops the server created by DialDefaultServer. +func stopDefaultServer() { + defaultServerMu.Lock() + defer defaultServerMu.Unlock() + if defaultServer != nil { + defaultServer.Stop() + defaultServer = nil + } +} + +// DefaultServerAddr starts the test server if not already started and returns +// the address of that server. +func DefaultServerAddr() (string, error) { + defaultServerMu.Lock() + defer defaultServerMu.Unlock() + addr := fmt.Sprintf("%v:%d", *serverAddress, *serverBasePort) + if defaultServer != nil || defaultServerErr != nil { + return addr, defaultServerErr + } + defaultServer, defaultServerErr = NewServer( + "default", + "--port", strconv.Itoa(*serverBasePort), + "--bind", *serverAddress, + "--save", "", + "--appendonly", "no") + return addr, defaultServerErr +} + +// DialDefaultServer starts the test server if not already started and dials a +// connection to the server. +func DialDefaultServer(options ...DialOption) (Conn, error) { + addr, err := DefaultServerAddr() + if err != nil { + return nil, err + } + c, err := Dial("tcp", addr, append([]DialOption{DialReadTimeout(1 * time.Second), DialWriteTimeout(1 * time.Second)}, options...)...) + if err != nil { + return nil, err + } + c.Do("FLUSHDB") + return c, nil +} + +func TestMain(m *testing.M) { + os.Exit(func() int { + flag.Parse() + + var f *os.File + if *serverLogName != "" { + var err error + f, err = os.OpenFile(*serverLogName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + fmt.Fprintf(os.Stderr, "Error opening redis-log: %v\n", err) + return 1 + } + defer f.Close() + serverLog = f + } + + defer stopDefaultServer() + + return m.Run() + }()) +} diff --git a/src/go/redigo/redis/zpop_example_test.go b/src/go/redigo/redis/zpop_example_test.go new file mode 100644 index 0000000..9cc2372 --- /dev/null +++ b/src/go/redigo/redis/zpop_example_test.go @@ -0,0 +1,114 @@ +// Copyright 2013 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis_test + +import ( + "fmt" + + "github.com/cea-hpc/pdwfs/redigo/redis" +) + +// zpop pops a value from the ZSET key using WATCH/MULTI/EXEC commands. +func zpop(c redis.Conn, key string) (result string, err error) { + + defer func() { + // Return connection to normal state on error. + if err != nil { + c.Do("DISCARD") + } + }() + + // Loop until transaction is successful. + for { + if _, err := c.Do("WATCH", key); err != nil { + return "", err + } + + members, err := redis.Strings(c.Do("ZRANGE", key, 0, 0)) + if err != nil { + return "", err + } + if len(members) != 1 { + return "", redis.ErrNil + } + + c.Send("MULTI") + c.Send("ZREM", key, members[0]) + queued, err := c.Do("EXEC") + if err != nil { + return "", err + } + + if queued != nil { + result = members[0] + break + } + } + + return result, nil +} + +// zpopScript pops a value from a ZSET. +var zpopScript = redis.NewScript(1, ` + local r = redis.call('ZRANGE', KEYS[1], 0, 0) + if r ~= nil then + r = r[1] + redis.call('ZREM', KEYS[1], r) + end + return r +`) + +// This example implements ZPOP as described at +// http://redis.io/topics/transactions using WATCH/MULTI/EXEC and scripting. +func Example_zpop() { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + + // Add test data using a pipeline. + + for i, member := range []string{"red", "blue", "green"} { + c.Send("ZADD", "zset", i, member) + } + if _, err := c.Do(""); err != nil { + fmt.Println(err) + return + } + + // Pop using WATCH/MULTI/EXEC + + v, err := zpop(c, "zset") + if err != nil { + fmt.Println(err) + return + } + fmt.Println(v) + + // Pop using a script. + + v, err = redis.String(zpopScript.Do(c, "zset")) + if err != nil { + fmt.Println(err) + return + } + fmt.Println(v) + + // Output: + // red + // blue +} diff --git a/src/go/redisfs/blockedbuffer.go b/src/go/redisfs/blockedbuffer.go deleted file mode 100644 index b5a2279..0000000 --- a/src/go/redisfs/blockedbuffer.go +++ /dev/null @@ -1,401 +0,0 @@ -// Copyright 2019 CEA -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package redisfs - -import ( - "errors" - "fmt" - "math/bits" - "sync" - - "github.com/go-redis/redis" - "github.com/cea-hpc/pdwfs/config" -) - -var ( - errBlockMustExists = errors.New("Getting a block for reading that does not exist") -) - -// RedisBlockedBuffer is a composite Buffer that writes/reads data in a set fixed-sized Buffers (or blocks). -type RedisBlockedBuffer struct { - conf *config.Mount - client IRedisClient - blocks map[int]Buffer - blockSize int - key string - keyNbBlocks string - mtx *sync.RWMutex -} - -// NewRedisBlockedBuffer creates a new data volume based on a Buffer -func NewRedisBlockedBuffer(conf *config.Mount, client IRedisClient, key string) *RedisBlockedBuffer { - if conf.BlockSize == 0 { - panic(fmt.Errorf("BlockSize in configuration is not set")) - } - return &RedisBlockedBuffer{ - conf: conf, - client: client, - blocks: make(map[int]Buffer), - blockSize: conf.BlockSize, - key: key, - keyNbBlocks: key + ":NbBlocks", - mtx: &sync.RWMutex{}, - } -} - -func (b *RedisBlockedBuffer) metaAddBlock(id int) { - err := b.client.SetBit(b.keyNbBlocks, int64(id), 1).Err() - if err != nil { - panic(err) - } -} - -func (b *RedisBlockedBuffer) metaRemoveBlock(id int) { - err := b.client.SetBit(b.keyNbBlocks, int64(id), 0).Err() - if err != nil { - panic(err) - } -} - -//NbBlocks returns the total number of blocks taken -func (b *RedisBlockedBuffer) NbBlocks() int { - blocksBitmap, err := b.client.Get(b.keyNbBlocks).Bytes() - if err == redis.Nil { - return 0 - } - if err != nil { - panic(err) - } - // get last non-null byte in bitmap - i := len(blocksBitmap) - 1 - for { - if blocksBitmap[i] != 0 || i == 0 { - break - } - i-- - } - nbBlocks := 8*i + (8 - bits.TrailingZeros8(uint8(blocksBitmap[i]))) - return nbBlocks -} - -func (b *RedisBlockedBuffer) getBlockedBufferByID(id int) Buffer { - if _, ok := b.blocks[id]; !ok { - b.metaAddBlock(id) - b.blocks[id] = NewRedisBuffer(b.conf, b.client, fmt.Sprintf("%s:%d", b.key, id)) - } - return b.blocks[id] -} - -func (b *RedisBlockedBuffer) removeBlockedBufferByID(id int) error { - err := b.getBlockedBufferByID(id).Clear() - if err != nil { - return err - } - b.metaRemoveBlock(id) - delete(b.blocks, id) - return nil -} - -// Size returns the total length of data in the RedisBlockedBuffer -func (b *RedisBlockedBuffer) Size() int64 { - n := b.NbBlocks() - if n == 0 { - return 0 - } - return int64(b.blockSize*(n-1)) + b.getBlockedBufferByID(n-1).Size() -} - -type blockInfo struct { - id int - off int64 // offset relative to beginning of Buffer - data [][]byte // slice of data slices - len int // length of data in Buffer - buf Buffer -} - -func (b *RedisBlockedBuffer) newBlocksLayout(off, size int64) []blockInfo { - - startID := int(off / int64(b.blockSize)) - endID := int((off + size - 1) / int64(b.blockSize)) // last block inclusive - nBlocks := endID - startID + 1 - - rb := make([]blockInfo, nBlocks) - - // first block - rb[0].id = startID - rb[0].off = off % int64(b.blockSize) - if nBlocks == 1 { - rb[0].len = int(size) - } else { - rb[0].len = int(int64(b.blockSize) - rb[0].off) - } - rb[0].buf = b.getBlockedBufferByID(rb[0].id) - - if nBlocks > 1 { - //last block, inclusive - rb[nBlocks-1].id = endID - rb[nBlocks-1].off = 0 - rb[nBlocks-1].len = int((off+size-1)%int64(b.blockSize) + 1) - rb[nBlocks-1].buf = b.getBlockedBufferByID(rb[nBlocks-1].id) - - // other blocks (nBlocks > 2) - for i := 1; i < nBlocks-1; i++ { - rb[i].id = startID + i - rb[i].off = 0 - rb[i].len = b.blockSize - rb[i].buf = b.getBlockedBufferByID(rb[i].id) - } - } - return rb -} - -func (b *RedisBlockedBuffer) relevantBlocks(datav [][]byte, off, size int64) []blockInfo { - - blockInfos := b.newBlocksLayout(off, size) - - nBlocks := len(blockInfos) - nData := len(datav) - iBlock := 0 - block := &blockInfos[iBlock] - offsetInBlock := int((*block).off) - iData := 0 - data := datav[iData] - offsetInData := 0 - for { - remainBlockSize := b.blockSize - offsetInBlock - remainDataSize := len(data) - offsetInData - - if remainDataSize <= remainBlockSize { - (*block).data = append((*block).data, data[offsetInData:offsetInData+remainDataSize]) - offsetInBlock += remainDataSize - // move to next data - iData++ - if iData >= nData { - break - } - data = datav[iData] - offsetInData = 0 - continue - } else { - (*block).data = append((*block).data, data[offsetInData:offsetInData+remainBlockSize]) - offsetInData += remainBlockSize - // move to next block - iBlock++ - if iBlock >= nBlocks { - break - } - block = &blockInfos[iBlock] - offsetInBlock = 0 - continue - } - } - return blockInfos -} - -type chanReturnData struct { - n int - err error -} - -func (b *RedisBlockedBuffer) writeBlocksParallel(blockInfos []blockInfo) (int, error) { - - retChan := make(chan chanReturnData) - for _, blockInfo := range blockInfos { - go func(buf Buffer, datav [][]byte, off int64, retChan chan<- chanReturnData) { - wrote, err := buf.WriteVecAt(datav, off) - retChan <- chanReturnData{wrote, err} - }(blockInfo.buf, blockInfo.data, blockInfo.off, retChan) - } - - var n int - var err error - for range blockInfos { - retData := <-retChan - if retData.err != nil { - err = retData.err - } - n += retData.n - } - return n, err -} - -func (b *RedisBlockedBuffer) writeBlocksSequential(blockInfos []blockInfo) (int, error) { - - var n int - for _, blockInfo := range blockInfos { - wrote, err := blockInfo.buf.WriteVecAt(blockInfo.data, blockInfo.off) - n += wrote - if err != nil { - return n, err - } - } - return n, nil -} - -//WriteAt writes a byte slices starting at byte offset off. -func (b *RedisBlockedBuffer) WriteAt(data []byte, off int64) (int, error) { - blockInfos := b.relevantBlocks([][]byte{data}, off, int64(len(data))) - if b.conf.WriteParallel { - return b.writeBlocksParallel(blockInfos) - } - return b.writeBlocksSequential(blockInfos) -} - -//WriteVecAt writes a vector of byte slices starting at byte offset off. -func (b *RedisBlockedBuffer) WriteVecAt(datav [][]byte, off int64) (int, error) { - - var size int - for _, data := range datav { - size += len(data) - } - - blockInfos := b.relevantBlocks(datav, off, int64(size)) - if b.conf.WriteParallel { - return b.writeBlocksParallel(blockInfos) - } - return b.writeBlocksSequential(blockInfos) -} - -func (b *RedisBlockedBuffer) readBlocksParallel(blockInfos []blockInfo) (int, error) { - - retChan := make(chan chanReturnData) - for _, blockInfo := range blockInfos { - go func(buf Buffer, datav [][]byte, off int64, retChan chan<- chanReturnData) { - read, err := buf.ReadVecAt(datav, off) - retChan <- chanReturnData{read, err} - }(blockInfo.buf, blockInfo.data, blockInfo.off, retChan) - } - - var n int - var err error - for range blockInfos { - retData := <-retChan - if retData.err != nil { - err = retData.err - } - n += retData.n - } - return n, err -} - -func (b *RedisBlockedBuffer) readBlocksSequential(blockInfos []blockInfo) (int, error) { - var n int - for _, blockInfo := range blockInfos { - read, err := blockInfo.buf.ReadVecAt(blockInfo.data, blockInfo.off) - n += read - if err != nil { - return n, err - } - } - return n, nil -} - -//ReadAt reads a vector of byte slices starting at byte offset off -func (b *RedisBlockedBuffer) ReadAt(data []byte, off int64) (int, error) { - blockInfos := b.relevantBlocks([][]byte{data}, off, int64(len(data))) - if b.conf.ReadParallel { - return b.readBlocksParallel(blockInfos) - } - return b.readBlocksSequential(blockInfos) -} - -//ReadVecAt reads a vector of byte slices from the Buffer starting at byte offset off -func (b *RedisBlockedBuffer) ReadVecAt(datav [][]byte, off int64) (int, error) { - var size int - for _, data := range datav { - size += len(data) - } - blockInfos := b.relevantBlocks(datav, off, int64(size)) - if b.conf.ReadParallel { - return b.readBlocksParallel(blockInfos) - } - return b.readBlocksSequential(blockInfos) -} - -// Resize resizes the Buffer to a given size. -// It returns an error if the given size is negative. -// If the Buffer is larger than the specified size, the extra data is lost. -// If the Buffer is smaller, it is extended and the extended part (hole) -// reads as zero bytes. -func (b *RedisBlockedBuffer) Resize(size int64) error { - if size < 0 { - return errors.New("Resize: size must be non-negative") - } - if storeSize := b.Size(); size == storeSize { - return nil - } else if size < storeSize { - err := b.shrink(size) - if err != nil { - return err - } - } else { - err := b.grow(size) - if err != nil { - return err - } - } - return nil -} - -func (b *RedisBlockedBuffer) shrink(size int64) error { - newBlocksInfo := b.newBlocksLayout(0, size) - - newLastBlockInfo := newBlocksInfo[len(newBlocksInfo)-1] - // remove all existing blocks after this new last block - for id, max := newLastBlockInfo.id+1, b.NbBlocks(); id < max; id++ { - err := b.removeBlockedBufferByID(id) - if err != nil { - return err - } - } - // resize the last block to correct size - err := newLastBlockInfo.buf.Resize(int64(newLastBlockInfo.len)) - if err != nil { - return err - } - return nil -} - -func (b *RedisBlockedBuffer) grow(size int64) error { - - var lastBlockID int // ID of existing last block - if l := b.NbBlocks(); l == 0 { - // empty RedisBlockedBuffer, no block was ever created - lastBlockID = 0 - } else { - lastBlockID = l - 1 - } - - newBlocksInfo := b.newBlocksLayout(0, size) - - oldLastBlockInfo := newBlocksInfo[lastBlockID] - // get all blocks including and after the existing last one and size them correctly - for id := oldLastBlockInfo.id; id < len(newBlocksInfo); id++ { - blockInfo := newBlocksInfo[id] - blockInfo.buf.Resize(int64(blockInfo.len)) - } - return nil -} - -// Clear the Buffers -func (b *RedisBlockedBuffer) Clear() error { - var err error - for id, max := 0, b.NbBlocks(); id < max; id++ { - err = b.removeBlockedBufferByID(id) - } - // FIXME: wrap or stack all errors in one error ? - b.client.Del(b.keyNbBlocks) - return err -} diff --git a/src/go/redisfs/blockedbuffer_test.go b/src/go/redisfs/blockedbuffer_test.go deleted file mode 100644 index ae7bdd8..0000000 --- a/src/go/redisfs/blockedbuffer_test.go +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright 2019 CEA -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package redisfs - -import ( - "fmt" - "strings" - "testing" -) - -func TestBlocksLayout(t *testing.T) { - client, _ := GetRedisClient() - defer client.FlushAll() - - conf := GetMountPathConf() - conf.BlockSize = 1024 - - b := NewRedisBlockedBuffer(conf, client, "Key") - - // all in one block, starting at 0 - rb := b.newBlocksLayout(0, 500) - Equals(t, len(rb), 1, "Nb of block error") - - if rb[0].id != 0 || rb[0].off != 0 || rb[0].len != 500 { - t.Errorf("error in block data: id %d, off %d, len %d", rb[0].id, rb[0].off, rb[0].len) - } - - // all in one block, starting at 500 - rb = b.newBlocksLayout(500, 500) - Equals(t, len(rb), 1, "Nb of block error") - - if rb[0].id != 0 || rb[0].off != 500 || rb[0].len != 500 { - t.Errorf("error in block data: id %d, off %d, len %d", rb[0].id, rb[0].off, rb[0].len) - } - - // taking exactly one block - rb = b.newBlocksLayout(0, 1024) - Equals(t, len(rb), 1, "Nb of block error") - - if rb[0].id != 0 || rb[0].off != 0 || rb[0].len != 1024 { - t.Errorf("error in block data: id %d, off %d, len %d", rb[0].id, rb[0].off, rb[0].len) - } - - // taking one block + 1 byte - rb = b.newBlocksLayout(0, 1025) - Equals(t, len(rb), 2, "Nb of block error") - - if rb[0].id != 0 || rb[0].off != 0 || rb[0].len != 1024 { - t.Errorf("error in block data: id %d, off %d, len %d", rb[0].id, rb[0].off, rb[0].len) - } - if rb[1].id != 1 || rb[1].off != 0 || rb[1].len != 1 { - t.Errorf("error in block data: id %d, off %d, len %d", rb[1].id, rb[1].off, rb[1].len) - } - - // taking exactly two block - rb = b.newBlocksLayout(0, 2048) - Equals(t, len(rb), 2, "Nb of block error") - - if rb[0].id != 0 || rb[0].off != 0 || rb[0].len != 1024 { - t.Errorf("error in block data: id %d, off %d, len %d", rb[0].id, rb[0].off, rb[0].len) - } - if rb[1].id != 1 || rb[1].off != 0 || rb[1].len != 1024 { - t.Errorf("error in block data: id %d, off %d, len %d", rb[1].id, rb[1].off, rb[1].len) - } - - // spanning two blocks - rb = b.newBlocksLayout(500, 1000) - Equals(t, len(rb), 2, "Nb of block error") - - if rb[0].id != 0 || rb[0].off != 500 || rb[0].len != 524 { - t.Errorf("error in block data: id %d, off %d, len %d", rb[0].id, rb[0].off, rb[0].len) - } - if rb[1].id != 1 || rb[1].off != 0 || rb[1].len != 476 { - t.Errorf("error in block data: id %d, off %d, len %d", rb[1].id, rb[1].off, rb[1].len) - } - - // spanning three blocks, starting on second one, one byte on fourth block - rb = b.newBlocksLayout(1024, 2049) - Equals(t, len(rb), 3, "Nb of block error") - - if rb[0].id != 1 || rb[0].off != 0 || rb[0].len != 1024 { - t.Errorf("error in block data: id %d, off %d, len %d", rb[0].id, rb[0].off, rb[0].len) - } - if rb[1].id != 2 || rb[1].off != 0 || rb[1].len != 1024 { - t.Errorf("error in block data: id %d, off %d, len %d", rb[1].id, rb[1].off, rb[1].len) - } - if rb[2].id != 3 || rb[2].off != 0 || rb[2].len != 1 { - t.Errorf("error in block data: id %d, off %d, len %d", rb[2].id, rb[2].off, rb[2].len) - } -} - -func writeBlockedBuffer(t *testing.T, blockSize int, datav [][]byte, offset int64) (*RedisBlockedBuffer, error) { - client, _ := GetRedisClient() - defer client.FlushAll() - - conf := GetMountPathConf() - conf.BlockSize = blockSize - - b := NewRedisBlockedBuffer(conf, client, "Key") - - var ntot int - for _, data := range datav { - ntot += len(data) - } - - if n, err := b.WriteVecAt(datav, offset); err != nil { - return b, fmt.Errorf("Unexpected error: %b", err) - } else if n != ntot { - return b, fmt.Errorf("Invalid write count: %d, expecetd %d", n, ntot) - } - - readData := make([][]byte, len(datav)) - for i, data := range datav { - readData[i] = make([]byte, len(data)) - } - if n, err := b.ReadVecAt(readData, offset); err != nil && err != ErrEndOfBuffer || n == 0 { - return b, fmt.Errorf("Error in read: %d, %b", n, err) - } - - for i, read := range readData { - if string(read) != string(datav[i]) { - return b, fmt.Errorf("Read data does not match written data: %b vs %b", read, datav[i]) - } - } - - return b, nil -} - -func TestWriteBlockedBuffer(t *testing.T) { - - // Single value data vector - - // Data fits within a single block, start at 0 offset - blockSize := 1024 - data := strings.Repeat("0123456789", 100) // 1000 bytes - offset := 0 - b, err := writeBlockedBuffer(t, blockSize, [][]byte{[]byte(data)}, int64(offset)) - Ok(t, err) - Equals(t, len(b.blocks), 1, "Nb of block error") - - // Data fits within a single block, start at non-zero offset - blockSize = 1024 - data = strings.Repeat("0123456789", 50) // 500 bytes - offset = 500 - b, err = writeBlockedBuffer(t, blockSize, [][]byte{[]byte(data)}, int64(offset)) - Ok(t, err) - Equals(t, len(b.blocks), 1, "Nb of block error") - - // Data fits exactly within a single block - blockSize = 1000 - data = strings.Repeat("0123456789", 100) // 1000 bytes - offset = 0 - b, err = writeBlockedBuffer(t, blockSize, [][]byte{[]byte(data)}, int64(offset)) - if err != nil && err != ErrEndOfBuffer { - t.Errorf("WriteBlockedBuffer error: %b", err) - } - Equals(t, len(b.blocks), 1, "Nb of block error") - - // Data fits within a block + 1 byte in next block - blockSize = 999 - data = strings.Repeat("0123456789", 100) // 1000 bytes - offset = 0 - b, err = writeBlockedBuffer(t, blockSize, [][]byte{[]byte(data)}, int64(offset)) - if err != nil && err != ErrEndOfBuffer { - t.Errorf("WriteBlockedBuffer error: %b", err) - } - Equals(t, len(b.blocks), 2, "Nb of block error") - - // Data fits in two blocks - blockSize = 1000 - data = strings.Repeat("0123456789", 100) // 1000 bytes - offset = 500 - b, err = writeBlockedBuffer(t, blockSize, [][]byte{[]byte(data)}, int64(offset)) - Ok(t, err) - Equals(t, len(b.blocks), 2, "Nb of block error") - - // Data fits in three blocks starting on second - blockSize = 1000 - data = strings.Repeat("0123456789", 200) // 2000 bytes - offset = 1500 - b, err = writeBlockedBuffer(t, blockSize, [][]byte{[]byte(data)}, int64(offset)) - Ok(t, err) - Equals(t, len(b.blocks), 3, "Nb of block error") - - // Multiple value data vector - - // Data vector fits within a single block - blockSize = 1000 - data = strings.Repeat("0123456789", 10) // 100 bytes - offset = 500 - b, err = writeBlockedBuffer(t, blockSize, [][]byte{[]byte(data), []byte(data)}, int64(offset)) - Ok(t, err) - Equals(t, len(b.blocks), 1, "Nb of block error") - - // Data vector fits exactly two blocks - blockSize = 1000 - data = strings.Repeat("0123456789", 100) // 1000 bytes - offset = 0 - b, err = writeBlockedBuffer(t, blockSize, [][]byte{[]byte(data), []byte(data)}, int64(offset)) - Ok(t, err) - Equals(t, len(b.blocks), 2, "Nb of block error") - -} - -func TestEndOfBlockedBuffer(t *testing.T) { - client, _ := GetRedisClient() - defer client.FlushAll() - - conf := GetMountPathConf() - conf.BlockSize = 20 // 20 bytes block size - - b := NewRedisBlockedBuffer(conf, client, "Key") - - // test reading empty BlockedBuffer - rdata1 := make([]byte, 30) // read over two blocks - read, err := b.ReadAt(rdata1, int64(0)) - if err != nil && err != ErrEndOfBuffer { - t.Errorf("Error in read, %d, %s", read, err) - } - if read != 0 { - t.Errorf("Different number of bytes read (%d) and written (%d)", read, 0) - } - - data := strings.Repeat("0123456789", 3) // 30 bytes to write - wrote, err := b.WriteAt([]byte(data), int64(0)) - Ok(t, err) - - rdata := make([]byte, len(data)+10) // read more than what was written - read, err = b.ReadAt(rdata, int64(0)) - if err != nil && err != ErrEndOfBuffer { - t.Errorf("Error in read, %d, %b", read, err) - } - if read != wrote { - t.Errorf("Different number of bytes read (%d) and written (%d)", read, wrote) - } -} - -func TestResizeBlockedBuffer(t *testing.T) { - client, _ := GetRedisClient() - defer client.FlushAll() - - conf := GetMountPathConf() - conf.BlockSize = 100 // 100 bytes block size - - b := NewRedisBlockedBuffer(conf, client, "Key") - - Equals(t, len(b.blocks), 0, "Nb of block error") - - err := b.Resize(100) - Ok(t, err) - if len(b.blocks) != 1 || b.blocks[0].Size() != 100 { - t.Errorf("Error in blocks, n blocks %d, len: %d", len(b.blocks), b.blocks[0].Size()) - } - - err = b.Resize(100) // no op - Ok(t, err) - if len(b.blocks) != 1 || b.blocks[0].Size() != 100 { - t.Errorf("Error in blocks, n blocks %d, len: %d", len(b.blocks), b.blocks[0].Size()) - } - - err = b.Resize(250) - Ok(t, err) - if len(b.blocks) != 3 || b.blocks[0].Size() != 100 || b.blocks[1].Size() != 100 || b.blocks[2].Size() != 50 { - t.Errorf("Error in blocks, n blocks %d, len: %d, %d, %d", len(b.blocks), b.blocks[0].Size(), b.blocks[1].Size(), b.blocks[2].Size()) - } - - err = b.Resize(200) - Ok(t, err) - if len(b.blocks) != 2 || b.blocks[0].Size() != 100 || b.blocks[1].Size() != 100 { - t.Errorf("Error in blocks, n blocks %d, len: %d, %d", len(b.blocks), b.blocks[0].Size(), b.blocks[1].Size()) - } - - err = b.Resize(150) - Ok(t, err) - if len(b.blocks) != 2 || b.blocks[0].Size() != 100 || b.blocks[1].Size() != 50 { - t.Errorf("Error in blocks, n blocks %d, len: %d, %d", len(b.blocks), b.blocks[0].Size(), b.blocks[1].Size()) - } - - err = b.Resize(0) - Ok(t, err) - if len(b.blocks) != 1 || b.blocks[0].Size() != 0 { - t.Errorf("Error in blocks, n blocks %d, len: %d", len(b.blocks), b.blocks[0].Size()) - } - -} - -func TestTruncate(t *testing.T) { - client, _ := GetRedisClient() - defer client.FlushAll() - - conf := GetMountPathConf() - conf.BlockSize = 20 // 20 bytes block size - - b := NewRedisBlockedBuffer(conf, client, "Key") - - data := strings.Repeat("0123456789", 3) // 30 bytes to write - wrote, err := b.WriteAt([]byte(data), int64(0)) - Ok(t, err) - Equals(t, wrote, 30, "Error in WriteAt") - - newLen := 15 - err = b.Resize(int64(newLen)) - Ok(t, err) - - rdata := make([]byte, len(data)+10) - read, err := b.ReadAt(rdata, 0) - if err != nil && err != ErrEndOfBuffer { - t.Errorf("Error in read, %d, %b", read, err) - } - if read != newLen { - t.Errorf("Different number of bytes read (%d) and truncated (%d)", read, newLen) - } - -} - -func TestMetaBlock(t *testing.T) { - client, _ := GetRedisClient() - defer client.FlushAll() - - conf := GetMountPathConf() - conf.BlockSize = 20 // 20 bytes block size - - b := NewRedisBlockedBuffer(conf, client, "Key") - - n := b.NbBlocks() - Equals(t, 0, n, "Wrong number of blocks") - - b.metaAddBlock(4) - n = b.NbBlocks() - Equals(t, 5, n, "Wrong number of blocks") - - b.metaAddBlock(9) - n = b.NbBlocks() - Equals(t, 10, n, "Wrong number of blocks") - - b.metaAddBlock(16) - n = b.NbBlocks() - Equals(t, 17, n, "Wrong number of blocks") - - b.metaRemoveBlock(16) - n = b.NbBlocks() - Equals(t, 10, n, "Wrong number of blocks") - -} diff --git a/src/go/redisfs/buffer.go b/src/go/redisfs/buffer.go deleted file mode 100644 index 3e38445..0000000 --- a/src/go/redisfs/buffer.go +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2019 CEA -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package redisfs - -import ( - "errors" - "fmt" - "unsafe" - - "github.com/cea-hpc/pdwfs/config" - "github.com/go-redis/redis" -) - -const maxRedisString = 512 * 1024 * 1024 // 512MB - -var ( - // ErrEndOfBuffer is thrown when read past the buffer size - ErrEndOfBuffer = errors.New("End of Buffer") - // ErrMaxRedisString is thrown when a string larger than 512MB is being written - ErrMaxRedisString = errors.New("Maximum Redis String of 512MB reached") -) - -// RedisBuffer is a Buffer in memory working on a slice of bytes. -type RedisBuffer struct { - //FIXME: conf is currently only use for the BlockSize, consider passing the BlockSize directly - // instead of MountPathConfig which is irrelevant for the most part of RedisBuffer - conf *config.Mount - redis IRedisClient - key string - bufSize int -} - -// NewRedisBuffer creates a new data volume based on a Buffer -func NewRedisBuffer(conf *config.Mount, client IRedisClient, key string) *RedisBuffer { - if conf.BlockSize == 0 { - panic(fmt.Errorf("BlockSize in configuration is not set")) - } - return &RedisBuffer{ - conf: conf, - redis: client, - key: key, - bufSize: conf.BlockSize, - } -} - -// Size returns the length of the Buffer -func (b *RedisBuffer) Size() int64 { - n, err := b.redis.StrLen(b.key).Result() - if err != nil { - panic(err) - } - return n -} - -func byte2StringNoCopy(b []byte) string { - // is this really not copying b ?? - return *(*string)(unsafe.Pointer(&b)) -} - -func string2byteNoCopy(s string) []byte { - // is this really not copying s ?? - return *(*[]byte)(unsafe.Pointer(&s)) -} - -func (b *RedisBuffer) writeString(off int64, data string) (int, error) { - newLen, err := b.redis.SetRange(b.key, off, data).Result() - if err != nil { - return 0, err - } - dataLen := len(data) - if (off + int64(dataLen)) > newLen { - // FIXME: to be tested - return int(newLen - off), ErrMaxRedisString - } - return dataLen, nil - -} - -//WriteAt writes data to the Buffer starting at byte offset off. -func (b *RedisBuffer) WriteAt(data []byte, off int64) (int, error) { - return b.writeString(off, byte2StringNoCopy(data)) -} - -//WriteVecAt writes a vector of byte slices to the Buffer starting at byte offset off. -func (b *RedisBuffer) WriteVecAt(datav [][]byte, off int64) (int, error) { - var n int - for _, data := range datav { - wrote, err := b.writeString(off, byte2StringNoCopy(data)) - if err != nil { - return n, err - } - off += int64(wrote) - n += wrote - } - return n, nil -} - -/* -//WriteVecAt writes a vector of byte slices to the Buffer starting at byte offset off. -func (b *RedisBuffer) WriteVecAt(datav [][]byte, off int64) (int, error) { - var n int - pipeRet := make([]*redis.IntCmd, len(datav)) - pipe := b.redis.Pipeline() - for i, data := range datav { - pipeRet[i] = pipe.SetRange(b.key, off+int64(n), byte2StringNoCopy(data)) - n += len(data) - } - _, err := pipe.Exec() - if err != nil { - return -1, err - } - - newLen := pipeRet[len(datav)-1].Val() // total length of string after pipe execution - if (off + int64(n)) > newLen { - // all data has not been written, Redis string limit may be been reached - // FIXME: to be tested - return int(newLen - off), ErrMaxRedisString - } - return n, nil -} -*/ - -//Clear resets the Buffer to default capacity and zero length -func (b *RedisBuffer) Clear() error { - return b.redis.Unlink(b.key).Err() -} - -//ReadAt reads in dst from the Buffer starting at byte offset off -func (b *RedisBuffer) ReadAt(dst []byte, off int64) (int, error) { - if off >= b.Size() { - return 0, ErrEndOfBuffer - } - val, err := b.redis.GetRange(b.key, off, off+int64(len(dst))-1).Result() - n := copy(dst, val) // can we avoid this copy ? - if err != nil { - return n, err - } - if n < len(dst) { - return n, ErrEndOfBuffer - } - return n, nil -} - -//ReadVecAt reads a vector of byte slices from the Buffer starting at byte offset off -func (b *RedisBuffer) ReadVecAt(dstv [][]byte, off int64) (int, error) { - if off >= b.Size() { - return 0, ErrEndOfBuffer - } - var n, ldstv int - for _, dst := range dstv { - val, err := b.redis.GetRange(b.key, off, off+int64(len(dst))-1).Result() - n += copy(dst, val) // can we avoid this copy ? - if err != nil { - return n, err - } - off += int64(n) - ldstv += len(dst) - } - if n < ldstv { - return n, ErrEndOfBuffer - } - return n, nil -} - -var trimStringScript = redis.NewScript(` - local str = redis.call("GETRANGE", KEYS[1], 0, ARGV[1]) - return redis.call("SET", KEYS[1], str) - `) - -// Resize resizes the Buffer to a given size. -// It returns an error if the given size is negative. -// If the Buffer is larger than the specified size, the extra data is lost. -// If the Buffer is smaller, it is extended and the extended part (hole) -// reads as zero bytes. -func (b *RedisBuffer) Resize(size int64) error { - if size < 0 { - return errors.New("Resize: size must be non-negative") - } - bufSize := b.Size() - if size == bufSize { - return nil - } else if size == 0 { - return b.redis.Set(b.key, "", 0).Err() - } else if size < bufSize { - return trimStringScript.Run(b.redis, []string{b.key}, size-1).Err() - } - /* else size > bufSize */ - return b.redis.SetRange(b.key, size-1, "\x00").Err() -} diff --git a/src/go/redisfs/buffer_test.go b/src/go/redisfs/buffer_test.go deleted file mode 100644 index 966e197..0000000 --- a/src/go/redisfs/buffer_test.go +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2019 CEA -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package redisfs - -import ( - "reflect" - "strings" - "testing" -) - -func TestWriteBuffer(t *testing.T) { - client, _ := GetRedisClient() - defer client.FlushAll() - - conf := GetMountPathConf() - - key := "key" - - b := NewRedisBuffer(conf, client, key) - - // Write first dots - if n, err := b.WriteVecAt([][]byte{[]byte(dots)}, 0); err != nil { - t.Errorf("Unexpected error: %s", err) - } else if n != len(dots) { - t.Errorf("Invalid write count: %d", n) - } - if s, err := client.GetRange(key, 0, int64(len(dots))-1).Result(); err != nil || s != dots { - t.Errorf("Error in GetRange: %s (val: %q)", err, s) - } - - // Write second time: abc - Buffer must grow - if n, err := b.WriteVecAt([][]byte{[]byte(abc)}, int64(len(dots))); err != nil { - t.Errorf("Unexpected error: %s", err) - } else if n != len(abc) { - t.Errorf("Invalid write count: %d", n) - } - if s, err := client.GetRange(key, 0, int64(len(dots+abc))-1).Result(); err != nil || s != dots+abc { - t.Errorf("Error in GetRange: %s (val: %q)", err, s) - } - - if s := client.StrLen(key).Val(); s != int64(len(dots)+len(abc)) { - t.Errorf("Origin Buffer did not grow: len=%d", s) - } - - // Test on case when no Buffer grow is needed - // Write dots on start of the Buffer - if n, err := b.WriteVecAt([][]byte{[]byte(dots)}, 0); err != nil { - t.Errorf("Unexpected error: %s", err) - } else if n != len(dots) { - t.Errorf("Invalid write count: %d", n) - } - if s, err := client.GetRange(key, 0, int64(len(dots))-1).Result(); err != nil || s != dots { - t.Errorf("Error in GetRange: %s (val: %q)", err, s) - } - - if s := client.StrLen(key).Val(); s != int64(len(dots)+len(abc)) { - t.Errorf("Origin Buffer should not grow: len=%d", s) - } - - // Can not read, ptr at the end - p := make([]byte, len(dots)) - end := len(dots) + len(abc) - if n, err := b.ReadVecAt([][]byte{p}, int64(end)); err == nil || n > 0 { - t.Errorf("Expected read error: %d %s", n, err) - } - - // Read dots - if n, err := b.ReadVecAt([][]byte{p}, 0); err != nil || n != len(dots) || string(p) != dots { - t.Errorf("Unexpected read error: %d %s, res: %s", n, err, string(p)) - } - - // Read abc - if n, err := b.ReadVecAt([][]byte{p}, int64(len(dots))); err != nil || n != len(abc) || string(p) != abc { - t.Errorf("Unexpected read error: %d %s, res: %s", n, err, string(p)) - } - - // Write so that Buffer must expand more than 2x - if n, err := b.WriteVecAt([][]byte{[]byte(large)}, int64(end)); err != nil { - t.Errorf("Unexpected error: %s", err) - } else if n != len(large) { - t.Errorf("Invalid write count: %d", n) - } - if s, err := client.GetRange(key, 0, int64(len(dots+abc+large))-1).Result(); err != nil || s != dots+abc+large { - t.Errorf("Error in GetRange: %s (val: %q)", err, s) - } - - if s := client.StrLen(key).Val(); s != int64(len(dots)+len(abc)+len(large)) { - t.Errorf("Origin Buffer did not grow: len=%d", s) - } - -} - -// TestBufferGrowWriteAndSeek tests if Write and Seek inside the -// Buffers boundaries result in invalid growth -func TestBufferGrowWriteAndSeek(t *testing.T) { - client, _ := GetRedisClient() - defer client.FlushAll() - - conf := GetMountPathConf() - - key := "key" - - b := NewRedisBuffer(conf, client, key) - - writeByte := func(bt byte, off int64) { - n, err := b.WriteVecAt([][]byte{[]byte{bt}}, off) - if err != nil { - t.Fatalf("Error on write: %s", err) - } else if n != 1 { - t.Fatalf("Unexpected num of bytes written: %d", n) - } - } - - // Buffer: [][XXX] - writeByte(0x01, 0) - // Buffer: [1][XX] - writeByte(0x02, 1) - // Buffer: [1,2][X] - writeByte(0x03, 0) - // Buffer: [3,2][X] - writeByte(0x01, 2) // write to end - // Buffer: [3,2,1][] - - // Check content of buf - data, err := client.Get(key).Result() - Ok(t, err) - - buf := []byte(data) - if !reflect.DeepEqual([]byte{0x03, 0x02, 0x01}, buf) { - t.Fatalf("Invalid Buffer: %s, len=%d, cap=%d", buf, len(buf), cap(buf)) - } -} - -func TestEndOfBuffer(t *testing.T) { - client, _ := GetRedisClient() - defer client.FlushAll() - - conf := GetMountPathConf() - conf.BlockSize = 20 // 20 bytes capacity Buffer - - b := NewRedisBuffer(conf, client, "Key") - - data := strings.Repeat("0123456789", 3) // 30 bytes to write - wrote, err := b.WriteAt([]byte(data), int64(0)) - Ok(t, err) - - rdata := make([]byte, len(data)+10) // read more than what was written - read, err := b.ReadAt(rdata, int64(0)) - if err != nil && err != ErrEndOfBuffer { - t.Errorf("Error in read, %d, %s", read, err) - } - if read != wrote { - t.Errorf("Different number of bytes read (%d) and written (%d)", read, wrote) - } -} diff --git a/src/go/redisfs/client.go b/src/go/redisfs/client.go deleted file mode 100644 index 6b2ccc4..0000000 --- a/src/go/redisfs/client.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2019 CEA -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package redisfs - -import ( - "fmt" - "time" - - "github.com/cea-hpc/pdwfs/config" - "github.com/go-redis/redis" -) - -// IRedisClient interface to allow multiple client implementations (client, ring, cluster) -type IRedisClient interface { - StrLen(string) *redis.IntCmd - SetRange(string, int64, string) *redis.IntCmd - GetRange(string, int64, int64) *redis.StringCmd - Exists(keys ...string) *redis.IntCmd - Set(string, interface{}, time.Duration) *redis.StatusCmd - Get(string) *redis.StringCmd - Del(...string) *redis.IntCmd - Unlink(...string) *redis.IntCmd - SetNX(string, interface{}, time.Duration) *redis.BoolCmd - SetBit(key string, offset int64, value int) *redis.IntCmd - SAdd(key string, members ...interface{}) *redis.IntCmd - SRem(key string, members ...interface{}) *redis.IntCmd - SMembers(key string) *redis.StringSliceCmd - Eval(script string, keys []string, args ...interface{}) *redis.Cmd - EvalSha(sha1 string, keys []string, args ...interface{}) *redis.Cmd - ScriptExists(hashes ...string) *redis.BoolSliceCmd - ScriptLoad(script string) *redis.StringCmd - Pipeline() redis.Pipeliner - Ping() *redis.StatusCmd - Close() error - PTTL(key string) *redis.DurationCmd - PExpire(key string, expiration time.Duration) *redis.BoolCmd - FlushAll() *redis.StatusCmd -} - -// NewRedisClient returns a redis client matching the IRedisClient interface -func NewRedisClient(conf *config.Redis) IRedisClient { - var client IRedisClient - /* - // single Redis client - client := redis.NewClient(&redis.Options{ - Addr: conf.RedisAddrs[0], - PoolSize: 50, - Password: "", // no password set - DB: 0, // use default DB - }) - */ - if conf.RedisCluster { - // Redis cluster client - //FIXME: cluter is not working... - client = redis.NewClusterClient(&redis.ClusterOptions{ - Addrs: conf.RedisClusterAddrs, - ReadOnly: true, - }) - } else { - // Ring client - addrs := make(map[string]string) - for i, addr := range conf.RedisAddrs { - addrs[fmt.Sprintf("shard%d", i)] = addr - } - opt := &redis.RingOptions{ - Addrs: addrs, - PoolTimeout: 1 * time.Hour, // deactivate timeouts - ReadTimeout: 1 * time.Hour, - IdleTimeout: 1 * time.Hour, - } - client = redis.NewRing(opt) - } - return client -} - -// Buffer is a linear addressable abstract structure to store data -type Buffer interface { - WriteAt([]byte, int64) (int, error) - ReadAt([]byte, int64) (int, error) - WriteVecAt([][]byte, int64) (int, error) - ReadVecAt([][]byte, int64) (int, error) - Clear() error - Size() int64 - Resize(int64) error -} diff --git a/src/go/redisfs/file.go b/src/go/redisfs/file.go index b7a6ecf..14c7f90 100644 --- a/src/go/redisfs/file.go +++ b/src/go/redisfs/file.go @@ -24,6 +24,8 @@ import ( var ( // ErrNegativeOffset is returned if the offset is negative. ErrNegativeOffset = errors.New("Negative offset") + // ErrNegativeTruncateSize is returned if the truncating size is negative. + ErrNegativeTruncateSize = errors.New("Negative truncate size") // ErrInvalidSeekWhence is returned if the whence argument of a seek is not a proper value. ErrInvalidSeekWhence = errors.New("Seek whence is not a proper value") // ErrNegativeSeekLocation is returned if the seek location is negative. @@ -32,7 +34,7 @@ var ( // MemFile represents a file backed by a Store which is secured from concurrent access. type MemFile struct { - buf Buffer + store *DataStore path string offset int64 mtx *sync.RWMutex @@ -40,11 +42,11 @@ type MemFile struct { // NewMemFile creates a file which byte slice is safe from concurrent access, // the file itself is not thread-safe. -func NewMemFile(buf Buffer, path string, mtx *sync.RWMutex) *MemFile { +func NewMemFile(store *DataStore, path string, mtx *sync.RWMutex) *MemFile { return &MemFile{ - buf: buf, - path: path, - mtx: mtx, + store: store, + path: path, + mtx: mtx, } } @@ -55,7 +57,7 @@ func (f MemFile) Name() string { // Size of file func (f MemFile) Size() int64 { - return f.buf.Size() + return f.store.GetSize(f.path) } // Sync has no effect @@ -65,9 +67,13 @@ func (f MemFile) Sync() error { // Truncate changes the size of the file func (f MemFile) Truncate(size int64) error { + if size < 0 { + return ErrNegativeTruncateSize + } f.mtx.Lock() defer f.mtx.Unlock() - return f.buf.Resize(size) + f.store.Resize(f.path, size) + return nil } // Close the file (no op) @@ -77,20 +83,17 @@ func (f MemFile) Close() error { func (f MemFile) readAt(dst []byte, off int64) (int, error) { if off < 0 { - return 0, ErrNegativeOffset + panic(ErrNegativeOffset) } - - read, err := f.buf.ReadAt(dst, off) - if err != nil { - if err == ErrEndOfBuffer { - return read, io.EOF - } - return read, err + if len(dst) == 0 { + return 0, nil } - if read < len(dst) { - return read, io.EOF + n := int(f.store.ReadAt(f.path, off, dst)) + //FIXME: should use int64 for all written/read lengths + if n < len(dst) { + return n, io.EOF } - return read, nil + return n, nil } // Read reads len(dst) byte starting at the current offset. @@ -110,25 +113,19 @@ func (f MemFile) ReadAt(dst []byte, off int64) (int, error) { } func (f MemFile) readVecAt(dstv [][]byte, off int64) (int, error) { - if off < 0 { - return 0, ErrNegativeOffset - } - - var size int + var n int for _, dst := range dstv { - size += len(dst) - } - read, err := f.buf.ReadVecAt(dstv, off) - if err != nil { - if err == ErrEndOfBuffer { - return read, io.EOF + read, err := f.readAt(dst, off) + n += read + if err != nil { + if err == io.EOF { + return n, io.EOF + } + panic(err) } - return read, err + off += int64(read) } - if read < size { - return read, io.EOF - } - return read, nil + return n, nil } // ReadVec reads a vector of byte slices starting at the current offset. @@ -149,9 +146,10 @@ func (f MemFile) ReadVecAt(dstv [][]byte, off int64) (int, error) { func (f MemFile) writeAt(data []byte, off int64) (int, error) { if off < 0 { - return 0, ErrNegativeOffset + panic(ErrNegativeOffset) } - return f.buf.WriteAt(data, off) + f.store.WriteAt(f.path, off, data) + return len(data), nil } // Write writes len(data) byte starting at the current offset @@ -171,10 +169,13 @@ func (f MemFile) WriteAt(data []byte, off int64) (int, error) { } func (f MemFile) writeVecAt(datav [][]byte, off int64) (int, error) { - if off < 0 { - return 0, ErrNegativeOffset + var n int + for _, data := range datav { + wrote, _ := f.writeAt(data, off) + off += int64(wrote) + n += wrote } - return f.buf.WriteVecAt(datav, off) + return n, nil } // WriteVec writes a vector of byte slices starting at the current offset diff --git a/src/go/redisfs/file_test.go b/src/go/redisfs/file_test.go index 6e016de..a5bb7aa 100644 --- a/src/go/redisfs/file_test.go +++ b/src/go/redisfs/file_test.go @@ -20,6 +20,9 @@ import ( "strings" "sync" "testing" + + "github.com/cea-hpc/pdwfs/config" + "github.com/cea-hpc/pdwfs/util" ) const ( @@ -28,28 +31,29 @@ const ( ) var ( - large = strings.Repeat("0123456789", 200) // 2000 bytes + large = strings.Repeat("0123456789", 200) // 2000 bytes + huge = strings.Repeat("0123456789", 2000000) // 20 MB ) -func setupMemFile(t *testing.T) (*MemFile, IRedisClient) { - client, _ := GetRedisClient() - conf := GetMountPathConf() - - buf := NewRedisBuffer(conf, client, "Key") - f := NewMemFile(buf, "/path/to/file", &sync.RWMutex{}) - return f, client +func setupMemFile(t *testing.T) (*MemFile, *util.RedisTestServer, *DataStore) { + redis, conf := util.InitRedisTestServer() + store := NewDataStore(NewRedisRing(conf), config.DefaultStripeSize) + f := NewMemFile(store, "/path/to/file", &sync.RWMutex{}) + return f, redis, store } func TestFileInterface(t *testing.T) { - f, client := setupMemFile(t) - defer client.FlushAll() + f, redis, client := setupMemFile(t) + defer redis.Stop() + defer client.Close() _ = File(f) } func TestWrite(t *testing.T) { - f, client := setupMemFile(t) - defer client.FlushAll() + f, redis, client := setupMemFile(t) + defer redis.Stop() + defer client.Close() // Write first dots if n, err := f.Write([]byte(dots)); err != nil { @@ -123,11 +127,19 @@ func TestWrite(t *testing.T) { } else if n != len(large) { t.Errorf("Invalid write count: %d", n) } + + // Write huge + if n, err := f.Write([]byte(huge)); err != nil { + t.Errorf("Unexpected error: %s", err) + } else if n != len(huge) { + t.Errorf("Invalid write count: %d", n) + } } func TestSeek(t *testing.T) { - f, client := setupMemFile(t) - defer client.FlushAll() + f, redis, client := setupMemFile(t) + defer redis.Stop() + defer client.Close() // write dots if n, err := f.Write([]byte(dots)); err != nil || n != len(dots) { @@ -155,8 +167,9 @@ func TestSeek(t *testing.T) { } func TestRead(t *testing.T) { - f, client := setupMemFile(t) - defer client.FlushAll() + f, redis, client := setupMemFile(t) + defer redis.Stop() + defer client.Close() // write dots if n, err := f.Write([]byte(dots)); err != nil || n != len(dots) { @@ -176,8 +189,9 @@ func TestRead(t *testing.T) { } func TestReadAt(t *testing.T) { - f, client := setupMemFile(t) - defer client.FlushAll() + f, redis, client := setupMemFile(t) + defer redis.Stop() + defer client.Close() // write dots if n, err := f.Write([]byte(dots)); err != nil || n != len(dots) { @@ -213,8 +227,9 @@ func TestReadAt(t *testing.T) { } func TestSize(t *testing.T) { - f, client := setupMemFile(t) - defer client.FlushAll() + f, redis, client := setupMemFile(t) + defer redis.Stop() + defer client.Close() // write dots if n, err := f.Write([]byte(dots)); err != nil || n != len(dots) { diff --git a/src/go/redisfs/fs.go b/src/go/redisfs/fs.go index 33ab858..0e7d414 100644 --- a/src/go/redisfs/fs.go +++ b/src/go/redisfs/fs.go @@ -38,6 +38,8 @@ var ( ErrNotDirectory = errors.New("Is not a directory") // ErrDirNotEmpty is returned if a directory is not empty (rmdir) ErrDirNotEmpty = errors.New("Directory is not empty") + // ErrParentDirNotExist is returned if the parent directory does not exist + ErrParentDirNotExist = errors.New("Parent directory does not exist") ) // File represents a File with common operations. @@ -66,20 +68,35 @@ const PathSeparator = "/" // RedisFS is a in-memory filesystem type RedisFS struct { mountConf *config.Mount - inodes *InodeRegister + dataStore *DataStore + redisRing *RedisRing + inodes map[string]*Inode + root *Inode } // NewRedisFS a new RedisFS filesystem which entirely resides in memory func NewRedisFS(redisConf *config.Redis, mountConf *config.Mount) *RedisFS { + redisRing := NewRedisRing(redisConf) + dataStore := NewDataStore(redisRing, int64(mountConf.StripeSize)) + + // create root inode + //FIXME: mount path (root) should only be created it it exists on the FS at startup + root := NewInode(dataStore, redisRing, mountConf.Path) + root.initMeta(true, 0600) + return &RedisFS{ mountConf: mountConf, - inodes: NewInodeRegister(redisConf, mountConf), + redisRing: redisRing, + dataStore: dataStore, + inodes: map[string]*Inode{root.Path(): root}, + root: root, } } // Finalize performs close up actions on the virtual file system -func (fs *RedisFS) Finalize() error { - return fs.inodes.Finalize() +func (fs *RedisFS) Finalize() { + fs.redisRing.Close() + fs.dataStore.Close() } // ValidatePath ensures path belongs to a filesystem tree catched by pdwfs @@ -94,13 +111,41 @@ func (fs *RedisFS) ValidatePath(path string) error { return nil } +func (fs *RedisFS) createInode(path string, dir bool, mode os.FileMode, parent *Inode) *Inode { + i := NewInode(fs.dataStore, fs.redisRing, path) + i.initMeta(dir, mode) + parent.setChild(i) + fs.inodes[i.Path()] = i + return i +} + +func (fs *RedisFS) getInode(path string) (*Inode, bool) { + if i, ok := fs.inodes[path]; ok { + return i, true + } + i := NewInode(fs.dataStore, fs.redisRing, path) + if ok := i.exists(); !ok { + return nil, false + } + fs.inodes[i.Path()] = i + return i, true +} + +func (fs *RedisFS) removeInode(i *Inode) { + i.remove() + delete(fs.inodes, i.Path()) +} + func (fs *RedisFS) fileInfo(abspath string) (parent, node *Inode, err error) { + if abspath == fs.root.Path() { + return nil, fs.root, nil + } parentPath := filepath.Dir(abspath) - fiParent, _ := fs.inodes.getInode(parentPath) + fiParent, _ := fs.getInode(parentPath) if fiParent == nil || !fiParent.IsDir() { - return nil, nil, os.ErrNotExist + return nil, nil, ErrParentDirNotExist } - fiNode, _ := fs.inodes.getInode(abspath) + fiNode, _ := fs.getInode(abspath) return fiParent, fiNode, nil } @@ -110,17 +155,21 @@ func (fs *RedisFS) Mkdir(name string, perm os.FileMode) error { return &os.PathError{Op: "mkdir", Path: name, Err: err} } path, err := filepath.Abs(name) - if err != nil { - panic(err) - } + Check(err) fiParent, fiNode, err := fs.fileInfo(path) if err != nil { return &os.PathError{Op: "mkdir", Path: name, Err: err} } + if fiNode == fs.root { + //FIXME: hack to cover the case the app creates the mount path directory, + // because we create the root dir at initialization of RedisFS, it should fail with ErrExist. + // Proper way to do this is to create the root dir in RedisFS only if it already exists on FS at startup + return nil + } if fiNode != nil { return &os.PathError{Op: "mkdir", Path: name, Err: os.ErrExist} } - fs.inodes.CreateInode(path, true, perm, fiParent) + fs.createInode(path, true, perm, fiParent) return nil } @@ -142,9 +191,7 @@ func (fs *RedisFS) ReadDir(path string) ([]os.FileInfo, error) { return nil, &os.PathError{Op: "readdir", Path: path, Err: err} } path, err := filepath.Abs(path) - if err != nil { - panic(err) - } + Check(err) _, fi, err := fs.fileInfo(path) if err != nil { return nil, &os.PathError{Op: "readdir", Path: path, Err: err} @@ -153,7 +200,7 @@ func (fs *RedisFS) ReadDir(path string) ([]os.FileInfo, error) { return nil, &os.PathError{Op: "readdir", Path: path, Err: ErrNotDirectory} } - fis, err := fs.inodes.getChildren(fi) + fis, err := fi.getChildren() if err != nil { return nil, &os.PathError{Op: "readdir", Path: path, Err: err} } @@ -193,9 +240,7 @@ func (fs *RedisFS) OpenFile(name string, flag int, perm os.FileMode) (File, erro return nil, &os.PathError{Op: "open", Path: name, Err: err} } path, err := filepath.Abs(name) - if err != nil { - panic(err) - } + Check(err) fiParent, fiNode, err := fs.fileInfo(path) if err != nil { return nil, &os.PathError{Op: "open", Path: name, Err: err} @@ -205,7 +250,7 @@ func (fs *RedisFS) OpenFile(name string, flag int, perm os.FileMode) (File, erro if !hasFlag(os.O_CREATE, flag) { return nil, &os.PathError{Op: "open", Path: name, Err: os.ErrNotExist} } - fiNode = fs.inodes.CreateInode(path, false, perm, fiParent) + fiNode = fs.createInode(path, false, perm, fiParent) } else { // file exists if hasFlag(os.O_CREATE|os.O_EXCL, flag) { return nil, &os.PathError{Op: "open", Path: name, Err: os.ErrExist} @@ -214,7 +259,7 @@ func (fs *RedisFS) OpenFile(name string, flag int, perm os.FileMode) (File, erro return nil, &os.PathError{Op: "open", Path: name, Err: ErrIsDirectory} } } - return fs.inodes.getFile(fiNode, flag) + return fiNode.getFile(flag) } // roFile wraps the given file and disables Write(..) operation. @@ -244,9 +289,7 @@ func (fs *RedisFS) Remove(name string) error { return &os.PathError{Op: "remove", Path: name, Err: err} } path, err := filepath.Abs(name) - if err != nil { - panic(err) - } + Check(err) fiParent, fiNode, err := fs.fileInfo(path) if err != nil { return &os.PathError{Op: "remove", Path: name, Err: err} @@ -255,7 +298,7 @@ func (fs *RedisFS) Remove(name string) error { return &os.PathError{Op: "remove", Path: name, Err: os.ErrNotExist} } fiParent.removeChild(fiNode) - fs.inodes.delete(fiNode) + fs.removeInode(fiNode) return nil } @@ -266,9 +309,7 @@ func (fs *RedisFS) Stat(name string) (os.FileInfo, error) { return nil, &os.PathError{Op: "stat", Path: name, Err: err} } path, err := filepath.Abs(name) - if err != nil { - panic(err) - } + Check(err) _, fi, err := fs.fileInfo(path) if err != nil { return nil, &os.PathError{Op: "stat", Path: name, Err: err} diff --git a/src/go/redisfs/fs_test.go b/src/go/redisfs/fs_test.go index ccff968..c549324 100644 --- a/src/go/redisfs/fs_test.go +++ b/src/go/redisfs/fs_test.go @@ -15,38 +15,46 @@ package redisfs import ( + "bytes" + "fmt" "io/ioutil" "os" "path/filepath" "testing" + "time" + "reflect" + + "github.com/cea-hpc/pdwfs/util" ) func TestCreate(t *testing.T) { - client, redisConf := GetRedisClient() - defer client.FlushAll() + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() - mountConf := GetMountPathConf() + mountConf := util.GetMountPathConf() mountConf.Path = "/" fs := NewRedisFS(redisConf, mountConf) + defer fs.Finalize() + // NewRedisFS file with absolute path { f, err := fs.OpenFile("/testfile", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) - Ok(t, err) - Equals(t, "/testfile", f.Name(), "Wrong name") + util.Ok(t, err) + util.Equals(t, "/testfile", f.Name(), "Wrong name") } // NewRedisFS same file again { _, err := fs.OpenFile("/testfile", os.O_RDWR|os.O_CREATE, 0666) - Ok(t, err) + util.Ok(t, err) } // NewRedisFS same file again, but truncate it { _, err := fs.OpenFile("/testfile", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) - Ok(t, err) + util.Ok(t, err) } // NewRedisFS same file again with O_CREATE|O_EXCL, which is an error @@ -67,36 +75,39 @@ func TestCreate(t *testing.T) { } func TestCreateRelative(t *testing.T) { - client, redisConf := GetRedisClient() - defer client.FlushAll() + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() - mountConf := GetMountPathConf() + mountConf := util.GetMountPathConf() cwd, err := os.Getwd() - Ok(t, err) + util.Ok(t, err) mountConf.Path = cwd fs := NewRedisFS(redisConf, mountConf) + defer fs.Finalize() + // NewRedisFS file with relative path (workingDir == root) { f, err := fs.OpenFile("relFile", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) - Ok(t, err) - Equals(t, filepath.Join(cwd, "relFile"), f.Name(), "Wrong name") + util.Ok(t, err) + util.Equals(t, filepath.Join(cwd, "relFile"), f.Name(), "Wrong name") } } func TestMkdirAbs(t *testing.T) { - client, redisConf := GetRedisClient() - defer client.FlushAll() + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() - mountConf := GetMountPathConf() + mountConf := util.GetMountPathConf() mountConf.Path = "/" fs := NewRedisFS(redisConf, mountConf) + defer fs.Finalize() // NewRedisFS dir with absolute path { err := fs.Mkdir("/usr", 0) - Ok(t, err) + util.Ok(t, err) } // NewRedisFS dir twice @@ -109,40 +120,42 @@ func TestMkdirAbs(t *testing.T) { } func TestMkdirRel(t *testing.T) { - client, redisConf := GetRedisClient() - defer client.FlushAll() + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() - mountConf := GetMountPathConf() + mountConf := util.GetMountPathConf() cwd, err := os.Getwd() - Ok(t, err) + util.Ok(t, err) mountConf.Path = cwd fs := NewRedisFS(redisConf, mountConf) + defer fs.Finalize() // NewRedisFS dir with relative path { err := fs.Mkdir("home", 0) - Ok(t, err) + util.Ok(t, err) } } func TestMkdirTree(t *testing.T) { - client, redisConf := GetRedisClient() - defer client.FlushAll() + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() - mountConf := GetMountPathConf() + mountConf := util.GetMountPathConf() mountConf.Path = "/" fs := NewRedisFS(redisConf, mountConf) + defer fs.Finalize() err := fs.Mkdir("/home", 0) - Ok(t, err) + util.Ok(t, err) err = fs.Mkdir("/home/blang", 0) - Ok(t, err) + util.Ok(t, err) err = fs.Mkdir("/home/blang/goprojects", 0) - Ok(t, err) + util.Ok(t, err) err = fs.Mkdir("/home/johndoe/goprojects", 0) if err == nil { @@ -152,33 +165,51 @@ func TestMkdirTree(t *testing.T) { //TODO: Subdir of file } +func TestMkdirMountPath(t *testing.T) { + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() + + mountConf := util.GetMountPathConf() + mountConf.Path = "/foo" + + fs := NewRedisFS(redisConf, mountConf) + defer fs.Finalize() + + err := fs.Mkdir("/foo", 0) + util.Ok(t, err) + + err = fs.Mkdir("/foo/bar", 0) + util.Ok(t, err) +} + func TestReadDir(t *testing.T) { - client, redisConf := GetRedisClient() - defer client.FlushAll() + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() - mountConf := GetMountPathConf() + mountConf := util.GetMountPathConf() mountConf.Path = "/" fs := NewRedisFS(redisConf, mountConf) + defer fs.Finalize() dirs := []string{"/home", "/home/linus", "/home/rob", "/home/pike", "/home/blang"} expectNames := []string{"/home/README.txt", "/home/blang", "/home/linus", "/home/pike", "/home/rob"} for _, dir := range dirs { err := fs.Mkdir(dir, 0777) - Ok(t, err) + util.Ok(t, err) } f, err := fs.OpenFile("/home/README.txt", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) - Ok(t, err) + util.Ok(t, err) f.Close() fis, err := fs.ReadDir("/home") - Ok(t, err) + util.Ok(t, err) - Equals(t, len(fis), len(expectNames), "Wrong size") + util.Equals(t, len(fis), len(expectNames), "Wrong size") for i, n := range expectNames { - Equals(t, n, fis[i].Name(), "Wrong name") + util.Equals(t, n, fis[i].Name(), "Wrong name") } // Readdir empty directory @@ -206,22 +237,23 @@ func TestReadDir(t *testing.T) { } func TestRemove(t *testing.T) { - client, redisConf := GetRedisClient() - defer client.FlushAll() + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() - mountConf := GetMountPathConf() + mountConf := util.GetMountPathConf() mountConf.Path = "/" fs := NewRedisFS(redisConf, mountConf) + defer fs.Finalize() err := fs.Mkdir("/tmp", 0777) - Ok(t, err) + util.Ok(t, err) f, err := fs.OpenFile("/tmp/README.txt", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) - Ok(t, err) + util.Ok(t, err) _, err = f.Write([]byte("test")) - Ok(t, err) + util.Ok(t, err) f.Close() @@ -237,7 +269,7 @@ func TestRemove(t *testing.T) { // remove created file err = fs.Remove("/tmp/README.txt") - Ok(t, err) + util.Ok(t, err) if _, err = fs.OpenFile("/tmp/README.txt", os.O_RDWR, 0666); err == nil { t.Errorf("Could open removed file!") @@ -248,14 +280,14 @@ func TestRemove(t *testing.T) { t.Errorf("Could not create removed file!") } fi, err := fs.Stat(f.Name()) - Ok(t, err) + util.Ok(t, err) if fi.Size() != 0 { t.Errorf("Error in Size, got %d (expecting 0)", fi.Size()) } err = fs.Remove("/tmp") - Ok(t, err) + util.Ok(t, err) if fis, err := fs.ReadDir("/"); err != nil { t.Errorf("Readdir error: %s", err) @@ -265,16 +297,17 @@ func TestRemove(t *testing.T) { } func TestReadWrite(t *testing.T) { - client, redisConf := GetRedisClient() - defer client.FlushAll() + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() - mountConf := GetMountPathConf() + mountConf := util.GetMountPathConf() mountConf.Path = "/" fs := NewRedisFS(redisConf, mountConf) + defer fs.Finalize() f, err := fs.OpenFile("/readme.txt", os.O_CREATE|os.O_RDWR, 0666) - Ok(t, err) + util.Ok(t, err) // Write first dots if n, err := f.Write([]byte(dots)); err != nil { @@ -321,16 +354,17 @@ func TestReadWrite(t *testing.T) { } func TestOpenRO(t *testing.T) { - client, redisConf := GetRedisClient() - defer client.FlushAll() + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() - mountConf := GetMountPathConf() + mountConf := util.GetMountPathConf() mountConf.Path = "/" fs := NewRedisFS(redisConf, mountConf) + defer fs.Finalize() f, err := fs.OpenFile("/readme.txt", os.O_CREATE|os.O_RDONLY, 0666) - Ok(t, err) + util.Ok(t, err) // Write first dots if _, err := f.Write([]byte(dots)); err == nil { @@ -340,16 +374,16 @@ func TestOpenRO(t *testing.T) { } func TestOpenWO(t *testing.T) { - client, redisConf := GetRedisClient() - defer client.FlushAll() + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() - mountConf := GetMountPathConf() + mountConf := util.GetMountPathConf() mountConf.Path = "/" fs := NewRedisFS(redisConf, mountConf) f, err := fs.OpenFile("/readme.txt", os.O_CREATE|os.O_WRONLY, 0666) - Ok(t, err) + util.Ok(t, err) // Write first dots if n, err := f.Write([]byte(dots)); err != nil { @@ -373,16 +407,17 @@ func TestOpenWO(t *testing.T) { } func TestOpenAppend(t *testing.T) { - client, redisConf := GetRedisClient() - defer client.FlushAll() + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() - mountConf := GetMountPathConf() + mountConf := util.GetMountPathConf() mountConf.Path = "/" fs := NewRedisFS(redisConf, mountConf) + defer fs.Finalize() f, err := fs.OpenFile("/readme.txt", os.O_CREATE|os.O_RDWR, 0666) - Ok(t, err) + util.Ok(t, err) // Write first dots if n, err := f.Write([]byte(dots)); err != nil { @@ -420,10 +455,10 @@ func TestOpenAppend(t *testing.T) { } func TestTruncateToLength(t *testing.T) { - client, redisConf := GetRedisClient() - defer client.FlushAll() + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() - mountConf := GetMountPathConf() + mountConf := util.GetMountPathConf() mountConf.Path = "/" var params = []struct { @@ -439,7 +474,7 @@ func TestTruncateToLength(t *testing.T) { for _, param := range params { fs := NewRedisFS(redisConf, mountConf) f, err := fs.OpenFile("/readme.txt", os.O_CREATE|os.O_RDWR, 0666) - Ok(t, err) + util.Ok(t, err) if n, err := f.Write([]byte(dots)); err != nil { t.Errorf("Unexpected error: %s", err) } else if n != len(dots) { @@ -459,7 +494,7 @@ func TestTruncateToLength(t *testing.T) { } b, err := readFile(fs, "/readme.txt") - Ok(t, err) + util.Ok(t, err) if int64(len(b)) != newSize { t.Errorf("File should be empty after truncation: %d", len(b)) @@ -469,17 +504,19 @@ func TestTruncateToLength(t *testing.T) { } else if fi.Size() != newSize { t.Errorf("Filesize should be %d after truncation", newSize) } + fs.Finalize() } } func TestTruncateToZero(t *testing.T) { - client, redisConf := GetRedisClient() - defer client.FlushAll() + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() - mountConf := GetMountPathConf() + mountConf := util.GetMountPathConf() mountConf.Path = "/" fs := NewRedisFS(redisConf, mountConf) + defer fs.Finalize() const content = "read me" @@ -488,12 +525,12 @@ func TestTruncateToZero(t *testing.T) { } f, err := fs.OpenFile("/readme.txt", os.O_RDWR|os.O_TRUNC, 0666) - Ok(t, err) + util.Ok(t, err) f.Close() b, err := readFile(fs, "/readme.txt") - Ok(t, err) + util.Ok(t, err) if len(b) != 0 { t.Errorf("File should be empty after truncation") @@ -506,19 +543,20 @@ func TestTruncateToZero(t *testing.T) { } func TestStat(t *testing.T) { - client, redisConf := GetRedisClient() - defer client.FlushAll() + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() - mountConf := GetMountPathConf() + mountConf := util.GetMountPathConf() mountConf.Path = "/" fs := NewRedisFS(redisConf, mountConf) + defer fs.Finalize() f, err := fs.OpenFile("/readme.txt", os.O_CREATE|os.O_RDWR, 0666) - Ok(t, err) + util.Ok(t, err) fi, err := fs.Stat(f.Name()) - Ok(t, err) + util.Ok(t, err) if s := fi.Size(); s != int64(0) { t.Errorf("Invalid size: %d", s) @@ -537,7 +575,7 @@ func TestStat(t *testing.T) { } fi, err = fs.Stat(f.Name()) - Ok(t, err) + util.Ok(t, err) // File name is abs name if name := f.Name(); name != "/readme.txt" { @@ -572,19 +610,20 @@ func readFile(fs *RedisFS, name string) ([]byte, error) { } func TestVolumesConcurrentAccess(t *testing.T) { - client, redisConf := GetRedisClient() - defer client.FlushAll() + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() - mountConf := GetMountPathConf() + mountConf := util.GetMountPathConf() mountConf.Path = "/" fs := NewRedisFS(redisConf, mountConf) + defer fs.Finalize() f1, err := fs.OpenFile("/testfile", os.O_RDWR|os.O_CREATE, 0666) - Ok(t, err) + util.Ok(t, err) f2, err := fs.OpenFile("/testfile", os.O_RDWR, 0666) - Ok(t, err) + util.Ok(t, err) // f1 write dots if n, err := f1.Write([]byte(dots)); err != nil || n != len(dots) { @@ -609,3 +648,66 @@ func TestVolumesConcurrentAccess(t *testing.T) { } } + +var ( + bigdata = bytes.Repeat([]byte("0123456789"), 200000) // 2MB +) + +func TestBenchOpenWriteClose(t *testing.T) { + redis, redisConf := util.InitRedisTestServer() + defer redis.Stop() + + mountConf := util.GetMountPathConf() + mountConf.Path = "/" + mountConf.StripeSize = 10 * 1024 * 1024 + + fs := NewRedisFS(redisConf, mountConf) + defer fs.Finalize() + + allstart := time.Now() + + f, err := fs.OpenFile("/testfile", os.O_RDWR|os.O_CREATE, 0666) + util.Ok(t, err) + + openelapsed := time.Since(allstart) + fmt.Println("Open took: ", openelapsed) + + writestart := time.Now() + // f write bigdata + if n, err := f.Write(bigdata); err != nil || n != len(bigdata) { + t.Errorf("Unexpected write error: %d %s", n, err) + } + + writeelapsed := time.Since(writestart) + fmt.Println("Write took: ", writeelapsed) + + closestart := time.Now() + if err := f.Close(); err != nil { + t.Errorf("Unexpected close error: %s", err) + } + closeelapsed := time.Since(closestart) + fmt.Println("Close took: ", closeelapsed) + + reopen := time.Now() + f, err = fs.OpenFile("/testfile", os.O_RDONLY, 0666) + util.Ok(t, err) + reopenelapsed := time.Since(reopen) + fmt.Println("Reopen took: ", reopenelapsed) + + readstart := time.Now() + // f read bigdata + readBuf := make([]byte, len(bigdata), len(bigdata)) + if n, err := f.Read(readBuf); err != nil || n != len(bigdata) { + t.Errorf("Unexpected read error: %d %s", n, err) + } + readelapsed := time.Since(readstart) + fmt.Println("Read took: ", readelapsed) + + allelapsed := time.Since(allstart) + fmt.Println("TestOpenWriteClose took: ", allelapsed) + + if !reflect.DeepEqual(bigdata, readBuf) { + fmt.Println("read data does not match written data: exp: ", bigdata[:10], "act: ", readBuf[:10]) + t.FailNow() + } +} diff --git a/src/go/redisfs/inodes.go b/src/go/redisfs/inodes.go index 4800744..1135b38 100644 --- a/src/go/redisfs/inodes.go +++ b/src/go/redisfs/inodes.go @@ -11,6 +11,9 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +// +// The Inode layer manages inodes as either regular files or a directories and associated metadata. +// The metadata is reduced the bare minimum on purpose (to reduce bottlenecks of handling metadata). package redisfs @@ -19,203 +22,79 @@ import ( "strconv" "sync" "time" - - "github.com/cea-hpc/pdwfs/config" ) -//InodeRegister ... -type InodeRegister struct { - mountConf *config.Mount - client IRedisClient - fiMap map[string]*Inode - root *Inode -} - //Inode object type Inode struct { - mountConf *config.Mount - client IRedisClient - id string - redisBuf *RedisBlockedBuffer - mtx *sync.RWMutex - metaBaseKey string - isDir *bool - mode *os.FileMode -} - -//NewInodeRegister constructor -func NewInodeRegister(redisConf *config.Redis, mountConf *config.Mount) *InodeRegister { - client := NewRedisClient(redisConf) - - // create root inode - root := NewInode(mountConf, client, mountConf.Path) - root.initMeta(true, 0600) - - return &InodeRegister{ - mountConf: mountConf, - client: client, - fiMap: map[string]*Inode{mountConf.Path: root}, - root: root, - } -} - -//Finalize ... -func (ir *InodeRegister) Finalize() error { - return ir.client.Close() -} - -//CreateInode ... -func (ir *InodeRegister) CreateInode(path string, dir bool, mode os.FileMode, parent *Inode) *Inode { - - inode := NewInode(ir.mountConf, ir.client, path) - inode.initMeta(dir, mode) - - parent.setChild(inode) - ir.fiMap[inode.ID()] = inode - return inode + dataStore *DataStore + redisRing *RedisRing + path string + keyPrefix string + mtx *sync.RWMutex } -func (ir *InodeRegister) getInode(id string) (*Inode, bool) { - if i, ok := ir.fiMap[id]; ok { - return i, true - } - i := NewInode(ir.mountConf, ir.client, id) - - if ok, _ := i.exists(); !ok { - return nil, false - } - ir.fiMap[id] = i - return i, true -} - -func (ir *InodeRegister) getChildren(inode *Inode) ([]*Inode, error) { - IDs := inode.childrenID() - children := make([]*Inode, 0, len(IDs)) - for _, id := range IDs { - child, ok := ir.getInode(id) - if !ok { - panic("Inode not found") - } - children = append(children, child) - } - return children, nil -} - -func (ir *InodeRegister) getFile(inode *Inode, flag int) (File, error) { - if inode.IsDir() { - return nil, ErrIsDirectory - } - inodeBuf := inode.getBuffer() - - if hasFlag(os.O_TRUNC, flag) { - inodeBuf.Clear() - } - - var f File = NewMemFile(inodeBuf, inode.Name(), inode.mtx) - - if hasFlag(os.O_APPEND, flag) { - f.Seek(0, os.SEEK_END) - } else { - f.Seek(0, os.SEEK_SET) - } - if hasFlag(os.O_RDWR, flag) { - return f, nil - } else if hasFlag(os.O_WRONLY, flag) { - f = &woFile{f} - } else { - f = &roFile{f} +//NewInode returns a new Inode object +func NewInode(dataStore *DataStore, ring *RedisRing, path string) *Inode { + return &Inode{ + dataStore: dataStore, + redisRing: ring, + path: path, + mtx: &sync.RWMutex{}, + keyPrefix: "{" + path + "}", // key prefix is in curly braces to ensure all metadata keys goes on the same instance (see RedisRing) } - - return f, nil } -func (ir *InodeRegister) delete(inode *Inode) { - if buf := inode.getBuffer(); buf != nil { - buf.Clear() - } - children, _ := ir.getChildren(inode) - for _, child := range children { - ir.delete(child) - } - delete(ir.fiMap, inode.Name()) - err := inode.delMeta() - if err != nil { - panic(err) - } +// check if the inode object already exists in pdwfs (check in Redis) +func (i *Inode) exists() bool { + client := i.redisRing.GetClient(i.keyPrefix) + ret, err := client.Exists(i.keyPrefix + ":mode") + Check(err) + return ret } -//NewInode ... -func NewInode(mountConf *config.Mount, client IRedisClient, id string) *Inode { - return &Inode{ - mountConf: mountConf, - client: client, - id: id, - mtx: &sync.RWMutex{}, - metaBaseKey: "{" + id + "}", // hastag to ensure all metadata keys goes on the same instance - } -} -func (i *Inode) getBuffer() *RedisBlockedBuffer { - if i.redisBuf == nil && !i.IsDir() { - i.redisBuf = NewRedisBlockedBuffer(i.mountConf, i.client, i.ID()) - } - return i.redisBuf -} -func (i *Inode) exists() (bool, error) { - if i.mode != nil { - return true, nil +// creates the metadata in Redis of a newly created Inode in pdwfs +func (i *Inode) initMeta(isDir bool, mode os.FileMode) { + pipeline := i.redisRing.GetClient(i.keyPrefix).Pipeline() + if isDir { + pipeline.Do("SADD", i.keyPrefix+":children", "") } - ret, err := i.client.Exists(i.metaBaseKey + ":mode").Result() - return ret != 0, err + pipeline.Do("SETNX", i.keyPrefix+":mode", []byte(strconv.FormatInt(int64(mode), 10))) + pipeline.Flush() } -func (i *Inode) initMeta(isDir bool, mode os.FileMode) error { - pipeline := i.client.Pipeline() - pipeline.SetNX(i.metaBaseKey+":isDir", isDir, 0) - pipeline.SetNX(i.metaBaseKey+":mode", uint32(mode), 0) - _, err := pipeline.Exec() - return err +// delete the metadata from Redis +func (i *Inode) delMeta() { + client := i.redisRing.GetClient(i.keyPrefix) + Try(client.Unlink(i.keyPrefix+":children", i.keyPrefix+":mode")) } -func (i *Inode) delMeta() error { - pipeline := i.client.Pipeline() - pipeline.Del(i.metaBaseKey + ":children") - pipeline.Del(i.metaBaseKey + ":isDir") - pipeline.Del(i.metaBaseKey + ":mode") - _, err := pipeline.Exec() - return err -} - -//IsDir ... +//IsDir returns true if inode is a directory func (i *Inode) IsDir() bool { - if i.isDir == nil { - res := i.client.Get(i.metaBaseKey+":isDir").Val() == "1" - i.isDir = &res - } - return (*i.isDir) + //FIXME: cache the query + client := i.redisRing.GetClient(i.keyPrefix) + res, err := client.Exists(i.keyPrefix + ":children") + Check(err) + return res } //Mode returns the inode access mode func (i *Inode) Mode() os.FileMode { - if i.mode == nil { - val := i.client.Get(i.metaBaseKey + ":mode").Val() - res, err := strconv.Atoi(val) - if err != nil { - panic(err) - } - m := os.FileMode(res) - i.mode = &m - } - return (*i.mode) + client := i.redisRing.GetClient(i.keyPrefix) + val, err := client.Get(i.keyPrefix + ":mode") + Check(err) + res, err := strconv.ParseInt(string(val), 10, 64) + Check(err) + return os.FileMode(res) } -//ID returns the ID of the file -func (i *Inode) ID() string { - return i.id +//Path returns the Path of the file +func (i *Inode) Path() string { + return i.path } -//Name returns the inode base name +//Name returns the inode base name (for os.FileInfo interface) func (i *Inode) Name() string { - return i.ID() + return i.path } //Sys no op (to fulfill os.FileMode interface) @@ -233,17 +112,76 @@ func (i *Inode) Size() int64 { if i.IsDir() { return 0 } - return i.getBuffer().Size() + return i.dataStore.GetSize(i.path) } -func (i *Inode) childrenID() []string { - return i.client.SMembers(i.metaBaseKey + ":children").Val() +// records a child inode to the current inode +func (i *Inode) setChild(child *Inode) { + client := i.redisRing.GetClient(i.keyPrefix) + Try(client.SAdd(i.keyPrefix+":children", child.Path())) } -func (i *Inode) setChild(child *Inode) error { - return i.client.SAdd(i.metaBaseKey+":children", child.ID()).Err() +// removes a child inode from the current inode children list +func (i *Inode) removeChild(child *Inode) { + client := i.redisRing.GetClient(i.keyPrefix) + Try(client.SRem(i.keyPrefix+":children", child.Path())) +} + +// returns a list of children inodes +func (i *Inode) getChildren() ([]*Inode, error) { + if !i.IsDir() { + return nil, ErrNotDirectory + } + client := i.redisRing.GetClient(i.keyPrefix) + paths, err := client.SMembers(i.keyPrefix + ":children") + Check(err) + children := make([]*Inode, 0, len(paths)-1) + for _, path := range paths { + if path != "" { + children = append(children, NewInode(i.dataStore, i.redisRing, path)) + } + } + return children, nil +} + +// returns a File object wrapping the current inode +func (i *Inode) getFile(flag int) (File, error) { + if i.IsDir() { + return nil, ErrIsDirectory + } + + if hasFlag(os.O_TRUNC, flag) { + i.dataStore.Remove(i.path) + } + + var f File = NewMemFile(i.dataStore, i.path, i.mtx) + + if hasFlag(os.O_APPEND, flag) { + f.Seek(0, os.SEEK_END) + } else { + f.Seek(0, os.SEEK_SET) + } + if hasFlag(os.O_RDWR, flag) { + return f, nil + } else if hasFlag(os.O_WRONLY, flag) { + f = &woFile{f} + } else { + f = &roFile{f} + } + + return f, nil } -func (i *Inode) removeChild(child *Inode) error { - return i.client.SRem(i.metaBaseKey+":children", child.ID()).Err() +// removes the current inode (file content, children, metadata) +func (i *Inode) remove() { + if !i.IsDir() { + i.dataStore.Remove(i.path) + } else { + if children, _ := i.getChildren(); children != nil { + for _, child := range children { + child.remove() + } + } + } + i.delMeta() } diff --git a/src/go/redisfs/inodes_test.go b/src/go/redisfs/inodes_test.go index 64b6e3f..855387d 100644 --- a/src/go/redisfs/inodes_test.go +++ b/src/go/redisfs/inodes_test.go @@ -17,38 +17,38 @@ package redisfs import ( "os" "testing" + + "github.com/cea-hpc/pdwfs/util" ) func TestInodeMeta(t *testing.T) { - client, _ := GetRedisClient() - //defer client.FlushAll() + redis, confRedis := util.InitRedisTestServer() + defer redis.Stop() + + confMount := util.GetMountPathConf() - mountConf := GetMountPathConf() + ring := NewRedisRing(confRedis) + defer ring.Close() + store := NewDataStore(ring, int64(confMount.StripeSize)) + defer store.Close() - i := NewInode(mountConf, client, "id") + i := NewInode(store, ring, "id") - res, err := i.exists() - Ok(t, err) - Equals(t, false, res, "no metadata expected") + res := i.exists() + util.Equals(t, false, res, "no metadata expected") - err = i.initMeta(true, 0600) - Ok(t, err) + i.initMeta(true, 0600) - res, err = i.exists() - Ok(t, err) - Equals(t, true, res, "metadata expected") + res = i.exists() + util.Equals(t, true, res, "metadata expected") - err = i.initMeta(false, 0777) // should be a no op - Ok(t, err) + i.initMeta(false, 0777) // should be a no op d := i.IsDir() - Ok(t, err) - Equals(t, d, true, "should be a dir") + util.Equals(t, d, true, "should be a dir") m := i.Mode() - Ok(t, err) - Equals(t, m, os.FileMode(0600), "should be 0600 mode") + util.Equals(t, m, os.FileMode(0600), "should be 0600 mode") - err = i.delMeta() - Ok(t, err) + i.delMeta() } diff --git a/src/go/redisfs/redis.go b/src/go/redisfs/redis.go new file mode 100644 index 0000000..1d7edb9 --- /dev/null +++ b/src/go/redisfs/redis.go @@ -0,0 +1,257 @@ +// Copyright 2019 CEA +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Redis client and clients ring implementations + +package redisfs + +import ( + "errors" + "fmt" + "strings" + "time" + + "github.com/cea-hpc/pdwfs/config" + "github.com/cea-hpc/pdwfs/redigo/redis" + "github.com/cea-hpc/pdwfs/util" +) + +var ( + // ErrRedisKeyNotFound is returned if a queried key in Redis is not found + ErrRedisKeyNotFound = errors.New("Redis key not found") +) + +// Try ... +func Try(err error) { + if err != nil { + panic(err) + } +} + +// Check is an alias for Try +var Check = Try + +func err(a interface{}, err error) error { + return err +} + +// RedisClient is a client to a single Redis instance, safe to use by multiple goroutines +type RedisClient struct { + pool *redis.Pool +} + +// NewRedisClient creates a new RedisClient instance +func NewRedisClient(addr string) *RedisClient { + return &RedisClient{ + pool: &redis.Pool{ + MaxIdle: 5, + MaxActive: 50, // max active connection at the same time + Wait: true, // throttles goroutines to MaxActive goroutines + IdleTimeout: 240 * time.Second, + Dial: func() (redis.Conn, error) { + return redis.Dial("tcp", addr) + }, + }, + } +} + +// Close the connection pool +func (c *RedisClient) Close() error { + return c.pool.Close() +} + +// the following methods implements some type-safe and concurrent-safe Redis commands + +// SetRange command +func (c *RedisClient) SetRange(key string, offset int64, data []byte) error { + conn := c.pool.Get() + defer conn.Close() + return err(conn.Do("SETRANGE", key, offset, data)) +} + +// GetRange command +func (c *RedisClient) GetRange(key string, start, end int64) ([]byte, error) { + conn := c.pool.Get() + defer conn.Close() + b, err := redis.Bytes(conn.Do("GETRANGE", key, start, end)) + if err == redis.ErrNil { + return b, ErrRedisKeyNotFound + } + return b, err +} + +// GetRangeInto implements GETRANGE redis command with an input destination buffer to read bytes into. +// The actual number of bytes read is returned. +// If dst is too small, it will panic. +func (c *RedisClient) GetRangeInto(key string, start, end int64, dst []byte) (int, error) { + conn := c.pool.Get() + defer conn.Close() + conn.SetReadBuffer(dst) + defer conn.UnsetReadBuffer() + read, err := redis.ReadBytes(conn.Do("GETRANGE", key, start, end)) + if err == redis.ErrNil { + return 0, ErrRedisKeyNotFound + } + return read, err +} + +// Exists command +func (c *RedisClient) Exists(key string) (bool, error) { + conn := c.pool.Get() + defer conn.Close() + return redis.Bool(conn.Do("EXISTS", key)) +} + +// Set command +func (c *RedisClient) Set(key string, data []byte) error { + conn := c.pool.Get() + defer conn.Close() + return err(conn.Do("SET", key, data)) +} + +// SetNX command +func (c *RedisClient) SetNX(key string, data []byte) error { + conn := c.pool.Get() + defer conn.Close() + return err(conn.Do("SETNX", key, data)) +} + +// Get command +func (c *RedisClient) Get(key string) ([]byte, error) { + conn := c.pool.Get() + defer conn.Close() + b, err := redis.Bytes(conn.Do("GET", key)) + if err == redis.ErrNil { + return b, ErrRedisKeyNotFound + } + return b, err +} + +// GetInto implements GET redis command with an input destination buffer to read bytes into. +// The actual number of bytes read is returned. +// If dst is too small, it will panic. +func (c *RedisClient) GetInto(key string, dst []byte) (int, error) { + conn := c.pool.Get() + defer conn.Close() + conn.SetReadBuffer(dst) + defer conn.UnsetReadBuffer() + read, err := redis.ReadBytes(conn.Do("GET", key)) + if err == redis.ErrNil { + return 0, ErrRedisKeyNotFound + } + return read, err +} + +// Unlink command +func (c *RedisClient) Unlink(keys ...string) error { + // convert slice of string in slice of interface{} ref: https://golang.org/doc/faq#convert_slice_of_interface + k := make([]interface{}, len(keys)) + for i, v := range keys { + k[i] = v + } + conn := c.pool.Get() + defer conn.Close() + return err(conn.Do("UNLINK", k...)) +} + +// SAdd command +func (c *RedisClient) SAdd(key string, member string) error { + conn := c.pool.Get() + defer conn.Close() + return err(conn.Do("SADD", key, member)) +} + +// SRem command +func (c *RedisClient) SRem(key string, member string) error { + conn := c.pool.Get() + defer conn.Close() + return err(conn.Do("SREM", key, member)) +} + +// SMembers command +func (c *RedisClient) SMembers(key string) ([]string, error) { + conn := c.pool.Get() + defer conn.Close() + return redis.Strings(conn.Do("SMEMBERS", key)) +} + +// Pipe wraps the Redis pipeline feature of redigo +type Pipe struct { + conn redis.Conn +} + +// Do registers a new command in the pipeline +func (p Pipe) Do(cmd string, args ...interface{}) { + Try(p.conn.Send(cmd, args...)) +} + +// Flush flushes all pipeline commands to Redis +func (p Pipe) Flush() { + defer p.conn.Close() + _, err := p.conn.Do("EXEC") + Check(err) +} + +// Pipeline returns a Pipe instance +func (c *RedisClient) Pipeline() *Pipe { + conn := c.pool.Get() + conn.Send("MULTI") + return &Pipe{conn} +} + +// RedisRing manages multiple Redis instances and use consistent hashing to distribute the load +type RedisRing struct { + clients map[string]*RedisClient + hash *util.ConsistentHash +} + +// NewRedisRing returns a new RedisRing instance +func NewRedisRing(conf *config.Redis) *RedisRing { + + ids := make([]string, len(conf.Addrs)) + clients := make(map[string]*RedisClient) + for i, addr := range conf.Addrs { + ids[i] = fmt.Sprintf("%d", i) + clients[ids[i]] = NewRedisClient(addr) + } + hash := util.NewConsistentHash(100, nil) + hash.Add(ids...) + + return &RedisRing{ + clients: clients, + hash: hash, + } +} + +// GetClient returns a client from the ring based on a key +// if the key has curly braces in it (e.g "{mydirectory}/file"), only the string within the braces is used +// in the hasing process to get a client +func (r *RedisRing) GetClient(key string) *RedisClient { + k := key + if s := strings.IndexByte(key, '{'); s > -1 { + if e := strings.IndexByte(key[s+1:], '}'); e > 0 { + k = key[s+1 : s+e+1] + } + } + return r.clients[r.hash.Get(k)] +} + +// Close all clients in the ring +func (r *RedisRing) Close() error { + var err error + for _, client := range r.clients { + err = client.Close() + } + return err +} diff --git a/src/go/redisfs/redis_test.go b/src/go/redisfs/redis_test.go new file mode 100644 index 0000000..e1ff258 --- /dev/null +++ b/src/go/redisfs/redis_test.go @@ -0,0 +1,107 @@ +// Copyright 2019 CEA +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package redisfs + +import ( + "strings" + "testing" + + "github.com/cea-hpc/pdwfs/util" +) + +func TestUnlinkMultiKeys(t *testing.T) { + server, conf := util.InitRedisTestServer() + defer server.Stop() + + client := NewRedisClient(conf.Addrs[0]) + defer client.Close() + + err := client.Set("foo", []byte("bar")) + util.Ok(t, err) + + err = client.Set("foo2", []byte("bar2")) + util.Ok(t, err) + + ok, err := client.Exists("foo") + util.Ok(t, err) + util.Assert(t, ok, "key should exist") + + ok, err = client.Exists("foo2") + util.Ok(t, err) + util.Assert(t, ok, "key should exist") + + err = client.Unlink("foo", "foo2") + util.Ok(t, err) + + ok, err = client.Exists("foo") + util.Ok(t, err) + util.Assert(t, !ok, "key should not exist") + + ok, err = client.Exists("foo2") + util.Ok(t, err) + util.Assert(t, !ok, "key should not exist") +} + +func TestGetInto(t *testing.T) { + server, conf := util.InitRedisTestServer() + defer server.Stop() + + client := NewRedisClient(conf.Addrs[0]) + defer client.Close() + + data := []byte("0123456789") + + err := client.Set("foo", data) + util.Ok(t, err) + + // Destination buffer has same size as data + b := make([]byte, 10) + read, err := client.GetInto("foo", b) + util.Ok(t, err) + util.Equals(t, len(data), read, "wrong number of bytes read") + util.Equals(t, data, b, "read data does not match written data") + + // Destination buffer has larger size + b = make([]byte, 20) + read, err = client.GetInto("foo", b) + util.Ok(t, err) + util.Equals(t, len(data), read, "wrong number of bytes read") + util.Equals(t, data, b[:len(data)], "read data does not match written data") + + // Destination buffer with smaller size returns an error + b = make([]byte, 5) + read, err = client.GetInto("foo", b) + util.Assert(t, err != nil, "should raise an error") + util.Assert(t, strings.Contains(err.Error(), "destination buffer is too small"), "a different error is expected") +} + +func TestGetRangeInto(t *testing.T) { + server, conf := util.InitRedisTestServer() + defer server.Stop() + + client := NewRedisClient(conf.Addrs[0]) + defer client.Close() + + data := []byte("0123456789") + + err := client.Set("foo", data) + util.Ok(t, err) + + b := make([]byte, 5) + read, err := client.GetRangeInto("foo", 4, 8, b) + util.Ok(t, err) + util.Equals(t, len(b), read, "wrong number of bytes read") + util.Equals(t, data[4:9], b, "read data does not match written data") +} diff --git a/src/go/redisfs/store.go b/src/go/redisfs/store.go new file mode 100644 index 0000000..cfe9573 --- /dev/null +++ b/src/go/redisfs/store.go @@ -0,0 +1,291 @@ +// Copyright 2019 CEA +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// DataStore component that uses multiple Redis instances in a "ring" +// to store the content of files stripped accross the Redis instances + +package redisfs + +import ( + "fmt" + "sync" + "sync/atomic" + + "github.com/cea-hpc/pdwfs/redigo/redis" +) + +// helpers functions + +// builds the key string use to address stripes in Redis +func key(name string, id int64) string { + return fmt.Sprintf("%s:%d", name, id) +} + +func divmod(n, d int64) (q, r int64) { + q = n / d + r = n % d + return +} + +// DataStore uses multiple Redis instances (ring) to store flat sequences of bytes stripped accross instances +type DataStore struct { + redisRing *RedisRing + stripeSize int64 +} + +// NewDataStore returns a DataStore struct instance +func NewDataStore(ring *RedisRing, stripeSize int64) *DataStore { + return &DataStore{ + redisRing: ring, + stripeSize: stripeSize, + } +} + +// Close the Redis clients in the ring +func (s DataStore) Close() error { + return s.redisRing.Close() +} + +type stripeInfo struct { + id int64 + off int64 // offset relative to beginning of stripe + data []byte // data in stripe starting at offset off +} + +// returns a slice of stripeInfo struct describing the stripping of a sequence of bytes 'data' +// written or read from offset 'off' +func stripeLayout(stripeSize, off int64, data []byte) []stripeInfo { + stripes := make([]stripeInfo, 0, 100) + var id, offset, pos, stripeLen int64 +loop: + for { + if pos >= int64(len(data)) { + break loop + } + if len(stripes) >= cap(stripes) { + // grow stripes + newStripes := make([]stripeInfo, 2*cap(stripes)) + copy(newStripes, stripes) + stripes = newStripes + } + id, offset = divmod(off+pos, stripeSize) + if stripeLen = int64(len(data[pos:])); stripeLen > stripeSize-offset { + stripeLen = stripeSize - offset + } + stripes = append(stripes, stripeInfo{id, offset, data[pos : pos+stripeLen]}) + pos += stripeLen + } + return stripes +} + +// writes a single stripe in the store +// Note: each Redis instance in the store contains a set of all the stripes stored by that instance for a specific file +// this is used when searching the last stripe of a file to compute its size, see next methods +func (s DataStore) writeStripe(name string, stripe stripeInfo, wg *sync.WaitGroup) { + defer wg.Done() + stripeKey := key(name, stripe.id) + pipeline := s.redisRing.GetClient(stripeKey).Pipeline() + pipeline.Do("SADD", name+":stripes", stripe.id) + if stripe.off == 0 && int64(len(stripe.data)) == s.stripeSize { + // SET is faster than SETRANGE + pipeline.Do("SET", stripeKey, stripe.data) + } else { + pipeline.Do("SETRANGE", stripeKey, stripe.off, stripe.data) + } + pipeline.Flush() +} + +// erases the stripe from its instance +func (s DataStore) removeStripe(name string, id int64, wg *sync.WaitGroup) { + defer wg.Done() + stripeKey := key(name, id) + pipeline := s.redisRing.GetClient(stripeKey).Pipeline() + pipeline.Do("SREM", name+":stripes", id) + pipeline.Do("UNLINK", stripeKey) + pipeline.Flush() +} + +// reads stripe data from its Redis instance, copy the data into the destination buffer +// and returns the number of bytes read +func (s DataStore) readStripe(name string, stripe stripeInfo, wg *sync.WaitGroup, read *int64) { + defer wg.Done() + stripeKey := key(name, stripe.id) + client := s.redisRing.GetClient(stripeKey) + + var n int + var err error + size := int64(len(stripe.data)) + if stripe.off == 0 && size == s.stripeSize { + n, err = client.GetInto(stripeKey, stripe.data) + } else { + n, err = client.GetRangeInto(stripeKey, stripe.off, stripe.off+size-1, stripe.data) + } + if err != nil && err != ErrRedisKeyNotFound { + panic(err) + } + // copy res into destination data buffer and atomically increment the number of bytes read + atomic.AddInt64(read, int64(n)) +} + +var trimStripeScript = redis.NewScript(1, ` + local str = redis.call("GETRANGE", KEYS[1], 0, ARGV[1]) + return redis.call("SET", KEYS[1], str) + `) + +func (s DataStore) trimStripe(name string, id int64, size int64, wg *sync.WaitGroup) { + defer wg.Done() + stripeKey := key(name, id) + client := s.redisRing.GetClient(stripeKey) + conn := client.pool.Get() + defer conn.Close() + + if size == 0 { + _, err := conn.Do("SET", stripeKey, []byte("")) + Check(err) + } else { + Try(err(trimStripeScript.Do(conn, stripeKey, size-1))) + } +} + +// main DataStore public API + +// WriteAt writes the content of 'data' keyed by 'name' at offset 'off' into the DataStore +// the content is stripped and each stripe is written concurrently in its own goroutine +// Note: goroutines are throttled by the limited connection pools of each Redis instance +func (s DataStore) WriteAt(name string, off int64, data []byte) { + wg := sync.WaitGroup{} + for _, stripe := range stripeLayout(s.stripeSize, off, data) { + wg.Add(1) + go s.writeStripe(name, stripe, &wg) + } + wg.Wait() +} + +// ReadAt reads data into 'dst' byte slice and returns the number of read bytes +func (s DataStore) ReadAt(name string, off int64, dst []byte) int64 { + var read int64 + wg := sync.WaitGroup{} + for _, stripe := range stripeLayout(s.stripeSize, off, dst) { + wg.Add(1) + go s.readStripe(name, stripe, &wg, &read) + } + wg.Wait() + return read +} + +// Remove all stripes keyed by 'name' +func (s DataStore) Remove(name string) { + wg := sync.WaitGroup{} + lastStripe := s.searchLastStripe(name) + for i := int64(0); i <= lastStripe; i++ { + wg.Add(1) + go s.removeStripe(name, i, &wg) + } + wg.Wait() +} + +// gather from all Redis instances the list of stripes keyed by 'name' and returns the highest stripe ID +func (s DataStore) searchLastStripe(name string) int64 { + retChan := make(chan int64, len(s.redisRing.clients)) + wg := sync.WaitGroup{} + for _, client := range s.redisRing.clients { + wg.Add(1) + go func(c *RedisClient, wg *sync.WaitGroup, ch chan int64) { + defer wg.Done() + conn := c.pool.Get() + defer conn.Close() + ids, err := redis.Int64s(conn.Do("SMEMBERS", name+":stripes")) + Check(err) + max := int64(-1) + for _, id := range ids { + if id > max { + max = id + } + } + ch <- max + }(client, &wg, retChan) + } + wg.Wait() + var n int64 + max := int64(-1) + for i := 0; i < len(s.redisRing.clients); i++ { + n = <-retChan + if n > max { + max = n + } + } + return max +} + +// GetSize returns the total size in bytes of data stored keyed by 'name' (all stripes). +func (s DataStore) GetSize(name string) int64 { + ilast := s.searchLastStripe(name) + if ilast < 0 { + return 0 + } + key := key(name, ilast) + lastStripe, err := s.redisRing.GetClient(key).Get(key) + Check(err) + return ilast*s.stripeSize + int64(len(lastStripe)) +} + +// helper to obtain the last stripe ID and length based on the total size and stripe size +func lastStripeInfo(size, stripeSize int64) (stripeID, stripeLen int64) { + stripeID, stripeLen = divmod(size, stripeSize) + if stripeLen == 0 { + stripeID-- + stripeLen = stripeSize + } + return +} + +// Resize (grow or shrink) the data content keyed by 'name' +func (s DataStore) Resize(name string, newSize int64) { + if newSize < 0 { + panic(fmt.Errorf("size must be non-negative")) + } + curSize := s.GetSize(name) + curLastStripeID, curLastStripeLen := lastStripeInfo(curSize, s.stripeSize) + newLastStripeID, newLastStripeLen := lastStripeInfo(newSize, s.stripeSize) + switch { + case newSize < curSize: // shrink + // remove all existing stripes after this new last stripe + wg := sync.WaitGroup{} + for id := newLastStripeID + 1; id <= curLastStripeID; id++ { + wg.Add(1) + go s.removeStripe(name, id, &wg) + } + // resize the last stripe + wg.Add(1) + go s.trimStripe(name, newLastStripeID, newLastStripeLen, &wg) + wg.Wait() + + case newSize > curSize: // grow + // write new stripes but the last + wg := sync.WaitGroup{} + for id := curLastStripeID + 1; id < newLastStripeID; id++ { + wg.Add(1) + go s.writeStripe(name, stripeInfo{id, s.stripeSize - 1, []byte("\x00")}, &wg) + } + // write last stripe + wg.Add(1) + go s.writeStripe(name, stripeInfo{newLastStripeID, newLastStripeLen - 1, []byte("\x00")}, &wg) + // fill current last stripe with null bytes if needed + if curLastStripeLen < s.stripeSize { + wg.Add(1) + go s.writeStripe(name, stripeInfo{curLastStripeID, s.stripeSize - curLastStripeLen - 1, []byte("\x00")}, &wg) + } + wg.Wait() + } +} diff --git a/src/go/redisfs/store_test.go b/src/go/redisfs/store_test.go new file mode 100644 index 0000000..46128c9 --- /dev/null +++ b/src/go/redisfs/store_test.go @@ -0,0 +1,232 @@ +// Copyright 2019 CEA +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package redisfs + +import ( + "bytes" + "testing" + + "github.com/cea-hpc/pdwfs/util" +) + +func TestLayout(t *testing.T) { + + stripeSize := int64(1024) + + // all in one stripe, starting at 0 + data := make([]byte, 500) + s := stripeLayout(stripeSize, 0, data) + util.Equals(t, len(s), 1, "Nb of stripe error") + + if s[0].id != 0 || s[0].off != 0 || len(s[0].data) != 500 { + t.Errorf("error in stripe data: id %d, off %d, len %d", s[0].id, s[0].off, len(s[0].data)) + } + + // all in one stripe, starting at 500 + data = make([]byte, 500) + s = stripeLayout(stripeSize, 500, data) + util.Equals(t, len(s), 1, "Nb of stripe error") + + if s[0].id != 0 || s[0].off != 500 || len(s[0].data) != 500 { + t.Errorf("error in stripe data: id %d, off %d, len %d", s[0].id, s[0].off, len(s[0].data)) + } + + // taking exactly one stripe + data = make([]byte, 1024) + s = stripeLayout(stripeSize, 0, data) + util.Equals(t, len(s), 1, "Nb of stripe error") + + if s[0].id != 0 || s[0].off != 0 || len(s[0].data) != 1024 { + t.Errorf("error in stripe data: id %d, off %d, len %d", s[0].id, s[0].off, len(s[0].data)) + } + + // taking one stripe + 1 byte + data = make([]byte, 1025) + s = stripeLayout(stripeSize, 0, data) + util.Equals(t, len(s), 2, "Nb of stripe error") + + if s[0].id != 0 || s[0].off != 0 || len(s[0].data) != 1024 { + t.Errorf("error in stripe data: id %d, off %d, len %d", s[0].id, s[0].off, len(s[0].data)) + } + if s[1].id != 1 || s[1].off != 0 || len(s[1].data) != 1 { + t.Errorf("error in stripe data: id %d, off %d, len %d", s[1].id, s[1].off, len(s[1].data)) + } + + // taking exactly two stripe + data = make([]byte, 2048) + s = stripeLayout(stripeSize, 0, data) + util.Equals(t, len(s), 2, "Nb of stripe error") + + if s[0].id != 0 || s[0].off != 0 || len(s[0].data) != 1024 { + t.Errorf("error in stripe data: id %d, off %d, len %d", s[0].id, s[0].off, len(s[0].data)) + } + if s[1].id != 1 || s[1].off != 0 || len(s[1].data) != 1024 { + t.Errorf("error in stripe data: id %d, off %d, len %d", s[1].id, s[1].off, len(s[1].data)) + } + + // spanning two stripes + data = make([]byte, 1000) + s = stripeLayout(stripeSize, 500, data) + util.Equals(t, len(s), 2, "Nb of stripe error") + + if s[0].id != 0 || s[0].off != 500 || len(s[0].data) != 524 { + t.Errorf("error in stripe data: id %d, off %d, len %d", s[0].id, s[0].off, len(s[0].data)) + } + if s[1].id != 1 || s[1].off != 0 || len(s[1].data) != 476 { + t.Errorf("error in stripe data: id %d, off %d, len %d", s[1].id, s[1].off, len(s[1].data)) + } + + // spanning three stripes, starting on second one, one byte on fourth stripe + data = make([]byte, 2049) + s = stripeLayout(stripeSize, 1024, data) + util.Equals(t, len(s), 3, "Nb of stripe error") + + if s[0].id != 1 || s[0].off != 0 || len(s[0].data) != 1024 { + t.Errorf("error in stripe data: id %d, off %d, len %d", s[0].id, s[0].off, len(s[0].data)) + } + if s[1].id != 2 || s[1].off != 0 || len(s[1].data) != 1024 { + t.Errorf("error in stripe data: id %d, off %d, len %d", s[1].id, s[1].off, len(s[1].data)) + } + if s[2].id != 3 || s[2].off != 0 || len(s[2].data) != 1 { + t.Errorf("error in stripe data: id %d, off %d, len %d", s[2].id, s[2].off, len(s[2].data)) + } +} + +func writeData(t *testing.T, stripeSize int64, data []byte, off int64) { + redis, conf := util.InitRedisTestServer() + defer redis.Stop() + + store := NewDataStore(NewRedisRing(conf), stripeSize) + defer store.Close() + + store.WriteAt("myfile", off, data) + readData := make([]byte, len(data), len(data)) + n := store.ReadAt("myfile", off, readData) + util.Equals(t, int64(len(data)), n, "number of bytes read does not match input") + util.Equals(t, data, readData, "read data does not match written data") +} + +func TestWriteData(t *testing.T) { + + var stripeSize int64 + + // Data fits within a single stripe, start at 0 offset + stripeSize = 1024 + data := bytes.Repeat([]byte("0123456789"), 100) // 1000 bytes + offset := 0 + writeData(t, stripeSize, data, int64(offset)) + + // Data fits within a single stripe, start at non-zero offset + stripeSize = 1024 + data = bytes.Repeat([]byte("0123456789"), 50) // 500 bytes + offset = 500 + writeData(t, stripeSize, data, int64(offset)) + + // Data fits exactly within a single stripe + stripeSize = 1000 + data = bytes.Repeat([]byte("0123456789"), 100) // 1000 bytes + offset = 0 + writeData(t, stripeSize, data, int64(offset)) + + // Data fits within a stripe + 1 byte in next stripe + stripeSize = 999 + data = bytes.Repeat([]byte("0123456789"), 100) // 1000 bytes + offset = 0 + writeData(t, stripeSize, data, int64(offset)) + + // Data fits in two stripes + stripeSize = 1000 + data = bytes.Repeat([]byte("0123456789"), 100) // 1000 bytes + offset = 500 + writeData(t, stripeSize, data, int64(offset)) + + // Data fits in three stripes starting on second + stripeSize = 1000 + data = bytes.Repeat([]byte("0123456789"), 200) // 2000 bytes + offset = 1500 + writeData(t, stripeSize, data, int64(offset)) +} + +func TestReadEmpty(t *testing.T) { + redis, conf := util.InitRedisTestServer() + defer redis.Stop() + + store := NewDataStore(NewRedisRing(conf), 100) + defer store.Close() + + readData := make([]byte, 1000, 1000) + n := store.ReadAt("myfile", 0, readData) + util.Equals(t, int64(0), n, "number of byte read should be 0") +} + +func TestGetSize(t *testing.T) { + redis, conf := util.InitRedisTestServer() + defer redis.Stop() + + store := NewDataStore(NewRedisRing(conf), 100) + defer store.Close() + + data := bytes.Repeat([]byte("0123456789"), 500) // 5000 bytes + store.WriteAt("myfile", 0, data) + + readData := make([]byte, len(data), len(data)) + store.ReadAt("myfile", 0, readData) + util.Equals(t, data, readData, "read data is different from written data") + + s := store.GetSize("myfile") + util.Equals(t, int64(len(data)), s, "size is incorrect") +} + +func TestResize(t *testing.T) { + redis, conf := util.InitRedisTestServer() + defer redis.Stop() + + store := NewDataStore(NewRedisRing(conf), 100) + defer store.Close() + + store.Resize("myfile", 100) + util.Equals(t, int64(100), store.GetSize("myfile"), "resize error") + + store.Resize("myfile", 100) // no op + util.Equals(t, int64(100), store.GetSize("myfile"), "resize error") + + store.Resize("myfile", 250) + util.Equals(t, int64(250), store.GetSize("myfile"), "resize error") + + store.Resize("myfile", 150) + util.Equals(t, int64(150), store.GetSize("myfile"), "resize error") + + store.Resize("myfile", 0) + util.Equals(t, int64(0), store.GetSize("myfile"), "resize error") +} + +func TestTruncate(t *testing.T) { + redis, conf := util.InitRedisTestServer() + defer redis.Stop() + + store := NewDataStore(NewRedisRing(conf), 20) // 20 bytes stripes + defer store.Close() + + data := bytes.Repeat([]byte("0123456789"), 3) // 30 bytes to write + store.WriteAt("myfile", 0, data) + + store.Resize("myfile", 15) + + readSize := int64(len(data) + 10) + readData := make([]byte, readSize, readSize) + n := store.ReadAt("myfile", 0, readData) + util.Equals(t, int64(15), n, "read error") + util.Equals(t, data[:15], readData[:n], "data read does not match data written") +} diff --git a/src/go/redisfs/utils_test.go b/src/go/redisfs/utils_test.go deleted file mode 100644 index 26a010f..0000000 --- a/src/go/redisfs/utils_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2019 CEA -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package redisfs - -import ( - "reflect" - "testing" -) - -func TestSplitPath(t *testing.T) { - const PathSeperator = "/" - if p := SplitPath("/", PathSeperator); !reflect.DeepEqual(p, []string{""}) { - t.Errorf("Invalid path: %q", p) - } - if p := SplitPath("./test", PathSeperator); !reflect.DeepEqual(p, []string{".", "test"}) { - t.Errorf("Invalid path: %q", p) - } - if p := SplitPath(".", PathSeperator); !reflect.DeepEqual(p, []string{"."}) { - t.Errorf("Invalid path: %q", p) - } - if p := SplitPath("test", PathSeperator); !reflect.DeepEqual(p, []string{".", "test"}) { - t.Errorf("Invalid path: %q", p) - } - if p := SplitPath("/usr/src/linux/", PathSeperator); !reflect.DeepEqual(p, []string{"", "usr", "src", "linux"}) { - t.Errorf("Invalid path: %q", p) - } - if p := SplitPath("usr/src/linux/", PathSeperator); !reflect.DeepEqual(p, []string{".", "usr", "src", "linux"}) { - t.Errorf("Invalid path: %q", p) - } -} diff --git a/src/go/vendor/github.com/go-redis/redis/internal/consistenthash/consistenthash.go b/src/go/util/consistenthash.go similarity index 73% rename from src/go/vendor/github.com/go-redis/redis/internal/consistenthash/consistenthash.go rename to src/go/util/consistenthash.go index a9c56f0..c95c622 100644 --- a/src/go/vendor/github.com/go-redis/redis/internal/consistenthash/consistenthash.go +++ b/src/go/util/consistenthash.go @@ -14,8 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package consistenthash provides an implementation of a ring hash. -package consistenthash +// provides an implementation of a ring hash. + +package util import ( "hash/crc32" @@ -23,17 +24,20 @@ import ( "strconv" ) +// Hash ... type Hash func(data []byte) uint32 -type Map struct { +// ConsistentHash ... +type ConsistentHash struct { hash Hash replicas int keys []int // Sorted hashMap map[int]string } -func New(replicas int, fn Hash) *Map { - m := &Map{ +// NewConsistentHash ... +func NewConsistentHash(replicas int, fn Hash) *ConsistentHash { + m := &ConsistentHash{ replicas: replicas, hash: fn, hashMap: make(map[int]string), @@ -44,13 +48,13 @@ func New(replicas int, fn Hash) *Map { return m } -// Returns true if there are no items available. -func (m *Map) IsEmpty() bool { +// IsEmpty returns true if there are no items available. +func (m *ConsistentHash) IsEmpty() bool { return len(m.keys) == 0 } -// Adds some keys to the hash. -func (m *Map) Add(keys ...string) { +// Add some keys to the hash. +func (m *ConsistentHash) Add(keys ...string) { for _, key := range keys { for i := 0; i < m.replicas; i++ { hash := int(m.hash([]byte(strconv.Itoa(i) + key))) @@ -61,8 +65,8 @@ func (m *Map) Add(keys ...string) { sort.Ints(m.keys) } -// Gets the closest item in the hash to the provided key. -func (m *Map) Get(key string) string { +// Get the closest item in the hash to the provided key. +func (m *ConsistentHash) Get(key string) string { if m.IsEmpty() { return "" } diff --git a/src/go/util/consistenthash_test.go b/src/go/util/consistenthash_test.go new file mode 100644 index 0000000..92125e0 --- /dev/null +++ b/src/go/util/consistenthash_test.go @@ -0,0 +1,110 @@ +/* +Copyright 2013 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "strconv" + "testing" +) + +func TestHashing(t *testing.T) { + + // Override the hash function to return easier to reason about values. Assumes + // the keys can be converted to an integer. + hash := NewConsistentHash(3, func(key []byte) uint32 { + i, err := strconv.Atoi(string(key)) + if err != nil { + panic(err) + } + return uint32(i) + }) + + // Given the above hash function, this will give replicas with "hashes": + // 2, 4, 6, 12, 14, 16, 22, 24, 26 + hash.Add("6", "4", "2") + + testCases := map[string]string{ + "2": "2", + "11": "2", + "23": "4", + "27": "2", + } + + for k, v := range testCases { + if hash.Get(k) != v { + t.Errorf("Asking for %s, should have yielded %s", k, v) + } + } + + // Adds 8, 18, 28 + hash.Add("8") + + // 27 should now map to 8. + testCases["27"] = "8" + + for k, v := range testCases { + if hash.Get(k) != v { + t.Errorf("Asking for %s, should have yielded %s", k, v) + } + } + +} + +func TestConsistency(t *testing.T) { + hash1 := NewConsistentHash(1, nil) + hash2 := NewConsistentHash(1, nil) + + hash1.Add("Bill", "Bob", "Bonny") + hash2.Add("Bob", "Bonny", "Bill") + + if hash1.Get("Ben") != hash2.Get("Ben") { + t.Errorf("Fetching 'Ben' from both hashes should be the same") + } + + hash2.Add("Becky", "Ben", "Bobby") + + if hash1.Get("Ben") != hash2.Get("Ben") || + hash1.Get("Bob") != hash2.Get("Bob") || + hash1.Get("Bonny") != hash2.Get("Bonny") { + t.Errorf("Direct matches should always return the same entry") + } + +} + +func BenchmarkGet8(b *testing.B) { benchmarkGet(b, 8) } +func BenchmarkGet32(b *testing.B) { benchmarkGet(b, 32) } +func BenchmarkGet128(b *testing.B) { benchmarkGet(b, 128) } +func BenchmarkGet512(b *testing.B) { benchmarkGet(b, 512) } + +func benchmarkGet(b *testing.B, shards int) { + + hash := NewConsistentHash(50, nil) + + var buckets []string + for i := 0; i < shards; i++ { + buckets = append(buckets, fmt.Sprintf("shard-%d", i)) + } + + hash.Add(buckets...) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + hash.Get(buckets[i&(shards-1)]) + } +} diff --git a/src/go/redisfs/utils.go b/src/go/util/util.go similarity index 50% rename from src/go/redisfs/utils.go rename to src/go/util/util.go index 2b01542..a84736c 100644 --- a/src/go/redisfs/utils.go +++ b/src/go/util/util.go @@ -12,22 +12,33 @@ // See the License for the specific language governing permissions and // limitations under the License. -package redisfs +package util import ( - "crypto/rand" - "encoding/base64" "fmt" "os" + "os/exec" "path/filepath" "reflect" "runtime" - "strings" + "strconv" "testing" + "time" "github.com/cea-hpc/pdwfs/config" + "github.com/cea-hpc/pdwfs/redigo/redis" ) +// error checking helpers + +func try(err error) { + if err != nil { + panic(err) + } +} + +var check = try + // testing utilities // assert, ok and equals are from https://github.com/benbjohnson/testing @@ -59,60 +70,72 @@ func Equals(tb testing.TB, exp, act interface{}, msg string) { } } -//GetRedisClient returns a new Redis client and the configuration object -func GetRedisClient() (IRedisClient, *config.Redis) { - redisConf := &config.Redis{RedisAddrs: []string{":6379"}} - return NewRedisClient(redisConf), redisConf +// RedisTestServer used to running tests +type RedisTestServer struct { + cmd *exec.Cmd + port int } -//GetMountPathConf returns a default configuration (for testing) -func GetMountPathConf() *config.Mount { - cwd, err := filepath.Abs(".") - if err != nil { - panic(err) +// NewRedisTestServer returns a RedisTestServer instance with an available port (not yet started) +func NewRedisTestServer() *RedisTestServer { + // check redis-server binary is in PATH + _, err := exec.LookPath("redis-server") + check(err) + port := 6379 + for { + // find a free port + if _, err := redis.Dial("tcp", fmt.Sprintf(":%d", port)); err == nil { + port++ + } + break } - return &config.Mount{ - Path: cwd, - BlockSize: 1024 * 1024, // 1Mo - WriteParallel: true, - ReadParallel: true, + return &RedisTestServer{ + cmd: exec.Command("redis-server", "--save", "\"\"", "--port", strconv.Itoa(port)), + port: port, } } -//Log ... -func Log(data ...interface{}) { - fmt.Printf("\033[35m[PDWFS][LOGS]\033[39m[Go][MPI_RANK %s]", os.Getenv("PMIX_RANK")) - fmt.Println(data...) -} - -// SplitPath splits the given path in segments: -// "/" -> []string{""} -// "./file" -> []string{".", "file"} -// "file" -> []string{".", "file"} -// "/usr/src/linux/" -> []string{"", "usr", "src", "linux"} -// The returned slice of path segments consists of one more more segments. -func SplitPath(path string, sep string) []string { - path = strings.TrimSpace(path) - path = strings.TrimSuffix(path, sep) - if path == "" { // was "/" - return []string{""} - } - if path == "." { - return []string{"."} +// Start the Redis server, make sure its up and running +func (r *RedisTestServer) Start() { + try(r.cmd.Start()) + time.Sleep(50 * time.Millisecond) + for { + if conn, err := redis.Dial("tcp", fmt.Sprintf(":%d", r.port)); err == nil { + conn.Close() + return + } } +} - if len(path) > 0 && !strings.HasPrefix(path, sep) && !strings.HasPrefix(path, "."+sep) { - path = "./" + path +// Stop the Redis server process +// FIXME: this call is used in defer statements in all tests to make sure the server is stopped +// after each test, however this is not always happening and the server stays up after some test failures +// which is annoying for the subsequent tests as it leads to uncontrolled behaviour...probably better to revert +// to miniredis +func (r *RedisTestServer) Stop() { + if err := r.cmd.Process.Signal(os.Interrupt); err != nil { + panic(err) } - parts := strings.Split(path, sep) + r.cmd.Wait() +} - return parts +//InitRedisTestServer returns a new Redis test server with its configuration for clients +func InitRedisTestServer() (*RedisTestServer, *config.Redis) { + server := NewRedisTestServer() + server.Start() + conf := config.NewRedisConf() + conf.Addrs = []string{fmt.Sprintf(":%d", server.port)} + return server, conf } -func randomToken() (string, error) { - buf := make([]byte, 16) - if _, err := rand.Read(buf); err != nil { - return "", err +//GetMountPathConf returns a default configuration (for testing) +func GetMountPathConf() *config.Mount { + cwd, err := filepath.Abs(".") + if err != nil { + panic(err) + } + return &config.Mount{ + Path: cwd, + StripeSize: config.DefaultStripeSize, } - return base64.URLEncoding.EncodeToString(buf), nil } diff --git a/src/go/util/util_test.go b/src/go/util/util_test.go new file mode 100644 index 0000000..43ff857 --- /dev/null +++ b/src/go/util/util_test.go @@ -0,0 +1,40 @@ +// Copyright 2019 CEA +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "testing" + + "github.com/cea-hpc/pdwfs/redigo/redis" +) + +func TestRedisTestServer(t *testing.T) { + server := NewRedisTestServer() + server.Start() + defer server.Stop() + conn, err := redis.Dial("tcp", ":6379") + if err != nil { + panic(err) + } + _, err = conn.Do("SET", "foo", "bar") + if err != nil { + panic(err) + } + s, err := redis.String(conn.Do("GET", "foo")) + if err != nil { + panic(err) + } + Equals(t, "bar", s, "reply should be bar") +} diff --git a/src/go/vendor/github.com/go-redis/redis/.gitignore b/src/go/vendor/github.com/go-redis/redis/.gitignore deleted file mode 100644 index ebfe903..0000000 --- a/src/go/vendor/github.com/go-redis/redis/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.rdb -testdata/*/ diff --git a/src/go/vendor/github.com/go-redis/redis/.travis.yml b/src/go/vendor/github.com/go-redis/redis/.travis.yml deleted file mode 100644 index 6b110b4..0000000 --- a/src/go/vendor/github.com/go-redis/redis/.travis.yml +++ /dev/null @@ -1,19 +0,0 @@ -sudo: false -language: go - -services: - - redis-server - -go: - - 1.9.x - - 1.10.x - - 1.11.x - - tip - -matrix: - allow_failures: - - go: tip - -install: - - go get github.com/onsi/ginkgo - - go get github.com/onsi/gomega diff --git a/src/go/vendor/github.com/go-redis/redis/CHANGELOG.md b/src/go/vendor/github.com/go-redis/redis/CHANGELOG.md deleted file mode 100644 index 1964566..0000000 --- a/src/go/vendor/github.com/go-redis/redis/CHANGELOG.md +++ /dev/null @@ -1,25 +0,0 @@ -# Changelog - -## Unreleased - -- Cluster and Ring pipelines process commands for each node in its own goroutine. - -## 6.14 - -- Added Options.MinIdleConns. -- Added Options.MaxConnAge. -- PoolStats.FreeConns is renamed to PoolStats.IdleConns. -- Add Client.Do to simplify creating custom commands. -- Add Cmd.String, Cmd.Int, Cmd.Int64, Cmd.Uint64, Cmd.Float64, and Cmd.Bool helpers. -- Lower memory usage. - -## v6.13 - -- Ring got new options called `HashReplicas` and `Hash`. It is recommended to set `HashReplicas = 1000` for better keys distribution between shards. -- Cluster client was optimized to use much less memory when reloading cluster state. -- PubSub.ReceiveMessage is re-worked to not use ReceiveTimeout so it does not lose data when timeout occurres. In most cases it is recommended to use PubSub.Channel instead. -- Dialer.KeepAlive is set to 5 minutes by default. - -## v6.12 - -- ClusterClient got new option called `ClusterSlots` which allows to build cluster of normal Redis Servers that don't have cluster mode enabled. See https://godoc.org/github.com/go-redis/redis#example-NewClusterClient--ManualSetup diff --git a/src/go/vendor/github.com/go-redis/redis/LICENSE b/src/go/vendor/github.com/go-redis/redis/LICENSE deleted file mode 100644 index 298bed9..0000000 --- a/src/go/vendor/github.com/go-redis/redis/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -Copyright (c) 2013 The github.com/go-redis/redis Authors. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/go/vendor/github.com/go-redis/redis/Makefile b/src/go/vendor/github.com/go-redis/redis/Makefile deleted file mode 100644 index 187c746..0000000 --- a/src/go/vendor/github.com/go-redis/redis/Makefile +++ /dev/null @@ -1,22 +0,0 @@ -all: testdeps - go test ./... - go test ./... -short -race - env GOOS=linux GOARCH=386 go test ./... - go vet - go get github.com/gordonklaus/ineffassign - ineffassign . - -testdeps: testdata/redis/src/redis-server - -bench: testdeps - go test ./... -test.run=NONE -test.bench=. -test.benchmem - -.PHONY: all test testdeps bench - -testdata/redis: - mkdir -p $@ - wget -qO- https://github.com/antirez/redis/archive/unstable.tar.gz | tar xvz --strip-components=1 -C $@ - -testdata/redis/src/redis-server: testdata/redis - sed -i.bak 's/libjemalloc.a/libjemalloc.a -lrt/g' $ -} - -func ExampleClient() { - err := client.Set("key", "value", 0).Err() - if err != nil { - panic(err) - } - - val, err := client.Get("key").Result() - if err != nil { - panic(err) - } - fmt.Println("key", val) - - val2, err := client.Get("key2").Result() - if err == redis.Nil { - fmt.Println("key2 does not exist") - } else if err != nil { - panic(err) - } else { - fmt.Println("key2", val2) - } - // Output: key value - // key2 does not exist -} -``` - -## Howto - -Please go through [examples](https://godoc.org/github.com/go-redis/redis#pkg-examples) to get an idea how to use this package. - -## Look and feel - -Some corner cases: - -```go -// SET key value EX 10 NX -set, err := client.SetNX("key", "value", 10*time.Second).Result() - -// SORT list LIMIT 0 2 ASC -vals, err := client.Sort("list", redis.Sort{Offset: 0, Count: 2, Order: "ASC"}).Result() - -// ZRANGEBYSCORE zset -inf +inf WITHSCORES LIMIT 0 2 -vals, err := client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ - Min: "-inf", - Max: "+inf", - Offset: 0, - Count: 2, -}).Result() - -// ZINTERSTORE out 2 zset1 zset2 WEIGHTS 2 3 AGGREGATE SUM -vals, err := client.ZInterStore("out", redis.ZStore{Weights: []int64{2, 3}}, "zset1", "zset2").Result() - -// EVAL "return {KEYS[1],ARGV[1]}" 1 "key" "hello" -vals, err := client.Eval("return {KEYS[1],ARGV[1]}", []string{"key"}, "hello").Result() -``` - -## Benchmark - -go-redis vs redigo: - -``` -BenchmarkSetGoRedis10Conns64Bytes-4 200000 7621 ns/op 210 B/op 6 allocs/op -BenchmarkSetGoRedis100Conns64Bytes-4 200000 7554 ns/op 210 B/op 6 allocs/op -BenchmarkSetGoRedis10Conns1KB-4 200000 7697 ns/op 210 B/op 6 allocs/op -BenchmarkSetGoRedis100Conns1KB-4 200000 7688 ns/op 210 B/op 6 allocs/op -BenchmarkSetGoRedis10Conns10KB-4 200000 9214 ns/op 210 B/op 6 allocs/op -BenchmarkSetGoRedis100Conns10KB-4 200000 9181 ns/op 210 B/op 6 allocs/op -BenchmarkSetGoRedis10Conns1MB-4 2000 583242 ns/op 2337 B/op 6 allocs/op -BenchmarkSetGoRedis100Conns1MB-4 2000 583089 ns/op 2338 B/op 6 allocs/op -BenchmarkSetRedigo10Conns64Bytes-4 200000 7576 ns/op 208 B/op 7 allocs/op -BenchmarkSetRedigo100Conns64Bytes-4 200000 7782 ns/op 208 B/op 7 allocs/op -BenchmarkSetRedigo10Conns1KB-4 200000 7958 ns/op 208 B/op 7 allocs/op -BenchmarkSetRedigo100Conns1KB-4 200000 7725 ns/op 208 B/op 7 allocs/op -BenchmarkSetRedigo10Conns10KB-4 100000 18442 ns/op 208 B/op 7 allocs/op -BenchmarkSetRedigo100Conns10KB-4 100000 18818 ns/op 208 B/op 7 allocs/op -BenchmarkSetRedigo10Conns1MB-4 2000 668829 ns/op 226 B/op 7 allocs/op -BenchmarkSetRedigo100Conns1MB-4 2000 679542 ns/op 226 B/op 7 allocs/op -``` - -Redis Cluster: - -``` -BenchmarkRedisPing-4 200000 6983 ns/op 116 B/op 4 allocs/op -BenchmarkRedisClusterPing-4 100000 11535 ns/op 117 B/op 4 allocs/op -``` - -## See also - -- [Golang PostgreSQL ORM](https://github.com/go-pg/pg) -- [Golang msgpack](https://github.com/vmihailenco/msgpack) -- [Golang message task queue](https://github.com/go-msgqueue/msgqueue) diff --git a/src/go/vendor/github.com/go-redis/redis/cluster.go b/src/go/vendor/github.com/go-redis/redis/cluster.go deleted file mode 100644 index f283722..0000000 --- a/src/go/vendor/github.com/go-redis/redis/cluster.go +++ /dev/null @@ -1,1610 +0,0 @@ -package redis - -import ( - "context" - "crypto/tls" - "fmt" - "math" - "math/rand" - "net" - "runtime" - "sort" - "sync" - "sync/atomic" - "time" - - "github.com/go-redis/redis/internal" - "github.com/go-redis/redis/internal/hashtag" - "github.com/go-redis/redis/internal/pool" - "github.com/go-redis/redis/internal/proto" -) - -var errClusterNoNodes = fmt.Errorf("redis: cluster has no nodes") - -// ClusterOptions are used to configure a cluster client and should be -// passed to NewClusterClient. -type ClusterOptions struct { - // A seed list of host:port addresses of cluster nodes. - Addrs []string - - // The maximum number of retries before giving up. Command is retried - // on network errors and MOVED/ASK redirects. - // Default is 8 retries. - MaxRedirects int - - // Enables read-only commands on slave nodes. - ReadOnly bool - // Allows routing read-only commands to the closest master or slave node. - // It automatically enables ReadOnly. - RouteByLatency bool - // Allows routing read-only commands to the random master or slave node. - // It automatically enables ReadOnly. - RouteRandomly bool - - // Optional function that returns cluster slots information. - // It is useful to manually create cluster of standalone Redis servers - // and load-balance read/write operations between master and slaves. - // It can use service like ZooKeeper to maintain configuration information - // and Cluster.ReloadState to manually trigger state reloading. - ClusterSlots func() ([]ClusterSlot, error) - - // Optional hook that is called when a new node is created. - OnNewNode func(*Client) - - // Following options are copied from Options struct. - - OnConnect func(*Conn) error - - Password string - - MaxRetries int - MinRetryBackoff time.Duration - MaxRetryBackoff time.Duration - - DialTimeout time.Duration - ReadTimeout time.Duration - WriteTimeout time.Duration - - // PoolSize applies per cluster node and not for the whole cluster. - PoolSize int - MinIdleConns int - MaxConnAge time.Duration - PoolTimeout time.Duration - IdleTimeout time.Duration - IdleCheckFrequency time.Duration - - TLSConfig *tls.Config -} - -func (opt *ClusterOptions) init() { - if opt.MaxRedirects == -1 { - opt.MaxRedirects = 0 - } else if opt.MaxRedirects == 0 { - opt.MaxRedirects = 8 - } - - if (opt.RouteByLatency || opt.RouteRandomly) && opt.ClusterSlots == nil { - opt.ReadOnly = true - } - - if opt.PoolSize == 0 { - opt.PoolSize = 5 * runtime.NumCPU() - } - - switch opt.ReadTimeout { - case -1: - opt.ReadTimeout = 0 - case 0: - opt.ReadTimeout = 3 * time.Second - } - switch opt.WriteTimeout { - case -1: - opt.WriteTimeout = 0 - case 0: - opt.WriteTimeout = opt.ReadTimeout - } - - switch opt.MinRetryBackoff { - case -1: - opt.MinRetryBackoff = 0 - case 0: - opt.MinRetryBackoff = 8 * time.Millisecond - } - switch opt.MaxRetryBackoff { - case -1: - opt.MaxRetryBackoff = 0 - case 0: - opt.MaxRetryBackoff = 512 * time.Millisecond - } -} - -func (opt *ClusterOptions) clientOptions() *Options { - const disableIdleCheck = -1 - - return &Options{ - OnConnect: opt.OnConnect, - - MaxRetries: opt.MaxRetries, - MinRetryBackoff: opt.MinRetryBackoff, - MaxRetryBackoff: opt.MaxRetryBackoff, - Password: opt.Password, - readOnly: opt.ReadOnly, - - DialTimeout: opt.DialTimeout, - ReadTimeout: opt.ReadTimeout, - WriteTimeout: opt.WriteTimeout, - - PoolSize: opt.PoolSize, - MinIdleConns: opt.MinIdleConns, - MaxConnAge: opt.MaxConnAge, - PoolTimeout: opt.PoolTimeout, - IdleTimeout: opt.IdleTimeout, - IdleCheckFrequency: disableIdleCheck, - - TLSConfig: opt.TLSConfig, - } -} - -//------------------------------------------------------------------------------ - -type clusterNode struct { - Client *Client - - latency uint32 // atomic - generation uint32 // atomic - loading uint32 // atomic -} - -func newClusterNode(clOpt *ClusterOptions, addr string) *clusterNode { - opt := clOpt.clientOptions() - opt.Addr = addr - node := clusterNode{ - Client: NewClient(opt), - } - - node.latency = math.MaxUint32 - if clOpt.RouteByLatency { - go node.updateLatency() - } - - if clOpt.OnNewNode != nil { - clOpt.OnNewNode(node.Client) - } - - return &node -} - -func (n *clusterNode) String() string { - return n.Client.String() -} - -func (n *clusterNode) Close() error { - return n.Client.Close() -} - -func (n *clusterNode) updateLatency() { - const probes = 10 - - var latency uint32 - for i := 0; i < probes; i++ { - start := time.Now() - n.Client.Ping() - probe := uint32(time.Since(start) / time.Microsecond) - latency = (latency + probe) / 2 - } - atomic.StoreUint32(&n.latency, latency) -} - -func (n *clusterNode) Latency() time.Duration { - latency := atomic.LoadUint32(&n.latency) - return time.Duration(latency) * time.Microsecond -} - -func (n *clusterNode) MarkAsLoading() { - atomic.StoreUint32(&n.loading, uint32(time.Now().Unix())) -} - -func (n *clusterNode) Loading() bool { - const minute = int64(time.Minute / time.Second) - - loading := atomic.LoadUint32(&n.loading) - if loading == 0 { - return false - } - if time.Now().Unix()-int64(loading) < minute { - return true - } - atomic.StoreUint32(&n.loading, 0) - return false -} - -func (n *clusterNode) Generation() uint32 { - return atomic.LoadUint32(&n.generation) -} - -func (n *clusterNode) SetGeneration(gen uint32) { - for { - v := atomic.LoadUint32(&n.generation) - if gen < v || atomic.CompareAndSwapUint32(&n.generation, v, gen) { - break - } - } -} - -//------------------------------------------------------------------------------ - -type clusterNodes struct { - opt *ClusterOptions - - mu sync.RWMutex - allAddrs []string - allNodes map[string]*clusterNode - clusterAddrs []string - closed bool - - _generation uint32 // atomic -} - -func newClusterNodes(opt *ClusterOptions) *clusterNodes { - return &clusterNodes{ - opt: opt, - - allAddrs: opt.Addrs, - allNodes: make(map[string]*clusterNode), - } -} - -func (c *clusterNodes) Close() error { - c.mu.Lock() - defer c.mu.Unlock() - - if c.closed { - return nil - } - c.closed = true - - var firstErr error - for _, node := range c.allNodes { - if err := node.Client.Close(); err != nil && firstErr == nil { - firstErr = err - } - } - - c.allNodes = nil - c.clusterAddrs = nil - - return firstErr -} - -func (c *clusterNodes) Addrs() ([]string, error) { - var addrs []string - c.mu.RLock() - closed := c.closed - if !closed { - if len(c.clusterAddrs) > 0 { - addrs = c.clusterAddrs - } else { - addrs = c.allAddrs - } - } - c.mu.RUnlock() - - if closed { - return nil, pool.ErrClosed - } - if len(addrs) == 0 { - return nil, errClusterNoNodes - } - return addrs, nil -} - -func (c *clusterNodes) NextGeneration() uint32 { - return atomic.AddUint32(&c._generation, 1) -} - -// GC removes unused nodes. -func (c *clusterNodes) GC(generation uint32) { - var collected []*clusterNode - c.mu.Lock() - for addr, node := range c.allNodes { - if node.Generation() >= generation { - continue - } - - c.clusterAddrs = remove(c.clusterAddrs, addr) - delete(c.allNodes, addr) - collected = append(collected, node) - } - c.mu.Unlock() - - for _, node := range collected { - _ = node.Client.Close() - } -} - -func (c *clusterNodes) Get(addr string) (*clusterNode, error) { - var node *clusterNode - var err error - c.mu.RLock() - if c.closed { - err = pool.ErrClosed - } else { - node = c.allNodes[addr] - } - c.mu.RUnlock() - return node, err -} - -func (c *clusterNodes) GetOrCreate(addr string) (*clusterNode, error) { - node, err := c.Get(addr) - if err != nil { - return nil, err - } - if node != nil { - return node, nil - } - - c.mu.Lock() - defer c.mu.Unlock() - - if c.closed { - return nil, pool.ErrClosed - } - - node, ok := c.allNodes[addr] - if ok { - return node, err - } - - node = newClusterNode(c.opt, addr) - - c.allAddrs = appendIfNotExists(c.allAddrs, addr) - c.clusterAddrs = append(c.clusterAddrs, addr) - c.allNodes[addr] = node - - return node, err -} - -func (c *clusterNodes) All() ([]*clusterNode, error) { - c.mu.RLock() - defer c.mu.RUnlock() - - if c.closed { - return nil, pool.ErrClosed - } - - cp := make([]*clusterNode, 0, len(c.allNodes)) - for _, node := range c.allNodes { - cp = append(cp, node) - } - return cp, nil -} - -func (c *clusterNodes) Random() (*clusterNode, error) { - addrs, err := c.Addrs() - if err != nil { - return nil, err - } - - n := rand.Intn(len(addrs)) - return c.GetOrCreate(addrs[n]) -} - -//------------------------------------------------------------------------------ - -type clusterSlot struct { - start, end int - nodes []*clusterNode -} - -type clusterSlotSlice []*clusterSlot - -func (p clusterSlotSlice) Len() int { - return len(p) -} - -func (p clusterSlotSlice) Less(i, j int) bool { - return p[i].start < p[j].start -} - -func (p clusterSlotSlice) Swap(i, j int) { - p[i], p[j] = p[j], p[i] -} - -type clusterState struct { - nodes *clusterNodes - Masters []*clusterNode - Slaves []*clusterNode - - slots []*clusterSlot - - generation uint32 - createdAt time.Time -} - -func newClusterState( - nodes *clusterNodes, slots []ClusterSlot, origin string, -) (*clusterState, error) { - c := clusterState{ - nodes: nodes, - - slots: make([]*clusterSlot, 0, len(slots)), - - generation: nodes.NextGeneration(), - createdAt: time.Now(), - } - - originHost, _, _ := net.SplitHostPort(origin) - isLoopbackOrigin := isLoopback(originHost) - - for _, slot := range slots { - var nodes []*clusterNode - for i, slotNode := range slot.Nodes { - addr := slotNode.Addr - if !isLoopbackOrigin { - addr = replaceLoopbackHost(addr, originHost) - } - - node, err := c.nodes.GetOrCreate(addr) - if err != nil { - return nil, err - } - - node.SetGeneration(c.generation) - nodes = append(nodes, node) - - if i == 0 { - c.Masters = appendUniqueNode(c.Masters, node) - } else { - c.Slaves = appendUniqueNode(c.Slaves, node) - } - } - - c.slots = append(c.slots, &clusterSlot{ - start: slot.Start, - end: slot.End, - nodes: nodes, - }) - } - - sort.Sort(clusterSlotSlice(c.slots)) - - time.AfterFunc(time.Minute, func() { - nodes.GC(c.generation) - }) - - return &c, nil -} - -func replaceLoopbackHost(nodeAddr, originHost string) string { - nodeHost, nodePort, err := net.SplitHostPort(nodeAddr) - if err != nil { - return nodeAddr - } - - nodeIP := net.ParseIP(nodeHost) - if nodeIP == nil { - return nodeAddr - } - - if !nodeIP.IsLoopback() { - return nodeAddr - } - - // Use origin host which is not loopback and node port. - return net.JoinHostPort(originHost, nodePort) -} - -func isLoopback(host string) bool { - ip := net.ParseIP(host) - if ip == nil { - return true - } - return ip.IsLoopback() -} - -func (c *clusterState) slotMasterNode(slot int) (*clusterNode, error) { - nodes := c.slotNodes(slot) - if len(nodes) > 0 { - return nodes[0], nil - } - return c.nodes.Random() -} - -func (c *clusterState) slotSlaveNode(slot int) (*clusterNode, error) { - nodes := c.slotNodes(slot) - switch len(nodes) { - case 0: - return c.nodes.Random() - case 1: - return nodes[0], nil - case 2: - if slave := nodes[1]; !slave.Loading() { - return slave, nil - } - return nodes[0], nil - default: - var slave *clusterNode - for i := 0; i < 10; i++ { - n := rand.Intn(len(nodes)-1) + 1 - slave = nodes[n] - if !slave.Loading() { - return slave, nil - } - } - - // All slaves are loading - use master. - return nodes[0], nil - } -} - -func (c *clusterState) slotClosestNode(slot int) (*clusterNode, error) { - const threshold = time.Millisecond - - nodes := c.slotNodes(slot) - if len(nodes) == 0 { - return c.nodes.Random() - } - - var node *clusterNode - for _, n := range nodes { - if n.Loading() { - continue - } - if node == nil || node.Latency()-n.Latency() > threshold { - node = n - } - } - return node, nil -} - -func (c *clusterState) slotRandomNode(slot int) *clusterNode { - nodes := c.slotNodes(slot) - n := rand.Intn(len(nodes)) - return nodes[n] -} - -func (c *clusterState) slotNodes(slot int) []*clusterNode { - i := sort.Search(len(c.slots), func(i int) bool { - return c.slots[i].end >= slot - }) - if i >= len(c.slots) { - return nil - } - x := c.slots[i] - if slot >= x.start && slot <= x.end { - return x.nodes - } - return nil -} - -//------------------------------------------------------------------------------ - -type clusterStateHolder struct { - load func() (*clusterState, error) - - state atomic.Value - reloading uint32 // atomic -} - -func newClusterStateHolder(fn func() (*clusterState, error)) *clusterStateHolder { - return &clusterStateHolder{ - load: fn, - } -} - -func (c *clusterStateHolder) Reload() (*clusterState, error) { - state, err := c.load() - if err != nil { - return nil, err - } - c.state.Store(state) - return state, nil -} - -func (c *clusterStateHolder) LazyReload() { - if !atomic.CompareAndSwapUint32(&c.reloading, 0, 1) { - return - } - go func() { - defer atomic.StoreUint32(&c.reloading, 0) - - _, err := c.Reload() - if err != nil { - return - } - time.Sleep(100 * time.Millisecond) - }() -} - -func (c *clusterStateHolder) Get() (*clusterState, error) { - v := c.state.Load() - if v != nil { - state := v.(*clusterState) - if time.Since(state.createdAt) > time.Minute { - c.LazyReload() - } - return state, nil - } - return c.Reload() -} - -func (c *clusterStateHolder) ReloadOrGet() (*clusterState, error) { - state, err := c.Reload() - if err == nil { - return state, nil - } - return c.Get() -} - -//------------------------------------------------------------------------------ - -// ClusterClient is a Redis Cluster client representing a pool of zero -// or more underlying connections. It's safe for concurrent use by -// multiple goroutines. -type ClusterClient struct { - cmdable - - ctx context.Context - - opt *ClusterOptions - nodes *clusterNodes - state *clusterStateHolder - cmdsInfoCache *cmdsInfoCache - - process func(Cmder) error - processPipeline func([]Cmder) error - processTxPipeline func([]Cmder) error -} - -// NewClusterClient returns a Redis Cluster client as described in -// http://redis.io/topics/cluster-spec. -func NewClusterClient(opt *ClusterOptions) *ClusterClient { - opt.init() - - c := &ClusterClient{ - opt: opt, - nodes: newClusterNodes(opt), - } - c.state = newClusterStateHolder(c.loadState) - c.cmdsInfoCache = newCmdsInfoCache(c.cmdsInfo) - - c.process = c.defaultProcess - c.processPipeline = c.defaultProcessPipeline - c.processTxPipeline = c.defaultProcessTxPipeline - - c.init() - if opt.IdleCheckFrequency > 0 { - go c.reaper(opt.IdleCheckFrequency) - } - - return c -} - -func (c *ClusterClient) init() { - c.cmdable.setProcessor(c.Process) -} - -// ReloadState reloads cluster state. If available it calls ClusterSlots func -// to get cluster slots information. -func (c *ClusterClient) ReloadState() error { - _, err := c.state.Reload() - return err -} - -func (c *ClusterClient) Context() context.Context { - if c.ctx != nil { - return c.ctx - } - return context.Background() -} - -func (c *ClusterClient) WithContext(ctx context.Context) *ClusterClient { - if ctx == nil { - panic("nil context") - } - c2 := c.copy() - c2.ctx = ctx - return c2 -} - -func (c *ClusterClient) copy() *ClusterClient { - cp := *c - cp.init() - return &cp -} - -// Options returns read-only Options that were used to create the client. -func (c *ClusterClient) Options() *ClusterOptions { - return c.opt -} - -func (c *ClusterClient) retryBackoff(attempt int) time.Duration { - return internal.RetryBackoff(attempt, c.opt.MinRetryBackoff, c.opt.MaxRetryBackoff) -} - -func (c *ClusterClient) cmdsInfo() (map[string]*CommandInfo, error) { - addrs, err := c.nodes.Addrs() - if err != nil { - return nil, err - } - - var firstErr error - for _, addr := range addrs { - node, err := c.nodes.Get(addr) - if err != nil { - return nil, err - } - if node == nil { - continue - } - - info, err := node.Client.Command().Result() - if err == nil { - return info, nil - } - if firstErr == nil { - firstErr = err - } - } - return nil, firstErr -} - -func (c *ClusterClient) cmdInfo(name string) *CommandInfo { - cmdsInfo, err := c.cmdsInfoCache.Get() - if err != nil { - return nil - } - - info := cmdsInfo[name] - if info == nil { - internal.Logf("info for cmd=%s not found", name) - } - return info -} - -func cmdSlot(cmd Cmder, pos int) int { - if pos == 0 { - return hashtag.RandomSlot() - } - firstKey := cmd.stringArg(pos) - return hashtag.Slot(firstKey) -} - -func (c *ClusterClient) cmdSlot(cmd Cmder) int { - cmdInfo := c.cmdInfo(cmd.Name()) - return cmdSlot(cmd, cmdFirstKeyPos(cmd, cmdInfo)) -} - -func (c *ClusterClient) cmdSlotAndNode(cmd Cmder) (int, *clusterNode, error) { - state, err := c.state.Get() - if err != nil { - return 0, nil, err - } - - cmdInfo := c.cmdInfo(cmd.Name()) - slot := cmdSlot(cmd, cmdFirstKeyPos(cmd, cmdInfo)) - - if c.opt.ReadOnly && cmdInfo != nil && cmdInfo.ReadOnly { - if c.opt.RouteByLatency { - node, err := state.slotClosestNode(slot) - return slot, node, err - } - - if c.opt.RouteRandomly { - node := state.slotRandomNode(slot) - return slot, node, nil - } - - node, err := state.slotSlaveNode(slot) - return slot, node, err - } - - node, err := state.slotMasterNode(slot) - return slot, node, err -} - -func (c *ClusterClient) slotMasterNode(slot int) (*clusterNode, error) { - state, err := c.state.Get() - if err != nil { - return nil, err - } - - nodes := state.slotNodes(slot) - if len(nodes) > 0 { - return nodes[0], nil - } - return c.nodes.Random() -} - -func (c *ClusterClient) Watch(fn func(*Tx) error, keys ...string) error { - if len(keys) == 0 { - return fmt.Errorf("redis: Watch requires at least one key") - } - - slot := hashtag.Slot(keys[0]) - for _, key := range keys[1:] { - if hashtag.Slot(key) != slot { - err := fmt.Errorf("redis: Watch requires all keys to be in the same slot") - return err - } - } - - node, err := c.slotMasterNode(slot) - if err != nil { - return err - } - - for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { - if attempt > 0 { - time.Sleep(c.retryBackoff(attempt)) - } - - err = node.Client.Watch(fn, keys...) - if err == nil { - break - } - - if internal.IsRetryableError(err, true) { - c.state.LazyReload() - continue - } - - moved, ask, addr := internal.IsMovedError(err) - if moved || ask { - c.state.LazyReload() - node, err = c.nodes.GetOrCreate(addr) - if err != nil { - return err - } - continue - } - - if err == pool.ErrClosed { - node, err = c.slotMasterNode(slot) - if err != nil { - return err - } - continue - } - - return err - } - - return err -} - -// Close closes the cluster client, releasing any open resources. -// -// It is rare to Close a ClusterClient, as the ClusterClient is meant -// to be long-lived and shared between many goroutines. -func (c *ClusterClient) Close() error { - return c.nodes.Close() -} - -// Do creates a Cmd from the args and processes the cmd. -func (c *ClusterClient) Do(args ...interface{}) *Cmd { - cmd := NewCmd(args...) - c.Process(cmd) - return cmd -} - -func (c *ClusterClient) WrapProcess( - fn func(oldProcess func(Cmder) error) func(Cmder) error, -) { - c.process = fn(c.process) -} - -func (c *ClusterClient) Process(cmd Cmder) error { - return c.process(cmd) -} - -func (c *ClusterClient) defaultProcess(cmd Cmder) error { - var node *clusterNode - var ask bool - for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { - if attempt > 0 { - time.Sleep(c.retryBackoff(attempt)) - } - - if node == nil { - var err error - _, node, err = c.cmdSlotAndNode(cmd) - if err != nil { - cmd.setErr(err) - break - } - } - - var err error - if ask { - pipe := node.Client.Pipeline() - _ = pipe.Process(NewCmd("ASKING")) - _ = pipe.Process(cmd) - _, err = pipe.Exec() - _ = pipe.Close() - ask = false - } else { - err = node.Client.Process(cmd) - } - - // If there is no error - we are done. - if err == nil { - break - } - - // If slave is loading - pick another node. - if c.opt.ReadOnly && internal.IsLoadingError(err) { - node.MarkAsLoading() - node = nil - continue - } - - if internal.IsRetryableError(err, true) { - c.state.LazyReload() - - // First retry the same node. - if attempt == 0 { - continue - } - - // Second try random node. - node, err = c.nodes.Random() - if err != nil { - break - } - continue - } - - var moved bool - var addr string - moved, ask, addr = internal.IsMovedError(err) - if moved || ask { - c.state.LazyReload() - - node, err = c.nodes.GetOrCreate(addr) - if err != nil { - break - } - continue - } - - if err == pool.ErrClosed { - node = nil - continue - } - - break - } - - return cmd.Err() -} - -// ForEachMaster concurrently calls the fn on each master node in the cluster. -// It returns the first error if any. -func (c *ClusterClient) ForEachMaster(fn func(client *Client) error) error { - state, err := c.state.ReloadOrGet() - if err != nil { - return err - } - - var wg sync.WaitGroup - errCh := make(chan error, 1) - for _, master := range state.Masters { - wg.Add(1) - go func(node *clusterNode) { - defer wg.Done() - err := fn(node.Client) - if err != nil { - select { - case errCh <- err: - default: - } - } - }(master) - } - wg.Wait() - - select { - case err := <-errCh: - return err - default: - return nil - } -} - -// ForEachSlave concurrently calls the fn on each slave node in the cluster. -// It returns the first error if any. -func (c *ClusterClient) ForEachSlave(fn func(client *Client) error) error { - state, err := c.state.ReloadOrGet() - if err != nil { - return err - } - - var wg sync.WaitGroup - errCh := make(chan error, 1) - for _, slave := range state.Slaves { - wg.Add(1) - go func(node *clusterNode) { - defer wg.Done() - err := fn(node.Client) - if err != nil { - select { - case errCh <- err: - default: - } - } - }(slave) - } - wg.Wait() - - select { - case err := <-errCh: - return err - default: - return nil - } -} - -// ForEachNode concurrently calls the fn on each known node in the cluster. -// It returns the first error if any. -func (c *ClusterClient) ForEachNode(fn func(client *Client) error) error { - state, err := c.state.ReloadOrGet() - if err != nil { - return err - } - - var wg sync.WaitGroup - errCh := make(chan error, 1) - worker := func(node *clusterNode) { - defer wg.Done() - err := fn(node.Client) - if err != nil { - select { - case errCh <- err: - default: - } - } - } - - for _, node := range state.Masters { - wg.Add(1) - go worker(node) - } - for _, node := range state.Slaves { - wg.Add(1) - go worker(node) - } - - wg.Wait() - select { - case err := <-errCh: - return err - default: - return nil - } -} - -// PoolStats returns accumulated connection pool stats. -func (c *ClusterClient) PoolStats() *PoolStats { - var acc PoolStats - - state, _ := c.state.Get() - if state == nil { - return &acc - } - - for _, node := range state.Masters { - s := node.Client.connPool.Stats() - acc.Hits += s.Hits - acc.Misses += s.Misses - acc.Timeouts += s.Timeouts - - acc.TotalConns += s.TotalConns - acc.IdleConns += s.IdleConns - acc.StaleConns += s.StaleConns - } - - for _, node := range state.Slaves { - s := node.Client.connPool.Stats() - acc.Hits += s.Hits - acc.Misses += s.Misses - acc.Timeouts += s.Timeouts - - acc.TotalConns += s.TotalConns - acc.IdleConns += s.IdleConns - acc.StaleConns += s.StaleConns - } - - return &acc -} - -func (c *ClusterClient) loadState() (*clusterState, error) { - if c.opt.ClusterSlots != nil { - slots, err := c.opt.ClusterSlots() - if err != nil { - return nil, err - } - return newClusterState(c.nodes, slots, "") - } - - addrs, err := c.nodes.Addrs() - if err != nil { - return nil, err - } - - var firstErr error - for _, addr := range addrs { - node, err := c.nodes.GetOrCreate(addr) - if err != nil { - if firstErr == nil { - firstErr = err - } - continue - } - - slots, err := node.Client.ClusterSlots().Result() - if err != nil { - if firstErr == nil { - firstErr = err - } - continue - } - - return newClusterState(c.nodes, slots, node.Client.opt.Addr) - } - - return nil, firstErr -} - -// reaper closes idle connections to the cluster. -func (c *ClusterClient) reaper(idleCheckFrequency time.Duration) { - ticker := time.NewTicker(idleCheckFrequency) - defer ticker.Stop() - - for range ticker.C { - nodes, err := c.nodes.All() - if err != nil { - break - } - - for _, node := range nodes { - _, err := node.Client.connPool.(*pool.ConnPool).ReapStaleConns() - if err != nil { - internal.Logf("ReapStaleConns failed: %s", err) - } - } - } -} - -func (c *ClusterClient) Pipeline() Pipeliner { - pipe := Pipeline{ - exec: c.processPipeline, - } - pipe.statefulCmdable.setProcessor(pipe.Process) - return &pipe -} - -func (c *ClusterClient) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) { - return c.Pipeline().Pipelined(fn) -} - -func (c *ClusterClient) WrapProcessPipeline( - fn func(oldProcess func([]Cmder) error) func([]Cmder) error, -) { - c.processPipeline = fn(c.processPipeline) -} - -func (c *ClusterClient) defaultProcessPipeline(cmds []Cmder) error { - cmdsMap := newCmdsMap() - err := c.mapCmdsByNode(cmds, cmdsMap) - if err != nil { - setCmdsErr(cmds, err) - return err - } - - for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { - if attempt > 0 { - time.Sleep(c.retryBackoff(attempt)) - } - - failedCmds := newCmdsMap() - var wg sync.WaitGroup - - for node, cmds := range cmdsMap.m { - wg.Add(1) - go func(node *clusterNode, cmds []Cmder) { - defer wg.Done() - - cn, err := node.Client.getConn() - if err != nil { - if err == pool.ErrClosed { - c.mapCmdsByNode(cmds, failedCmds) - } else { - setCmdsErr(cmds, err) - } - return - } - - err = c.pipelineProcessCmds(node, cn, cmds, failedCmds) - node.Client.releaseConnStrict(cn, err) - }(node, cmds) - } - - wg.Wait() - if len(failedCmds.m) == 0 { - break - } - cmdsMap = failedCmds - } - - return cmdsFirstErr(cmds) -} - -type cmdsMap struct { - mu sync.Mutex - m map[*clusterNode][]Cmder -} - -func newCmdsMap() *cmdsMap { - return &cmdsMap{ - m: make(map[*clusterNode][]Cmder), - } -} - -func (c *ClusterClient) mapCmdsByNode(cmds []Cmder, cmdsMap *cmdsMap) error { - state, err := c.state.Get() - if err != nil { - setCmdsErr(cmds, err) - return err - } - - cmdsAreReadOnly := c.cmdsAreReadOnly(cmds) - for _, cmd := range cmds { - var node *clusterNode - var err error - if cmdsAreReadOnly { - _, node, err = c.cmdSlotAndNode(cmd) - } else { - slot := c.cmdSlot(cmd) - node, err = state.slotMasterNode(slot) - } - if err != nil { - return err - } - cmdsMap.mu.Lock() - cmdsMap.m[node] = append(cmdsMap.m[node], cmd) - cmdsMap.mu.Unlock() - } - return nil -} - -func (c *ClusterClient) cmdsAreReadOnly(cmds []Cmder) bool { - for _, cmd := range cmds { - cmdInfo := c.cmdInfo(cmd.Name()) - if cmdInfo == nil || !cmdInfo.ReadOnly { - return false - } - } - return true -} - -func (c *ClusterClient) pipelineProcessCmds( - node *clusterNode, cn *pool.Conn, cmds []Cmder, failedCmds *cmdsMap, -) error { - err := cn.WithWriter(c.opt.WriteTimeout, func(wr *proto.Writer) error { - return writeCmd(wr, cmds...) - }) - if err != nil { - setCmdsErr(cmds, err) - failedCmds.mu.Lock() - failedCmds.m[node] = cmds - failedCmds.mu.Unlock() - return err - } - - err = cn.WithReader(c.opt.ReadTimeout, func(rd *proto.Reader) error { - return c.pipelineReadCmds(node, rd, cmds, failedCmds) - }) - return err -} - -func (c *ClusterClient) pipelineReadCmds( - node *clusterNode, rd *proto.Reader, cmds []Cmder, failedCmds *cmdsMap, -) error { - var firstErr error - for _, cmd := range cmds { - err := cmd.readReply(rd) - if err == nil { - continue - } - - if c.checkMovedErr(cmd, err, failedCmds) { - continue - } - - if internal.IsRedisError(err) { - continue - } - - failedCmds.mu.Lock() - failedCmds.m[node] = append(failedCmds.m[node], cmd) - failedCmds.mu.Unlock() - if firstErr == nil { - firstErr = err - } - } - return firstErr -} - -func (c *ClusterClient) checkMovedErr( - cmd Cmder, err error, failedCmds *cmdsMap, -) bool { - moved, ask, addr := internal.IsMovedError(err) - - if moved { - c.state.LazyReload() - - node, err := c.nodes.GetOrCreate(addr) - if err != nil { - return false - } - - failedCmds.mu.Lock() - failedCmds.m[node] = append(failedCmds.m[node], cmd) - failedCmds.mu.Unlock() - return true - } - - if ask { - node, err := c.nodes.GetOrCreate(addr) - if err != nil { - return false - } - - failedCmds.mu.Lock() - failedCmds.m[node] = append(failedCmds.m[node], NewCmd("ASKING"), cmd) - failedCmds.mu.Unlock() - return true - } - - return false -} - -// TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC. -func (c *ClusterClient) TxPipeline() Pipeliner { - pipe := Pipeline{ - exec: c.processTxPipeline, - } - pipe.statefulCmdable.setProcessor(pipe.Process) - return &pipe -} - -func (c *ClusterClient) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) { - return c.TxPipeline().Pipelined(fn) -} - -func (c *ClusterClient) defaultProcessTxPipeline(cmds []Cmder) error { - state, err := c.state.Get() - if err != nil { - return err - } - - cmdsMap := c.mapCmdsBySlot(cmds) - for slot, cmds := range cmdsMap { - node, err := state.slotMasterNode(slot) - if err != nil { - setCmdsErr(cmds, err) - continue - } - cmdsMap := map[*clusterNode][]Cmder{node: cmds} - - for attempt := 0; attempt <= c.opt.MaxRedirects; attempt++ { - if attempt > 0 { - time.Sleep(c.retryBackoff(attempt)) - } - - failedCmds := newCmdsMap() - var wg sync.WaitGroup - - for node, cmds := range cmdsMap { - wg.Add(1) - go func(node *clusterNode, cmds []Cmder) { - defer wg.Done() - - cn, err := node.Client.getConn() - if err != nil { - if err == pool.ErrClosed { - c.mapCmdsByNode(cmds, failedCmds) - } else { - setCmdsErr(cmds, err) - } - return - } - - err = c.txPipelineProcessCmds(node, cn, cmds, failedCmds) - node.Client.releaseConnStrict(cn, err) - }(node, cmds) - } - - wg.Wait() - if len(failedCmds.m) == 0 { - break - } - cmdsMap = failedCmds.m - } - } - - return cmdsFirstErr(cmds) -} - -func (c *ClusterClient) mapCmdsBySlot(cmds []Cmder) map[int][]Cmder { - cmdsMap := make(map[int][]Cmder) - for _, cmd := range cmds { - slot := c.cmdSlot(cmd) - cmdsMap[slot] = append(cmdsMap[slot], cmd) - } - return cmdsMap -} - -func (c *ClusterClient) txPipelineProcessCmds( - node *clusterNode, cn *pool.Conn, cmds []Cmder, failedCmds *cmdsMap, -) error { - err := cn.WithWriter(c.opt.WriteTimeout, func(wr *proto.Writer) error { - return txPipelineWriteMulti(wr, cmds) - }) - if err != nil { - setCmdsErr(cmds, err) - failedCmds.mu.Lock() - failedCmds.m[node] = cmds - failedCmds.mu.Unlock() - return err - } - - err = cn.WithReader(c.opt.ReadTimeout, func(rd *proto.Reader) error { - err := c.txPipelineReadQueued(rd, cmds, failedCmds) - if err != nil { - setCmdsErr(cmds, err) - return err - } - return pipelineReadCmds(rd, cmds) - }) - return err -} - -func (c *ClusterClient) txPipelineReadQueued( - rd *proto.Reader, cmds []Cmder, failedCmds *cmdsMap, -) error { - // Parse queued replies. - var statusCmd StatusCmd - if err := statusCmd.readReply(rd); err != nil { - return err - } - - for _, cmd := range cmds { - err := statusCmd.readReply(rd) - if err == nil { - continue - } - - if c.checkMovedErr(cmd, err, failedCmds) || internal.IsRedisError(err) { - continue - } - - return err - } - - // Parse number of replies. - line, err := rd.ReadLine() - if err != nil { - if err == Nil { - err = TxFailedErr - } - return err - } - - switch line[0] { - case proto.ErrorReply: - err := proto.ParseErrorReply(line) - for _, cmd := range cmds { - if !c.checkMovedErr(cmd, err, failedCmds) { - break - } - } - return err - case proto.ArrayReply: - // ok - default: - err := fmt.Errorf("redis: expected '*', but got line %q", line) - return err - } - - return nil -} - -func (c *ClusterClient) pubSub(channels []string) *PubSub { - var node *clusterNode - pubsub := &PubSub{ - opt: c.opt.clientOptions(), - - newConn: func(channels []string) (*pool.Conn, error) { - if node == nil { - var slot int - if len(channels) > 0 { - slot = hashtag.Slot(channels[0]) - } else { - slot = -1 - } - - masterNode, err := c.slotMasterNode(slot) - if err != nil { - return nil, err - } - node = masterNode - } - return node.Client.newConn() - }, - closeConn: func(cn *pool.Conn) error { - return node.Client.connPool.CloseConn(cn) - }, - } - pubsub.init() - return pubsub -} - -// Subscribe subscribes the client to the specified channels. -// Channels can be omitted to create empty subscription. -func (c *ClusterClient) Subscribe(channels ...string) *PubSub { - pubsub := c.pubSub(channels) - if len(channels) > 0 { - _ = pubsub.Subscribe(channels...) - } - return pubsub -} - -// PSubscribe subscribes the client to the given patterns. -// Patterns can be omitted to create empty subscription. -func (c *ClusterClient) PSubscribe(channels ...string) *PubSub { - pubsub := c.pubSub(channels) - if len(channels) > 0 { - _ = pubsub.PSubscribe(channels...) - } - return pubsub -} - -func appendUniqueNode(nodes []*clusterNode, node *clusterNode) []*clusterNode { - for _, n := range nodes { - if n == node { - return nodes - } - } - return append(nodes, node) -} - -func appendIfNotExists(ss []string, es ...string) []string { -loop: - for _, e := range es { - for _, s := range ss { - if s == e { - continue loop - } - } - ss = append(ss, e) - } - return ss -} - -func remove(ss []string, es ...string) []string { - if len(es) == 0 { - return ss[:0] - } - for _, e := range es { - for i, s := range ss { - if s == e { - ss = append(ss[:i], ss[i+1:]...) - break - } - } - } - return ss -} diff --git a/src/go/vendor/github.com/go-redis/redis/cluster_commands.go b/src/go/vendor/github.com/go-redis/redis/cluster_commands.go deleted file mode 100644 index dff62c9..0000000 --- a/src/go/vendor/github.com/go-redis/redis/cluster_commands.go +++ /dev/null @@ -1,22 +0,0 @@ -package redis - -import "sync/atomic" - -func (c *ClusterClient) DBSize() *IntCmd { - cmd := NewIntCmd("dbsize") - var size int64 - err := c.ForEachMaster(func(master *Client) error { - n, err := master.DBSize().Result() - if err != nil { - return err - } - atomic.AddInt64(&size, n) - return nil - }) - if err != nil { - cmd.setErr(err) - return cmd - } - cmd.val = size - return cmd -} diff --git a/src/go/vendor/github.com/go-redis/redis/command.go b/src/go/vendor/github.com/go-redis/redis/command.go deleted file mode 100644 index cb4f94b..0000000 --- a/src/go/vendor/github.com/go-redis/redis/command.go +++ /dev/null @@ -1,1936 +0,0 @@ -package redis - -import ( - "fmt" - "net" - "strconv" - "strings" - "time" - - "github.com/go-redis/redis/internal" - "github.com/go-redis/redis/internal/proto" -) - -type Cmder interface { - Name() string - Args() []interface{} - stringArg(int) string - - readReply(rd *proto.Reader) error - setErr(error) - - readTimeout() *time.Duration - - Err() error -} - -func setCmdsErr(cmds []Cmder, e error) { - for _, cmd := range cmds { - if cmd.Err() == nil { - cmd.setErr(e) - } - } -} - -func cmdsFirstErr(cmds []Cmder) error { - for _, cmd := range cmds { - if err := cmd.Err(); err != nil { - return err - } - } - return nil -} - -func writeCmd(wr *proto.Writer, cmds ...Cmder) error { - for _, cmd := range cmds { - err := wr.WriteArgs(cmd.Args()) - if err != nil { - return err - } - } - return nil -} - -func cmdString(cmd Cmder, val interface{}) string { - var ss []string - for _, arg := range cmd.Args() { - ss = append(ss, fmt.Sprint(arg)) - } - s := strings.Join(ss, " ") - if err := cmd.Err(); err != nil { - return s + ": " + err.Error() - } - if val != nil { - switch vv := val.(type) { - case []byte: - return s + ": " + string(vv) - default: - return s + ": " + fmt.Sprint(val) - } - } - return s - -} - -func cmdFirstKeyPos(cmd Cmder, info *CommandInfo) int { - switch cmd.Name() { - case "eval", "evalsha": - if cmd.stringArg(2) != "0" { - return 3 - } - - return 0 - case "publish": - return 1 - } - if info == nil { - return 0 - } - return int(info.FirstKeyPos) -} - -//------------------------------------------------------------------------------ - -type baseCmd struct { - _args []interface{} - err error - - _readTimeout *time.Duration -} - -var _ Cmder = (*Cmd)(nil) - -func (cmd *baseCmd) Err() error { - return cmd.err -} - -func (cmd *baseCmd) Args() []interface{} { - return cmd._args -} - -func (cmd *baseCmd) stringArg(pos int) string { - if pos < 0 || pos >= len(cmd._args) { - return "" - } - s, _ := cmd._args[pos].(string) - return s -} - -func (cmd *baseCmd) Name() string { - if len(cmd._args) > 0 { - // Cmd name must be lower cased. - s := internal.ToLower(cmd.stringArg(0)) - cmd._args[0] = s - return s - } - return "" -} - -func (cmd *baseCmd) readTimeout() *time.Duration { - return cmd._readTimeout -} - -func (cmd *baseCmd) setReadTimeout(d time.Duration) { - cmd._readTimeout = &d -} - -func (cmd *baseCmd) setErr(e error) { - cmd.err = e -} - -//------------------------------------------------------------------------------ - -type Cmd struct { - baseCmd - - val interface{} -} - -func NewCmd(args ...interface{}) *Cmd { - return &Cmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *Cmd) Val() interface{} { - return cmd.val -} - -func (cmd *Cmd) Result() (interface{}, error) { - return cmd.val, cmd.err -} - -func (cmd *Cmd) String() (string, error) { - if cmd.err != nil { - return "", cmd.err - } - switch val := cmd.val.(type) { - case string: - return val, nil - default: - err := fmt.Errorf("redis: unexpected type=%T for String", val) - return "", err - } -} - -func (cmd *Cmd) Int() (int, error) { - if cmd.err != nil { - return 0, cmd.err - } - switch val := cmd.val.(type) { - case int64: - return int(val), nil - case string: - return strconv.Atoi(val) - default: - err := fmt.Errorf("redis: unexpected type=%T for Int", val) - return 0, err - } -} - -func (cmd *Cmd) Int64() (int64, error) { - if cmd.err != nil { - return 0, cmd.err - } - switch val := cmd.val.(type) { - case int64: - return val, nil - case string: - return strconv.ParseInt(val, 10, 64) - default: - err := fmt.Errorf("redis: unexpected type=%T for Int64", val) - return 0, err - } -} - -func (cmd *Cmd) Uint64() (uint64, error) { - if cmd.err != nil { - return 0, cmd.err - } - switch val := cmd.val.(type) { - case int64: - return uint64(val), nil - case string: - return strconv.ParseUint(val, 10, 64) - default: - err := fmt.Errorf("redis: unexpected type=%T for Uint64", val) - return 0, err - } -} - -func (cmd *Cmd) Float64() (float64, error) { - if cmd.err != nil { - return 0, cmd.err - } - switch val := cmd.val.(type) { - case int64: - return float64(val), nil - case string: - return strconv.ParseFloat(val, 64) - default: - err := fmt.Errorf("redis: unexpected type=%T for Float64", val) - return 0, err - } -} - -func (cmd *Cmd) Bool() (bool, error) { - if cmd.err != nil { - return false, cmd.err - } - switch val := cmd.val.(type) { - case int64: - return val != 0, nil - case string: - return strconv.ParseBool(val) - default: - err := fmt.Errorf("redis: unexpected type=%T for Bool", val) - return false, err - } -} - -func (cmd *Cmd) readReply(rd *proto.Reader) error { - cmd.val, cmd.err = rd.ReadReply(sliceParser) - return cmd.err -} - -// Implements proto.MultiBulkParse -func sliceParser(rd *proto.Reader, n int64) (interface{}, error) { - vals := make([]interface{}, 0, n) - for i := int64(0); i < n; i++ { - v, err := rd.ReadReply(sliceParser) - if err != nil { - if err == Nil { - vals = append(vals, nil) - continue - } - if err, ok := err.(proto.RedisError); ok { - vals = append(vals, err) - continue - } - return nil, err - } - - switch v := v.(type) { - case string: - vals = append(vals, v) - default: - vals = append(vals, v) - } - } - return vals, nil -} - -//------------------------------------------------------------------------------ - -type SliceCmd struct { - baseCmd - - val []interface{} -} - -var _ Cmder = (*SliceCmd)(nil) - -func NewSliceCmd(args ...interface{}) *SliceCmd { - return &SliceCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *SliceCmd) Val() []interface{} { - return cmd.val -} - -func (cmd *SliceCmd) Result() ([]interface{}, error) { - return cmd.val, cmd.err -} - -func (cmd *SliceCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *SliceCmd) readReply(rd *proto.Reader) error { - var v interface{} - v, cmd.err = rd.ReadArrayReply(sliceParser) - if cmd.err != nil { - return cmd.err - } - cmd.val = v.([]interface{}) - return nil -} - -//------------------------------------------------------------------------------ - -type StatusCmd struct { - baseCmd - - val string -} - -var _ Cmder = (*StatusCmd)(nil) - -func NewStatusCmd(args ...interface{}) *StatusCmd { - return &StatusCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *StatusCmd) Val() string { - return cmd.val -} - -func (cmd *StatusCmd) Result() (string, error) { - return cmd.val, cmd.err -} - -func (cmd *StatusCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *StatusCmd) readReply(rd *proto.Reader) error { - cmd.val, cmd.err = rd.ReadString() - return cmd.err -} - -//------------------------------------------------------------------------------ - -type IntCmd struct { - baseCmd - - val int64 -} - -var _ Cmder = (*IntCmd)(nil) - -func NewIntCmd(args ...interface{}) *IntCmd { - return &IntCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *IntCmd) Val() int64 { - return cmd.val -} - -func (cmd *IntCmd) Result() (int64, error) { - return cmd.val, cmd.err -} - -func (cmd *IntCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *IntCmd) readReply(rd *proto.Reader) error { - cmd.val, cmd.err = rd.ReadIntReply() - return cmd.err -} - -//------------------------------------------------------------------------------ - -type DurationCmd struct { - baseCmd - - val time.Duration - precision time.Duration -} - -var _ Cmder = (*DurationCmd)(nil) - -func NewDurationCmd(precision time.Duration, args ...interface{}) *DurationCmd { - return &DurationCmd{ - baseCmd: baseCmd{_args: args}, - precision: precision, - } -} - -func (cmd *DurationCmd) Val() time.Duration { - return cmd.val -} - -func (cmd *DurationCmd) Result() (time.Duration, error) { - return cmd.val, cmd.err -} - -func (cmd *DurationCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *DurationCmd) readReply(rd *proto.Reader) error { - var n int64 - n, cmd.err = rd.ReadIntReply() - if cmd.err != nil { - return cmd.err - } - cmd.val = time.Duration(n) * cmd.precision - return nil -} - -//------------------------------------------------------------------------------ - -type TimeCmd struct { - baseCmd - - val time.Time -} - -var _ Cmder = (*TimeCmd)(nil) - -func NewTimeCmd(args ...interface{}) *TimeCmd { - return &TimeCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *TimeCmd) Val() time.Time { - return cmd.val -} - -func (cmd *TimeCmd) Result() (time.Time, error) { - return cmd.val, cmd.err -} - -func (cmd *TimeCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *TimeCmd) readReply(rd *proto.Reader) error { - var v interface{} - v, cmd.err = rd.ReadArrayReply(timeParser) - if cmd.err != nil { - return cmd.err - } - cmd.val = v.(time.Time) - return nil -} - -// Implements proto.MultiBulkParse -func timeParser(rd *proto.Reader, n int64) (interface{}, error) { - if n != 2 { - return nil, fmt.Errorf("got %d elements, expected 2", n) - } - - sec, err := rd.ReadInt() - if err != nil { - return nil, err - } - - microsec, err := rd.ReadInt() - if err != nil { - return nil, err - } - - return time.Unix(sec, microsec*1000), nil -} - -//------------------------------------------------------------------------------ - -type BoolCmd struct { - baseCmd - - val bool -} - -var _ Cmder = (*BoolCmd)(nil) - -func NewBoolCmd(args ...interface{}) *BoolCmd { - return &BoolCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *BoolCmd) Val() bool { - return cmd.val -} - -func (cmd *BoolCmd) Result() (bool, error) { - return cmd.val, cmd.err -} - -func (cmd *BoolCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *BoolCmd) readReply(rd *proto.Reader) error { - var v interface{} - v, cmd.err = rd.ReadReply(nil) - // `SET key value NX` returns nil when key already exists. But - // `SETNX key value` returns bool (0/1). So convert nil to bool. - // TODO: is this okay? - if cmd.err == Nil { - cmd.val = false - cmd.err = nil - return nil - } - if cmd.err != nil { - return cmd.err - } - switch v := v.(type) { - case int64: - cmd.val = v == 1 - return nil - case string: - cmd.val = v == "OK" - return nil - default: - cmd.err = fmt.Errorf("got %T, wanted int64 or string", v) - return cmd.err - } -} - -//------------------------------------------------------------------------------ - -type StringCmd struct { - baseCmd - - val string -} - -var _ Cmder = (*StringCmd)(nil) - -func NewStringCmd(args ...interface{}) *StringCmd { - return &StringCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *StringCmd) Val() string { - return cmd.val -} - -func (cmd *StringCmd) Result() (string, error) { - return cmd.Val(), cmd.err -} - -func (cmd *StringCmd) Bytes() ([]byte, error) { - return []byte(cmd.val), cmd.err -} - -func (cmd *StringCmd) Int() (int, error) { - if cmd.err != nil { - return 0, cmd.err - } - return strconv.Atoi(cmd.Val()) -} - -func (cmd *StringCmd) Int64() (int64, error) { - if cmd.err != nil { - return 0, cmd.err - } - return strconv.ParseInt(cmd.Val(), 10, 64) -} - -func (cmd *StringCmd) Uint64() (uint64, error) { - if cmd.err != nil { - return 0, cmd.err - } - return strconv.ParseUint(cmd.Val(), 10, 64) -} - -func (cmd *StringCmd) Float64() (float64, error) { - if cmd.err != nil { - return 0, cmd.err - } - return strconv.ParseFloat(cmd.Val(), 64) -} - -func (cmd *StringCmd) Scan(val interface{}) error { - if cmd.err != nil { - return cmd.err - } - return proto.Scan([]byte(cmd.val), val) -} - -func (cmd *StringCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *StringCmd) readReply(rd *proto.Reader) error { - cmd.val, cmd.err = rd.ReadString() - return cmd.err -} - -//------------------------------------------------------------------------------ - -type FloatCmd struct { - baseCmd - - val float64 -} - -var _ Cmder = (*FloatCmd)(nil) - -func NewFloatCmd(args ...interface{}) *FloatCmd { - return &FloatCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *FloatCmd) Val() float64 { - return cmd.val -} - -func (cmd *FloatCmd) Result() (float64, error) { - return cmd.Val(), cmd.Err() -} - -func (cmd *FloatCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *FloatCmd) readReply(rd *proto.Reader) error { - cmd.val, cmd.err = rd.ReadFloatReply() - return cmd.err -} - -//------------------------------------------------------------------------------ - -type StringSliceCmd struct { - baseCmd - - val []string -} - -var _ Cmder = (*StringSliceCmd)(nil) - -func NewStringSliceCmd(args ...interface{}) *StringSliceCmd { - return &StringSliceCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *StringSliceCmd) Val() []string { - return cmd.val -} - -func (cmd *StringSliceCmd) Result() ([]string, error) { - return cmd.Val(), cmd.Err() -} - -func (cmd *StringSliceCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *StringSliceCmd) ScanSlice(container interface{}) error { - return proto.ScanSlice(cmd.Val(), container) -} - -func (cmd *StringSliceCmd) readReply(rd *proto.Reader) error { - var v interface{} - v, cmd.err = rd.ReadArrayReply(stringSliceParser) - if cmd.err != nil { - return cmd.err - } - cmd.val = v.([]string) - return nil -} - -// Implements proto.MultiBulkParse -func stringSliceParser(rd *proto.Reader, n int64) (interface{}, error) { - ss := make([]string, 0, n) - for i := int64(0); i < n; i++ { - s, err := rd.ReadString() - if err == Nil { - ss = append(ss, "") - } else if err != nil { - return nil, err - } else { - ss = append(ss, s) - } - } - return ss, nil -} - -//------------------------------------------------------------------------------ - -type BoolSliceCmd struct { - baseCmd - - val []bool -} - -var _ Cmder = (*BoolSliceCmd)(nil) - -func NewBoolSliceCmd(args ...interface{}) *BoolSliceCmd { - return &BoolSliceCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *BoolSliceCmd) Val() []bool { - return cmd.val -} - -func (cmd *BoolSliceCmd) Result() ([]bool, error) { - return cmd.val, cmd.err -} - -func (cmd *BoolSliceCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *BoolSliceCmd) readReply(rd *proto.Reader) error { - var v interface{} - v, cmd.err = rd.ReadArrayReply(boolSliceParser) - if cmd.err != nil { - return cmd.err - } - cmd.val = v.([]bool) - return nil -} - -// Implements proto.MultiBulkParse -func boolSliceParser(rd *proto.Reader, n int64) (interface{}, error) { - bools := make([]bool, 0, n) - for i := int64(0); i < n; i++ { - n, err := rd.ReadIntReply() - if err != nil { - return nil, err - } - bools = append(bools, n == 1) - } - return bools, nil -} - -//------------------------------------------------------------------------------ - -type StringStringMapCmd struct { - baseCmd - - val map[string]string -} - -var _ Cmder = (*StringStringMapCmd)(nil) - -func NewStringStringMapCmd(args ...interface{}) *StringStringMapCmd { - return &StringStringMapCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *StringStringMapCmd) Val() map[string]string { - return cmd.val -} - -func (cmd *StringStringMapCmd) Result() (map[string]string, error) { - return cmd.val, cmd.err -} - -func (cmd *StringStringMapCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *StringStringMapCmd) readReply(rd *proto.Reader) error { - var v interface{} - v, cmd.err = rd.ReadArrayReply(stringStringMapParser) - if cmd.err != nil { - return cmd.err - } - cmd.val = v.(map[string]string) - return nil -} - -// Implements proto.MultiBulkParse -func stringStringMapParser(rd *proto.Reader, n int64) (interface{}, error) { - m := make(map[string]string, n/2) - for i := int64(0); i < n; i += 2 { - key, err := rd.ReadString() - if err != nil { - return nil, err - } - - value, err := rd.ReadString() - if err != nil { - return nil, err - } - - m[key] = value - } - return m, nil -} - -//------------------------------------------------------------------------------ - -type StringIntMapCmd struct { - baseCmd - - val map[string]int64 -} - -var _ Cmder = (*StringIntMapCmd)(nil) - -func NewStringIntMapCmd(args ...interface{}) *StringIntMapCmd { - return &StringIntMapCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *StringIntMapCmd) Val() map[string]int64 { - return cmd.val -} - -func (cmd *StringIntMapCmd) Result() (map[string]int64, error) { - return cmd.val, cmd.err -} - -func (cmd *StringIntMapCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *StringIntMapCmd) readReply(rd *proto.Reader) error { - var v interface{} - v, cmd.err = rd.ReadArrayReply(stringIntMapParser) - if cmd.err != nil { - return cmd.err - } - cmd.val = v.(map[string]int64) - return nil -} - -// Implements proto.MultiBulkParse -func stringIntMapParser(rd *proto.Reader, n int64) (interface{}, error) { - m := make(map[string]int64, n/2) - for i := int64(0); i < n; i += 2 { - key, err := rd.ReadString() - if err != nil { - return nil, err - } - - n, err := rd.ReadIntReply() - if err != nil { - return nil, err - } - - m[key] = n - } - return m, nil -} - -//------------------------------------------------------------------------------ - -type StringStructMapCmd struct { - baseCmd - - val map[string]struct{} -} - -var _ Cmder = (*StringStructMapCmd)(nil) - -func NewStringStructMapCmd(args ...interface{}) *StringStructMapCmd { - return &StringStructMapCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *StringStructMapCmd) Val() map[string]struct{} { - return cmd.val -} - -func (cmd *StringStructMapCmd) Result() (map[string]struct{}, error) { - return cmd.val, cmd.err -} - -func (cmd *StringStructMapCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *StringStructMapCmd) readReply(rd *proto.Reader) error { - var v interface{} - v, cmd.err = rd.ReadArrayReply(stringStructMapParser) - if cmd.err != nil { - return cmd.err - } - cmd.val = v.(map[string]struct{}) - return nil -} - -// Implements proto.MultiBulkParse -func stringStructMapParser(rd *proto.Reader, n int64) (interface{}, error) { - m := make(map[string]struct{}, n) - for i := int64(0); i < n; i++ { - key, err := rd.ReadString() - if err != nil { - return nil, err - } - - m[key] = struct{}{} - } - return m, nil -} - -//------------------------------------------------------------------------------ - -type XMessage struct { - ID string - Values map[string]interface{} -} - -type XMessageSliceCmd struct { - baseCmd - - val []XMessage -} - -var _ Cmder = (*XMessageSliceCmd)(nil) - -func NewXMessageSliceCmd(args ...interface{}) *XMessageSliceCmd { - return &XMessageSliceCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *XMessageSliceCmd) Val() []XMessage { - return cmd.val -} - -func (cmd *XMessageSliceCmd) Result() ([]XMessage, error) { - return cmd.val, cmd.err -} - -func (cmd *XMessageSliceCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *XMessageSliceCmd) readReply(rd *proto.Reader) error { - var v interface{} - v, cmd.err = rd.ReadArrayReply(xMessageSliceParser) - if cmd.err != nil { - return cmd.err - } - cmd.val = v.([]XMessage) - return nil -} - -// Implements proto.MultiBulkParse -func xMessageSliceParser(rd *proto.Reader, n int64) (interface{}, error) { - msgs := make([]XMessage, 0, n) - for i := int64(0); i < n; i++ { - _, err := rd.ReadArrayReply(func(rd *proto.Reader, n int64) (interface{}, error) { - id, err := rd.ReadString() - if err != nil { - return nil, err - } - - v, err := rd.ReadArrayReply(stringInterfaceMapParser) - if err != nil { - return nil, err - } - - msgs = append(msgs, XMessage{ - ID: id, - Values: v.(map[string]interface{}), - }) - return nil, nil - }) - if err != nil { - return nil, err - } - } - return msgs, nil -} - -// Implements proto.MultiBulkParse -func stringInterfaceMapParser(rd *proto.Reader, n int64) (interface{}, error) { - m := make(map[string]interface{}, n/2) - for i := int64(0); i < n; i += 2 { - key, err := rd.ReadString() - if err != nil { - return nil, err - } - - value, err := rd.ReadString() - if err != nil { - return nil, err - } - - m[key] = value - } - return m, nil -} - -//------------------------------------------------------------------------------ - -type XStream struct { - Stream string - Messages []XMessage -} - -type XStreamSliceCmd struct { - baseCmd - - val []XStream -} - -var _ Cmder = (*XStreamSliceCmd)(nil) - -func NewXStreamSliceCmd(args ...interface{}) *XStreamSliceCmd { - return &XStreamSliceCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *XStreamSliceCmd) Val() []XStream { - return cmd.val -} - -func (cmd *XStreamSliceCmd) Result() ([]XStream, error) { - return cmd.val, cmd.err -} - -func (cmd *XStreamSliceCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *XStreamSliceCmd) readReply(rd *proto.Reader) error { - var v interface{} - v, cmd.err = rd.ReadArrayReply(xStreamSliceParser) - if cmd.err != nil { - return cmd.err - } - cmd.val = v.([]XStream) - return nil -} - -// Implements proto.MultiBulkParse -func xStreamSliceParser(rd *proto.Reader, n int64) (interface{}, error) { - ret := make([]XStream, 0, n) - for i := int64(0); i < n; i++ { - _, err := rd.ReadArrayReply(func(rd *proto.Reader, n int64) (interface{}, error) { - if n != 2 { - return nil, fmt.Errorf("got %d, wanted 2", n) - } - - stream, err := rd.ReadString() - if err != nil { - return nil, err - } - - v, err := rd.ReadArrayReply(xMessageSliceParser) - if err != nil { - return nil, err - } - - ret = append(ret, XStream{ - Stream: stream, - Messages: v.([]XMessage), - }) - return nil, nil - }) - if err != nil { - return nil, err - } - } - return ret, nil -} - -//------------------------------------------------------------------------------ - -type XPending struct { - Count int64 - Lower string - Higher string - Consumers map[string]int64 -} - -type XPendingCmd struct { - baseCmd - val *XPending -} - -var _ Cmder = (*XPendingCmd)(nil) - -func NewXPendingCmd(args ...interface{}) *XPendingCmd { - return &XPendingCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *XPendingCmd) Val() *XPending { - return cmd.val -} - -func (cmd *XPendingCmd) Result() (*XPending, error) { - return cmd.val, cmd.err -} - -func (cmd *XPendingCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *XPendingCmd) readReply(rd *proto.Reader) error { - var info interface{} - info, cmd.err = rd.ReadArrayReply(xPendingParser) - if cmd.err != nil { - return cmd.err - } - cmd.val = info.(*XPending) - return nil -} - -func xPendingParser(rd *proto.Reader, n int64) (interface{}, error) { - if n != 4 { - return nil, fmt.Errorf("got %d, wanted 4", n) - } - - count, err := rd.ReadIntReply() - if err != nil { - return nil, err - } - - lower, err := rd.ReadString() - if err != nil && err != Nil { - return nil, err - } - - higher, err := rd.ReadString() - if err != nil && err != Nil { - return nil, err - } - - pending := &XPending{ - Count: count, - Lower: lower, - Higher: higher, - } - _, err = rd.ReadArrayReply(func(rd *proto.Reader, n int64) (interface{}, error) { - for i := int64(0); i < n; i++ { - _, err = rd.ReadArrayReply(func(rd *proto.Reader, n int64) (interface{}, error) { - if n != 2 { - return nil, fmt.Errorf("got %d, wanted 2", n) - } - - consumerName, err := rd.ReadString() - if err != nil { - return nil, err - } - - consumerPending, err := rd.ReadInt() - if err != nil { - return nil, err - } - - if pending.Consumers == nil { - pending.Consumers = make(map[string]int64) - } - pending.Consumers[consumerName] = consumerPending - - return nil, nil - }) - if err != nil { - return nil, err - } - } - return nil, nil - }) - if err != nil && err != Nil { - return nil, err - } - - return pending, nil -} - -//------------------------------------------------------------------------------ - -type XPendingExt struct { - Id string - Consumer string - Idle time.Duration - RetryCount int64 -} - -type XPendingExtCmd struct { - baseCmd - val []XPendingExt -} - -var _ Cmder = (*XPendingExtCmd)(nil) - -func NewXPendingExtCmd(args ...interface{}) *XPendingExtCmd { - return &XPendingExtCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *XPendingExtCmd) Val() []XPendingExt { - return cmd.val -} - -func (cmd *XPendingExtCmd) Result() ([]XPendingExt, error) { - return cmd.val, cmd.err -} - -func (cmd *XPendingExtCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *XPendingExtCmd) readReply(rd *proto.Reader) error { - var info interface{} - info, cmd.err = rd.ReadArrayReply(xPendingExtSliceParser) - if cmd.err != nil { - return cmd.err - } - cmd.val = info.([]XPendingExt) - return nil -} - -func xPendingExtSliceParser(rd *proto.Reader, n int64) (interface{}, error) { - ret := make([]XPendingExt, 0, n) - for i := int64(0); i < n; i++ { - _, err := rd.ReadArrayReply(func(rd *proto.Reader, n int64) (interface{}, error) { - if n != 4 { - return nil, fmt.Errorf("got %d, wanted 4", n) - } - - id, err := rd.ReadString() - if err != nil { - return nil, err - } - - consumer, err := rd.ReadString() - if err != nil && err != Nil { - return nil, err - } - - idle, err := rd.ReadIntReply() - if err != nil && err != Nil { - return nil, err - } - - retryCount, err := rd.ReadIntReply() - if err != nil && err != Nil { - return nil, err - } - - ret = append(ret, XPendingExt{ - Id: id, - Consumer: consumer, - Idle: time.Duration(idle) * time.Millisecond, - RetryCount: retryCount, - }) - return nil, nil - }) - if err != nil { - return nil, err - } - } - return ret, nil -} - -//------------------------------------------------------------------------------ - -//------------------------------------------------------------------------------ - -type ZSliceCmd struct { - baseCmd - - val []Z -} - -var _ Cmder = (*ZSliceCmd)(nil) - -func NewZSliceCmd(args ...interface{}) *ZSliceCmd { - return &ZSliceCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *ZSliceCmd) Val() []Z { - return cmd.val -} - -func (cmd *ZSliceCmd) Result() ([]Z, error) { - return cmd.val, cmd.err -} - -func (cmd *ZSliceCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *ZSliceCmd) readReply(rd *proto.Reader) error { - var v interface{} - v, cmd.err = rd.ReadArrayReply(zSliceParser) - if cmd.err != nil { - return cmd.err - } - cmd.val = v.([]Z) - return nil -} - -// Implements proto.MultiBulkParse -func zSliceParser(rd *proto.Reader, n int64) (interface{}, error) { - zz := make([]Z, n/2) - for i := int64(0); i < n; i += 2 { - var err error - - z := &zz[i/2] - - z.Member, err = rd.ReadString() - if err != nil { - return nil, err - } - - z.Score, err = rd.ReadFloatReply() - if err != nil { - return nil, err - } - } - return zz, nil -} - -//------------------------------------------------------------------------------ - -type ZWithKeyCmd struct { - baseCmd - - val ZWithKey -} - -var _ Cmder = (*ZWithKeyCmd)(nil) - -func NewZWithKeyCmd(args ...interface{}) *ZWithKeyCmd { - return &ZWithKeyCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *ZWithKeyCmd) Val() ZWithKey { - return cmd.val -} - -func (cmd *ZWithKeyCmd) Result() (ZWithKey, error) { - return cmd.Val(), cmd.Err() -} - -func (cmd *ZWithKeyCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *ZWithKeyCmd) readReply(rd *proto.Reader) error { - var v interface{} - v, cmd.err = rd.ReadArrayReply(zWithKeyParser) - if cmd.err != nil { - return cmd.err - } - cmd.val = v.(ZWithKey) - return nil -} - -// Implements proto.MultiBulkParse -func zWithKeyParser(rd *proto.Reader, n int64) (interface{}, error) { - if n != 3 { - return nil, fmt.Errorf("got %d elements, expected 3", n) - } - - var z ZWithKey - var err error - - z.Key, err = rd.ReadString() - if err != nil { - return nil, err - } - z.Member, err = rd.ReadString() - if err != nil { - return nil, err - } - z.Score, err = rd.ReadFloatReply() - if err != nil { - return nil, err - } - return z, nil -} - -//------------------------------------------------------------------------------ - -type ScanCmd struct { - baseCmd - - page []string - cursor uint64 - - process func(cmd Cmder) error -} - -var _ Cmder = (*ScanCmd)(nil) - -func NewScanCmd(process func(cmd Cmder) error, args ...interface{}) *ScanCmd { - return &ScanCmd{ - baseCmd: baseCmd{_args: args}, - process: process, - } -} - -func (cmd *ScanCmd) Val() (keys []string, cursor uint64) { - return cmd.page, cmd.cursor -} - -func (cmd *ScanCmd) Result() (keys []string, cursor uint64, err error) { - return cmd.page, cmd.cursor, cmd.err -} - -func (cmd *ScanCmd) String() string { - return cmdString(cmd, cmd.page) -} - -func (cmd *ScanCmd) readReply(rd *proto.Reader) error { - cmd.page, cmd.cursor, cmd.err = rd.ReadScanReply() - return cmd.err -} - -// Iterator creates a new ScanIterator. -func (cmd *ScanCmd) Iterator() *ScanIterator { - return &ScanIterator{ - cmd: cmd, - } -} - -//------------------------------------------------------------------------------ - -type ClusterNode struct { - Id string - Addr string -} - -type ClusterSlot struct { - Start int - End int - Nodes []ClusterNode -} - -type ClusterSlotsCmd struct { - baseCmd - - val []ClusterSlot -} - -var _ Cmder = (*ClusterSlotsCmd)(nil) - -func NewClusterSlotsCmd(args ...interface{}) *ClusterSlotsCmd { - return &ClusterSlotsCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *ClusterSlotsCmd) Val() []ClusterSlot { - return cmd.val -} - -func (cmd *ClusterSlotsCmd) Result() ([]ClusterSlot, error) { - return cmd.Val(), cmd.Err() -} - -func (cmd *ClusterSlotsCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *ClusterSlotsCmd) readReply(rd *proto.Reader) error { - var v interface{} - v, cmd.err = rd.ReadArrayReply(clusterSlotsParser) - if cmd.err != nil { - return cmd.err - } - cmd.val = v.([]ClusterSlot) - return nil -} - -// Implements proto.MultiBulkParse -func clusterSlotsParser(rd *proto.Reader, n int64) (interface{}, error) { - slots := make([]ClusterSlot, n) - for i := 0; i < len(slots); i++ { - n, err := rd.ReadArrayLen() - if err != nil { - return nil, err - } - if n < 2 { - err := fmt.Errorf("redis: got %d elements in cluster info, expected at least 2", n) - return nil, err - } - - start, err := rd.ReadIntReply() - if err != nil { - return nil, err - } - - end, err := rd.ReadIntReply() - if err != nil { - return nil, err - } - - nodes := make([]ClusterNode, n-2) - for j := 0; j < len(nodes); j++ { - n, err := rd.ReadArrayLen() - if err != nil { - return nil, err - } - if n != 2 && n != 3 { - err := fmt.Errorf("got %d elements in cluster info address, expected 2 or 3", n) - return nil, err - } - - ip, err := rd.ReadString() - if err != nil { - return nil, err - } - - port, err := rd.ReadString() - if err != nil { - return nil, err - } - - nodes[j].Addr = net.JoinHostPort(ip, port) - - if n == 3 { - id, err := rd.ReadString() - if err != nil { - return nil, err - } - nodes[j].Id = id - } - } - - slots[i] = ClusterSlot{ - Start: int(start), - End: int(end), - Nodes: nodes, - } - } - return slots, nil -} - -//------------------------------------------------------------------------------ - -// GeoLocation is used with GeoAdd to add geospatial location. -type GeoLocation struct { - Name string - Longitude, Latitude, Dist float64 - GeoHash int64 -} - -// GeoRadiusQuery is used with GeoRadius to query geospatial index. -type GeoRadiusQuery struct { - Radius float64 - // Can be m, km, ft, or mi. Default is km. - Unit string - WithCoord bool - WithDist bool - WithGeoHash bool - Count int - // Can be ASC or DESC. Default is no sort order. - Sort string - Store string - StoreDist string -} - -type GeoLocationCmd struct { - baseCmd - - q *GeoRadiusQuery - locations []GeoLocation -} - -var _ Cmder = (*GeoLocationCmd)(nil) - -func NewGeoLocationCmd(q *GeoRadiusQuery, args ...interface{}) *GeoLocationCmd { - args = append(args, q.Radius) - if q.Unit != "" { - args = append(args, q.Unit) - } else { - args = append(args, "km") - } - if q.WithCoord { - args = append(args, "withcoord") - } - if q.WithDist { - args = append(args, "withdist") - } - if q.WithGeoHash { - args = append(args, "withhash") - } - if q.Count > 0 { - args = append(args, "count", q.Count) - } - if q.Sort != "" { - args = append(args, q.Sort) - } - if q.Store != "" { - args = append(args, "store") - args = append(args, q.Store) - } - if q.StoreDist != "" { - args = append(args, "storedist") - args = append(args, q.StoreDist) - } - return &GeoLocationCmd{ - baseCmd: baseCmd{_args: args}, - q: q, - } -} - -func (cmd *GeoLocationCmd) Val() []GeoLocation { - return cmd.locations -} - -func (cmd *GeoLocationCmd) Result() ([]GeoLocation, error) { - return cmd.locations, cmd.err -} - -func (cmd *GeoLocationCmd) String() string { - return cmdString(cmd, cmd.locations) -} - -func (cmd *GeoLocationCmd) readReply(rd *proto.Reader) error { - var v interface{} - v, cmd.err = rd.ReadArrayReply(newGeoLocationSliceParser(cmd.q)) - if cmd.err != nil { - return cmd.err - } - cmd.locations = v.([]GeoLocation) - return nil -} - -func newGeoLocationParser(q *GeoRadiusQuery) proto.MultiBulkParse { - return func(rd *proto.Reader, n int64) (interface{}, error) { - var loc GeoLocation - var err error - - loc.Name, err = rd.ReadString() - if err != nil { - return nil, err - } - if q.WithDist { - loc.Dist, err = rd.ReadFloatReply() - if err != nil { - return nil, err - } - } - if q.WithGeoHash { - loc.GeoHash, err = rd.ReadIntReply() - if err != nil { - return nil, err - } - } - if q.WithCoord { - n, err := rd.ReadArrayLen() - if err != nil { - return nil, err - } - if n != 2 { - return nil, fmt.Errorf("got %d coordinates, expected 2", n) - } - - loc.Longitude, err = rd.ReadFloatReply() - if err != nil { - return nil, err - } - loc.Latitude, err = rd.ReadFloatReply() - if err != nil { - return nil, err - } - } - - return &loc, nil - } -} - -func newGeoLocationSliceParser(q *GeoRadiusQuery) proto.MultiBulkParse { - return func(rd *proto.Reader, n int64) (interface{}, error) { - locs := make([]GeoLocation, 0, n) - for i := int64(0); i < n; i++ { - v, err := rd.ReadReply(newGeoLocationParser(q)) - if err != nil { - return nil, err - } - switch vv := v.(type) { - case string: - locs = append(locs, GeoLocation{ - Name: vv, - }) - case *GeoLocation: - locs = append(locs, *vv) - default: - return nil, fmt.Errorf("got %T, expected string or *GeoLocation", v) - } - } - return locs, nil - } -} - -//------------------------------------------------------------------------------ - -type GeoPos struct { - Longitude, Latitude float64 -} - -type GeoPosCmd struct { - baseCmd - - positions []*GeoPos -} - -var _ Cmder = (*GeoPosCmd)(nil) - -func NewGeoPosCmd(args ...interface{}) *GeoPosCmd { - return &GeoPosCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *GeoPosCmd) Val() []*GeoPos { - return cmd.positions -} - -func (cmd *GeoPosCmd) Result() ([]*GeoPos, error) { - return cmd.Val(), cmd.Err() -} - -func (cmd *GeoPosCmd) String() string { - return cmdString(cmd, cmd.positions) -} - -func (cmd *GeoPosCmd) readReply(rd *proto.Reader) error { - var v interface{} - v, cmd.err = rd.ReadArrayReply(geoPosSliceParser) - if cmd.err != nil { - return cmd.err - } - cmd.positions = v.([]*GeoPos) - return nil -} - -func geoPosSliceParser(rd *proto.Reader, n int64) (interface{}, error) { - positions := make([]*GeoPos, 0, n) - for i := int64(0); i < n; i++ { - v, err := rd.ReadReply(geoPosParser) - if err != nil { - if err == Nil { - positions = append(positions, nil) - continue - } - return nil, err - } - switch v := v.(type) { - case *GeoPos: - positions = append(positions, v) - default: - return nil, fmt.Errorf("got %T, expected *GeoPos", v) - } - } - return positions, nil -} - -func geoPosParser(rd *proto.Reader, n int64) (interface{}, error) { - var pos GeoPos - var err error - - pos.Longitude, err = rd.ReadFloatReply() - if err != nil { - return nil, err - } - - pos.Latitude, err = rd.ReadFloatReply() - if err != nil { - return nil, err - } - - return &pos, nil -} - -//------------------------------------------------------------------------------ - -type CommandInfo struct { - Name string - Arity int8 - Flags []string - FirstKeyPos int8 - LastKeyPos int8 - StepCount int8 - ReadOnly bool -} - -type CommandsInfoCmd struct { - baseCmd - - val map[string]*CommandInfo -} - -var _ Cmder = (*CommandsInfoCmd)(nil) - -func NewCommandsInfoCmd(args ...interface{}) *CommandsInfoCmd { - return &CommandsInfoCmd{ - baseCmd: baseCmd{_args: args}, - } -} - -func (cmd *CommandsInfoCmd) Val() map[string]*CommandInfo { - return cmd.val -} - -func (cmd *CommandsInfoCmd) Result() (map[string]*CommandInfo, error) { - return cmd.Val(), cmd.Err() -} - -func (cmd *CommandsInfoCmd) String() string { - return cmdString(cmd, cmd.val) -} - -func (cmd *CommandsInfoCmd) readReply(rd *proto.Reader) error { - var v interface{} - v, cmd.err = rd.ReadArrayReply(commandInfoSliceParser) - if cmd.err != nil { - return cmd.err - } - cmd.val = v.(map[string]*CommandInfo) - return nil -} - -// Implements proto.MultiBulkParse -func commandInfoSliceParser(rd *proto.Reader, n int64) (interface{}, error) { - m := make(map[string]*CommandInfo, n) - for i := int64(0); i < n; i++ { - v, err := rd.ReadReply(commandInfoParser) - if err != nil { - return nil, err - } - vv := v.(*CommandInfo) - m[vv.Name] = vv - - } - return m, nil -} - -func commandInfoParser(rd *proto.Reader, n int64) (interface{}, error) { - var cmd CommandInfo - var err error - - if n != 6 { - return nil, fmt.Errorf("redis: got %d elements in COMMAND reply, wanted 6", n) - } - - cmd.Name, err = rd.ReadString() - if err != nil { - return nil, err - } - - arity, err := rd.ReadIntReply() - if err != nil { - return nil, err - } - cmd.Arity = int8(arity) - - flags, err := rd.ReadReply(stringSliceParser) - if err != nil { - return nil, err - } - cmd.Flags = flags.([]string) - - firstKeyPos, err := rd.ReadIntReply() - if err != nil { - return nil, err - } - cmd.FirstKeyPos = int8(firstKeyPos) - - lastKeyPos, err := rd.ReadIntReply() - if err != nil { - return nil, err - } - cmd.LastKeyPos = int8(lastKeyPos) - - stepCount, err := rd.ReadIntReply() - if err != nil { - return nil, err - } - cmd.StepCount = int8(stepCount) - - for _, flag := range cmd.Flags { - if flag == "readonly" { - cmd.ReadOnly = true - break - } - } - - return &cmd, nil -} - -//------------------------------------------------------------------------------ - -type cmdsInfoCache struct { - fn func() (map[string]*CommandInfo, error) - - once internal.Once - cmds map[string]*CommandInfo -} - -func newCmdsInfoCache(fn func() (map[string]*CommandInfo, error)) *cmdsInfoCache { - return &cmdsInfoCache{ - fn: fn, - } -} - -func (c *cmdsInfoCache) Get() (map[string]*CommandInfo, error) { - err := c.once.Do(func() error { - cmds, err := c.fn() - if err != nil { - return err - } - c.cmds = cmds - return nil - }) - return c.cmds, err -} diff --git a/src/go/vendor/github.com/go-redis/redis/commands.go b/src/go/vendor/github.com/go-redis/redis/commands.go deleted file mode 100644 index f5e34d4..0000000 --- a/src/go/vendor/github.com/go-redis/redis/commands.go +++ /dev/null @@ -1,2575 +0,0 @@ -package redis - -import ( - "errors" - "io" - "time" - - "github.com/go-redis/redis/internal" -) - -func usePrecise(dur time.Duration) bool { - return dur < time.Second || dur%time.Second != 0 -} - -func formatMs(dur time.Duration) int64 { - if dur > 0 && dur < time.Millisecond { - internal.Logf( - "specified duration is %s, but minimal supported value is %s", - dur, time.Millisecond, - ) - } - return int64(dur / time.Millisecond) -} - -func formatSec(dur time.Duration) int64 { - if dur > 0 && dur < time.Second { - internal.Logf( - "specified duration is %s, but minimal supported value is %s", - dur, time.Second, - ) - } - return int64(dur / time.Second) -} - -func appendArgs(dst, src []interface{}) []interface{} { - if len(src) == 1 { - if ss, ok := src[0].([]string); ok { - for _, s := range ss { - dst = append(dst, s) - } - return dst - } - } - - for _, v := range src { - dst = append(dst, v) - } - return dst -} - -type Cmdable interface { - Pipeline() Pipeliner - Pipelined(fn func(Pipeliner) error) ([]Cmder, error) - - TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) - TxPipeline() Pipeliner - - Command() *CommandsInfoCmd - ClientGetName() *StringCmd - Echo(message interface{}) *StringCmd - Ping() *StatusCmd - Quit() *StatusCmd - Del(keys ...string) *IntCmd - Unlink(keys ...string) *IntCmd - Dump(key string) *StringCmd - Exists(keys ...string) *IntCmd - Expire(key string, expiration time.Duration) *BoolCmd - ExpireAt(key string, tm time.Time) *BoolCmd - Keys(pattern string) *StringSliceCmd - Migrate(host, port, key string, db int64, timeout time.Duration) *StatusCmd - Move(key string, db int64) *BoolCmd - ObjectRefCount(key string) *IntCmd - ObjectEncoding(key string) *StringCmd - ObjectIdleTime(key string) *DurationCmd - Persist(key string) *BoolCmd - PExpire(key string, expiration time.Duration) *BoolCmd - PExpireAt(key string, tm time.Time) *BoolCmd - PTTL(key string) *DurationCmd - RandomKey() *StringCmd - Rename(key, newkey string) *StatusCmd - RenameNX(key, newkey string) *BoolCmd - Restore(key string, ttl time.Duration, value string) *StatusCmd - RestoreReplace(key string, ttl time.Duration, value string) *StatusCmd - Sort(key string, sort *Sort) *StringSliceCmd - SortStore(key, store string, sort *Sort) *IntCmd - SortInterfaces(key string, sort *Sort) *SliceCmd - Touch(keys ...string) *IntCmd - TTL(key string) *DurationCmd - Type(key string) *StatusCmd - Scan(cursor uint64, match string, count int64) *ScanCmd - SScan(key string, cursor uint64, match string, count int64) *ScanCmd - HScan(key string, cursor uint64, match string, count int64) *ScanCmd - ZScan(key string, cursor uint64, match string, count int64) *ScanCmd - Append(key, value string) *IntCmd - BitCount(key string, bitCount *BitCount) *IntCmd - BitOpAnd(destKey string, keys ...string) *IntCmd - BitOpOr(destKey string, keys ...string) *IntCmd - BitOpXor(destKey string, keys ...string) *IntCmd - BitOpNot(destKey string, key string) *IntCmd - BitPos(key string, bit int64, pos ...int64) *IntCmd - Decr(key string) *IntCmd - DecrBy(key string, decrement int64) *IntCmd - Get(key string) *StringCmd - GetBit(key string, offset int64) *IntCmd - GetRange(key string, start, end int64) *StringCmd - GetSet(key string, value interface{}) *StringCmd - Incr(key string) *IntCmd - IncrBy(key string, value int64) *IntCmd - IncrByFloat(key string, value float64) *FloatCmd - MGet(keys ...string) *SliceCmd - MSet(pairs ...interface{}) *StatusCmd - MSetNX(pairs ...interface{}) *BoolCmd - Set(key string, value interface{}, expiration time.Duration) *StatusCmd - SetBit(key string, offset int64, value int) *IntCmd - SetNX(key string, value interface{}, expiration time.Duration) *BoolCmd - SetXX(key string, value interface{}, expiration time.Duration) *BoolCmd - SetRange(key string, offset int64, value string) *IntCmd - StrLen(key string) *IntCmd - HDel(key string, fields ...string) *IntCmd - HExists(key, field string) *BoolCmd - HGet(key, field string) *StringCmd - HGetAll(key string) *StringStringMapCmd - HIncrBy(key, field string, incr int64) *IntCmd - HIncrByFloat(key, field string, incr float64) *FloatCmd - HKeys(key string) *StringSliceCmd - HLen(key string) *IntCmd - HMGet(key string, fields ...string) *SliceCmd - HMSet(key string, fields map[string]interface{}) *StatusCmd - HSet(key, field string, value interface{}) *BoolCmd - HSetNX(key, field string, value interface{}) *BoolCmd - HVals(key string) *StringSliceCmd - BLPop(timeout time.Duration, keys ...string) *StringSliceCmd - BRPop(timeout time.Duration, keys ...string) *StringSliceCmd - BRPopLPush(source, destination string, timeout time.Duration) *StringCmd - LIndex(key string, index int64) *StringCmd - LInsert(key, op string, pivot, value interface{}) *IntCmd - LInsertBefore(key string, pivot, value interface{}) *IntCmd - LInsertAfter(key string, pivot, value interface{}) *IntCmd - LLen(key string) *IntCmd - LPop(key string) *StringCmd - LPush(key string, values ...interface{}) *IntCmd - LPushX(key string, value interface{}) *IntCmd - LRange(key string, start, stop int64) *StringSliceCmd - LRem(key string, count int64, value interface{}) *IntCmd - LSet(key string, index int64, value interface{}) *StatusCmd - LTrim(key string, start, stop int64) *StatusCmd - RPop(key string) *StringCmd - RPopLPush(source, destination string) *StringCmd - RPush(key string, values ...interface{}) *IntCmd - RPushX(key string, value interface{}) *IntCmd - SAdd(key string, members ...interface{}) *IntCmd - SCard(key string) *IntCmd - SDiff(keys ...string) *StringSliceCmd - SDiffStore(destination string, keys ...string) *IntCmd - SInter(keys ...string) *StringSliceCmd - SInterStore(destination string, keys ...string) *IntCmd - SIsMember(key string, member interface{}) *BoolCmd - SMembers(key string) *StringSliceCmd - SMembersMap(key string) *StringStructMapCmd - SMove(source, destination string, member interface{}) *BoolCmd - SPop(key string) *StringCmd - SPopN(key string, count int64) *StringSliceCmd - SRandMember(key string) *StringCmd - SRandMemberN(key string, count int64) *StringSliceCmd - SRem(key string, members ...interface{}) *IntCmd - SUnion(keys ...string) *StringSliceCmd - SUnionStore(destination string, keys ...string) *IntCmd - XAdd(a *XAddArgs) *StringCmd - XDel(stream string, ids ...string) *IntCmd - XLen(stream string) *IntCmd - XRange(stream, start, stop string) *XMessageSliceCmd - XRangeN(stream, start, stop string, count int64) *XMessageSliceCmd - XRevRange(stream string, start, stop string) *XMessageSliceCmd - XRevRangeN(stream string, start, stop string, count int64) *XMessageSliceCmd - XRead(a *XReadArgs) *XStreamSliceCmd - XReadStreams(streams ...string) *XStreamSliceCmd - XGroupCreate(stream, group, start string) *StatusCmd - XGroupCreateMkStream(stream, group, start string) *StatusCmd - XGroupSetID(stream, group, start string) *StatusCmd - XGroupDestroy(stream, group string) *IntCmd - XGroupDelConsumer(stream, group, consumer string) *IntCmd - XReadGroup(a *XReadGroupArgs) *XStreamSliceCmd - XAck(stream, group string, ids ...string) *IntCmd - XPending(stream, group string) *XPendingCmd - XPendingExt(a *XPendingExtArgs) *XPendingExtCmd - XClaim(a *XClaimArgs) *XMessageSliceCmd - XClaimJustID(a *XClaimArgs) *StringSliceCmd - XTrim(key string, maxLen int64) *IntCmd - XTrimApprox(key string, maxLen int64) *IntCmd - BZPopMax(timeout time.Duration, keys ...string) *ZWithKeyCmd - BZPopMin(timeout time.Duration, keys ...string) *ZWithKeyCmd - ZAdd(key string, members ...Z) *IntCmd - ZAddNX(key string, members ...Z) *IntCmd - ZAddXX(key string, members ...Z) *IntCmd - ZAddCh(key string, members ...Z) *IntCmd - ZAddNXCh(key string, members ...Z) *IntCmd - ZAddXXCh(key string, members ...Z) *IntCmd - ZIncr(key string, member Z) *FloatCmd - ZIncrNX(key string, member Z) *FloatCmd - ZIncrXX(key string, member Z) *FloatCmd - ZCard(key string) *IntCmd - ZCount(key, min, max string) *IntCmd - ZLexCount(key, min, max string) *IntCmd - ZIncrBy(key string, increment float64, member string) *FloatCmd - ZInterStore(destination string, store ZStore, keys ...string) *IntCmd - ZPopMax(key string, count ...int64) *ZSliceCmd - ZPopMin(key string, count ...int64) *ZSliceCmd - ZRange(key string, start, stop int64) *StringSliceCmd - ZRangeWithScores(key string, start, stop int64) *ZSliceCmd - ZRangeByScore(key string, opt ZRangeBy) *StringSliceCmd - ZRangeByLex(key string, opt ZRangeBy) *StringSliceCmd - ZRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd - ZRank(key, member string) *IntCmd - ZRem(key string, members ...interface{}) *IntCmd - ZRemRangeByRank(key string, start, stop int64) *IntCmd - ZRemRangeByScore(key, min, max string) *IntCmd - ZRemRangeByLex(key, min, max string) *IntCmd - ZRevRange(key string, start, stop int64) *StringSliceCmd - ZRevRangeWithScores(key string, start, stop int64) *ZSliceCmd - ZRevRangeByScore(key string, opt ZRangeBy) *StringSliceCmd - ZRevRangeByLex(key string, opt ZRangeBy) *StringSliceCmd - ZRevRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd - ZRevRank(key, member string) *IntCmd - ZScore(key, member string) *FloatCmd - ZUnionStore(dest string, store ZStore, keys ...string) *IntCmd - PFAdd(key string, els ...interface{}) *IntCmd - PFCount(keys ...string) *IntCmd - PFMerge(dest string, keys ...string) *StatusCmd - BgRewriteAOF() *StatusCmd - BgSave() *StatusCmd - ClientKill(ipPort string) *StatusCmd - ClientKillByFilter(keys ...string) *IntCmd - ClientList() *StringCmd - ClientPause(dur time.Duration) *BoolCmd - ClientID() *IntCmd - ConfigGet(parameter string) *SliceCmd - ConfigResetStat() *StatusCmd - ConfigSet(parameter, value string) *StatusCmd - ConfigRewrite() *StatusCmd - DBSize() *IntCmd - FlushAll() *StatusCmd - FlushAllAsync() *StatusCmd - FlushDB() *StatusCmd - FlushDBAsync() *StatusCmd - Info(section ...string) *StringCmd - LastSave() *IntCmd - Save() *StatusCmd - Shutdown() *StatusCmd - ShutdownSave() *StatusCmd - ShutdownNoSave() *StatusCmd - SlaveOf(host, port string) *StatusCmd - Time() *TimeCmd - Eval(script string, keys []string, args ...interface{}) *Cmd - EvalSha(sha1 string, keys []string, args ...interface{}) *Cmd - ScriptExists(hashes ...string) *BoolSliceCmd - ScriptFlush() *StatusCmd - ScriptKill() *StatusCmd - ScriptLoad(script string) *StringCmd - DebugObject(key string) *StringCmd - Publish(channel string, message interface{}) *IntCmd - PubSubChannels(pattern string) *StringSliceCmd - PubSubNumSub(channels ...string) *StringIntMapCmd - PubSubNumPat() *IntCmd - ClusterSlots() *ClusterSlotsCmd - ClusterNodes() *StringCmd - ClusterMeet(host, port string) *StatusCmd - ClusterForget(nodeID string) *StatusCmd - ClusterReplicate(nodeID string) *StatusCmd - ClusterResetSoft() *StatusCmd - ClusterResetHard() *StatusCmd - ClusterInfo() *StringCmd - ClusterKeySlot(key string) *IntCmd - ClusterCountFailureReports(nodeID string) *IntCmd - ClusterCountKeysInSlot(slot int) *IntCmd - ClusterDelSlots(slots ...int) *StatusCmd - ClusterDelSlotsRange(min, max int) *StatusCmd - ClusterSaveConfig() *StatusCmd - ClusterSlaves(nodeID string) *StringSliceCmd - ClusterFailover() *StatusCmd - ClusterAddSlots(slots ...int) *StatusCmd - ClusterAddSlotsRange(min, max int) *StatusCmd - GeoAdd(key string, geoLocation ...*GeoLocation) *IntCmd - GeoPos(key string, members ...string) *GeoPosCmd - GeoRadius(key string, longitude, latitude float64, query *GeoRadiusQuery) *GeoLocationCmd - GeoRadiusRO(key string, longitude, latitude float64, query *GeoRadiusQuery) *GeoLocationCmd - GeoRadiusByMember(key, member string, query *GeoRadiusQuery) *GeoLocationCmd - GeoRadiusByMemberRO(key, member string, query *GeoRadiusQuery) *GeoLocationCmd - GeoDist(key string, member1, member2, unit string) *FloatCmd - GeoHash(key string, members ...string) *StringSliceCmd - ReadOnly() *StatusCmd - ReadWrite() *StatusCmd - MemoryUsage(key string, samples ...int) *IntCmd -} - -type StatefulCmdable interface { - Cmdable - Auth(password string) *StatusCmd - Select(index int) *StatusCmd - SwapDB(index1, index2 int) *StatusCmd - ClientSetName(name string) *BoolCmd -} - -var _ Cmdable = (*Client)(nil) -var _ Cmdable = (*Tx)(nil) -var _ Cmdable = (*Ring)(nil) -var _ Cmdable = (*ClusterClient)(nil) - -type cmdable struct { - process func(cmd Cmder) error -} - -func (c *cmdable) setProcessor(fn func(Cmder) error) { - c.process = fn -} - -type statefulCmdable struct { - cmdable - process func(cmd Cmder) error -} - -func (c *statefulCmdable) setProcessor(fn func(Cmder) error) { - c.process = fn - c.cmdable.setProcessor(fn) -} - -//------------------------------------------------------------------------------ - -func (c *statefulCmdable) Auth(password string) *StatusCmd { - cmd := NewStatusCmd("auth", password) - c.process(cmd) - return cmd -} - -func (c *cmdable) Echo(message interface{}) *StringCmd { - cmd := NewStringCmd("echo", message) - c.process(cmd) - return cmd -} - -func (c *cmdable) Ping() *StatusCmd { - cmd := NewStatusCmd("ping") - c.process(cmd) - return cmd -} - -func (c *cmdable) Wait(numSlaves int, timeout time.Duration) *IntCmd { - cmd := NewIntCmd("wait", numSlaves, int(timeout/time.Millisecond)) - c.process(cmd) - return cmd -} - -func (c *cmdable) Quit() *StatusCmd { - panic("not implemented") -} - -func (c *statefulCmdable) Select(index int) *StatusCmd { - cmd := NewStatusCmd("select", index) - c.process(cmd) - return cmd -} - -func (c *statefulCmdable) SwapDB(index1, index2 int) *StatusCmd { - cmd := NewStatusCmd("swapdb", index1, index2) - c.process(cmd) - return cmd -} - -//------------------------------------------------------------------------------ - -func (c *cmdable) Command() *CommandsInfoCmd { - cmd := NewCommandsInfoCmd("command") - c.process(cmd) - return cmd -} - -func (c *cmdable) Del(keys ...string) *IntCmd { - args := make([]interface{}, 1+len(keys)) - args[0] = "del" - for i, key := range keys { - args[1+i] = key - } - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) Unlink(keys ...string) *IntCmd { - args := make([]interface{}, 1+len(keys)) - args[0] = "unlink" - for i, key := range keys { - args[1+i] = key - } - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) Dump(key string) *StringCmd { - cmd := NewStringCmd("dump", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) Exists(keys ...string) *IntCmd { - args := make([]interface{}, 1+len(keys)) - args[0] = "exists" - for i, key := range keys { - args[1+i] = key - } - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) Expire(key string, expiration time.Duration) *BoolCmd { - cmd := NewBoolCmd("expire", key, formatSec(expiration)) - c.process(cmd) - return cmd -} - -func (c *cmdable) ExpireAt(key string, tm time.Time) *BoolCmd { - cmd := NewBoolCmd("expireat", key, tm.Unix()) - c.process(cmd) - return cmd -} - -func (c *cmdable) Keys(pattern string) *StringSliceCmd { - cmd := NewStringSliceCmd("keys", pattern) - c.process(cmd) - return cmd -} - -func (c *cmdable) Migrate(host, port, key string, db int64, timeout time.Duration) *StatusCmd { - cmd := NewStatusCmd( - "migrate", - host, - port, - key, - db, - formatMs(timeout), - ) - cmd.setReadTimeout(timeout) - c.process(cmd) - return cmd -} - -func (c *cmdable) Move(key string, db int64) *BoolCmd { - cmd := NewBoolCmd("move", key, db) - c.process(cmd) - return cmd -} - -func (c *cmdable) ObjectRefCount(key string) *IntCmd { - cmd := NewIntCmd("object", "refcount", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) ObjectEncoding(key string) *StringCmd { - cmd := NewStringCmd("object", "encoding", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) ObjectIdleTime(key string) *DurationCmd { - cmd := NewDurationCmd(time.Second, "object", "idletime", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) Persist(key string) *BoolCmd { - cmd := NewBoolCmd("persist", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) PExpire(key string, expiration time.Duration) *BoolCmd { - cmd := NewBoolCmd("pexpire", key, formatMs(expiration)) - c.process(cmd) - return cmd -} - -func (c *cmdable) PExpireAt(key string, tm time.Time) *BoolCmd { - cmd := NewBoolCmd( - "pexpireat", - key, - tm.UnixNano()/int64(time.Millisecond), - ) - c.process(cmd) - return cmd -} - -func (c *cmdable) PTTL(key string) *DurationCmd { - cmd := NewDurationCmd(time.Millisecond, "pttl", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) RandomKey() *StringCmd { - cmd := NewStringCmd("randomkey") - c.process(cmd) - return cmd -} - -func (c *cmdable) Rename(key, newkey string) *StatusCmd { - cmd := NewStatusCmd("rename", key, newkey) - c.process(cmd) - return cmd -} - -func (c *cmdable) RenameNX(key, newkey string) *BoolCmd { - cmd := NewBoolCmd("renamenx", key, newkey) - c.process(cmd) - return cmd -} - -func (c *cmdable) Restore(key string, ttl time.Duration, value string) *StatusCmd { - cmd := NewStatusCmd( - "restore", - key, - formatMs(ttl), - value, - ) - c.process(cmd) - return cmd -} - -func (c *cmdable) RestoreReplace(key string, ttl time.Duration, value string) *StatusCmd { - cmd := NewStatusCmd( - "restore", - key, - formatMs(ttl), - value, - "replace", - ) - c.process(cmd) - return cmd -} - -type Sort struct { - By string - Offset, Count int64 - Get []string - Order string - Alpha bool -} - -func (sort *Sort) args(key string) []interface{} { - args := []interface{}{"sort", key} - if sort.By != "" { - args = append(args, "by", sort.By) - } - if sort.Offset != 0 || sort.Count != 0 { - args = append(args, "limit", sort.Offset, sort.Count) - } - for _, get := range sort.Get { - args = append(args, "get", get) - } - if sort.Order != "" { - args = append(args, sort.Order) - } - if sort.Alpha { - args = append(args, "alpha") - } - return args -} - -func (c *cmdable) Sort(key string, sort *Sort) *StringSliceCmd { - cmd := NewStringSliceCmd(sort.args(key)...) - c.process(cmd) - return cmd -} - -func (c *cmdable) SortStore(key, store string, sort *Sort) *IntCmd { - args := sort.args(key) - if store != "" { - args = append(args, "store", store) - } - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) SortInterfaces(key string, sort *Sort) *SliceCmd { - cmd := NewSliceCmd(sort.args(key)...) - c.process(cmd) - return cmd -} - -func (c *cmdable) Touch(keys ...string) *IntCmd { - args := make([]interface{}, len(keys)+1) - args[0] = "touch" - for i, key := range keys { - args[i+1] = key - } - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) TTL(key string) *DurationCmd { - cmd := NewDurationCmd(time.Second, "ttl", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) Type(key string) *StatusCmd { - cmd := NewStatusCmd("type", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) Scan(cursor uint64, match string, count int64) *ScanCmd { - args := []interface{}{"scan", cursor} - if match != "" { - args = append(args, "match", match) - } - if count > 0 { - args = append(args, "count", count) - } - cmd := NewScanCmd(c.process, args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) SScan(key string, cursor uint64, match string, count int64) *ScanCmd { - args := []interface{}{"sscan", key, cursor} - if match != "" { - args = append(args, "match", match) - } - if count > 0 { - args = append(args, "count", count) - } - cmd := NewScanCmd(c.process, args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) HScan(key string, cursor uint64, match string, count int64) *ScanCmd { - args := []interface{}{"hscan", key, cursor} - if match != "" { - args = append(args, "match", match) - } - if count > 0 { - args = append(args, "count", count) - } - cmd := NewScanCmd(c.process, args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZScan(key string, cursor uint64, match string, count int64) *ScanCmd { - args := []interface{}{"zscan", key, cursor} - if match != "" { - args = append(args, "match", match) - } - if count > 0 { - args = append(args, "count", count) - } - cmd := NewScanCmd(c.process, args...) - c.process(cmd) - return cmd -} - -//------------------------------------------------------------------------------ - -func (c *cmdable) Append(key, value string) *IntCmd { - cmd := NewIntCmd("append", key, value) - c.process(cmd) - return cmd -} - -type BitCount struct { - Start, End int64 -} - -func (c *cmdable) BitCount(key string, bitCount *BitCount) *IntCmd { - args := []interface{}{"bitcount", key} - if bitCount != nil { - args = append( - args, - bitCount.Start, - bitCount.End, - ) - } - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) bitOp(op, destKey string, keys ...string) *IntCmd { - args := make([]interface{}, 3+len(keys)) - args[0] = "bitop" - args[1] = op - args[2] = destKey - for i, key := range keys { - args[3+i] = key - } - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) BitOpAnd(destKey string, keys ...string) *IntCmd { - return c.bitOp("and", destKey, keys...) -} - -func (c *cmdable) BitOpOr(destKey string, keys ...string) *IntCmd { - return c.bitOp("or", destKey, keys...) -} - -func (c *cmdable) BitOpXor(destKey string, keys ...string) *IntCmd { - return c.bitOp("xor", destKey, keys...) -} - -func (c *cmdable) BitOpNot(destKey string, key string) *IntCmd { - return c.bitOp("not", destKey, key) -} - -func (c *cmdable) BitPos(key string, bit int64, pos ...int64) *IntCmd { - args := make([]interface{}, 3+len(pos)) - args[0] = "bitpos" - args[1] = key - args[2] = bit - switch len(pos) { - case 0: - case 1: - args[3] = pos[0] - case 2: - args[3] = pos[0] - args[4] = pos[1] - default: - panic("too many arguments") - } - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) Decr(key string) *IntCmd { - cmd := NewIntCmd("decr", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) DecrBy(key string, decrement int64) *IntCmd { - cmd := NewIntCmd("decrby", key, decrement) - c.process(cmd) - return cmd -} - -// Redis `GET key` command. It returns redis.Nil error when key does not exist. -func (c *cmdable) Get(key string) *StringCmd { - cmd := NewStringCmd("get", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) GetBit(key string, offset int64) *IntCmd { - cmd := NewIntCmd("getbit", key, offset) - c.process(cmd) - return cmd -} - -func (c *cmdable) GetRange(key string, start, end int64) *StringCmd { - cmd := NewStringCmd("getrange", key, start, end) - c.process(cmd) - return cmd -} - -func (c *cmdable) GetSet(key string, value interface{}) *StringCmd { - cmd := NewStringCmd("getset", key, value) - c.process(cmd) - return cmd -} - -func (c *cmdable) Incr(key string) *IntCmd { - cmd := NewIntCmd("incr", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) IncrBy(key string, value int64) *IntCmd { - cmd := NewIntCmd("incrby", key, value) - c.process(cmd) - return cmd -} - -func (c *cmdable) IncrByFloat(key string, value float64) *FloatCmd { - cmd := NewFloatCmd("incrbyfloat", key, value) - c.process(cmd) - return cmd -} - -func (c *cmdable) MGet(keys ...string) *SliceCmd { - args := make([]interface{}, 1+len(keys)) - args[0] = "mget" - for i, key := range keys { - args[1+i] = key - } - cmd := NewSliceCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) MSet(pairs ...interface{}) *StatusCmd { - args := make([]interface{}, 1, 1+len(pairs)) - args[0] = "mset" - args = appendArgs(args, pairs) - cmd := NewStatusCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) MSetNX(pairs ...interface{}) *BoolCmd { - args := make([]interface{}, 1, 1+len(pairs)) - args[0] = "msetnx" - args = appendArgs(args, pairs) - cmd := NewBoolCmd(args...) - c.process(cmd) - return cmd -} - -// Redis `SET key value [expiration]` command. -// -// Use expiration for `SETEX`-like behavior. -// Zero expiration means the key has no expiration time. -func (c *cmdable) Set(key string, value interface{}, expiration time.Duration) *StatusCmd { - args := make([]interface{}, 3, 4) - args[0] = "set" - args[1] = key - args[2] = value - if expiration > 0 { - if usePrecise(expiration) { - args = append(args, "px", formatMs(expiration)) - } else { - args = append(args, "ex", formatSec(expiration)) - } - } - cmd := NewStatusCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) SetBit(key string, offset int64, value int) *IntCmd { - cmd := NewIntCmd( - "setbit", - key, - offset, - value, - ) - c.process(cmd) - return cmd -} - -// Redis `SET key value [expiration] NX` command. -// -// Zero expiration means the key has no expiration time. -func (c *cmdable) SetNX(key string, value interface{}, expiration time.Duration) *BoolCmd { - var cmd *BoolCmd - if expiration == 0 { - // Use old `SETNX` to support old Redis versions. - cmd = NewBoolCmd("setnx", key, value) - } else { - if usePrecise(expiration) { - cmd = NewBoolCmd("set", key, value, "px", formatMs(expiration), "nx") - } else { - cmd = NewBoolCmd("set", key, value, "ex", formatSec(expiration), "nx") - } - } - c.process(cmd) - return cmd -} - -// Redis `SET key value [expiration] XX` command. -// -// Zero expiration means the key has no expiration time. -func (c *cmdable) SetXX(key string, value interface{}, expiration time.Duration) *BoolCmd { - var cmd *BoolCmd - if expiration == 0 { - cmd = NewBoolCmd("set", key, value, "xx") - } else { - if usePrecise(expiration) { - cmd = NewBoolCmd("set", key, value, "px", formatMs(expiration), "xx") - } else { - cmd = NewBoolCmd("set", key, value, "ex", formatSec(expiration), "xx") - } - } - c.process(cmd) - return cmd -} - -func (c *cmdable) SetRange(key string, offset int64, value string) *IntCmd { - cmd := NewIntCmd("setrange", key, offset, value) - c.process(cmd) - return cmd -} - -func (c *cmdable) StrLen(key string) *IntCmd { - cmd := NewIntCmd("strlen", key) - c.process(cmd) - return cmd -} - -//------------------------------------------------------------------------------ - -func (c *cmdable) HDel(key string, fields ...string) *IntCmd { - args := make([]interface{}, 2+len(fields)) - args[0] = "hdel" - args[1] = key - for i, field := range fields { - args[2+i] = field - } - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) HExists(key, field string) *BoolCmd { - cmd := NewBoolCmd("hexists", key, field) - c.process(cmd) - return cmd -} - -func (c *cmdable) HGet(key, field string) *StringCmd { - cmd := NewStringCmd("hget", key, field) - c.process(cmd) - return cmd -} - -func (c *cmdable) HGetAll(key string) *StringStringMapCmd { - cmd := NewStringStringMapCmd("hgetall", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) HIncrBy(key, field string, incr int64) *IntCmd { - cmd := NewIntCmd("hincrby", key, field, incr) - c.process(cmd) - return cmd -} - -func (c *cmdable) HIncrByFloat(key, field string, incr float64) *FloatCmd { - cmd := NewFloatCmd("hincrbyfloat", key, field, incr) - c.process(cmd) - return cmd -} - -func (c *cmdable) HKeys(key string) *StringSliceCmd { - cmd := NewStringSliceCmd("hkeys", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) HLen(key string) *IntCmd { - cmd := NewIntCmd("hlen", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) HMGet(key string, fields ...string) *SliceCmd { - args := make([]interface{}, 2+len(fields)) - args[0] = "hmget" - args[1] = key - for i, field := range fields { - args[2+i] = field - } - cmd := NewSliceCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) HMSet(key string, fields map[string]interface{}) *StatusCmd { - args := make([]interface{}, 2+len(fields)*2) - args[0] = "hmset" - args[1] = key - i := 2 - for k, v := range fields { - args[i] = k - args[i+1] = v - i += 2 - } - cmd := NewStatusCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) HSet(key, field string, value interface{}) *BoolCmd { - cmd := NewBoolCmd("hset", key, field, value) - c.process(cmd) - return cmd -} - -func (c *cmdable) HSetNX(key, field string, value interface{}) *BoolCmd { - cmd := NewBoolCmd("hsetnx", key, field, value) - c.process(cmd) - return cmd -} - -func (c *cmdable) HVals(key string) *StringSliceCmd { - cmd := NewStringSliceCmd("hvals", key) - c.process(cmd) - return cmd -} - -//------------------------------------------------------------------------------ - -func (c *cmdable) BLPop(timeout time.Duration, keys ...string) *StringSliceCmd { - args := make([]interface{}, 1+len(keys)+1) - args[0] = "blpop" - for i, key := range keys { - args[1+i] = key - } - args[len(args)-1] = formatSec(timeout) - cmd := NewStringSliceCmd(args...) - cmd.setReadTimeout(timeout) - c.process(cmd) - return cmd -} - -func (c *cmdable) BRPop(timeout time.Duration, keys ...string) *StringSliceCmd { - args := make([]interface{}, 1+len(keys)+1) - args[0] = "brpop" - for i, key := range keys { - args[1+i] = key - } - args[len(keys)+1] = formatSec(timeout) - cmd := NewStringSliceCmd(args...) - cmd.setReadTimeout(timeout) - c.process(cmd) - return cmd -} - -func (c *cmdable) BRPopLPush(source, destination string, timeout time.Duration) *StringCmd { - cmd := NewStringCmd( - "brpoplpush", - source, - destination, - formatSec(timeout), - ) - cmd.setReadTimeout(timeout) - c.process(cmd) - return cmd -} - -func (c *cmdable) LIndex(key string, index int64) *StringCmd { - cmd := NewStringCmd("lindex", key, index) - c.process(cmd) - return cmd -} - -func (c *cmdable) LInsert(key, op string, pivot, value interface{}) *IntCmd { - cmd := NewIntCmd("linsert", key, op, pivot, value) - c.process(cmd) - return cmd -} - -func (c *cmdable) LInsertBefore(key string, pivot, value interface{}) *IntCmd { - cmd := NewIntCmd("linsert", key, "before", pivot, value) - c.process(cmd) - return cmd -} - -func (c *cmdable) LInsertAfter(key string, pivot, value interface{}) *IntCmd { - cmd := NewIntCmd("linsert", key, "after", pivot, value) - c.process(cmd) - return cmd -} - -func (c *cmdable) LLen(key string) *IntCmd { - cmd := NewIntCmd("llen", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) LPop(key string) *StringCmd { - cmd := NewStringCmd("lpop", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) LPush(key string, values ...interface{}) *IntCmd { - args := make([]interface{}, 2, 2+len(values)) - args[0] = "lpush" - args[1] = key - args = appendArgs(args, values) - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) LPushX(key string, value interface{}) *IntCmd { - cmd := NewIntCmd("lpushx", key, value) - c.process(cmd) - return cmd -} - -func (c *cmdable) LRange(key string, start, stop int64) *StringSliceCmd { - cmd := NewStringSliceCmd( - "lrange", - key, - start, - stop, - ) - c.process(cmd) - return cmd -} - -func (c *cmdable) LRem(key string, count int64, value interface{}) *IntCmd { - cmd := NewIntCmd("lrem", key, count, value) - c.process(cmd) - return cmd -} - -func (c *cmdable) LSet(key string, index int64, value interface{}) *StatusCmd { - cmd := NewStatusCmd("lset", key, index, value) - c.process(cmd) - return cmd -} - -func (c *cmdable) LTrim(key string, start, stop int64) *StatusCmd { - cmd := NewStatusCmd( - "ltrim", - key, - start, - stop, - ) - c.process(cmd) - return cmd -} - -func (c *cmdable) RPop(key string) *StringCmd { - cmd := NewStringCmd("rpop", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) RPopLPush(source, destination string) *StringCmd { - cmd := NewStringCmd("rpoplpush", source, destination) - c.process(cmd) - return cmd -} - -func (c *cmdable) RPush(key string, values ...interface{}) *IntCmd { - args := make([]interface{}, 2, 2+len(values)) - args[0] = "rpush" - args[1] = key - args = appendArgs(args, values) - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) RPushX(key string, value interface{}) *IntCmd { - cmd := NewIntCmd("rpushx", key, value) - c.process(cmd) - return cmd -} - -//------------------------------------------------------------------------------ - -func (c *cmdable) SAdd(key string, members ...interface{}) *IntCmd { - args := make([]interface{}, 2, 2+len(members)) - args[0] = "sadd" - args[1] = key - args = appendArgs(args, members) - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) SCard(key string) *IntCmd { - cmd := NewIntCmd("scard", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) SDiff(keys ...string) *StringSliceCmd { - args := make([]interface{}, 1+len(keys)) - args[0] = "sdiff" - for i, key := range keys { - args[1+i] = key - } - cmd := NewStringSliceCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) SDiffStore(destination string, keys ...string) *IntCmd { - args := make([]interface{}, 2+len(keys)) - args[0] = "sdiffstore" - args[1] = destination - for i, key := range keys { - args[2+i] = key - } - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) SInter(keys ...string) *StringSliceCmd { - args := make([]interface{}, 1+len(keys)) - args[0] = "sinter" - for i, key := range keys { - args[1+i] = key - } - cmd := NewStringSliceCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) SInterStore(destination string, keys ...string) *IntCmd { - args := make([]interface{}, 2+len(keys)) - args[0] = "sinterstore" - args[1] = destination - for i, key := range keys { - args[2+i] = key - } - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) SIsMember(key string, member interface{}) *BoolCmd { - cmd := NewBoolCmd("sismember", key, member) - c.process(cmd) - return cmd -} - -// Redis `SMEMBERS key` command output as a slice -func (c *cmdable) SMembers(key string) *StringSliceCmd { - cmd := NewStringSliceCmd("smembers", key) - c.process(cmd) - return cmd -} - -// Redis `SMEMBERS key` command output as a map -func (c *cmdable) SMembersMap(key string) *StringStructMapCmd { - cmd := NewStringStructMapCmd("smembers", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) SMove(source, destination string, member interface{}) *BoolCmd { - cmd := NewBoolCmd("smove", source, destination, member) - c.process(cmd) - return cmd -} - -// Redis `SPOP key` command. -func (c *cmdable) SPop(key string) *StringCmd { - cmd := NewStringCmd("spop", key) - c.process(cmd) - return cmd -} - -// Redis `SPOP key count` command. -func (c *cmdable) SPopN(key string, count int64) *StringSliceCmd { - cmd := NewStringSliceCmd("spop", key, count) - c.process(cmd) - return cmd -} - -// Redis `SRANDMEMBER key` command. -func (c *cmdable) SRandMember(key string) *StringCmd { - cmd := NewStringCmd("srandmember", key) - c.process(cmd) - return cmd -} - -// Redis `SRANDMEMBER key count` command. -func (c *cmdable) SRandMemberN(key string, count int64) *StringSliceCmd { - cmd := NewStringSliceCmd("srandmember", key, count) - c.process(cmd) - return cmd -} - -func (c *cmdable) SRem(key string, members ...interface{}) *IntCmd { - args := make([]interface{}, 2, 2+len(members)) - args[0] = "srem" - args[1] = key - args = appendArgs(args, members) - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) SUnion(keys ...string) *StringSliceCmd { - args := make([]interface{}, 1+len(keys)) - args[0] = "sunion" - for i, key := range keys { - args[1+i] = key - } - cmd := NewStringSliceCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) SUnionStore(destination string, keys ...string) *IntCmd { - args := make([]interface{}, 2+len(keys)) - args[0] = "sunionstore" - args[1] = destination - for i, key := range keys { - args[2+i] = key - } - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -//------------------------------------------------------------------------------ - -type XAddArgs struct { - Stream string - MaxLen int64 // MAXLEN N - MaxLenApprox int64 // MAXLEN ~ N - ID string - Values map[string]interface{} -} - -func (c *cmdable) XAdd(a *XAddArgs) *StringCmd { - args := make([]interface{}, 0, 6+len(a.Values)*2) - args = append(args, "xadd") - args = append(args, a.Stream) - if a.MaxLen > 0 { - args = append(args, "maxlen", a.MaxLen) - } else if a.MaxLenApprox > 0 { - args = append(args, "maxlen", "~", a.MaxLenApprox) - } - if a.ID != "" { - args = append(args, a.ID) - } else { - args = append(args, "*") - } - for k, v := range a.Values { - args = append(args, k) - args = append(args, v) - } - - cmd := NewStringCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) XDel(stream string, ids ...string) *IntCmd { - args := []interface{}{"xdel", stream} - for _, id := range ids { - args = append(args, id) - } - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) XLen(stream string) *IntCmd { - cmd := NewIntCmd("xlen", stream) - c.process(cmd) - return cmd -} - -func (c *cmdable) XRange(stream, start, stop string) *XMessageSliceCmd { - cmd := NewXMessageSliceCmd("xrange", stream, start, stop) - c.process(cmd) - return cmd -} - -func (c *cmdable) XRangeN(stream, start, stop string, count int64) *XMessageSliceCmd { - cmd := NewXMessageSliceCmd("xrange", stream, start, stop, "count", count) - c.process(cmd) - return cmd -} - -func (c *cmdable) XRevRange(stream, start, stop string) *XMessageSliceCmd { - cmd := NewXMessageSliceCmd("xrevrange", stream, start, stop) - c.process(cmd) - return cmd -} - -func (c *cmdable) XRevRangeN(stream, start, stop string, count int64) *XMessageSliceCmd { - cmd := NewXMessageSliceCmd("xrevrange", stream, start, stop, "count", count) - c.process(cmd) - return cmd -} - -type XReadArgs struct { - Streams []string - Count int64 - Block time.Duration -} - -func (c *cmdable) XRead(a *XReadArgs) *XStreamSliceCmd { - args := make([]interface{}, 0, 5+len(a.Streams)) - args = append(args, "xread") - if a.Count > 0 { - args = append(args, "count") - args = append(args, a.Count) - } - if a.Block >= 0 { - args = append(args, "block") - args = append(args, int64(a.Block/time.Millisecond)) - } - args = append(args, "streams") - for _, s := range a.Streams { - args = append(args, s) - } - - cmd := NewXStreamSliceCmd(args...) - if a.Block >= 0 { - cmd.setReadTimeout(a.Block) - } - c.process(cmd) - return cmd -} - -func (c *cmdable) XReadStreams(streams ...string) *XStreamSliceCmd { - return c.XRead(&XReadArgs{ - Streams: streams, - Block: -1, - }) -} - -func (c *cmdable) XGroupCreate(stream, group, start string) *StatusCmd { - cmd := NewStatusCmd("xgroup", "create", stream, group, start) - c.process(cmd) - return cmd -} - -func (c *cmdable) XGroupCreateMkStream(stream, group, start string) *StatusCmd { - cmd := NewStatusCmd("xgroup", "create", stream, group, start, "mkstream") - c.process(cmd) - return cmd -} - -func (c *cmdable) XGroupSetID(stream, group, start string) *StatusCmd { - cmd := NewStatusCmd("xgroup", "setid", stream, group, start) - c.process(cmd) - return cmd -} - -func (c *cmdable) XGroupDestroy(stream, group string) *IntCmd { - cmd := NewIntCmd("xgroup", "destroy", stream, group) - c.process(cmd) - return cmd -} - -func (c *cmdable) XGroupDelConsumer(stream, group, consumer string) *IntCmd { - cmd := NewIntCmd("xgroup", "delconsumer", stream, group, consumer) - c.process(cmd) - return cmd -} - -type XReadGroupArgs struct { - Group string - Consumer string - Streams []string - Count int64 - Block time.Duration - NoAck bool -} - -func (c *cmdable) XReadGroup(a *XReadGroupArgs) *XStreamSliceCmd { - args := make([]interface{}, 0, 8+len(a.Streams)) - args = append(args, "xreadgroup", "group", a.Group, a.Consumer) - if a.Count > 0 { - args = append(args, "count", a.Count) - } - if a.Block >= 0 { - args = append(args, "block", int64(a.Block/time.Millisecond)) - } - if a.NoAck { - args = append(args, "noack") - } - args = append(args, "streams") - for _, s := range a.Streams { - args = append(args, s) - } - - cmd := NewXStreamSliceCmd(args...) - if a.Block >= 0 { - cmd.setReadTimeout(a.Block) - } - c.process(cmd) - return cmd -} - -func (c *cmdable) XAck(stream, group string, ids ...string) *IntCmd { - args := []interface{}{"xack", stream, group} - for _, id := range ids { - args = append(args, id) - } - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) XPending(stream, group string) *XPendingCmd { - cmd := NewXPendingCmd("xpending", stream, group) - c.process(cmd) - return cmd -} - -type XPendingExtArgs struct { - Stream string - Group string - Start string - End string - Count int64 - Consumer string -} - -func (c *cmdable) XPendingExt(a *XPendingExtArgs) *XPendingExtCmd { - args := make([]interface{}, 0, 7) - args = append(args, "xpending", a.Stream, a.Group, a.Start, a.End, a.Count) - if a.Consumer != "" { - args = append(args, a.Consumer) - } - cmd := NewXPendingExtCmd(args...) - c.process(cmd) - return cmd -} - -type XClaimArgs struct { - Stream string - Group string - Consumer string - MinIdle time.Duration - Messages []string -} - -func (c *cmdable) XClaim(a *XClaimArgs) *XMessageSliceCmd { - args := xClaimArgs(a) - cmd := NewXMessageSliceCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) XClaimJustID(a *XClaimArgs) *StringSliceCmd { - args := xClaimArgs(a) - args = append(args, "justid") - cmd := NewStringSliceCmd(args...) - c.process(cmd) - return cmd -} - -func xClaimArgs(a *XClaimArgs) []interface{} { - args := make([]interface{}, 0, 4+len(a.Messages)) - args = append(args, - "xclaim", - a.Stream, - a.Group, a.Consumer, - int64(a.MinIdle/time.Millisecond)) - for _, id := range a.Messages { - args = append(args, id) - } - return args -} - -func (c *cmdable) XTrim(key string, maxLen int64) *IntCmd { - cmd := NewIntCmd("xtrim", key, "maxlen", maxLen) - c.process(cmd) - return cmd -} - -func (c *cmdable) XTrimApprox(key string, maxLen int64) *IntCmd { - cmd := NewIntCmd("xtrim", key, "maxlen", "~", maxLen) - c.process(cmd) - return cmd -} - -//------------------------------------------------------------------------------ - -// Z represents sorted set member. -type Z struct { - Score float64 - Member interface{} -} - -// ZWithKey represents sorted set member including the name of the key where it was popped. -type ZWithKey struct { - Z - Key string -} - -// ZStore is used as an arg to ZInterStore and ZUnionStore. -type ZStore struct { - Weights []float64 - // Can be SUM, MIN or MAX. - Aggregate string -} - -// Redis `BZPOPMAX key [key ...] timeout` command. -func (c *cmdable) BZPopMax(timeout time.Duration, keys ...string) *ZWithKeyCmd { - args := make([]interface{}, 1+len(keys)+1) - args[0] = "bzpopmax" - for i, key := range keys { - args[1+i] = key - } - args[len(args)-1] = formatSec(timeout) - cmd := NewZWithKeyCmd(args...) - cmd.setReadTimeout(timeout) - c.process(cmd) - return cmd -} - -// Redis `BZPOPMIN key [key ...] timeout` command. -func (c *cmdable) BZPopMin(timeout time.Duration, keys ...string) *ZWithKeyCmd { - args := make([]interface{}, 1+len(keys)+1) - args[0] = "bzpopmin" - for i, key := range keys { - args[1+i] = key - } - args[len(args)-1] = formatSec(timeout) - cmd := NewZWithKeyCmd(args...) - cmd.setReadTimeout(timeout) - c.process(cmd) - return cmd -} - -func (c *cmdable) zAdd(a []interface{}, n int, members ...Z) *IntCmd { - for i, m := range members { - a[n+2*i] = m.Score - a[n+2*i+1] = m.Member - } - cmd := NewIntCmd(a...) - c.process(cmd) - return cmd -} - -// Redis `ZADD key score member [score member ...]` command. -func (c *cmdable) ZAdd(key string, members ...Z) *IntCmd { - const n = 2 - a := make([]interface{}, n+2*len(members)) - a[0], a[1] = "zadd", key - return c.zAdd(a, n, members...) -} - -// Redis `ZADD key NX score member [score member ...]` command. -func (c *cmdable) ZAddNX(key string, members ...Z) *IntCmd { - const n = 3 - a := make([]interface{}, n+2*len(members)) - a[0], a[1], a[2] = "zadd", key, "nx" - return c.zAdd(a, n, members...) -} - -// Redis `ZADD key XX score member [score member ...]` command. -func (c *cmdable) ZAddXX(key string, members ...Z) *IntCmd { - const n = 3 - a := make([]interface{}, n+2*len(members)) - a[0], a[1], a[2] = "zadd", key, "xx" - return c.zAdd(a, n, members...) -} - -// Redis `ZADD key CH score member [score member ...]` command. -func (c *cmdable) ZAddCh(key string, members ...Z) *IntCmd { - const n = 3 - a := make([]interface{}, n+2*len(members)) - a[0], a[1], a[2] = "zadd", key, "ch" - return c.zAdd(a, n, members...) -} - -// Redis `ZADD key NX CH score member [score member ...]` command. -func (c *cmdable) ZAddNXCh(key string, members ...Z) *IntCmd { - const n = 4 - a := make([]interface{}, n+2*len(members)) - a[0], a[1], a[2], a[3] = "zadd", key, "nx", "ch" - return c.zAdd(a, n, members...) -} - -// Redis `ZADD key XX CH score member [score member ...]` command. -func (c *cmdable) ZAddXXCh(key string, members ...Z) *IntCmd { - const n = 4 - a := make([]interface{}, n+2*len(members)) - a[0], a[1], a[2], a[3] = "zadd", key, "xx", "ch" - return c.zAdd(a, n, members...) -} - -func (c *cmdable) zIncr(a []interface{}, n int, members ...Z) *FloatCmd { - for i, m := range members { - a[n+2*i] = m.Score - a[n+2*i+1] = m.Member - } - cmd := NewFloatCmd(a...) - c.process(cmd) - return cmd -} - -// Redis `ZADD key INCR score member` command. -func (c *cmdable) ZIncr(key string, member Z) *FloatCmd { - const n = 3 - a := make([]interface{}, n+2) - a[0], a[1], a[2] = "zadd", key, "incr" - return c.zIncr(a, n, member) -} - -// Redis `ZADD key NX INCR score member` command. -func (c *cmdable) ZIncrNX(key string, member Z) *FloatCmd { - const n = 4 - a := make([]interface{}, n+2) - a[0], a[1], a[2], a[3] = "zadd", key, "incr", "nx" - return c.zIncr(a, n, member) -} - -// Redis `ZADD key XX INCR score member` command. -func (c *cmdable) ZIncrXX(key string, member Z) *FloatCmd { - const n = 4 - a := make([]interface{}, n+2) - a[0], a[1], a[2], a[3] = "zadd", key, "incr", "xx" - return c.zIncr(a, n, member) -} - -func (c *cmdable) ZCard(key string) *IntCmd { - cmd := NewIntCmd("zcard", key) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZCount(key, min, max string) *IntCmd { - cmd := NewIntCmd("zcount", key, min, max) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZLexCount(key, min, max string) *IntCmd { - cmd := NewIntCmd("zlexcount", key, min, max) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZIncrBy(key string, increment float64, member string) *FloatCmd { - cmd := NewFloatCmd("zincrby", key, increment, member) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZInterStore(destination string, store ZStore, keys ...string) *IntCmd { - args := make([]interface{}, 3+len(keys)) - args[0] = "zinterstore" - args[1] = destination - args[2] = len(keys) - for i, key := range keys { - args[3+i] = key - } - if len(store.Weights) > 0 { - args = append(args, "weights") - for _, weight := range store.Weights { - args = append(args, weight) - } - } - if store.Aggregate != "" { - args = append(args, "aggregate", store.Aggregate) - } - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZPopMax(key string, count ...int64) *ZSliceCmd { - args := []interface{}{ - "zpopmax", - key, - } - - switch len(count) { - case 0: - break - case 1: - args = append(args, count[0]) - default: - panic("too many arguments") - } - - cmd := NewZSliceCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZPopMin(key string, count ...int64) *ZSliceCmd { - args := []interface{}{ - "zpopmin", - key, - } - - switch len(count) { - case 0: - break - case 1: - args = append(args, count[0]) - default: - panic("too many arguments") - } - - cmd := NewZSliceCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) zRange(key string, start, stop int64, withScores bool) *StringSliceCmd { - args := []interface{}{ - "zrange", - key, - start, - stop, - } - if withScores { - args = append(args, "withscores") - } - cmd := NewStringSliceCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZRange(key string, start, stop int64) *StringSliceCmd { - return c.zRange(key, start, stop, false) -} - -func (c *cmdable) ZRangeWithScores(key string, start, stop int64) *ZSliceCmd { - cmd := NewZSliceCmd("zrange", key, start, stop, "withscores") - c.process(cmd) - return cmd -} - -type ZRangeBy struct { - Min, Max string - Offset, Count int64 -} - -func (c *cmdable) zRangeBy(zcmd, key string, opt ZRangeBy, withScores bool) *StringSliceCmd { - args := []interface{}{zcmd, key, opt.Min, opt.Max} - if withScores { - args = append(args, "withscores") - } - if opt.Offset != 0 || opt.Count != 0 { - args = append( - args, - "limit", - opt.Offset, - opt.Count, - ) - } - cmd := NewStringSliceCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { - return c.zRangeBy("zrangebyscore", key, opt, false) -} - -func (c *cmdable) ZRangeByLex(key string, opt ZRangeBy) *StringSliceCmd { - return c.zRangeBy("zrangebylex", key, opt, false) -} - -func (c *cmdable) ZRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { - args := []interface{}{"zrangebyscore", key, opt.Min, opt.Max, "withscores"} - if opt.Offset != 0 || opt.Count != 0 { - args = append( - args, - "limit", - opt.Offset, - opt.Count, - ) - } - cmd := NewZSliceCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZRank(key, member string) *IntCmd { - cmd := NewIntCmd("zrank", key, member) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZRem(key string, members ...interface{}) *IntCmd { - args := make([]interface{}, 2, 2+len(members)) - args[0] = "zrem" - args[1] = key - args = appendArgs(args, members) - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZRemRangeByRank(key string, start, stop int64) *IntCmd { - cmd := NewIntCmd( - "zremrangebyrank", - key, - start, - stop, - ) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZRemRangeByScore(key, min, max string) *IntCmd { - cmd := NewIntCmd("zremrangebyscore", key, min, max) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZRemRangeByLex(key, min, max string) *IntCmd { - cmd := NewIntCmd("zremrangebylex", key, min, max) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZRevRange(key string, start, stop int64) *StringSliceCmd { - cmd := NewStringSliceCmd("zrevrange", key, start, stop) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZRevRangeWithScores(key string, start, stop int64) *ZSliceCmd { - cmd := NewZSliceCmd("zrevrange", key, start, stop, "withscores") - c.process(cmd) - return cmd -} - -func (c *cmdable) zRevRangeBy(zcmd, key string, opt ZRangeBy) *StringSliceCmd { - args := []interface{}{zcmd, key, opt.Max, opt.Min} - if opt.Offset != 0 || opt.Count != 0 { - args = append( - args, - "limit", - opt.Offset, - opt.Count, - ) - } - cmd := NewStringSliceCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZRevRangeByScore(key string, opt ZRangeBy) *StringSliceCmd { - return c.zRevRangeBy("zrevrangebyscore", key, opt) -} - -func (c *cmdable) ZRevRangeByLex(key string, opt ZRangeBy) *StringSliceCmd { - return c.zRevRangeBy("zrevrangebylex", key, opt) -} - -func (c *cmdable) ZRevRangeByScoreWithScores(key string, opt ZRangeBy) *ZSliceCmd { - args := []interface{}{"zrevrangebyscore", key, opt.Max, opt.Min, "withscores"} - if opt.Offset != 0 || opt.Count != 0 { - args = append( - args, - "limit", - opt.Offset, - opt.Count, - ) - } - cmd := NewZSliceCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZRevRank(key, member string) *IntCmd { - cmd := NewIntCmd("zrevrank", key, member) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZScore(key, member string) *FloatCmd { - cmd := NewFloatCmd("zscore", key, member) - c.process(cmd) - return cmd -} - -func (c *cmdable) ZUnionStore(dest string, store ZStore, keys ...string) *IntCmd { - args := make([]interface{}, 3+len(keys)) - args[0] = "zunionstore" - args[1] = dest - args[2] = len(keys) - for i, key := range keys { - args[3+i] = key - } - if len(store.Weights) > 0 { - args = append(args, "weights") - for _, weight := range store.Weights { - args = append(args, weight) - } - } - if store.Aggregate != "" { - args = append(args, "aggregate", store.Aggregate) - } - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -//------------------------------------------------------------------------------ - -func (c *cmdable) PFAdd(key string, els ...interface{}) *IntCmd { - args := make([]interface{}, 2, 2+len(els)) - args[0] = "pfadd" - args[1] = key - args = appendArgs(args, els) - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) PFCount(keys ...string) *IntCmd { - args := make([]interface{}, 1+len(keys)) - args[0] = "pfcount" - for i, key := range keys { - args[1+i] = key - } - cmd := NewIntCmd(args...) - c.process(cmd) - return cmd -} - -func (c *cmdable) PFMerge(dest string, keys ...string) *StatusCmd { - args := make([]interface{}, 2+len(keys)) - args[0] = "pfmerge" - args[1] = dest - for i, key := range keys { - args[2+i] = key - } - cmd := NewStatusCmd(args...) - c.process(cmd) - return cmd -} - -//------------------------------------------------------------------------------ - -func (c *cmdable) BgRewriteAOF() *StatusCmd { - cmd := NewStatusCmd("bgrewriteaof") - c.process(cmd) - return cmd -} - -func (c *cmdable) BgSave() *StatusCmd { - cmd := NewStatusCmd("bgsave") - c.process(cmd) - return cmd -} - -func (c *cmdable) ClientKill(ipPort string) *StatusCmd { - cmd := NewStatusCmd("client", "kill", ipPort) - c.process(cmd) - return cmd -} - -// ClientKillByFilter is new style synx, while the ClientKill is old -// CLIENT KILL