Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add docker test for read range #57

Merged
merged 3 commits into from
Feb 14, 2024
Merged

Add docker test for read range #57

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3612175
add docker test for read range
jja725 Feb 7, 2024
da68cac
FixRangeRead
LuQQiu Feb 14, 2024
1e331c3
fix test
jja725 Feb 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 49 additions & 16 deletions tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import logging
import os
import shlex
import shutil
import subprocess
import time
from urllib.parse import urlparse
Expand All @@ -10,44 +11,74 @@

from alluxio.alluxio_file_system import AlluxioFileSystem

logger = logging.getLogger("alluxio_test")
TEST_ROOT = os.getenv("TEST_ROOT", "file://tmp/alluxio_test_root")
LOGGER = logging.getLogger("alluxio_test")
TEST_ROOT = os.getenv("TEST_ROOT", "file:///tmp/alluxio_test_root")
# This is the path to the file you want to access
LOCAL_FILE_PATH = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "test.csv"
)
ALLUXIO_FILE_PATH = "file://{}".format("/opt/alluxio/ufs/test.csv")


def stop_docker(container):
cmd = shlex.split('docker ps -a -q --filter "name=%s"' % container)
cid = subprocess.check_output(cmd).strip().decode()
if cid:
logger.debug("Stopping existing container %s" % cid)
LOGGER.debug("Stopping existing container %s" % cid)
subprocess.call(["docker", "rm", "-f", "-v", cid])


@pytest.fixture(scope="module")
def docker_alluxio():
def docker_alluxio(tmpdir_factory):
if "ALLUXIO_URL" in os.environ:
# assume we already have a server already set up
yield os.getenv("ALLUXIO_URL")
return
container = "alluxio-worker"
master_container = "alluxio-master"
worker_container = "alluxio-worker"
network_cmd = "docker network create alluxio_network"
run_cmd = (
"docker run --platform linux/amd64 -d --rm --net=alluxio_network -p 28080:28080 "
"--name=alluxio-worker --shm-size=1G -v /tmp/alluxio_ufs:/opt/alluxio/ufs "
'-e ALLUXIO_JAVA_OPTS=" -Dalluxio.worker.membership.manager.type=STATIC '
f'-Dalluxio.dora.client.ufs.root={TEST_ROOT} " alluxio/alluxio:308-SNAPSHOT worker'
test_root = tmpdir_factory.mktemp("alluxio_test")
LOGGER.debug(f"test_root: {test_root}")
test_file = test_root.join("test.csv")
shutil.copy(LOCAL_FILE_PATH, str(test_file))

run_cmd_master = (
"docker run --platform linux/amd64 -d --rm --net=alluxio_network -p 19999:19999 -p 19998:19998 "
f"--name=alluxio-master -v {test_root}:/opt/alluxio/ufs "
'-e ALLUXIO_JAVA_OPTS=" -Dalluxio.master.hostname=alluxio-master '
"-Dalluxio.security.authentication.type=NOSASL "
"-Dalluxio.security.authorization.permission.enabled=false "
"-Dalluxio.security.authorization.plugins.enabled=false "
"-Dalluxio.master.journal.type=NOOP "
"-Dalluxio.master.scheduler.initial.wait.time=1s "
"-Dalluxio.dora.client.ufs.root=file:/// "
'-Dalluxio.underfs.xattr.change.enabled=false " alluxio/alluxio:308-SNAPSHOT master'
)
run_cmd_worker = (
"docker run --platform linux/amd64 -d --rm --net=alluxio_network -p 28080:28080 -p 29999:29999 -p 29997:29997 "
f"--name=alluxio-worker --shm-size=1G -v {test_root}:/opt/alluxio/ufs "
'-e ALLUXIO_JAVA_OPTS=" -Dalluxio.master.hostname=alluxio-master '
"-Dalluxio.security.authentication.type=NOSASL "
"-Dalluxio.security.authorization.permission.enabled=false "
"-Dalluxio.security.authorization.plugins.enabled=false "
"-Dalluxio.dora.client.ufs.root=file:/// "
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why this path, or it does not matter?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it doesn't matter right now

'-Dalluxio.underfs.xattr.change.enabled=false " alluxio/alluxio:308-SNAPSHOT worker'
)
stop_docker(container)

stop_docker(worker_container)
stop_docker(master_container)
subprocess.run(
shlex.split(network_cmd)
) # could return error code if network already exists
subprocess.check_output(shlex.split(run_cmd))
subprocess.check_output(shlex.split(run_cmd_master))
subprocess.check_output(shlex.split(run_cmd_worker))
url = "http://127.0.0.1:28080"
timeout = 10
while True:
try:
logger.debug("trying to connected to alluxio")
LOGGER.debug("trying to connected to alluxio")
r = requests.get(url + "/v1/files?path=/")
logger.debug("successfullly connected to alluxio")
LOGGER.debug("successfullly connected to alluxio")
if r.ok:
yield url
break
Expand All @@ -56,12 +87,14 @@ def docker_alluxio():
if timeout < 0:
raise SystemError from e
time.sleep(10)
stop_docker(container)
stop_docker(worker_container)
stop_docker(master_container)


@pytest.fixture
def fs(docker_alluxio):
logger.debug(f"get AlluxioFileSystem connect to {docker_alluxio}")

LOGGER.debug(f"get AlluxioFileSystem connect to {docker_alluxio}")
parsed_url = urlparse(docker_alluxio)
host = parsed_url.hostname
port = parsed_url.port
Expand Down
Loading
Loading