Skip to content

Commit

Permalink
fix: flaky version testing
Browse files Browse the repository at this point in the history
  • Loading branch information
@therightstuff
therightstuff committed Oct 1, 2023
1 parent ac8c017 commit 68a0332
Show file tree
Hide file tree
Showing 7 changed files with 400 additions and 79 deletions.
16 changes: 15 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,21 @@ get_current_span().add_event('<error-message>', {'lumigo.type': '<error-type>'})
| --- | --- | :---: | :---: | :---: | :---: | :---: |
| | | 3.7 | 3.8 | 3.9 | 3.10 | 3.11 |
| botocore | [boto3](https://pypi.org/project/boto3) | 1.17.22~1.28.40|1.17.22~1.28.40|1.17.22~1.28.40|1.17.22~1.28.40|1.17.22~1.28.40|
| django | [django](https://pypi.org/project/django) | |4.2.5|4.2.5|4.2.5|4.2.5|
| django | [django](https://pypi.org/project/django) | 3.2.1~3.2.21|3.2.1~3.2.21|3.2.1~3.2.21|3.2.1~3.2.21|3.2.1~3.2.21|
| | | 4.0.1~4.2.5| 4.0.1~4.2.5| 4.0.1~4.2.5| 4.0.1~4.2.5| 4.0.1~4.2.5|
| | | 3.2| 3.2| 3.2| 3.2| 3.2|
| | | 4.0| 4.0| 4.0| 4.0| 4.0|
| | | 4.0.a1| 4.0.a1| 4.0.a1| 4.0.a1| 4.0.a1|
| | | 4.0.b1| 4.0.b1| 4.0.b1| 4.0.b1| 4.0.b1|
| | | 4.0.rc1| 4.0.rc1| 4.0.rc1| 4.0.rc1| 4.0.rc1|
| | | 4.1| 4.1| 4.1| 4.1| 4.1|
| | | 4.1.a1| 4.1.a1| 4.1.a1| 4.1.a1| 4.1.a1|
| | | 4.1.b1| 4.1.b1| 4.1.b1| 4.1.b1| 4.1.b1|
| | | 4.1.rc1| 4.1.rc1| 4.1.rc1| 4.1.rc1| 4.1.rc1|
| | | 4.2| 4.2| 4.2| 4.2| 4.2|
| | | 4.2.a1| 4.2.a1| 4.2.a1| 4.2.a1| 4.2.a1|
| | | 4.2.b1| 4.2.b1| 4.2.b1| 4.2.b1| 4.2.b1|
| | | 4.2.rc1| 4.2.rc1| 4.2.rc1| 4.2.rc1| 4.2.rc1|
| fastapi | [fastapi](https://pypi.org/project/fastapi) | 0.56.1~0.100.0|0.56.1~0.100.0|0.56.1~0.100.0|0.56.1~0.100.0|0.56.1~0.100.0|
| | | 0.100.0b2~0.103.1| 0.100.0b2~0.103.1| 0.100.0b2~0.103.1| 0.100.0b2~0.103.1| 0.100.0b2~0.103.1|
| | [uvicorn](https://pypi.org/project/uvicorn) | 0.11.3~0.22.0|0.11.3~0.22.0|0.11.3~0.22.0|0.11.3~0.22.0|0.12.0~0.22.0|
Expand Down
158 changes: 85 additions & 73 deletions noxfile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -498,33 +498,32 @@ def component_tests_execution_tags(

@nox.session()
@nox.parametrize(
"python,flask_version",
"python,django_version",
[
(python, flask_version)
(python, django_version)
for python in python_versions()
for flask_version in dependency_versions_to_be_tested(
for django_version in dependency_versions_to_be_tested(
python=python,
directory="flask",
dependency_name="flask",
directory="django",
dependency_name="django",
)
],
)
def integration_tests_flask(session, python, flask_version):
with TestedVersions.save_tests_result("flask", python, "flask", flask_version):
install_package("flask", flask_version, session)
def integration_tests_django(session, python, django_version):
with TestedVersions.save_tests_result("django", python, "django", django_version):
install_package("django", django_version, session)

session.install(".")

temp_file = create_it_tempfile("flask")
with session.chdir("src/test/integration/flask"):
temp_file = create_it_tempfile("django")
with session.chdir("src/test/integration/django"):
session.install("-r", OTHER_REQUIREMENTS)

try:
session.run(
"sh",
"./scripts/start_flask",
"./scripts/start_django",
env={
"AUTOWRAPT_BOOTSTRAP": "lumigo_opentelemetry",
"LUMIGO_DEBUG_SPANDUMP": temp_file,
"OTEL_SERVICE_NAME": "app",
},
Expand All @@ -540,43 +539,44 @@ def integration_tests_flask(session, python, flask_version):
"--log-cli-level=INFO",
"--color=yes",
"-v",
"./tests/test_flask.py",
"./tests/test_django.py",
env={
"LUMIGO_DEBUG_SPANDUMP": temp_file,
},
)
finally:
kill_process_and_clean_outputs(temp_file, "flask", session)
kill_process_and_clean_outputs(temp_file, "manage.py", session)


@nox.session()
@nox.parametrize(
"python,django_version",
"python,flask_version",
[
(python, django_version)
(python, flask_version)
for python in python_versions()
for django_version in dependency_versions_to_be_tested(
for flask_version in dependency_versions_to_be_tested(
python=python,
directory="django",
dependency_name="django",
directory="flask",
dependency_name="flask",
)
],
)
def integration_tests_django(session, python, django_version):
with TestedVersions.save_tests_result("django", python, "django", django_version):
install_package("django", django_version, session)
def integration_tests_flask(session, python, flask_version):
with TestedVersions.save_tests_result("flask", python, "flask", flask_version):
install_package("flask", flask_version, session)

session.install(".")

temp_file = create_it_tempfile("django")
with session.chdir("src/test/integration/django"):
temp_file = create_it_tempfile("flask")
with session.chdir("src/test/integration/flask"):
session.install("-r", OTHER_REQUIREMENTS)

try:
session.run(
"sh",
"./scripts/start_django",
"./scripts/start_flask",
env={
"AUTOWRAPT_BOOTSTRAP": "lumigo_opentelemetry",
"LUMIGO_DEBUG_SPANDUMP": temp_file,
"OTEL_SERVICE_NAME": "app",
},
Expand All @@ -592,13 +592,13 @@ def integration_tests_django(session, python, django_version):
"--log-cli-level=INFO",
"--color=yes",
"-v",
"./tests/test_django.py",
"./tests/test_flask.py",
env={
"LUMIGO_DEBUG_SPANDUMP": temp_file,
},
)
finally:
kill_process_and_clean_outputs(temp_file, "django", session)
kill_process_and_clean_outputs(temp_file, "flask", session)


@nox.session()
Expand Down Expand Up @@ -736,6 +736,49 @@ def integration_tests_kafka_python(
kill_process_and_clean_outputs(temp_file, "uvicorn", session)


@nox.session()
@nox.parametrize(
"python,motor_version",
[
(python, motor_version)
for python in python_versions()
for motor_version in dependency_versions_to_be_tested(
python=python,
directory="motor",
dependency_name="motor",
)
],
)
def integration_tests_motor(
session,
python,
motor_version,
):
with TestedVersions.save_tests_result("motor", python, "motor", motor_version):
install_package("motor", motor_version, session)

session.install(".")

temp_file = create_it_tempfile("motor")
with session.chdir("src/test/integration/motor"):
session.install("-r", OTHER_REQUIREMENTS)
try:
session.run(
"pytest",
"--tb",
"native",
"--log-cli-level=INFO",
"--color=yes",
"-v",
"./tests/test_motor.py",
env={
"LUMIGO_DEBUG_SPANDUMP": temp_file,
},
)
finally:
kill_process_and_clean_outputs(temp_file, "test_motor", session)


@nox.session()
@nox.parametrize(
"python,pika_version",
Expand Down Expand Up @@ -924,49 +967,6 @@ def integration_tests_pymongo(
kill_process_and_clean_outputs(temp_file, "uvicorn", session)


@nox.session()
@nox.parametrize(
"python,motor_version",
[
(python, motor_version)
for python in python_versions()
for motor_version in dependency_versions_to_be_tested(
python=python,
directory="motor",
dependency_name="motor",
)
],
)
def integration_tests_motor(
session,
python,
motor_version,
):
with TestedVersions.save_tests_result("motor", python, "motor", motor_version):
install_package("motor", motor_version, session)

session.install(".")

temp_file = create_it_tempfile("motor")
with session.chdir("src/test/integration/motor"):
session.install("-r", OTHER_REQUIREMENTS)
try:
session.run(
"pytest",
"--tb",
"native",
"--log-cli-level=INFO",
"--color=yes",
"-v",
"./tests/test_motor.py",
env={
"LUMIGO_DEBUG_SPANDUMP": temp_file,
},
)
finally:
kill_process_and_clean_outputs(temp_file, "test_motor", session)


@nox.session()
@nox.parametrize(
"python,pymysql_version",
Expand Down Expand Up @@ -1091,10 +1091,22 @@ def kill_process(process_name: str) -> None:
print(f"Killing process with name {proc_name}...")
proc.kill()
elif proc_name.lower().startswith("python"):
cmd_line = proc.cmdline()
if len(cmd_line) > 1 and cmd_line[1].endswith("/" + process_name):
# drop the first argument, which is the python executable
python_command_parts = proc.cmdline()[1:]
# the initial command part is the last part of the path
python_command_parts[0] = python_command_parts[0].split("/")[-1]
# combine the remaining arguments
command = " ".join(python_command_parts)
print(
f"Evaluating process with name '{proc_name}' and command '{command}'..."
)
if (
len(cmd_line) > 1
and "nox" not in command
and process_name in command
):
print(
f"Killing process with name {proc_name} and cmdline {cmd_line}..."
f"Killing process with name '{proc_name}' and command '{command}'..."
)
proc.kill()
except psutil.ZombieProcess as zp:
Expand Down
61 changes: 60 additions & 1 deletion src/lumigo_opentelemetry/instrumentations/django/tested_versions/3.10/django
View file
Original file line number Diff line number Diff line change
@@ -1 +1,60 @@
4.2.5
3.2
3.2.1
3.2.2
3.2.3
3.2.4
3.2.5
3.2.6
3.2.7
3.2.8
3.2.9
3.2.10
3.2.11
3.2.12
3.2.13
3.2.14
3.2.15
3.2.16
3.2.17
3.2.18
3.2.19
3.2.20
3.2.21
4.0.a1
4.0.b1
4.0.rc1
4.0
4.0.1
4.0.2
4.0.3
4.0.4
4.0.5
4.0.6
4.0.7
4.0.8
4.0.9
4.0.10
4.1.a1
4.1.b1
4.1.rc1
4.1
4.1.1
4.1.2
4.1.3
4.1.4
4.1.5
4.1.6
4.1.7
4.1.8
4.1.9
4.1.10
4.1.11
4.2.a1
4.2.b1
4.2.rc1
4.2
4.2.1
4.2.2
4.2.3
4.2.4
4.2.5
61 changes: 60 additions & 1 deletion src/lumigo_opentelemetry/instrumentations/django/tested_versions/3.11/django
View file
Original file line number Diff line number Diff line change
@@ -1 +1,60 @@
4.2.5
3.2
3.2.1
3.2.2
3.2.3
3.2.4
3.2.5
3.2.6
3.2.7
3.2.8
3.2.9
3.2.10
3.2.11
3.2.12
3.2.13
3.2.14
3.2.15
3.2.16
3.2.17
3.2.18
3.2.19
3.2.20
3.2.21
4.0.a1
4.0.b1
4.0.rc1
4.0
4.0.1
4.0.2
4.0.3
4.0.4
4.0.5
4.0.6
4.0.7
4.0.8
4.0.9
4.0.10
4.1.a1
4.1.b1
4.1.rc1
4.1
4.1.1
4.1.2
4.1.3
4.1.4
4.1.5
4.1.6
4.1.7
4.1.8
4.1.9
4.1.10
4.1.11
4.2.a1
4.2.b1
4.2.rc1
4.2
4.2.1
4.2.2
4.2.3
4.2.4
4.2.5
Loading

0 comments on commit 68a0332

Please sign in to comment.