Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor file structure and nsjail module #194

Merged
merged 7 commits into from
Nov 5, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Refactor modules into subpackages
  • Loading branch information
MarkKoz committed Oct 27, 2023
commit 56376361a941383555e2db129baf34250d409b06
2 changes: 1 addition & 1 deletion config/gunicorn.conf.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
workers = 2
bind = "0.0.0.0:8060"
logger_class = "snekbox.utils.gunicorn.GunicornLogger"
logger_class = "snekbox.logging.GunicornLogger"
access_logformat = "%(m)s %(U)s%(q)s %(s)s %(b)s %(L)ss"
access_logfile = "-"
wsgi_app = "snekbox:SnekAPI()"
2 changes: 1 addition & 1 deletion snekbox/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
__version__ = "0.0.0.0+unknown"

from snekbox.api import SnekAPI # noqa: E402
from snekbox.logging import init_logger, init_sentry # noqa: E402
from snekbox.nsjail import NsJail # noqa: E402
from snekbox.utils.logging import init_logger, init_sentry # noqa: E402

__all__ = ("NsJail", "SnekAPI", "DEBUG")

Expand Down
3 changes: 1 addition & 2 deletions snekbox/api/resources/eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,10 @@
from falcon.media.validators.jsonschema import validate

from snekbox.nsjail import NsJail
from snekbox.snekio import FileAttachment, ParsingError

__all__ = ("EvalResource",)

from snekbox.snekio import FileAttachment, ParsingError

log = logging.getLogger(__name__)


Expand Down
3 changes: 3 additions & 0 deletions snekbox/limits/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
from . import cgroup, swap, timed

__all__ = ("cgroup", "swap", "timed")
File renamed without changes.
File renamed without changes.
File renamed without changes.
4 changes: 4 additions & 0 deletions snekbox/logging/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
from .gunicorn import GunicornLogger
from .init import FORMAT, init_logger, init_sentry

__all__ = ("FORMAT", "init_logger", "init_sentry", "GunicornLogger")
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

from snekbox import DEBUG

from .logging import FORMAT
from .init import FORMAT

__all__ = ("GunicornLogger",)

Expand Down
File renamed without changes.
13 changes: 6 additions & 7 deletions snekbox/nsjail.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,12 @@

from google.protobuf import text_format

from snekbox import DEBUG, utils
from snekbox import DEBUG, limits
from snekbox.config_pb2 import NsJailConfig
from snekbox.filesystem import Size
from snekbox.memfs import MemFS
from snekbox.limits.timed import time_limit
from snekbox.process import EvalResult
from snekbox.snekio import FileAttachment
from snekbox.utils.timed import time_limit
from snekbox.snekio import FileAttachment, MemFS
from snekbox.snekio.filesystem import Size

__all__ = ("NsJail",)

Expand Down Expand Up @@ -89,8 +88,8 @@ def __init__(
self.files_pattern = files_pattern

self.config = self._read_config(config_path)
self.cgroup_version = utils.cgroup.init(self.config)
self.ignore_swap_limits = utils.swap.should_ignore_limit(self.config, self.cgroup_version)
self.cgroup_version = limits.cgroup.init(self.config)
self.ignore_swap_limits = limits.swap.should_ignore_limit(self.config, self.cgroup_version)

log.info(f"Assuming cgroup version {self.cgroup_version}.")

Expand Down
6 changes: 6 additions & 0 deletions snekbox/snekio/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
from . import filesystem
from .attachment import FileAttachment, safe_path
from .errors import IllegalPathError, ParsingError
from .memfs import MemFS

__all__ = ("filesystem", "safe_path", "FileAttachment", "IllegalPathError", "MemFS", "ParsingError")
12 changes: 4 additions & 8 deletions snekbox/snekio.py → snekbox/snekio/attachment.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@
from functools import cached_property
from pathlib import Path

from .errors import IllegalPathError, ParsingError

__all__ = ("safe_path", "FileAttachment")


def safe_path(path: str) -> str:
"""
Expand All @@ -28,14 +32,6 @@ def safe_path(path: str) -> str:
return path


class ParsingError(ValueError):
"""Raised when an incoming content cannot be parsed."""


class IllegalPathError(ParsingError):
"""Raised when a request file has an illegal path."""


@dataclass(frozen=True)
class FileAttachment:
"""A file attachment."""
Expand Down
6 changes: 6 additions & 0 deletions snekbox/snekio/errors.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
class ParsingError(ValueError):
"""Raised when an incoming content cannot be parsed."""


class IllegalPathError(ParsingError):
"""Raised when a request file has an illegal path."""
File renamed without changes.
2 changes: 1 addition & 1 deletion snekbox/memfs.py → snekbox/snekio/memfs.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@
from typing import Type
from uuid import uuid4

from snekbox.filesystem import mount, unmount
from snekbox.snekio import FileAttachment
from snekbox.snekio.filesystem import mount, unmount

log = logging.getLogger(__name__)

Expand Down
3 changes: 0 additions & 3 deletions snekbox/utils/__init__.py

This file was deleted.

Empty file added tests/limits/__init__.py
Empty file.
2 changes: 1 addition & 1 deletion tests/test_timed.py → tests/limits/test_timed.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import time
from unittest import TestCase

from snekbox.utils.timed import time_limit
from snekbox.limits.timed import time_limit


class TimedTests(TestCase):
Expand Down
Empty file added tests/snekio/__init__.py
Empty file.
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from unittest import TestCase
from uuid import uuid4

from snekbox.filesystem import UnmountFlags, mount, unmount
from snekbox.snekio.filesystem import UnmountFlags, mount, unmount


class LibMountTests(TestCase):
Expand Down
6 changes: 3 additions & 3 deletions tests/test_memfs.py → tests/snekio/test_memfs.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,18 @@
from unittest import TestCase, mock
from uuid import uuid4

from snekbox.memfs import MemFS
from snekbox.snekio import MemFS

UUID_TEST = uuid4()


class MemFSTests(TestCase):
def setUp(self):
super().setUp()
self.logger = logging.getLogger("snekbox.memfs")
self.logger = logging.getLogger("snekbox.snekio.memfs")
self.logger.setLevel(logging.WARNING)

@mock.patch("snekbox.memfs.uuid4", lambda: UUID_TEST)
@mock.patch("snekbox.snekio.memfs.uuid4", lambda: UUID_TEST)
def test_assignment_thread_safe(self):
"""Test concurrent mounting works in multi-thread environments."""
# Concurrently create MemFS in threads, check only 1 can be created
Expand Down
File renamed without changes.
4 changes: 2 additions & 2 deletions tests/test_nsjail.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
from pathlib import Path
from textwrap import dedent

from snekbox.filesystem import Size
from snekbox.nsjail import NsJail
from snekbox.snekio import FileAttachment
from snekbox.snekio.filesystem import Size


class NsJailTests(unittest.TestCase):
Expand Down Expand Up @@ -576,7 +576,7 @@ class NsJailCgroupTests(unittest.TestCase):
# This should still pass for v2, even if this test isn't relevant.
def test_cgroupv1(self):
logging.getLogger("snekbox.nsjail").setLevel(logging.ERROR)
logging.getLogger("snekbox.utils.swap").setLevel(logging.ERROR)
logging.getLogger("snekbox.limits.swap").setLevel(logging.ERROR)

config_base = dedent(
"""
Expand Down