Merge branch 'master' into dependabot/pip/bcrypt-gte-3.1-and-lt-4.3

.github/workflows/main.yml

@@ -25,12 +25,16 @@ jobs:
         run: |
           docker compose -p cms -f docker-compose.test.yml run --rm testcms
 
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v4
+        env:
+            CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           files: ./codecov/unittests.xml
           flags: unittests
 
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v4
+        env:
+            CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           files: ./codecov/functionaltests.xml
           flags: functionaltests

Dockerfile

@@ -1,27 +1,28 @@
 # syntax=docker/dockerfile:1
-FROM ubuntu:20.04
+FROM ubuntu:24.04
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     build-essential \
     cgroup-lite \
     cppreference-doc-en-html \
     fp-compiler \
     git \
-    haskell-platform \
+    ghc \
     libcap-dev \
     libcups2-dev \
     libffi-dev \
     libpq-dev \
     libyaml-dev \
     mono-mcs \
     openjdk-8-jdk-headless \
-    php7.4-cli \
+    php-cli \
     postgresql-client \
-    python2 \
+    pypy3 \
     python3-pip \
-    python3.8 \
-    python3.8-dev \
+    python3.12 \
+    python3.12-dev \
     rustc \
+    shared-mime-info \
     sudo \
     wait-for-it \
     zip
@@ -39,8 +40,8 @@ COPY --chown=cmsuser:cmsuser requirements.txt dev-requirements.txt /home/cmsuser
 
 WORKDIR /home/cmsuser/cms
 
-RUN sudo pip3 install -r requirements.txt
-RUN sudo pip3 install -r dev-requirements.txt
+RUN sudo pip3 install --break-system-packages -r requirements.txt
+RUN sudo pip3 install --break-system-packages -r dev-requirements.txt
 
 COPY --chown=cmsuser:cmsuser . /home/cmsuser/cms
 

_cms-test-internal.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+sudo chown cmsuser:cmsuser ./codecov
+
+dropdb --host=testdb --username=postgres cmsdbfortesting
+createdb --host=testdb --username=postgres cmsdbfortesting
+cmsInitDB
+
+pytest --cov . --cov-report xml:codecov/unittests.xml
+UNIT=$?
+
+dropdb --host=testdb --username=postgres cmsdbfortesting
+createdb --host=testdb --username=postgres cmsdbfortesting
+cmsInitDB
+
+cmsRunFunctionalTests -v --coverage codecov/functionaltests.xml
+FUNC=$?
+
+# This check is needed because otherwise failing unit tests aren't reported in
+# the CI as long as the functional tests are passing. Ideally we should get rid
+# of `cmsRunFunctionalTests` and make those tests work with pytest so they can
+# be auto-discovered and run in a single command.
+if [ $UNIT -ne 0 ] || [ $FUNC -ne 0 ]
+then
+    exit 1
+else
+    exit 0
+fi

cms/conf.py

@@ -167,7 +167,7 @@ def __init__(self):
         # the prefix (or real_prefix to accommodate virtualenvs).
         bin_path = os.path.join(os.getcwd(), sys.argv[0])
         bin_name = os.path.basename(bin_path)
-        bin_is_python = bin_name in ["ipython", "python", "python2", "python3"]
+        bin_is_python = bin_name in ["ipython", "python", "python3"]
         bin_in_installed_path = bin_path.startswith(sys.prefix) or (
             hasattr(sys, 'real_prefix')
             and bin_path.startswith(sys.real_prefix))

cms/db/contest.py

@@ -79,7 +79,7 @@ class Contest(Base):
     languages = Column(
         ARRAY(String),
         nullable=False,
-        default=["C11 / gcc", "C++17 / g++", "Pascal / fpc"])
+        default=["C11 / gcc", "C++20 / g++", "Pascal / fpc"])
 
     # Whether contestants allowed to download their submissions.
     submissions_download_allowed = Column(

cms/grading/languages/cpp20_gpp.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+
+# Contest Management System - http://cms-dev.github.io/
+# Copyright © 2024 Filippo Casarin <[email protected]>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""C++20 programming language definition."""
+
+from cms.grading import CompiledLanguage
+
+
+__all__ = ["Cpp20Gpp"]
+
+
+class Cpp20Gpp(CompiledLanguage):
+    """This defines the C++ programming language, compiled with g++ (the
+    version available on the system) using the C++20 standard.
+
+    """
+
+    @property
+    def name(self):
+        """See Language.name."""
+        return "C++20 / g++"
+
+    @property
+    def source_extensions(self):
+        """See Language.source_extensions."""
+        return [".cpp", ".cc", ".cxx", ".c++", ".C"]
+
+    @property
+    def header_extensions(self):
+        """See Language.header_extensions."""
+        return [".h"]
+
+    @property
+    def object_extensions(self):
+        """See Language.object_extensions."""
+        return [".o"]
+
+    def get_compilation_commands(self,
+                                 source_filenames, executable_filename,
+                                 for_evaluation=True):
+        """See Language.get_compilation_commands."""
+        command = ["/usr/bin/g++"]
+        if for_evaluation:
+            command += ["-DEVAL"]
+        command += ["-std=gnu++20", "-O2", "-pipe", "-static",
+                    "-s", "-o", executable_filename]
+        command += source_filenames
+        return [command]

cms/grading/languages/python2_cpython.py → cms/grading/languages/python3_pypy.py

@@ -16,20 +16,20 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Python programming language, version 2, definition."""
+"""Python programming language, version 3, definition."""
 
 import os
 
 from cms.grading import CompiledLanguage
 
 
-__all__ = ["Python2CPython"]
+__all__ = ["Python3PyPy"]
 
 
-class Python2CPython(CompiledLanguage):
-    """This defines the Python programming language, version 2 (more
-    precisely, the subversion of Python 2 available on the system,
-    usually 2.7) using the default interpeter in the system.
+class Python3PyPy(CompiledLanguage):
+    """This defines the Python programming language, version 3 (more
+    precisely, the subversion of Python 3 available on the system)
+    using the default PyPy interpeter in the system.
 
     """
 
@@ -38,7 +38,7 @@ class Python2CPython(CompiledLanguage):
     @property
     def name(self):
         """See Language.name."""
-        return "Python 2 / CPython"
+        return "Python 3 / PyPy"
 
     @property
     def source_extensions(self):
@@ -47,8 +47,9 @@ def source_extensions(self):
 
     @property
     def executable_extension(self):
-        """See Language.executable_extension."""
-        return ".zip"
+        """See Language.executable.extension."""
+        # Defined in PEP 441 (https://www.python.org/dev/peps/pep-0441/).
+        return ".pyz"
 
     def get_compilation_commands(self,
                                  source_filenames, executable_filename,
@@ -57,7 +58,7 @@ def get_compilation_commands(self,
 
         commands = []
         files_to_package = []
-        commands.append(["/usr/bin/python2", "-m", "compileall", "."])
+        commands.append(["/usr/bin/pypy3", "-m", "compileall", "-b", "."])
         for idx, source_filename in enumerate(source_filenames):
             basename = os.path.splitext(os.path.basename(source_filename))[0]
             pyc_filename = "%s.pyc" % basename
@@ -77,4 +78,4 @@ def get_evaluation_commands(
             self, executable_filename, main=None, args=None):
         """See Language.get_evaluation_commands."""
         args = args if args is not None else []
-        return [["/usr/bin/python2", executable_filename] + args]
+        return [["/usr/bin/pypy3", executable_filename] + args]

cms/io/web_service.py

@@ -21,6 +21,13 @@
 
 import logging
 
+import collections
+try:
+    collections.MutableMapping
+except:
+    # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default
+    collections.MutableMapping = collections.abc.MutableMapping
+
 try:
     import tornado4.wsgi as tornado_wsgi
 except ImportError:

cms/server/admin/handlers/base.py

@@ -34,6 +34,13 @@
 from datetime import datetime, timedelta
 from functools import wraps
 
+import collections
+try:
+    collections.MutableMapping
+except:
+    # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default
+    collections.MutableMapping = collections.abc.MutableMapping
+
 try:
     import tornado4.web as tornado_web
 except ImportError:

cms/server/admin/handlers/contestannouncement.py

@@ -26,6 +26,13 @@
 
 """
 
+import collections
+try:
+    collections.MutableMapping
+except:
+    # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default
+    collections.MutableMapping = collections.abc.MutableMapping
+
 try:
     import tornado4.web as tornado_web
 except ImportError:

cms/server/admin/handlers/contestquestion.py

@@ -27,6 +27,13 @@
 
 import logging
 
+import collections
+try:
+    collections.MutableMapping
+except:
+    # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default
+    collections.MutableMapping = collections.abc.MutableMapping
+
 try:
     import tornado4.web as tornado_web
 except ImportError:

cms/server/admin/handlers/contestuser.py

@@ -31,6 +31,13 @@
 
 import logging
 
+import collections
+try:
+    collections.MutableMapping
+except:
+    # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default
+    collections.MutableMapping = collections.abc.MutableMapping
+
 try:
     import tornado4.web as tornado_web
 except ImportError:

cms/server/admin/handlers/dataset.py

@@ -32,6 +32,13 @@
 import re
 import zipfile
 
+import collections
+try:
+    collections.MutableMapping
+except:
+    # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default
+    collections.MutableMapping = collections.abc.MutableMapping
+
 try:
     import tornado4.web as tornado_web
 except ImportError:

cms/server/admin/handlers/task.py

@@ -29,6 +29,13 @@
 import logging
 import traceback
 
+import collections
+try:
+    collections.MutableMapping
+except:
+    # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default
+    collections.MutableMapping = collections.abc.MutableMapping
+
 try:
     import tornado4.web as tornado_web
 except ImportError:

cms/server/admin/server.py

@@ -27,7 +27,7 @@
 
 import logging
 
-from sqlalchemy import func, not_
+from sqlalchemy import func, not_, literal_column
 
 from cms import config, ServiceCoord, get_service_shards
 from cms.db import SessionGen, Dataset, Submission, SubmissionResult, Task
@@ -176,13 +176,17 @@ def submissions_status(contest_id):
                     .filter(Task.contest_id == contest_id)
             queries['total'] = total_query
 
-            stats = {}
+            # Add a "key" column for keeping track of each stats, in case they
+            # get shuffled.
+            for key, query in queries.items():
+                key_column = literal_column(f"'{key}'").label("key")
+                queries[key] = query.add_columns(key_column)
+
             keys = list(queries.keys())
             results = queries[keys[0]].union_all(
                 *(queries[key] for key in keys[1:])).all()
 
-        for i, k in enumerate(keys):
-            stats[k] = results[i][0]
+        stats = {key: value for value, key in results}
         stats['compiling'] += 2 * stats['total'] - sum(stats.values())
 
         return stats

cms/server/contest/handlers/base.py

@@ -32,6 +32,13 @@
 import logging
 import traceback
 
+import collections
+try:
+    collections.MutableMapping
+except:
+    # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default
+    collections.MutableMapping = collections.abc.MutableMapping
+
 try:
     import tornado4.web as tornado_web
 except ImportError:

cms/server/contest/handlers/communication.py

@@ -29,6 +29,13 @@
 
 import logging
 
+import collections
+try:
+    collections.MutableMapping
+except:
+    # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default
+    collections.MutableMapping = collections.abc.MutableMapping
+
 try:
     import tornado4.web as tornado_web
 except ImportError: