Vendor libmaxminddb

.github/workflows/address-sanitizer.yml

@@ -33,6 +33,8 @@ jobs:
               pip install setuptools tox tox-gh-actions wheel
 
       - name: Test with tox
-        run: MM_FORCE_EXT_TESTS=1 tox
+        run: tox
         env:
           LD_PRELOAD: libasan.so.6
+          MAXMINDDB_REQUIRE_EXTENSION: 1
+          MM_FORCE_EXT_TESTS: 1

.github/workflows/clang-analyzer.yml

@@ -13,6 +13,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          submodules: true
 
       - name: Install libmaxminddb
         run: sudo apt install clang-tools libmaxminddb-dev
@@ -28,4 +30,7 @@ jobs:
               pip install setuptools wheel
 
       - name: Build and run analyzer
-        run: CFLAGS="-Werror -Wall -Wextra" scan-build --status-bugs python setup.py build
+        run: scan-build --status-bugs python setup.py build
+        env:
+          CFLAGS: "-Werror -Wall -Wextra"
+          MAXMINDDB_REQUIRE_EXTENSION: 1

.github/workflows/codeql-analysis.yml

@@ -20,6 +20,7 @@ jobs:
         # We must fetch at least the immediate parents so that if this is
         # a pull request then we can checkout the head.
         fetch-depth: 2
+        submodules: true
 
     # If this run was triggered by a pull request event, then checkout
     # the head of the pull request instead of the merge commit.
@@ -47,6 +48,8 @@ jobs:
             pip install setuptools wheel
 
     - run: python setup.py build
+      env:
+        MAXMINDDB_REQUIRE_EXTENSION: 1
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v2

.github/workflows/create-wheels.yml

@@ -14,9 +14,13 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
 
       - name: Build wheels
         uses: pypa/[email protected]
+        env:
+          MAXMINDDB_REQUIRE_EXTENSION: 1
 
       - uses: actions/upload-artifact@v2
         with:

.github/workflows/test.yml

@@ -44,7 +44,13 @@ jobs:
               pip install setuptools tox tox-gh-actions wheel
 
       - name: Build with Werror and Wall
-        run: CFLAGS="-Werror -Wall -Wextra" python setup.py build
+        run: python setup.py build
+        env:
+          CFLAGS: "-Werror -Wall -Wextra"
+          MAXMINDDB_REQUIRE_EXTENSION: 1
 
       - name: Test with tox
         run: MM_FORCE_EXT_TESTS=1 tox
+        env:
+          MAXMINDDB_REQUIRE_EXTENSION: 1
+          MM_FORCE_EXT_TESTS: 1

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "tests/data"]
 	path = tests/data
 	url = https://github.com/maxmind/MaxMind-DB
+[submodule "extension/libmaxminddb"]
+	path = extension/libmaxminddb
+	url = [email protected]:maxmind/libmaxminddb.git

MANIFEST.in

@@ -1,3 +1,3 @@
 include HISTORY.rst README.rst LICENSE maxminddb/py.typed maxminddb/extension.pyi
-recursive-include tests/ *.mmdb *.py *.raw
+recursive-include tests/ *.c *.h *.mmdb *.py *.raw
 graft docs/html

extension/maxminddb.c

@@ -1,7 +1,7 @@
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include <arpa/inet.h>
-#include <maxminddb.h>
+#include "libmaxminddb/include/maxminddb.h"
 #include <netinet/in.h>
 #include <structmember.h>
 #include <sys/socket.h>

pyproject.toml

@@ -42,9 +42,3 @@ Documentation = "https://maxminddb.readthedocs.org/"
 # src is showing up in our GitHub linting builds. It seems to
 # contain deps.
 extend-exclude = '^/src/'
-
-[tool.cibuildwheel.linux]
-before-all = "apt install libmaxminddb-dev"
-
-[tool.cibuildwheel.macos]
-before-all = "brew install libmaxminddb"

setup.py

@@ -21,13 +21,25 @@
 PYPY = hasattr(sys, "pypy_version_info")
 JYTHON = sys.platform.startswith("java")
 
-compile_args = ["-Wall", "-Wextra"]
+compile_args = ["-Wall", "-Wextra", "-Wno-unknown-pragmas"]
 
 ext_module = [
     Extension(
         "maxminddb.extension",
-        libraries=["maxminddb"],
-        sources=["extension/maxminddb.c"],
+        sources=[
+            "extension/maxminddb.c",
+            "extension/libmaxminddb/src/data-pool.c",
+            "extension/libmaxminddb/src/maxminddb.c",
+        ],
+        define_macros=[
+            # XXX - maybe we should get this from python somehow
+            ("MMDB_LITTLE_ENDIAN", 1),
+            ("PACKAGE_VERSION", '"maxminddb-python"'),
+        ],
+        include_dirs=[
+            "extension/libmaxminddb/include",
+            "extension/libmaxminddb/src",
+        ],
         extra_compile_args=compile_args,
     )
 ]
@@ -116,6 +128,8 @@ def run_setup(with_cext):
     try:
         run_setup(True)
     except BuildFailed as exc:
+        if os.getenv('MAXMINDDB_REQUIRE_EXTENSION'):
+            raise exc
         status_msgs(
             exc.cause,
             "WARNING: The C extension could not be compiled, "