Skip to content

Commit

Permalink
Fix Windows extension build
Browse files Browse the repository at this point in the history
  • Loading branch information
oschwald committed Nov 3, 2023
1 parent 7c3a93b commit 6af427d
Show file tree
Hide file tree
Showing 4 changed files with 68 additions and 45 deletions.
4 changes: 1 addition & 3 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@ jobs:

strategy:
matrix:
# We don't test on Windows currently due to issues
# build libmaxminddb there.
platform: [macos-latest, ubuntu-latest]
platform: [macos-latest, ubuntu-latest, windows-latest]
python-version: [3.8, 3.9, "3.10", 3.11, 3.12]

name: Python ${{ matrix.python-version }} on ${{ matrix.platform }}
Expand Down
30 changes: 23 additions & 7 deletions extension/maxminddb.c
Original file line number Diff line number Diff line change
@@ -1,11 +1,17 @@
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#ifdef MS_WINDOWS
#include <winsock2.h>
#include <ws2tcpip.h>
#else
#include <arpa/inet.h>
#include <maxminddb.h>
#include <netinet/in.h>
#include <structmember.h>
#include <sys/socket.h>
#include <unistd.h>
#endif
#include <maxminddb.h>
#include <stdbool.h>
#include <structmember.h>

#define __STDC_FORMAT_MACROS
#include <inttypes.h>
Expand Down Expand Up @@ -53,6 +59,7 @@ typedef struct {
} Metadata_obj;
// clang-format on

static bool can_read(const char *path);
static int get_record(PyObject *self, PyObject *args, PyObject **record);
static bool format_sockaddr(struct sockaddr *addr, char *dst);
static PyObject *from_entry_data_list(MMDB_entry_data_list_s **entry_data_list);
Expand Down Expand Up @@ -97,8 +104,7 @@ static int Reader_init(PyObject *self, PyObject *args, PyObject *kwds) {
return -1;
}

if (0 != access(filename, R_OK)) {

if (!can_read(filename)) {
PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, filepath);
Py_XDECREF(filepath);
return -1;
Expand Down Expand Up @@ -737,7 +743,7 @@ static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list) {

const uint32_t map_size = (*entry_data_list)->entry_data.data_size;

uint i;
uint32_t i;
// entry_data_list cannot start out NULL (see from_entry_data_list). We
// check it in the loop because it may become NULL.
// coverity[check_after_deref]
Expand Down Expand Up @@ -778,7 +784,7 @@ static PyObject *from_array(MMDB_entry_data_list_s **entry_data_list) {
return NULL;
}

uint i;
uint32_t i;
// entry_data_list cannot start out NULL (see from_entry_data_list). We
// check it in the loop because it may become NULL.
// coverity[check_after_deref]
Expand Down Expand Up @@ -826,6 +832,16 @@ static PyObject *from_uint128(const MMDB_entry_data_list_s *entry_data_list) {
return py_obj;
}

static bool can_read(const char *path) {
#ifdef MS_WINDOWS
int rv = _access(path, 04);
#else
int rv = access(path, R_OK);
#endif

return rv == 0;
}

static PyMethodDef Reader_methods[] = {
{"get",
Reader_get,
Expand Down Expand Up @@ -873,7 +889,7 @@ static PyMethodDef ReaderIter_methods[] = {{NULL, NULL, 0, NULL}};

// clang-format off
static PyTypeObject ReaderIter_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
PyVarObject_HEAD_INIT(NULL, 0)
.tp_basicsize = sizeof(ReaderIter_obj),
.tp_dealloc = ReaderIter_dealloc,
.tp_doc = "Iterator for Reader object",
Expand Down
8 changes: 6 additions & 2 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,19 @@
JYTHON = sys.platform.startswith("java")

if os.name == "nt":
compile_args = []
# Disable unknown pragma warning
compile_args = ["-wd4068"]
libraries = ["Ws2_32"]
else:
compile_args = ["-Wall", "-Wextra", "-Wno-unknown-pragmas"]
libraries = []


if os.getenv("MAXMINDDB_USE_SYSTEM_LIBMAXMINDDB"):
ext_module = [
Extension(
"maxminddb.extension",
libraries=["maxminddb"],
libraries=["maxminddb"] + libraries,
sources=["extension/maxminddb.c"],
extra_compile_args=compile_args,
)
Expand All @@ -40,6 +43,7 @@
ext_module = [
Extension(
"maxminddb.extension",
libraries=libraries,
sources=[
"extension/maxminddb.c",
"extension/libmaxminddb/src/data-pool.c",
Expand Down
71 changes: 38 additions & 33 deletions tests/reader_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,11 @@ class BaseTestReader(object):
Type["maxminddb.extension.Reader"], Type["maxminddb.reader.Reader"]
]
use_ip_objects = False
mp = multiprocessing.get_context("fork")

# fork doesn't work on Windows and spawn would involve pickling the reader,
# which isn't possible.
if os.name != "nt":
mp = multiprocessing.get_context("fork")

def ipf(self, ip):
if self.use_ip_objects:
Expand Down Expand Up @@ -490,41 +494,42 @@ def test_closed_metadata(self):
else:
self.assertIsNotNone(metadata, "pure Python implementation returns value")

def test_multiprocessing(self):
self._check_concurrency(self.mp.Process)

def test_threading(self):
self._check_concurrency(threading.Thread)

def _check_concurrency(self, worker_class):
reader = open_database(
"tests/data/test-data/GeoIP2-Domain-Test.mmdb", self.mode
)

def lookup(pipe):
try:
for i in range(32):
reader.get(self.ipf(f"65.115.240.{i}"))
pipe.send(1)
except:
pipe.send(0)
finally:
if worker_class is self.mp.Process:
reader.close()
pipe.close()

pipes = [self.mp.Pipe() for _ in range(32)]
procs = [worker_class(target=lookup, args=(c,)) for (p, c) in pipes]
for proc in procs:
proc.start()
for proc in procs:
proc.join()
if os.name != "nt":
def test_multiprocessing(self):
self._check_concurrency(self.mp.Process)

reader.close()
def test_threading(self):
self._check_concurrency(threading.Thread)

count = sum([p.recv() for (p, c) in pipes])
def _check_concurrency(self, worker_class):
reader = open_database(
"tests/data/test-data/GeoIP2-Domain-Test.mmdb", self.mode
)

self.assertEqual(count, 32, "expected number of successful lookups")
def lookup(pipe):
try:
for i in range(32):
reader.get(self.ipf(f"65.115.240.{i}"))
pipe.send(1)
except:
pipe.send(0)
finally:
if worker_class is self.mp.Process:
reader.close()
pipe.close()

pipes = [self.mp.Pipe() for _ in range(32)]
procs = [worker_class(target=lookup, args=(c,)) for (p, c) in pipes]
for proc in procs:
proc.start()
for proc in procs:
proc.join()

reader.close()

count = sum([p.recv() for (p, c) in pipes])

self.assertEqual(count, 32, "expected number of successful lookups")

def _check_metadata(self, reader, ip_version, record_size):
metadata = reader.metadata()
Expand Down

0 comments on commit 6af427d

Please sign in to comment.