[pull] master from OSGeo:master

.github/workflows/alpine/Dockerfile.ci

@@ -21,6 +21,8 @@ RUN apk add \
     qhull-dev \
     unixodbc-dev \
     libpq-dev \
+    apache-arrow-dev \
+    py3-pyarrow py3-pyarrow-pyc \
     libxml2-dev \
     podofo-dev
 

.github/workflows/coverage/build.sh

@@ -14,6 +14,8 @@ cmake ${GDAL_SOURCE_DIR:=..} \
     -DGDAL_USE_GEOTIFF_INTERNAL=ON \
     -DECW_ROOT=/opt/libecwj2-3.3 \
     -DMRSID_ROOT=/usr/local \
-    -DFileGDB_ROOT=/usr/local/FileGDB_API
+    -DFileGDB_ROOT=/usr/local/FileGDB_API \
+    -DSQLite3_INCLUDE_DIR=/usr/local/install-sqlite-trusted-schema-off/include \
+    -DSQLite3_LIBRARY=/usr/local/install-sqlite-trusted-schema-off/lib/libsqlite3.so
 
 make -j$(nproc)

.github/workflows/ubuntu_20.04/Dockerfile.ci

@@ -214,12 +214,15 @@ RUN mkdir odbc-cpp-wrapper \
   && rm -rf odbc-cpp-wrapper
 
 # Arrow
+# We should perhaps pin the pyarrow version to the one of libarrow...
+# Actually not: the libarrow embedded in pyarrow 13 doesn't work with libarrow 13 linked from GDAL...
 RUN wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \
  && dpkg --install apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \
  && apt-get update \
  && apt-get install -y -V libarrow-dev libparquet-dev libarrow-dataset-dev \
  && rm ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \
- && rm -rf /var/lib/apt/lists/*
+ && rm -rf /var/lib/apt/lists/* \
+ && python3 -m pip install pyarrow==11.0.0
 
 # Build libQB3
 # Used by the MRF driver
@@ -243,6 +246,16 @@ RUN git clone -b cmake https://github.com/rouault/basis_universal.git \
     && cd ../..\
     && rm -rf basis_universal
 
+# Build a SQLite version with -DSQLITE_TRUSTED_SCHEMA=0
+RUN mkdir sqlite \
+    && wget -q https://sqlite.org/2023/sqlite-autoconf-3430100.tar.gz -O - \
+        | tar xz -C sqlite --strip-components=1 \
+    && cd sqlite \
+    && CFLAGS="-O2 -DSQLITE_TRUSTED_SCHEMA=0 -DSQLITE_ENABLE_COLUMN_METADATA" ./configure --prefix=/usr/local/install-sqlite-trusted-schema-off \
+    && make -j$(nproc) \
+    && make -j$(nproc) install \
+    && cd .. \
+    && rm -rf sqlite
 
 RUN ldconfig
 

.github/workflows/ubuntu_20.04/build.sh

@@ -13,7 +13,9 @@ cmake ${GDAL_SOURCE_DIR:=..} \
     -DGDAL_USE_GEOTIFF_INTERNAL=ON \
     -DECW_ROOT=/opt/libecwj2-3.3 \
     -DMRSID_ROOT=/usr/local \
-    -DFileGDB_ROOT=/usr/local/FileGDB_API
+    -DFileGDB_ROOT=/usr/local/FileGDB_API \
+    -DSQLite3_INCLUDE_DIR=/usr/local/install-sqlite-trusted-schema-off/include \
+    -DSQLite3_LIBRARY=/usr/local/install-sqlite-trusted-schema-off/lib/libsqlite3.so
 
 unset CXXFLAGS
 unset CFLAGS

MIGRATION_GUIDE.TXT

@@ -1,3 +1,11 @@
+MIGRATION GUIDE FROM GDAL 3.7 to GDAL 3.8
+-----------------------------------------
+
+- Out-of-tree vector drivers:
+  * GDALDataset::ICreateLayer() now takes a const OGRSpatialReference* instead
+    of a OGRSpatialReference*. Drivers should clone the passed SRS if they need
+    to keep it.
+
 MIGRATION GUIDE FROM GDAL 3.6 to GDAL 3.7
 -----------------------------------------
 

alg/gdaltransformer.cpp

@@ -3504,14 +3504,40 @@ static CPLXMLNode *GDALSerializeReprojectionTransformer(void *pTransformArg)
     /* -------------------------------------------------------------------- */
     /*      Handle SourceCS.                                                */
     /* -------------------------------------------------------------------- */
-    char *pszWKT = nullptr;
+    const auto ExportToWkt = [](const OGRSpatialReference *poSRS)
+    {
+        // Try first in WKT1 for backward compat
+        {
+            char *pszWKT = nullptr;
+            const char *const apszOptions[] = {"FORMAT=WKT1", nullptr};
+            CPLErrorHandlerPusher oHandler(CPLQuietErrorHandler);
+            CPLErrorStateBackuper oBackuper;
+            if (poSRS->exportToWkt(&pszWKT, apszOptions) == OGRERR_NONE)
+            {
+                std::string osRet(pszWKT);
+                CPLFree(pszWKT);
+                return osRet;
+            }
+            CPLFree(pszWKT);
+        }
+
+        char *pszWKT = nullptr;
+        const char *const apszOptions[] = {"FORMAT=WKT2_2019", nullptr};
+        if (poSRS->exportToWkt(&pszWKT, apszOptions) == OGRERR_NONE)
+        {
+            std::string osRet(pszWKT);
+            CPLFree(pszWKT);
+            return osRet;
+        }
+        CPLFree(pszWKT);
+        return std::string();
+    };
 
     auto poSRS = psInfo->poForwardTransform->GetSourceCS();
     if (poSRS)
     {
-        poSRS->exportToWkt(&pszWKT);
-        CPLCreateXMLElementAndValue(psTree, "SourceSRS", pszWKT);
-        CPLFree(pszWKT);
+        const auto osWKT = ExportToWkt(poSRS);
+        CPLCreateXMLElementAndValue(psTree, "SourceSRS", osWKT.c_str());
     }
 
     /* -------------------------------------------------------------------- */
@@ -3520,9 +3546,8 @@ static CPLXMLNode *GDALSerializeReprojectionTransformer(void *pTransformArg)
     poSRS = psInfo->poForwardTransform->GetTargetCS();
     if (poSRS)
     {
-        poSRS->exportToWkt(&pszWKT);
-        CPLCreateXMLElementAndValue(psTree, "TargetSRS", pszWKT);
-        CPLFree(pszWKT);
+        const auto osWKT = ExportToWkt(poSRS);
+        CPLCreateXMLElementAndValue(psTree, "TargetSRS", osWKT.c_str());
     }
 
     /* -------------------------------------------------------------------- */

alg/gdalwarpkernel.cpp

@@ -408,14 +408,17 @@ static void ThreadFuncAdapter(void *pData)
             psThreadData->mapThreadToTransformerArg[nThreadId] =
                 pTransformerArg;
         }
+
+        if (pTransformerArg == nullptr)
+        {
+            CPLAssert(psThreadData->pTransformerArgInput != nullptr);
+            CPLAssert(!psThreadData->bTransformerArgInputAssignedToThread);
+        }
     }
 
     // If no transformer assigned to current thread, instantiate one
     if (pTransformerArg == nullptr)
     {
-        CPLAssert(psThreadData->pTransformerArgInput != nullptr);
-        CPLAssert(!psThreadData->bTransformerArgInputAssignedToThread);
-
         // This somehow assumes that GDALCloneTransformer() is thread-safe
         // which should normally be the case.
         pTransformerArg =

alg/gdalwarpkernel_opencl.cpp

@@ -28,6 +28,7 @@
  * DEALINGS IN THE SOFTWARE.
  ****************************************************************************/
 
+#include <algorithm>
 #if defined(HAVE_OPENCL)
 
 /* The following line may be uncommented for increased debugging traces and
@@ -294,11 +295,10 @@ static cl_device_id get_device(OCLVendor *peVendor)
     size_t returned_size = 0;
     cl_char vendor_name[1024] = {0};
     cl_char device_name[1024] = {0};
+    cl_device_type device_type;
 
     std::vector<cl_platform_id> platforms;
     cl_uint num_platforms = 0;
-    cl_device_id preferred_device_id = nullptr;
-    int preferred_is_gpu = FALSE;
 
     static bool gbBuggyOpenCL = false;
     if (gbBuggyOpenCL)
@@ -323,110 +323,129 @@ static cl_device_id get_device(OCLVendor *peVendor)
         return nullptr;
     }
 
-    bool bUseOpenCLCPU =
+    const bool bUseOpenCLCPU =
         CPLTestBool(CPLGetConfigOption("OPENCL_USE_CPU", "FALSE"));
 
-    // In case we have several implementations, pick up the non Intel one by
-    // default, unless the PREFERRED_OPENCL_VENDOR config option is specified.
-    for (cl_uint i = 0; i < num_platforms; i++)
-    {
-        cl_device_id device = nullptr;
-        const char *pszBlacklistedVendor;
-        const char *pszPreferredVendor;
-        int is_gpu;
+    const char *pszBlacklistedVendor = "";
+    const char *pszPreferredVendor = "";
 
-        // Find the GPU CL device, this is what we really want
-        // If there is no GPU device is CL capable, fall back to CPU
-        if (bUseOpenCLCPU)
-            err = CL_DEVICE_NOT_FOUND;
-        else
-            err = clGetDeviceIDs(platforms[i], CL_DEVICE_TYPE_GPU, 1, &device,
-                                 nullptr);
-        is_gpu = (err == CL_SUCCESS);
-        if (err != CL_SUCCESS)
-        {
-            // Find the CPU CL device, as a fallback
-            err = clGetDeviceIDs(platforms[i], CL_DEVICE_TYPE_CPU, 1, &device,
-                                 nullptr);
-            if (err != CL_SUCCESS || device == nullptr)
-                continue;
-        }
+    struct device_info
+    {
+        cl_device_id id;
+        cl_device_type device_type;
+        std::string vendor_name;
+        std::string device_name;
+    };
 
-        // Get some information about the returned device
-        err = clGetDeviceInfo(device, CL_DEVICE_VENDOR, sizeof(vendor_name),
-                              vendor_name, &returned_size);
-        err |= clGetDeviceInfo(device, CL_DEVICE_NAME, sizeof(device_name),
-                               device_name, &returned_size);
-        assert(err == CL_SUCCESS);
+    std::vector<device_info> openCLdevices;
 
-        if (num_platforms > 1)
-            CPLDebug("OpenCL",
-                     "Found vendor='%s' / device='%s' (%s implementation).",
-                     vendor_name, device_name, (is_gpu) ? "GPU" : "CPU");
+    // In case we have several implementations, pick up the non Intel one by
+    // default, unless the PREFERRED_OPENCL_VENDOR config option is specified.
+    if (num_platforms > 0)
+    {
 
+        pszPreferredVendor =
+            CPLGetConfigOption("PREFERRED_OPENCL_VENDOR", nullptr);
         pszBlacklistedVendor =
             CPLGetConfigOption("BLACKLISTED_OPENCL_VENDOR", nullptr);
-        if (pszBlacklistedVendor &&
-            EQUAL(reinterpret_cast<const char *>(vendor_name),
-                  pszBlacklistedVendor))
-        {
-            CPLDebug(
-                "OpenCL",
-                "Blacklisted vendor='%s' / device='%s' implementation skipped",
-                vendor_name, device_name);
-            continue;
-        }
 
-        if (preferred_device_id == nullptr || (is_gpu && !preferred_is_gpu))
+        for (cl_uint platformIndex = 0; platformIndex < num_platforms;
+             platformIndex++)
         {
-            preferred_device_id = device;
-            preferred_is_gpu = is_gpu;
-        }
 
-        pszPreferredVendor =
-            CPLGetConfigOption("PREFERRED_OPENCL_VENDOR", nullptr);
-        if (pszPreferredVendor)
-        {
-            if (EQUAL(reinterpret_cast<const char *>(vendor_name),
-                      pszPreferredVendor))
+            constexpr int max_devices{20};
+            cl_uint num_devices;
+            cl_device_id devices[max_devices];
+            err = clGetDeviceIDs(platforms[platformIndex],
+                                 bUseOpenCLCPU
+                                     ? CL_DEVICE_TYPE_CPU
+                                     : CL_DEVICE_TYPE_CPU | CL_DEVICE_TYPE_GPU,
+                                 max_devices, devices, &num_devices);
+
+            if (err != CL_SUCCESS)
             {
-                preferred_device_id = device;
-                preferred_is_gpu = is_gpu;
-                break;
+                continue;
+            }
+
+            cl_device_id device = nullptr;
+
+            for (cl_uint deviceIndex = 0; deviceIndex < num_devices;
+                 ++deviceIndex)
+            {
+
+                device = devices[deviceIndex];
+                err = clGetDeviceInfo(device, CL_DEVICE_VENDOR,
+                                      sizeof(vendor_name), vendor_name,
+                                      &returned_size);
+                err |=
+                    clGetDeviceInfo(device, CL_DEVICE_NAME, sizeof(device_name),
+                                    device_name, &returned_size);
+                err |=
+                    clGetDeviceInfo(device, CL_DEVICE_TYPE, sizeof(device_type),
+                                    &device_type, &returned_size);
+                assert(err == CL_SUCCESS);
+                if (pszBlacklistedVendor &&
+                    EQUAL(reinterpret_cast<const char *>(vendor_name),
+                          pszBlacklistedVendor))
+                {
+                    CPLDebug("OpenCL",
+                             "Blacklisted vendor='%s' / device='%s' "
+                             "implementation skipped",
+                             vendor_name, device_name);
+                }
+                else
+                {
+                    CPLDebug(
+                        "OpenCL",
+                        "Found vendor='%s' / device='%s' (%s implementation).",
+                        vendor_name, device_name,
+                        (device_type == CL_DEVICE_TYPE_GPU) ? "GPU" : "CPU");
+                    openCLdevices.push_back(
+                        {device, device_type,
+                         reinterpret_cast<const char *>(vendor_name),
+                         reinterpret_cast<const char *>(device_name)});
+                }
             }
         }
-        else if (is_gpu &&
-                 !STARTS_WITH(reinterpret_cast<const char *>(vendor_name),
-                              "Intel"))
-        {
-            preferred_device_id = device;
-            preferred_is_gpu = is_gpu;
-            break;
-        }
     }
-    if (preferred_device_id == nullptr)
+
+    if (!openCLdevices.empty())
+    {
+        // Sort, GPU first, then preferred vendor, Intel comes last (unless is the preferred of course)
+        const std::string preferredVendorName{pszPreferredVendor};
+        std::sort(
+            openCLdevices.begin(), openCLdevices.end(),
+            [&](const device_info first, const device_info second) -> int
+            {
+                return (first.device_type == CL_DEVICE_TYPE_GPU ? 4 : 0) +
+                           (first.vendor_name == preferredVendorName ? 2 : 0) +
+                           (first.vendor_name.find("Intel") == 0 ? 0 : 1) >
+                       (second.device_type == CL_DEVICE_TYPE_GPU ? 4 : 0) +
+                           (second.vendor_name == preferredVendorName ? 2 : 0) +
+                           (second.vendor_name.find("Intel") == 0 ? 0 : 1);
+            });
+    }
+    else
     {
         CPLDebug("OpenCL", "No implementation found");
         return nullptr;
     }
 
-    err = clGetDeviceInfo(preferred_device_id, CL_DEVICE_VENDOR,
-                          sizeof(vendor_name), vendor_name, &returned_size);
-    err |= clGetDeviceInfo(preferred_device_id, CL_DEVICE_NAME,
-                           sizeof(device_name), device_name, &returned_size);
+    const device_info &device{openCLdevices.front()};
+
     CPLDebug("OpenCL",
              "Connected to vendor='%s' / device='%s' (%s implementation).",
-             vendor_name, device_name, (preferred_is_gpu) ? "GPU" : "CPU");
+             device.vendor_name.c_str(), device.device_name.c_str(),
+             (device.device_type == CL_DEVICE_TYPE_GPU) ? "GPU" : "CPU");
 
-    if (STARTS_WITH(reinterpret_cast<const char *>(vendor_name),
-                    "Advanced Micro Devices"))
+    if (STARTS_WITH(device.vendor_name.c_str(), "Advanced Micro Devices"))
         *peVendor = VENDOR_AMD;
-    else if (STARTS_WITH(reinterpret_cast<const char *>(vendor_name), "Intel"))
+    else if (STARTS_WITH(device.vendor_name.c_str(), "Intel"))
         *peVendor = VENDOR_INTEL;
     else
         *peVendor = VENDOR_OTHER;
 
-    return preferred_device_id;
+    return device.id;
 }
 
 /*