From 6968d7fb6a2c0cb60c8463696d271929842805be Mon Sep 17 00:00:00 2001 From: Grok Compression Date: Sat, 11 Apr 2026 15:52:59 -0400 Subject: [PATCH 1/2] JP2Grok: support native Grok decompression of vsicurl files --- autotest/gdrivers/jp2grok.py | 206 +++++++++++++++++++++++++++ frmts/grok/grkdatasetbase.h | 269 +++++++++++++++++++++++++++++++++-- 2 files changed, 466 insertions(+), 9 deletions(-) diff --git a/autotest/gdrivers/jp2grok.py b/autotest/gdrivers/jp2grok.py index 0645e9b50c56..ba85195adad8 100644 --- a/autotest/gdrivers/jp2grok.py +++ b/autotest/gdrivers/jp2grok.py @@ -19,6 +19,7 @@ import gdaltest import pytest +import webserver from test_py_scripts import samples_path from osgeo import gdal, ogr, osr @@ -1549,6 +1550,45 @@ def test_jp2grok_multitile_overview_decode(): gdal.Unlink(fname) +############################################################################### +# Test reading a remote JP2 via /vsicurl/ (handled natively by Grok's libcurl +# backend when available). Uses a real URL, so it is only run when slow +# tests are enabled. + + +def test_jp2grok_vsicurl_remote(): + + if not gdaltest.run_slow_tests(): + pytest.skip("GDAL_RUN_SLOW_TESTS not set") + if "CURL_ENABLED=YES" not in gdal.VersionInfo("BUILD_INFO"): + pytest.skip("curl not enabled in this GDAL build") + + url = ( + "/vsicurl/https://www.opengeodata.nrw.de/produkte/geobasis/lusat/" + "akt/dop/dop_jp2_f10/dop10rgbi_32_280_5653_1_nw_2025.jp2" + ) + + gdal.VSICurlClearCache() + try: + ds = gdal.Open(url) + if ds is None: + pytest.skip("remote host unreachable: " + gdal.GetLastErrorMsg()) + assert ds.RasterXSize > 0 + assert ds.RasterYSize > 0 + assert ds.RasterCount >= 1 + # Read a small window from an overview (if any) or from the full-res + # upper-left corner to exercise the fetch path without pulling + # too much data. + band = ds.GetRasterBand(1) + w = min(64, ds.RasterXSize) + h = min(64, ds.RasterYSize) + data = band.ReadRaster(0, 0, w, h, w, h) + assert data is not None and len(data) > 0 + ds = None + finally: + gdal.VSICurlClearCache() + + ############################################################################### # Test driver metadata @@ -1562,3 +1602,169 @@ def test_jp2grok_driver_metadata(): assert drv.GetMetadataItem(gdal.DCAP_CREATECOPY) == "YES" assert "jp2" in drv.GetMetadataItem(gdal.DMD_EXTENSIONS) assert "j2k" in drv.GetMetadataItem(gdal.DMD_EXTENSIONS) + + +############################################################################### +# Webserver fixture for HTTP tests + + +@pytest.fixture(scope="module") +def server(): + + process, port = webserver.launch(handler=webserver.DispatcherHttpHandler) + if port == 0: + pytest.skip("cannot start HTTP server") + + import collections + + WebServer = collections.namedtuple("WebServer", "process port") + + yield WebServer(process, port) + + gdal.VSICurlClearCache() + webserver.server_stop(process, port) + + +############################################################################### +# Test: blocklisted HTTP settings force VSILFILE fallback. +# +# When an unsupported GDAL HTTP config option is set, GrokCanRead() should +# return false for /vsicurl/ paths, causing the driver to use GDAL's VSILFILE +# callbacks instead of Grok's native libcurl I/O. The dataset should still +# open successfully — just via the fallback path. + + +# Each entry is (config_option, value) that should trigger VSILFILE fallback. +_BLOCKLIST_CASES = [ + ("GDAL_HTTP_AUTH", "NTLM"), + ("GDAL_HTTP_AUTH", "NEGOTIATE"), + ("GDAL_HTTP_SSLCERT", "/path/to/cert.pem"), + ("GDAL_HTTP_SSLKEY", "/path/to/key.pem"), + ("GDAL_HTTP_SSLCERTTYPE", "PEM"), + ("GDAL_HTTP_KEYPASSWD", "secret"), + ("GDAL_HTTP_SSL_VERIFYSTATUS", "YES"), + ("GDAL_CURL_CA_BUNDLE", "/path/to/ca-bundle.crt"), + ("GDAL_HTTP_CAPATH", "/etc/ssl/certs"), + ("GDAL_HTTP_HEADER_FILE", "/tmp/headers.txt"), + ("GDAL_HTTPS_PROXY", "http://proxy:8443"), + ("GDAL_PROXY_AUTH", "NTLM"), + ("GDAL_HTTP_LOW_SPEED_TIME", "30"), + ("GDAL_HTTP_LOW_SPEED_LIMIT", "1024"), + ("GDAL_GSSAPI_DELEGATION", "POLICY"), +] + + +@pytest.mark.require_curl() +@pytest.mark.parametrize( + "option,value", _BLOCKLIST_CASES, ids=[c[0] for c in _BLOCKLIST_CASES] +) +def test_jp2grok_blocklist_fallback(server, option, value, tmp_path): + """Blocklisted HTTP settings should trigger VSILFILE fallback while still + allowing the dataset to open successfully via GDAL's VSI layer.""" + + # GDAL_HTTP_HEADER_FILE requires the file to actually exist, otherwise + # GDAL logs an error when it tries to read headers from it. + if option == "GDAL_HTTP_HEADER_FILE": + header_file = tmp_path / "headers.txt" + header_file.write_text("X-Test: FallbackValue\n") + value = str(header_file) + + jp2_data = open("data/jpeg2000/byte.jp2", "rb").read() + gdal.VSICurlClearCache() + + handler = webserver.FileHandler({"/byte.jp2": jp2_data}) + url = "/vsicurl/http://localhost:%d/byte.jp2" % server.port + + with gdal.config_option(option, value): + with webserver.install_http_handler(handler): + ds = gdal.Open(url) + # The dataset should open successfully via the VSILFILE callback + # path — GDAL's own curl handles the request. + assert ds is not None + assert ds.RasterXSize == 100 + assert ds.RasterYSize == 100 + ds = None + + gdal.VSICurlClearCache() + + +############################################################################### +# Test: BASIC and BEARER auth should NOT trigger fallback (Grok handles these). + + +@pytest.mark.require_curl() +@pytest.mark.parametrize("auth_scheme", ["BASIC", "BEARER"]) +def test_jp2grok_supported_auth_no_fallback(server, auth_scheme): + """BASIC and BEARER auth are handled by Grok natively and should not + trigger the VSILFILE fallback path.""" + + jp2_data = open("data/jpeg2000/byte.jp2", "rb").read() + gdal.VSICurlClearCache() + + handler = webserver.FileHandler({"/byte.jp2": jp2_data}) + url = "/vsicurl/http://localhost:%d/byte.jp2" % server.port + + with gdal.config_option("GDAL_HTTP_AUTH", auth_scheme): + with webserver.install_http_handler(handler): + ds = gdal.Open(url) + assert ds is not None + assert ds.RasterXSize == 100 + assert ds.RasterYSize == 100 + ds = None + + gdal.VSICurlClearCache() + + +############################################################################### +# Test: GDAL_HTTP_HEADERS with a single custom header should be forwarded +# to Grok's native I/O (no fallback). + + +@pytest.mark.require_curl() +def test_jp2grok_single_custom_header(server): + """A single GDAL_HTTP_HEADERS entry should be forwarded to Grok's + custom_headers[] without triggering VSILFILE fallback.""" + + jp2_data = open("data/jpeg2000/byte.jp2", "rb").read() + gdal.VSICurlClearCache() + + handler = webserver.FileHandler({"/byte.jp2": jp2_data}) + url = "/vsicurl/http://localhost:%d/byte.jp2" % server.port + + with gdal.config_option("GDAL_HTTP_HEADERS", "X-Custom: TestValue"): + with webserver.install_http_handler(handler): + ds = gdal.Open(url) + assert ds is not None + assert ds.RasterXSize == 100 + assert ds.RasterYSize == 100 + ds = None + + gdal.VSICurlClearCache() + + +############################################################################### +# Test: GDAL_HTTP_HEADERS with multiple headers should still work +# (forwarded to Grok's custom_headers[] array, up to GRK_MAX_CUSTOM_HEADERS). + + +@pytest.mark.require_curl() +def test_jp2grok_multiple_custom_headers(server): + """Multiple GDAL_HTTP_HEADERS entries should be forwarded to Grok's + custom_headers[] array.""" + + jp2_data = open("data/jpeg2000/byte.jp2", "rb").read() + gdal.VSICurlClearCache() + + handler = webserver.FileHandler({"/byte.jp2": jp2_data}) + url = "/vsicurl/http://localhost:%d/byte.jp2" % server.port + + headers = "X-Custom1: Value1, X-Custom2: Value2, X-Custom3: Value3" + with gdal.config_option("GDAL_HTTP_HEADERS", headers): + with webserver.install_http_handler(handler): + ds = gdal.Open(url) + assert ds is not None + assert ds.RasterXSize == 100 + assert ds.RasterYSize == 100 + ds = None + + gdal.VSICurlClearCache() diff --git a/frmts/grok/grkdatasetbase.h b/frmts/grok/grkdatasetbase.h index 067de5241fec..c1fa88cf5331 100644 --- a/frmts/grok/grkdatasetbase.h +++ b/frmts/grok/grkdatasetbase.h @@ -109,6 +109,220 @@ template void safe_strcpy(char (&dest)[N], const std::string &src) dest[len] = '\0'; } +#if defined(HAVE_CURL) && defined(GRK_HAS_LIBCURL) + +/** + * @brief True if @p filename is a network path handled natively by Grok. + * + * Both /vsis3/ and /vsicurl/ are fetched via libcurl inside Grok; /vsis3/ + * needs AWS credentials resolved by GDAL, /vsicurl/ only needs the shared + * HTTP auth options (.netrc, cookies, allow-insecure). + */ +static bool isGrokNetworkPath(const char *filename) +{ + return STARTS_WITH(filename, "/vsis3/") || + STARTS_WITH(filename, "/vsicurl/"); +} + +/** + * @brief Forward GDAL's shared HTTP auth config options to grk_stream_params. + * + * These options apply to any network transport (both /vsis3/ and /vsicurl/), + * so they are set exactly once on the shared path + * - GDAL_HTTP_UNSAFESSL -> s3_allow_insecure + * - GDAL_HTTP_NETRC -> netrc (default YES, matching GDAL) + * - GDAL_HTTP_NETRC_FILE -> netrc_file + * - GDAL_HTTP_COOKIE -> cookie + * - GDAL_HTTP_COOKIEFILE -> cookie_file + * - GDAL_HTTP_COOKIEJAR -> cookie_jar + * - GDAL_HTTP_USERPWD -> username / password + * - GDAL_HTTP_BEARER -> bearer_token + * - GDAL_HTTP_PROXY -> proxy + * - GDAL_HTTP_PROXYUSERPWD -> proxy_userpwd + * - GDAL_HTTP_USERAGENT -> user_agent + * - GDAL_HTTP_TIMEOUT -> timeout + * - GDAL_HTTP_CONNECTTIMEOUT -> connect_timeout + * - GDAL_HTTP_MAX_RETRY -> max_retry + * - GDAL_HTTP_RETRY_DELAY -> retry_delay + */ +static void forwardSharedHttpAuth(grk_stream_params &streamParams) +{ + streamParams.s3_allow_insecure = + CPLTestBool(CPLGetConfigOption("GDAL_HTTP_UNSAFESSL", "NO")); + + streamParams.netrc = + CPLTestBool(CPLGetConfigOption("GDAL_HTTP_NETRC", "YES")); + if (const char *pszNetrcFile = + CPLGetConfigOption("GDAL_HTTP_NETRC_FILE", nullptr)) + safe_strcpy(streamParams.netrc_file, pszNetrcFile); + + if (const char *pszCookie = CPLGetConfigOption("GDAL_HTTP_COOKIE", nullptr)) + safe_strcpy(streamParams.cookie, pszCookie); + if (const char *pszCookieFile = + CPLGetConfigOption("GDAL_HTTP_COOKIEFILE", nullptr)) + safe_strcpy(streamParams.cookie_file, pszCookieFile); + if (const char *pszCookieJar = + CPLGetConfigOption("GDAL_HTTP_COOKIEJAR", nullptr)) + safe_strcpy(streamParams.cookie_jar, pszCookieJar); + + // HTTP basic auth (user:password) + if (const char *pszUserPwd = + CPLGetConfigOption("GDAL_HTTP_USERPWD", nullptr)) + { + const std::string osUserPwd(pszUserPwd); + const size_t nColon = osUserPwd.find(':'); + if (nColon != std::string::npos) + { + safe_strcpy(streamParams.username, osUserPwd.substr(0, nColon)); + safe_strcpy(streamParams.password, osUserPwd.substr(nColon + 1)); + } + } + + // Bearer token for HTTP(S) endpoints + if (const char *pszBearer = CPLGetConfigOption("GDAL_HTTP_BEARER", nullptr)) + safe_strcpy(streamParams.bearer_token, pszBearer); + + // Proxy + if (const char *pszProxy = CPLGetConfigOption("GDAL_HTTP_PROXY", nullptr)) + safe_strcpy(streamParams.proxy, pszProxy); + if (const char *pszProxyUserPwd = + CPLGetConfigOption("GDAL_HTTP_PROXYUSERPWD", nullptr)) + safe_strcpy(streamParams.proxy_userpwd, pszProxyUserPwd); + + // User agent + if (const char *pszUserAgent = + CPLGetConfigOption("GDAL_HTTP_USERAGENT", nullptr)) + safe_strcpy(streamParams.user_agent, pszUserAgent); + + // Timeouts + const char *pszTimeout = CPLGetConfigOption("GDAL_HTTP_TIMEOUT", nullptr); + if (pszTimeout) + streamParams.timeout = atol(pszTimeout); + const char *pszConnectTimeout = + CPLGetConfigOption("GDAL_HTTP_CONNECTTIMEOUT", nullptr); + if (pszConnectTimeout) + streamParams.connect_timeout = atol(pszConnectTimeout); + + // Retry configuration + const char *pszMaxRetry = + CPLGetConfigOption("GDAL_HTTP_MAX_RETRY", nullptr); + if (pszMaxRetry) + streamParams.max_retry = static_cast(atol(pszMaxRetry)); + const char *pszRetryDelay = + CPLGetConfigOption("GDAL_HTTP_RETRY_DELAY", nullptr); + if (pszRetryDelay) + streamParams.retry_delay = static_cast(atol(pszRetryDelay)); + + // Custom HTTP headers (GDAL_HTTP_HEADERS) + if (const char *pszHeaders = + CPLGetConfigOption("GDAL_HTTP_HEADERS", nullptr)) + { + // Parse using the same tokenization logic as GDAL's cpl_http.cpp: + // \r\n-separated raw headers, or comma-separated with quoting. + bool bSingleHeader = false; + if (strstr(pszHeaders, "\r\n") == nullptr) + { + const char *pszComma = strchr(pszHeaders, ','); + if (pszComma != nullptr && strchr(pszComma, ':') == nullptr) + { + // Single header whose value contains a comma + // (e.g. "Accept: text/plain, application/json") + bSingleHeader = true; + } + } + + if (bSingleHeader) + { + safe_strcpy(streamParams.custom_headers[0], pszHeaders); + streamParams.num_custom_headers = 1; + } + else + { + const CPLStringList aosTokens( + strstr(pszHeaders, "\r\n") + ? CSLTokenizeString2(pszHeaders, "\r\n", 0) + : CSLTokenizeString2(pszHeaders, ",", CSLT_HONOURSTRINGS)); + const int nCount = std::min( + aosTokens.size(), static_cast(GRK_MAX_CUSTOM_HEADERS)); + for (int i = 0; i < nCount; ++i) + safe_strcpy(streamParams.custom_headers[i], aosTokens[i]); + streamParams.num_custom_headers = static_cast(nCount); + } + } +} + +/** + * @brief Check if any GDAL HTTP config options are set that Grok's native + * I/O does not support. + * + * When true, network paths should fall back to VSILFILE callback I/O so + * GDAL's full HTTP stack handles the requests. As Grok adds support for + * more curl options, entries can be removed from the blocklist below. + */ +static bool hasUnsupportedHttpSettings() +{ + // Auth schemes beyond BASIC/BEARER (NTLM, NEGOTIATE, Kerberos) + if (const char *v = CPLGetConfigOption("GDAL_HTTP_AUTH", nullptr)) + { + if (!EQUAL(v, "BASIC") && !EQUAL(v, "BEARER")) + return true; + } + + static const char *const apszUnsupported[] = { + "GDAL_GSSAPI_DELEGATION", // Kerberos delegation + "GDAL_HTTP_SSLCERT", // mTLS client certificate + "GDAL_HTTP_SSLKEY", // mTLS private key + "GDAL_HTTP_SSLCERTTYPE", // cert format (PEM/DER) + "GDAL_HTTP_KEYPASSWD", // private key passphrase + "GDAL_HTTP_SSL_VERIFYSTATUS", // OCSP stapling + "GDAL_CURL_CA_BUNDLE", // GDAL-specific CA bundle + "GDAL_HTTP_CAPATH", // CA certificate directory + "GDAL_HTTP_HEADER_FILE", // headers from file + "GDAL_HTTPS_PROXY", // HTTPS-specific proxy + "GDAL_PROXY_AUTH", // proxy auth scheme + "GDAL_HTTP_LOW_SPEED_TIME", // stall detection (time) + "GDAL_HTTP_LOW_SPEED_LIMIT", // stall detection (threshold) + }; + + for (const char *pszOpt : apszUnsupported) + { + if (CPLGetConfigOption(pszOpt, nullptr) != nullptr) + return true; + } + + // GDAL_HTTP_HEADERS is forwarded to Grok's custom_headers[] array, + // but Grok has a fixed limit. Fall back if there are too many. + if (const char *pszHeaders = + CPLGetConfigOption("GDAL_HTTP_HEADERS", nullptr)) + { + int nCount = 0; + if (strstr(pszHeaders, "\r\n") == nullptr) + { + const char *pszComma = strchr(pszHeaders, ','); + if (pszComma != nullptr && strchr(pszComma, ':') == nullptr) + nCount = 1; // single header with comma in value + else + { + const CPLStringList aosTokens( + CSLTokenizeString2(pszHeaders, ",", CSLT_HONOURSTRINGS)); + nCount = aosTokens.size(); + } + } + else + { + const CPLStringList aosTokens( + CSLTokenizeString2(pszHeaders, "\r\n", 0)); + nCount = aosTokens.size(); + } + if (nCount > GRK_MAX_CUSTOM_HEADERS) + return true; + } + + return false; +} + +#endif // HAVE_CURL && GRK_HAS_LIBCURL + /************************************************************************/ /* GrokCanRead() */ /************************************************************************/ @@ -117,7 +331,14 @@ template void safe_strcpy(char (&dest)[N], const std::string &src) * @brief Check if Grok can read the file directly (bypassing VSILFILE). * * When true, Grok opens the file via its own I/O (streamParams.file) - * which is more efficient. When false, GDAL's VSILFILE callbacks are used. + * which is more efficient. When false, GDAL's VSILFILE callbacks are + * used, which support the full set of GDAL HTTP config options. + * + * For network paths, this also checks whether any GDAL HTTP settings + * are active that Grok's native libcurl I/O does not support (e.g. + * mTLS client certificates, NTLM/NEGOTIATE auth, GDAL-specific CA + * bundles). If so, the path is routed through VSILFILE callbacks so + * GDAL's HTTP stack handles the requests correctly. * * @param filename file path * @return true if Grok can use native file I/O @@ -128,11 +349,23 @@ static bool GrokCanRead(const char *filename) return false; #if defined(HAVE_CURL) && defined(GRK_HAS_LIBCURL) - // Cloud storage: Grok handles S3 fetching natively via libcurl. - // GDAL resolves AWS credentials and passes them to Grok via - // grk_stream_params (username/password/bearer_token/region). - if (strncmp(filename, "/vsis3/", 7) == 0) + // Cloud storage: Grok handles fetching natively via libcurl. + // For /vsis3/, GDAL resolves AWS credentials and passes them to Grok + // via grk_stream_params. For /vsicurl/, Grok's HTTPFetcher strips the + // prefix and issues HTTP(S) range requests directly, honoring the + // shared .netrc/cookie options forwarded from GDAL. + if (isGrokNetworkPath(filename)) + { + if (hasUnsupportedHttpSettings()) + { + CPLDebug("GROK", + "Unsupported GDAL HTTP setting detected; " + "falling back to VSILFILE I/O for: %s", + filename); + return false; + } return true; + } #endif // Any other GDAL virtual filesystem must go through VSILFILE callbacks @@ -225,7 +458,7 @@ static void JP2_DebugCallback(const char *pszMsg, CPL_UNUSED void *unused) /** * @brief VSILFILE write callback for Grok stream I/O. * - * Used when GrokCanRead() returns false (e.g. /vsimem/, /vsicurl/). + * Used when GrokCanRead() returns false (e.g. /vsimem/, /vsitar/). * Also always used for compression output. */ static size_t JP2Dataset_Write(const uint8_t *pBuffer, size_t nBytes, @@ -539,9 +772,21 @@ struct GRKCodecWrapper safe_strcpy(streamParams.file, pszFilename); streamParams.initial_offset = psJP2File->nBaseOffset; #if defined(HAVE_CURL) && defined(GRK_HAS_LIBCURL) + // Shared HTTP auth options (.netrc, cookies, allow-insecure) + // apply to both /vsis3/ and /vsicurl/ and are set exactly once + // here so the S3-specific branch below does not duplicate them. + if (isGrokNetworkPath(pszFilename)) + forwardSharedHttpAuth(streamParams); + // For /vsis3/ paths, resolve AWS credentials through GDAL's // full authentication chain and pass them to Grok so it can - // handle S3 fetching natively via libcurl. + // handle S3 fetching natively via libcurl. For /vsicurl/, + // Grok's HTTPFetcher strips the prefix and honors the shared + // HTTP auth options forwarded above. + // + // S3 credentials override the generic USERPWD / BEARER values + // set by forwardSharedHttpAuth, since the S3 signer requires + // the access-key / secret / session-token triple. if (strncmp(pszFilename, "/vsis3/", 7) == 0) { auto poHelper = std::unique_ptr( @@ -573,8 +818,13 @@ struct GRKCodecWrapper streamParams.s3_no_sign_request = poHelper->GetCredentialsSource() == AWSCredentialsSource::NO_SIGN_REQUEST; - streamParams.s3_allow_insecure = CPLTestBool( - CPLGetConfigOption("GDAL_HTTP_UNSAFESSL", "NO")); + + // Requester pays + const char *pszRequestPayer = VSIGetPathSpecificOption( + pszFilename, "AWS_REQUEST_PAYER", ""); + if (pszRequestPayer[0]) + safe_strcpy(streamParams.request_payer, + pszRequestPayer); } else { @@ -1331,6 +1581,7 @@ struct GRKCodecWrapper * * For local files and /vsis3/, uses native file I/O (closing the * VSILFILE). For other VSI paths, uses VSILFILE callbacks. + * (/vsicurl/ is read-only so it never reaches the compression path.) */ bool initCodec(const char *pszFilename, VSIVirtualHandleUniquePtr &fpOwner) { From e10e6980551584a657c9ec4ba7e1e38a492cfd05 Mon Sep 17 00:00:00 2001 From: Grok Compression Date: Mon, 8 Jun 2026 19:23:02 -0400 Subject: [PATCH 2/2] JP2Grok: make grok native vsicurl support opt in User must set OPJLIKE_VSICURL_NATIVE=YES to opt in, otherwise GDAL virtual file system is used to read from network. --- frmts/grok/grkdatasetbase.h | 22 +++++++++++++++++----- frmts/opjlike/jp2opjlikedataset.cpp | 5 +++++ frmts/opjlike/jp2opjlikedataset.h | 9 +++++++++ 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/frmts/grok/grkdatasetbase.h b/frmts/grok/grkdatasetbase.h index c1fa88cf5331..c6d997837d3a 100644 --- a/frmts/grok/grkdatasetbase.h +++ b/frmts/grok/grkdatasetbase.h @@ -114,14 +114,26 @@ template void safe_strcpy(char (&dest)[N], const std::string &src) /** * @brief True if @p filename is a network path handled natively by Grok. * - * Both /vsis3/ and /vsicurl/ are fetched via libcurl inside Grok; /vsis3/ - * needs AWS credentials resolved by GDAL, /vsicurl/ only needs the shared - * HTTP auth options (.netrc, cookies, allow-insecure). + * /vsis3/ is always handed to Grok (AWS credentials are resolved by GDAL + * before the path reaches Grok). + * + * /vsicurl/ is only handed to Grok when the user explicitly opts in by + * setting the OPJLIKE_VSICURL_NATIVE_OPT config option (or open option) to YES. + * Without the flag GDAL reads via curl and streams the bytes to Grok, + * which is the safe default. */ static bool isGrokNetworkPath(const char *filename) { - return STARTS_WITH(filename, "/vsis3/") || - STARTS_WITH(filename, "/vsicurl/"); + if (STARTS_WITH(filename, "/vsis3/")) + return true; + + if (STARTS_WITH(filename, "/vsicurl/")) + { + return CPLTestBool( + CPLGetConfigOption(OPJLIKE_VSICURL_NATIVE_OPT, "NO")); + } + + return false; } /** diff --git a/frmts/opjlike/jp2opjlikedataset.cpp b/frmts/opjlike/jp2opjlikedataset.cpp index 758ee27f1c00..d82a0c200454 100644 --- a/frmts/opjlike/jp2opjlikedataset.cpp +++ b/frmts/opjlike/jp2opjlikedataset.cpp @@ -1381,6 +1381,11 @@ GDALDataset *JP2OPJLikeDataset::Open(GDALOpenInfo *poOpenInfo) if (!Identify(poOpenInfo) || poOpenInfo->fpL == nullptr) return nullptr; + const char *pszNative = CSLFetchNameValueDef( + poOpenInfo->papszOpenOptions, OPJLIKE_VSICURL_NATIVE_OPT, nullptr); + if (pszNative) + CPLSetThreadLocalConfigOption(OPJLIKE_VSICURL_NATIVE_OPT, pszNative); + /* Detect which codec to use : J2K or JP2 ? */ vsi_l_offset nCodeStreamLength = 0; vsi_l_offset nCodeStreamStart = diff --git a/frmts/opjlike/jp2opjlikedataset.h b/frmts/opjlike/jp2opjlikedataset.h index a525fb8eed53..acf5c2340678 100644 --- a/frmts/opjlike/jp2opjlikedataset.h +++ b/frmts/opjlike/jp2opjlikedataset.h @@ -23,6 +23,15 @@ #include "gdaljp2abstractdataset.h" #include "gdaljp2metadata.h" +/** Environment variable / open option that must be set to YES to let opjlike + * codec handle /vsicurl/ reads natively. + * + * Set OPJLIKE_VSICURL_NATIVE=YES as a GDAL config option, or pass it as an + * open option on the dataset, to opt in. + */ +static constexpr const char *OPJLIKE_VSICURL_NATIVE_OPT = + "OPJLIKE_VSICURL_NATIVE"; + typedef int JP2_COLOR_SPACE; typedef int JP2_PROG_ORDER;