Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Band name with "/" for netCDF output throws NPE #323

Open
bossie opened this issue Sep 20, 2024 · 0 comments
Open

Band name with "/" for netCDF output throws NPE #323

bossie opened this issue Sep 20, 2024 · 0 comments
Labels
bug Something isn't working enhancement New feature or request

Comments

@bossie
Copy link
Collaborator

bossie commented Sep 20, 2024

This process graph will throw a NPE and it's not immediately apparent what causes it:

{
  "process_graph": {
    "applydimension1": {
      "arguments": {
        "data": {
          "from_node": "loadcollection1"
        },
        "dimension": "bands",
        "process": {
          "process_graph": {
            "arrayappend1": {
              "arguments": {
                "data": {
                  "from_parameter": "data"
                },
                "value": {
                  "from_node": "divide1"
                }
              },
              "process_id": "array_append",
              "result": true
            },
            "arrayelement1": {
              "arguments": {
                "data": {
                  "from_parameter": "data"
                },
                "index": 1
              },
              "process_id": "array_element"
            },
            "arrayelement2": {
              "arguments": {
                "data": {
                  "from_parameter": "data"
                },
                "index": 0
              },
              "process_id": "array_element"
            },
            "divide1": {
              "arguments": {
                "x": {
                  "from_node": "arrayelement1"
                },
                "y": {
                  "from_node": "arrayelement2"
                }
              },
              "process_id": "divide"
            }
          }
        }
      },
      "process_id": "apply_dimension"
    },
    "loadcollection1": {
      "arguments": {
        "bands": [
          "VV",
          "VH"
        ],
        "id": "SENTINEL1_GLOBAL_MOSAICS",
        "spatial_extent": {
          "crs": "EPSG:32755",
          "east": 576780,
          "north": 5246260,
          "south": 5244980,
          "west": 575500
        },
        "temporal_extent": [
          "2019-12-31T23:59:59Z",
          "2021-01-01T00:00:00Z"
        ]
      },
      "process_id": "load_collection"
    },
    "renamelabels1": {
      "arguments": {
        "data": {
          "from_node": "applydimension1"
        },
        "dimension": "bands",
        "target": [
          "VV",
          "VH",
          "VH/VV"
        ]
      },
      "process_id": "rename_labels"
    },
    "saveresult1": {
      "arguments": {
        "data": {
          "from_node": "renamelabels1"
        },
        "format": "NetCDF",
        "options": {
          "filename_prefix": "lcfm-s1-55GEN_062_19"
        }
      },
      "process_id": "save_result",
      "result": true
    }
  }
}
Traceback (most recent call last):
  File "/opt/openeo/lib/python3.8/site-packages/openeogeotrellis/deploy/batch_job.py", line 518, in start_main
    main(sys.argv)
  File "/opt/openeo/lib/python3.8/site-packages/openeogeotrellis/deploy/batch_job.py", line 194, in main
    run_driver()
  File "/opt/openeo/lib/python3.8/site-packages/openeogeotrellis/deploy/batch_job.py", line 165, in run_driver
    run_job(
  File "/opt/openeo/lib/python3.8/site-packages/openeogeotrellis/utils.py", line 56, in memory_logging_wrapper
    return function(*args, **kwargs)
  File "/opt/openeo/lib/python3.8/site-packages/openeogeotrellis/deploy/batch_job.py", line 315, in run_job
    the_assets_metadata = result.write_assets(str(output_file))
  File "/opt/openeo/lib/python3.8/site-packages/openeo_driver/save_result.py", line 150, in write_assets
    return self.cube.write_assets(filename=directory, format=self.format, format_options=self.options)
  File "/opt/openeo/lib/python3.8/site-packages/openeogeotrellis/geopysparkdatacube.py", line 1966, in write_assets
    asset_paths = get_jvm().org.openeo.geotrellis.netcdf.NetCDFRDDWriter.writeRasters(
  File "/usr/local/spark/python/lib/py4j-0.10.9.7-src.zip/py4j/java_gateway.py", line 1322, in __call__
    return_value = get_return_value(
  File "/usr/local/spark/python/lib/py4j-0.10.9.7-src.zip/py4j/protocol.py", line 326, in get_return_value
    raise Py4JJavaError(
py4j.protocol.Py4JJavaError: An error occurred while calling z:org.openeo.geotrellis.netcdf.NetCDFRDDWriter.writeRasters.
: java.lang.NullPointerException
	at ucar.nc2.jni.netcdf.Nc4Iosp.writeData(Nc4Iosp.java:2986)
	at ucar.nc2.NetcdfFileWriter.write(NetcdfFileWriter.java:1066)
	at ucar.nc2.NetcdfFileWriter.write(NetcdfFileWriter.java:1050)
	at org.openeo.geotrellis.netcdf.NetCDFRDDWriter$.writeTile(NetCDFRDDWriter.scala:308)
	at org.openeo.geotrellis.netcdf.NetCDFRDDWriter$.$anonfun$saveSingleNetCDFGeneric$5(NetCDFRDDWriter.scala:217)
	at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:158)
	at org.openeo.geotrellis.netcdf.NetCDFRDDWriter$.$anonfun$saveSingleNetCDFGeneric$3(NetCDFRDDWriter.scala:195)
	at org.openeo.geotrellis.netcdf.NetCDFRDDWriter$.$anonfun$saveSingleNetCDFGeneric$3$adapted(NetCDFRDDWriter.scala:158)
	at scala.collection.Iterator.foreach(Iterator.scala:943)
	at scala.collection.Iterator.foreach$(Iterator.scala:943)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
	at org.openeo.geotrellis.netcdf.NetCDFRDDWriter$.saveSingleNetCDFGeneric(NetCDFRDDWriter.scala:158)
	at org.openeo.geotrellis.netcdf.NetCDFRDDWriter$.saveSingleNetCDFGeneric(NetCDFRDDWriter.scala:108)
	at org.openeo.geotrellis.netcdf.NetCDFRDDWriter$.writeRasters(NetCDFRDDWriter.scala:80)
	at org.openeo.geotrellis.netcdf.NetCDFRDDWriter.writeRasters(NetCDFRDDWriter.scala)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
	at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
	at py4j.Gateway.invoke(Gateway.java:282)
	at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
	at py4j.commands.CallCommand.execute(CallCommand.java:79)
	at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
	at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
	at java.base/java.lang.Thread.run(Thread.java:829)

The same process graph works fine for e.g. GeoTiff.

Maybe this one-way replace has something to do with it?

Possible courses of action:

  • silently replace illegal characters but make sure to avoid this error further on;
  • throw a clearer error.
@bossie bossie added enhancement New feature or request bug Something isn't working labels Sep 20, 2024
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
bug Something isn't working enhancement New feature or request
Projects
None yet
Development

No branches or pull requests

1 participant