Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Corrected GIS File Input Shapefile multiple steps + paths with spaces #30

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Corrected GIS File Input Shapefile multiple steps + paths with spaces
  • Loading branch information
anvaro98 committed Jun 15, 2022
commit 1164b21a0fc4ede85a242f11a2d18ce94dacf7f0
Original file line number Diff line number Diff line change
@@ -24,8 +24,9 @@


import java.io.IOException;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;

@@ -90,10 +91,10 @@ public void setLimit(long limit) {
}

// Vérification du nom de fichier
protected URL checkFilename(String filename) throws KettleException {
protected String checkFilename(String filename) throws KettleException {

try {
return KettleVFS.getFileObject(filename).getURL();
return URLDecoder.decode(KettleVFS.getFileObject(filename).getURL().getFile(), StandardCharsets.UTF_8.name());

} catch (IOException e) {

Original file line number Diff line number Diff line change
@@ -49,12 +49,12 @@ public DXFReader(String fileName, String geometryFieldName, String charsetName,
super(null, geometryFieldName, charsetName);

try {
this.dxfFileExist = new File(checkFilename(fileName).getFile()).exists();
this.dxfFileExist = new File(checkFilename(fileName)).exists();

if (!this.dxfFileExist) {
throw new KettleException("Missing " + fileName + " file");
} else {
this.dxfFileName = checkFilename(fileName).getFile();
this.dxfFileName = checkFilename(fileName);
}
this.readXData = readXData;

Original file line number Diff line number Diff line change
@@ -47,12 +47,12 @@ public GPXReader(String fileName, String geometryFieldName, String charsetName)
super(null, geometryFieldName, charsetName);

try {
this.gpxFileExist = new File(checkFilename(fileName).getFile()).exists();
this.gpxFileExist = new File(checkFilename(fileName)).exists();

if (!this.gpxFileExist) {
throw new KettleException("Missing " + fileName + " file");
} else {
this.gpxFileName = checkFilename(fileName).getFile();
this.gpxFileName = checkFilename(fileName);
}

this.fields.add(new Field(geometryFieldName, FieldType.GEOMETRY, null, null));
Original file line number Diff line number Diff line change
@@ -51,12 +51,12 @@ public GeoJSONReader(String fileName, String geometryFieldName, String charsetNa

super(null, geometryFieldName, charsetName);

this.geoJsonFileExist = new File(checkFilename(fileName).getFile()).exists();
this.geoJsonFileExist = new File(checkFilename(fileName)).exists();

if (!this.geoJsonFileExist) {
throw new KettleException("Missing " + fileName + " file");
} else {
this.geoJsonFileName = checkFilename(fileName).getFile();
this.geoJsonFileName = checkFilename(fileName);
}

this.fields.add(new Field(geometryFieldName, FieldType.GEOMETRY, null, null));
Original file line number Diff line number Diff line change
@@ -74,12 +74,12 @@ public GeoPackageReader(String fileName, String tableName, String geometryFieldN

super(tableName, null, charsetName);

this.gpkgFileExist = new File(checkFilename(fileName).getFile()).exists();
this.gpkgFileExist = new File(checkFilename(fileName)).exists();

if (!this.gpkgFileExist) {
throw new KettleException("Missing " + fileName + " file");
} else {
this.gpkgFileName = checkFilename(fileName).getFile();
this.gpkgFileName = checkFilename(fileName);
}

if(tableName.equalsIgnoreCase("*")){
Original file line number Diff line number Diff line change
@@ -1,27 +1,27 @@
package com.atolcd.pentaho.di.gis.io;

/*
* #%L
* Pentaho Data Integrator GIS Plugin
* %%
* Copyright (C) 2015 Atol CD
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Lesser Public License for more details.
*
* You should have received a copy of the GNU General Lesser Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/lgpl-3.0.html>.
* #L%
*/

package com.atolcd.pentaho.di.gis.io;
/*
* #%L
* Pentaho Data Integrator GIS Plugin
* %%
* Copyright (C) 2015 Atol CD
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Lesser Public License for more details.
*
* You should have received a copy of the GNU General Lesser Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/lgpl-3.0.html>.
* #L%
*/

import java.io.File;
import java.io.IOException;
@@ -50,13 +50,13 @@ public MapInfoReader(String fileName, String geometryFieldName, String charsetNa

try {

this.mifFileExist = new File(checkFilename(fileName).getFile()).exists();
this.midFileExist = new File(checkFilename(replaceFileExtension(fileName, ".mif", ".mid")).getFile()).exists();
this.mifFileExist = new File(checkFilename(fileName)).exists();
this.midFileExist = new File(checkFilename(replaceFileExtension(fileName, ".mif", ".mid"))).exists();

if (!this.mifFileExist) {
throw new KettleException("Missing " + fileName + " file");
} else {
this.mifFileName = checkFilename(fileName).getFile();
this.mifFileName = checkFilename(fileName);
}

if (!this.midFileExist) {
Original file line number Diff line number Diff line change
@@ -1,27 +1,27 @@
package com.atolcd.pentaho.di.gis.io;

/*
* #%L
* Pentaho Data Integrator GIS Plugin
* %%
* Copyright (C) 2015 Atol CD
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Lesser Public License for more details.
*
* You should have received a copy of the GNU General Lesser Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/lgpl-3.0.html>.
* #L%
*/

package com.atolcd.pentaho.di.gis.io;
/*
* #%L
* Pentaho Data Integrator GIS Plugin
* %%
* Copyright (C) 2015 Atol CD
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Lesser Public License for more details.
*
* You should have received a copy of the GNU General Lesser Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/lgpl-3.0.html>.
* #L%
*/

import java.io.File;
import java.io.FileInputStream;
@@ -34,6 +34,7 @@
import org.geotools.shapefile.Shapefile;
import org.geotools.shapefile.ShapefileHeader;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.trans.steps.xbaseinput.XBase;

import com.atolcd.pentaho.di.gis.io.features.Feature;
import com.atolcd.pentaho.di.gis.io.features.Field;
@@ -58,18 +59,17 @@ public ShapefileReader(String fileName, String geometryFieldName, String charset
super(null, geometryFieldName, charsetName);

try {

this.shpFileExist = new File(checkFilename(fileName).getFile()).exists();
this.dbfFileExist = new File(checkFilename(replaceFileExtension(fileName, ".shp", ".dbf")).getFile()).exists();
this.shpFileExist = new File(checkFilename(fileName)).exists();
this.dbfFileExist = new File(checkFilename(replaceFileExtension(fileName, ".shp", ".dbf"))).exists();

if (!this.shpFileExist) {
throw new KettleException("Missing " + fileName + " file");
} else {
this.shpFileName = checkFilename(fileName).getFile();
this.shpFileName = checkFilename(fileName);
}

if (this.dbfFileExist) {
this.dbfFileName = checkFilename(replaceFileExtension(fileName, ".shp", ".dbf")).getFile();
this.dbfFileName = checkFilename(replaceFileExtension(fileName, ".shp", ".dbf"));
}

// Entête shapefile
@@ -153,24 +153,43 @@ public List<Feature> getFeatures() throws KettleException {

if (this.dbfFileExist) {

DbfFile dbfFile = new DbfFile(this.dbfFileName, this.charset);
// DbfFile dbfFile = new DbfFile(this.dbfFileName, this.charset);
// for (int i = 0; i < features.size(); i++) {

// byte[] record = dbfFile.GetDbfRec(i);

// for (int j = 0; j < this.getFields().size() - 1; j++) {
// features.get(i).addValue(this.fields.get(j + 1), dbfFile.ParseRecordColumn(record, j));
// }

// }

// dbfFile.close();


XBase xbase = new XBase(null, dbfFileName);

xbase.setDbfFile(dbfFileName);
xbase.open();

xbase.getReader().setCharactersetName(this.charset.name());
for (int i = 0; i < features.size(); i++) {

byte[] record = dbfFile.GetDbfRec(i);
Object[] record = xbase.getReader().nextRecord();

for (int j = 0; j < this.getFields().size() - 1; j++) {
features.get(i).addValue(this.fields.get(j + 1), dbfFile.ParseRecordColumn(record, j));
features.get(i).addValue(this.fields.get(j + 1), record[j]);
}

}

dbfFile.close();
xbase.close();

}

} catch (IOException e) {
throw new KettleException("Error reading features" + this.shpFileName, e);
throw new KettleException("Error reading features" + this.shpFileName + this.dbfFileName, e);
} catch (Exception e) {
throw new KettleException("Error reading features" + this.shpFileName, e);
throw new KettleException("Error reading features" + this.shpFileName + this.dbfFileName, e);
}

return features;
Original file line number Diff line number Diff line change
@@ -49,12 +49,12 @@ public SpatialiteReader(String fileName, String tableName, String charsetName) t

super(tableName, null, charsetName);

this.spatialiteFileExist = new File(checkFilename(fileName).getFile()).exists();
this.spatialiteFileExist = new File(checkFilename(fileName)).exists();

if (!this.spatialiteFileExist) {
throw new KettleException("Missing " + fileName + " file");
} else {
this.spatialiteFileName = checkFilename(fileName).getFile();
this.spatialiteFileName = checkFilename(fileName);
}

if(tableName.equalsIgnoreCase("*")){
Original file line number Diff line number Diff line change
@@ -221,11 +221,14 @@ public static Object[] getRow(RowMetaInterface rowMeta, Feature feature) throws
value = valueMeta.getNumber(featureValue);

} else if (field.getType().equals(FieldType.LONG)) {
if(featureValue instanceof Double){
featureValue = Double.valueOf( (Double) featureValue).longValue();
}

value = valueMeta.getInteger(Long.parseLong(String.valueOf(featureValue)));

} else {
value = valueMeta.getString(String.valueOf(featureValue));
value = valueMeta.getString(String.valueOf(featureValue).trim());
}

}