Skip to content

Commit

Permalink
improved Merger to filter wavelengths (+redim arrays)
Browse files Browse the repository at this point in the history
  • Loading branch information
@bourgesl
bourgesl committed Apr 28, 2021
1 parent 29a6078 commit 9d10589
Show file tree
Hide file tree
Showing 9 changed files with 693 additions and 166 deletions.
194 changes: 137 additions & 57 deletions src/main/java/fr/jmmc/oitools/fits/FitsTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,14 @@
import fr.jmmc.oitools.meta.KeywordMeta;
import fr.jmmc.oitools.meta.Types;
import fr.jmmc.oitools.meta.Units;
import fr.jmmc.oitools.meta.WaveColumnMeta;
import fr.jmmc.oitools.model.ModelVisitor;
import fr.jmmc.oitools.model.OIFitsChecker;
import fr.jmmc.oitools.model.OITable;
import fr.jmmc.oitools.model.Rule;
import fr.jmmc.oitools.model.range.Range;
import fr.nom.tam.util.ArrayFuncs;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
Expand Down Expand Up @@ -188,13 +190,26 @@ protected final void copyTable(final FitsTable src) throws IllegalArgumentExcept
* @throws IllegalArgumentException if the number of rows is less than 1
*/
public final void resizeTable(final int nbKeepRows, final BitSet maskRows) throws IllegalArgumentException {
resizeTable(nbKeepRows, maskRows, null);
}

/**
* Resize the table ie column arrays + fix the Fits NAXIS2 keyword value
*
* @param nbKeepRows number of rows to keep i.e. the Fits NAXIS2 keyword value
* @param maskRows bit set indicating which rows to keep (true means keep row)
* @param maskWavelengths bit set indicating which wavelength to keep (true means keep index)
* @throws IllegalArgumentException if the number of rows is less than 1
*/
public final void resizeTable(final int nbKeepRows, final BitSet maskRows, final BitSet maskWavelengths) throws IllegalArgumentException {
final int nbRows = getNbRows();
if (nbKeepRows == nbRows) {
if ((nbKeepRows == nbRows) && (maskWavelengths == null)) {
return;
}
if (nbKeepRows < 1) {
throw new IllegalArgumentException("Invalid number of rows : the table must have at least 1 row !");
}
final int nbKeepWl = (maskWavelengths != null) ? maskWavelengths.cardinality() : -1;

// Resize column values:
for (ColumnMeta column : getColumnDescCollection()) {
Expand All @@ -203,11 +218,11 @@ public final void resizeTable(final int nbKeepRows, final BitSet maskRows) throw

// ignore optional columns (null):
if (columnValueOriginal != null) {
final int[] dims = getColumnArrayDims(column, nbKeepRows);
final int[] dims = getColumnArrayDims(column, nbKeepRows, nbKeepWl);
final Object columnValue = createColumnArray(column, dims);

// copy data
filterColumnArray(columnValueOriginal, maskRows, columnValue, dims);
// copy data (may filter wavelengths)
filterColumnArray(columnName, columnValueOriginal, columnValue, (dims.length == 1), maskRows, nbKeepWl, maskWavelengths);

if (logger.isLoggable(Level.FINE)) {
logger.log(Level.FINE, "COLUMN {0} = ''{1}''", new Object[]{columnName, columnValue});
Expand All @@ -231,7 +246,7 @@ public final void resizeTable(final int nbKeepRows, final BitSet maskRows) throw
* @return new column arrays
*/
public Object createColumnArray(final ColumnMeta column, final int nbRows) {
return createColumnArray(column, getColumnArrayDims(column, nbRows));
return createColumnArray(column, getColumnArrayDims(column, nbRows, -1));
}

protected static Object createColumnArray(final ColumnMeta column, final int[] dims) {
Expand All @@ -250,11 +265,13 @@ protected static Object createColumnArray(final ColumnMeta column, final int[] d
return value;
}

protected static int[] getColumnArrayDims(final ColumnMeta column, final int nbRows) {
protected static int[] getColumnArrayDims(final ColumnMeta column, final int nbRows, final int nbWl) {
final String name = column.getName();
final int repeat = column.getRepeat(); // repeat = row size
final Types type = column.getDataType(); // data type

// fixed array size:
final int repeat = ((nbWl != -1) && column instanceof WaveColumnMeta) ? nbWl : column.getRepeat();

if (logger.isLoggable(Level.FINE)) {
logger.log(Level.FINE, "COLUMN [{0}] [{1}{2}]", new Object[]{name, repeat, column.getType()});
}
Expand Down Expand Up @@ -320,7 +337,8 @@ protected static void fillUndefinedArrays(final Object output, final int[] dimen
/* character/date data type */
Arrays.fill((String[]) output, UNDEFINED_STRING);
} else {
logger.log(Level.INFO, "fillUndefinedArrays: Unsupported array type: {0}", output.getClass());
logger.log(Level.INFO, "fillUndefinedArrays: Unsupported array type: {0}",
(output != null) ? output.getClass() : null);
}
} else {
final Object[] oo = (Object[]) output;
Expand All @@ -330,71 +348,133 @@ protected static void fillUndefinedArrays(final Object output, final int[] dimen
}
}

protected static void filterColumnArray(final Object input, final BitSet keepMask, final Object output, final int[] dims) {
protected static void filterColumnArray(final String columnName, final Object input, final Object output, final boolean is1D,
final BitSet keepMaskRows, final int nbKeepWl, final BitSet keepMaskWavelengths) {
// no bound checks:
if (1 == dims.length) {
if (output instanceof double[]) {
/* double data type */
final double[] os = (double[]) input;
final double[] oo = (double[]) output;
if (is1D) {
filterColumnArray1D(columnName, input, output, keepMaskRows);
} else {
final Object[] os = (Object[]) input;
final Object[] oo = (Object[]) output;

for (int i = keepMask.nextSetBit(0), j = 0, len = oo.length; i >= 0 && j < len; i = keepMask.nextSetBit(i + 1), j++) {
oo[j] = os[i];
}
} else if (output instanceof float[]) {
/* real data type */
final float[] os = (float[]) input;
final float[] oo = (float[]) output;
for (int i = keepMaskRows.nextSetBit(0), j = 0, len = oo.length; i >= 0 && j < len; i = keepMaskRows.nextSetBit(i + 1), j++) {
// Array copy needed to ensure deep copy (and not mixing sub-arrays):
filterColumnArray(columnName, os[i], oo[j], nbKeepWl, keepMaskWavelengths);
}
}
}

for (int i = keepMask.nextSetBit(0), j = 0, len = oo.length; i >= 0 && j < len; i = keepMask.nextSetBit(i + 1), j++) {
oo[j] = os[i];
}
} else if (output instanceof short[]) {
/* integer data type */
final short[] os = (short[]) input;
final short[] oo = (short[]) output;
private static void filterColumnArray(final String columnName, final Object input, final Object output,
final int nbKeepWl, final BitSet keepMaskWavelengths) {

for (int i = keepMask.nextSetBit(0), j = 0, len = oo.length; i >= 0 && j < len; i = keepMask.nextSetBit(i + 1), j++) {
oo[j] = os[i];
}
} else if (output instanceof int[]) {
/* integer data type */
final int[] os = (int[]) input;
final int[] oo = (int[]) output;
final String oname = input.getClass().getName();
final String cname = output.getClass().getName();

for (int i = keepMask.nextSetBit(0), j = 0, len = oo.length; i >= 0 && j < len; i = keepMask.nextSetBit(i + 1), j++) {
oo[j] = os[i];
}
} else if (output instanceof boolean[]) {
/* logical data type */
final boolean[] os = (boolean[]) input;
final boolean[] oo = (boolean[]) output;
if (!oname.equals(cname)) {
return;
}

for (int i = keepMask.nextSetBit(0), j = 0, len = oo.length; i >= 0 && j < len; i = keepMask.nextSetBit(i + 1), j++) {
oo[j] = os[i];
}
} else if (output instanceof String[]) {
/* character/date data type */
final String[] os = (String[]) input;
final String[] oo = (String[]) output;
if (oname.charAt(0) != '[') {
return;
}

if (oname.charAt(1) == '[') {
// N dimensions
final Object[] os = (Object[]) input;
final Object[] oo = (Object[]) output;

for (int i = keepMask.nextSetBit(0), j = 0, len = oo.length; i >= 0 && j < len; i = keepMask.nextSetBit(i + 1), j++) {
oo[j] = os[i];
if (os.length == oo.length) {
// dimensions matches:
for (int i = 0; i < os.length; i += 1) {
filterColumnArray(columnName, os[i], oo[i], nbKeepWl, keepMaskWavelengths);
}
} else {
logger.log(Level.INFO, "filterColumnArray: Unsupported array type: {0}", output.getClass());
if (keepMaskWavelengths == null) {
logger.log(Level.WARNING, "filterColumnArray[{0}] invalid dimensions: {1} != {2}", new Object[]{columnName, os.length, oo.length});
} else if (nbKeepWl != oo.length) {
logger.log(Level.WARNING, "filterColumnArray[{0}] invalid dimensions: {1} != {2}", new Object[]{columnName, nbKeepWl, oo.length});
} else {
// dimensions matches:
filterColumnArray(columnName, input, output, false, keepMaskWavelengths, nbKeepWl, keepMaskWavelengths);
}
}
} else {
final Object[] os = (Object[]) input;
final Object[] oo = (Object[]) output;
// 1 dimension:
final int oLen = Array.getLength(input);
final int cLen = Array.getLength(output);

for (int i = keepMask.nextSetBit(0), j = 0, len = oo.length; i >= 0 && j < len; i = keepMask.nextSetBit(i + 1), j++) {
// Array copy needed to ensure deep copy (and not mixing sub-arrays):
ArrayFuncs.copyArray(os[i], oo[j]);
if (oLen == cLen) {
// dimensions matches:
System.arraycopy(input, 0, output, 0, oLen);
} else {
if (keepMaskWavelengths == null) {
logger.log(Level.WARNING, "filterColumnArray[{0}] invalid dimensions: {1} != {2}", new Object[]{columnName, oLen, cLen});
} else if (nbKeepWl != cLen) {
logger.log(Level.WARNING, "filterColumnArray[{0}] invalid dimensions: {1} != {2}", new Object[]{columnName, nbKeepWl, cLen});
} else {
// dimensions matches:
filterColumnArray1D(columnName, input, output, keepMaskWavelengths);
}
}
}
}

protected static void filterColumnArray1D(final String columnName, final Object input, final Object output, final BitSet keepMaskRows) {
// no bound checks:
if (output instanceof double[]) {
/* double data type */
final double[] os = (double[]) input;
final double[] oo = (double[]) output;

for (int i = keepMaskRows.nextSetBit(0), j = 0, len = oo.length; i >= 0 && j < len; i = keepMaskRows.nextSetBit(i + 1), j++) {
oo[j] = os[i];
}
} else if (output instanceof float[]) {
/* real data type */
final float[] os = (float[]) input;
final float[] oo = (float[]) output;

for (int i = keepMaskRows.nextSetBit(0), j = 0, len = oo.length; i >= 0 && j < len; i = keepMaskRows.nextSetBit(i + 1), j++) {
oo[j] = os[i];
}
} else if (output instanceof short[]) {
/* integer data type */
final short[] os = (short[]) input;
final short[] oo = (short[]) output;

for (int i = keepMaskRows.nextSetBit(0), j = 0, len = oo.length; i >= 0 && j < len; i = keepMaskRows.nextSetBit(i + 1), j++) {
oo[j] = os[i];
}
} else if (output instanceof int[]) {
/* integer data type */
final int[] os = (int[]) input;
final int[] oo = (int[]) output;

for (int i = keepMaskRows.nextSetBit(0), j = 0, len = oo.length; i >= 0 && j < len; i = keepMaskRows.nextSetBit(i + 1), j++) {
oo[j] = os[i];
}
} else if (output instanceof boolean[]) {
/* logical data type */
final boolean[] os = (boolean[]) input;
final boolean[] oo = (boolean[]) output;

for (int i = keepMaskRows.nextSetBit(0), j = 0, len = oo.length; i >= 0 && j < len; i = keepMaskRows.nextSetBit(i + 1), j++) {
oo[j] = os[i];
}
} else if (output instanceof String[]) {
/* character/date data type */
final String[] os = (String[]) input;
final String[] oo = (String[]) output;

for (int i = keepMaskRows.nextSetBit(0), j = 0, len = oo.length; i >= 0 && j < len; i = keepMaskRows.nextSetBit(i + 1), j++) {
oo[j] = os[i];
}
} else {
logger.log(Level.INFO, "filterColumnArray[{0}]: Unsupported array type: {1}",
new Object[]{columnName, (output != null) ? output.getClass() : null});
}
}

/**
* Indicate to clear any cached value (derived column ...)
*/
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/fr/jmmc/oitools/model/Granule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public boolean match(final Granule pattern, final Granule candidate) {
}
}
if (pattern.hasDistinctWavelengthRanges() && (candidate.getInsMode() != null)) {
if (!Range.matchRange(pattern.getDistinctMjdRanges(), candidate.getInsMode().getWavelengthRange())) {
if (!Range.matchRange(pattern.getDistinctWavelengthRanges(), candidate.getInsMode().getWavelengthRange())) {
return false;
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/fr/jmmc/oitools/model/OIFitsCollection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -502,7 +502,7 @@ private List<Granule> findGranules(final Selector selector) {
}

// Wavelength ranges criteria:
if (selector.hasWavelengthRange()) {
if (selector.hasWavelengthRanges()) {
pattern.getDistinctWavelengthRanges().addAll(selector.getWavelengthRanges());
}

Expand Down
67 changes: 59 additions & 8 deletions src/main/java/fr/jmmc/oitools/model/range/Range.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import fr.jmmc.jmcs.util.ObjectUtils;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

Expand Down Expand Up @@ -153,6 +154,10 @@ public boolean overlap(final Range other) {
return (getMin() <= other.getMax()) && (getMax() >= other.getMin());
}

public boolean overlapFully(final Range other) {
return (getMin() <= other.getMin()) && (getMax() >= other.getMax());
}

public static boolean matchRange(final Collection<Range> selected, final Range candidate) {
for (Range sel : selected) {
if (sel.overlap(candidate)) {
Expand All @@ -173,6 +178,59 @@ public static boolean matchRanges(final Collection<Range> selected, final Collec
return false;
}

public static boolean matchFully(final Range selected, final Collection<Range> candidates) {
for (Range cand : candidates) {
if (selected.overlapFully(cand)) {
return true;
}
}
return false;
}

public static boolean matchFully(final Collection<Range> selected, final Range candidate) {
final Set<Range> matchSelected = new HashSet<Range>();
for (Range sel : selected) {
if (sel.overlapFully(candidate)) {
matchSelected.add(sel);
}
}
return (selected.size() == matchSelected.size());
}

public static boolean matchFully(final Collection<Range> selected, final Collection<Range> candidates) {
final Set<Range> matchSelected = new HashSet<Range>();
for (Range cand : candidates) {
for (Range sel : selected) {
if (sel.overlapFully(cand)) {
matchSelected.add(sel);
}
}
}
return (selected.size() == matchSelected.size());
}

public static void getMatchingSelected(final Collection<Range> selected, final Range candidate, final Set<Range> matching) {
matching.clear();

for (Range sel : selected) {
if (sel.overlap(candidate)) {
matching.add(sel);
}
}
}

public static void getMatchingSelected(final Collection<Range> selected, final Collection<Range> candidates, final Set<Range> matching) {
matching.clear();

for (Range sel : selected) {
for (Range cand : candidates) {
if (sel.overlap(cand)) {
matching.add(sel);
}
}
}
}

public static void getMatchingRanges(final Collection<Range> selected, final Collection<Range> candidates, final Set<Range> matching) {
matching.clear();

Expand Down Expand Up @@ -269,14 +327,7 @@ public static boolean equals(final List<Range> ranges, final List<Range> otherRa
* @param value value to test
* @return true if the given value is inside given ranges
*/
public static boolean contains(final List<Range> ranges, final double value) {
if (ranges == null) {
return false;
}
return find(ranges, value) != null;
}

public static boolean contains(final Set<Range> ranges, final double value) {
public static boolean contains(final Collection<Range> ranges, final double value) {
if (ranges == null) {
return false;
}
Expand Down
Loading

0 comments on commit 9d10589

Please sign in to comment.