Skip to content

Commit

Permalink
Merge pull request #526 from apache/remove_duplicate_generic_sorted_v…
Browse files Browse the repository at this point in the history 
…iews

Combined classic ItemsSketchSortedView and KllItemsSketchSortedView.
  • Loading branch information
@leerho
leerho authored Mar 13, 2024
2 parents f365850 + 03efd5a commit c0c20ac
Show file tree
Hide file tree
Showing 26 changed files with 620 additions and 777 deletions.
2 changes: 1 addition & 1 deletion src/main/java/org/apache/datasketches/kll/KllHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,7 @@ private static String outputDataDetail(final KllSketch sketch) {
return sb.toString();
}

static String outputLevels(final int k, final int m, final int numLevels, final int[] levelsArr) {
private static String outputLevels(final int k, final int m, final int numLevels, final int[] levelsArr) {
final StringBuilder sb = new StringBuilder();
sb.append(LS + "### KLL Levels Array:").append(LS)
.append(" Level, Offset: Nominal Capacity, Actual Capacity").append(LS);
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/apache/datasketches/kll/KllItemsHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -434,7 +434,7 @@ private static <T> int[] generalItemsCompress(

// level zero might not be sorted, so we must sort it if we wish to compact it
if ((curLevel == 0) && !isLevelZeroSorted) {
Arrays.sort(inBuf, adjBeg, adjBeg + adjPop);
Arrays.sort((T[])inBuf, adjBeg, adjBeg + adjPop, comp);
}

if (popAbove == 0) { // Level above is empty, so halve up
Expand Down
58 changes: 32 additions & 26 deletions src/main/java/org/apache/datasketches/kll/KllItemsSketch.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
import org.apache.datasketches.memory.MemoryRequestServer;
import org.apache.datasketches.memory.WritableMemory;
import org.apache.datasketches.quantilescommon.GenericPartitionBoundaries;
import org.apache.datasketches.quantilescommon.PartitioningFeature;
import org.apache.datasketches.quantilescommon.ItemsSketchSortedView;
import org.apache.datasketches.quantilescommon.QuantileSearchCriteria;
import org.apache.datasketches.quantilescommon.QuantilesGenericAPI;
import org.apache.datasketches.quantilescommon.QuantilesGenericSketchIterator;
Expand All @@ -49,12 +49,12 @@
* @see org.apache.datasketches.kll.KllSketch
*/
@SuppressWarnings("unchecked")
public abstract class KllItemsSketch<T> extends KllSketch implements QuantilesGenericAPI<T>, PartitioningFeature<T> {
private KllItemsSketchSortedView<T> kllItemsSV = null;
public abstract class KllItemsSketch<T> extends KllSketch implements QuantilesGenericAPI<T> {
private ItemsSketchSortedView<T> itemsSV = null;
final Comparator<? super T> comparator;
final ArrayOfItemsSerDe<T> serDe;

KllItemsSketch(
KllItemsSketch( //pass-through constructor
final SketchStructure skStructure,
final Comparator<? super T> comparator,
final ArrayOfItemsSerDe<T> serDe) {
Expand All @@ -79,8 +79,8 @@ public abstract class KllItemsSketch<T> extends KllSketch implements QuantilesGe
public static <T> KllItemsSketch<T> newHeapInstance(
final Comparator<? super T> comparator,
final ArrayOfItemsSerDe<T> serDe) {
final KllItemsSketch<T> itmSk =
new KllHeapItemsSketch<>(DEFAULT_K, DEFAULT_M, comparator, serDe);
final KllItemsSketch<T> itmSk =
new KllHeapItemsSketch<>(DEFAULT_K, DEFAULT_M, comparator, serDe);
return itmSk;
}

Expand Down Expand Up @@ -145,33 +145,36 @@ public static <T> KllItemsSketch<T> wrap(

//END of Constructors

@Override
public Class<T> getClassOfT() { return serDe.getClassOfT(); }

@Override
public double[] getCDF(final T[] splitPoints, final QuantileSearchCriteria searchCrit) {
if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); }
refreshSortedView();
return kllItemsSV.getCDF(splitPoints, searchCrit);
return itemsSV.getCDF(splitPoints, searchCrit);
}

@Override
public GenericPartitionBoundaries<T> getPartitionBoundaries(final int numEquallySized,
final QuantileSearchCriteria searchCrit) {
if (isEmpty()) { throw new IllegalArgumentException(EMPTY_MSG); }
refreshSortedView();
return kllItemsSV.getPartitionBoundaries(numEquallySized, searchCrit);
return itemsSV.getPartitionBoundaries(numEquallySized, searchCrit);
}

@Override
public double[] getPMF(final T[] splitPoints, final QuantileSearchCriteria searchCrit) {
if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); }
refreshSortedView();
return kllItemsSV.getPMF(splitPoints, searchCrit);
return itemsSV.getPMF(splitPoints, searchCrit);
}

@Override
public T getQuantile(final double rank, final QuantileSearchCriteria searchCrit) {
if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); }
refreshSortedView();
return kllItemsSV.getQuantile(rank, searchCrit);
return itemsSV.getQuantile(rank, searchCrit);
}

@Override
Expand All @@ -181,7 +184,7 @@ public T[] getQuantiles(final double[] ranks, final QuantileSearchCriteria searc
final int len = ranks.length;
final T[] quantiles = (T[]) Array.newInstance(getMinItem().getClass(), len);
for (int i = 0; i < len; i++) {
quantiles[i] = kllItemsSV.getQuantile(ranks[i], searchCrit);
quantiles[i] = itemsSV.getQuantile(ranks[i], searchCrit);
}
return quantiles;
}
Expand All @@ -200,7 +203,7 @@ public T getQuantileUpperBound(final double rank) {
public double getRank(final T quantile, final QuantileSearchCriteria searchCrit) {
if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); }
refreshSortedView();
return kllItemsSV.getRank(quantile, searchCrit);
return itemsSV.getRank(quantile, searchCrit);
}

/**
Expand Down Expand Up @@ -230,16 +233,16 @@ public double[] getRanks(final T[] quantiles, final QuantileSearchCriteria searc
final int len = quantiles.length;
final double[] ranks = new double[len];
for (int i = 0; i < len; i++) {
ranks[i] = kllItemsSV.getRank(quantiles[i], searchCrit);
ranks[i] = itemsSV.getRank(quantiles[i], searchCrit);
}
return ranks;
}

@Override
public final KllItemsSketchSortedView<T> getSortedView() {
public final ItemsSketchSortedView<T> getSortedView() {
if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); }
return refreshSortedView();
//return kllItemsSV; //SpotBugs EI_EXPOSE_REP, Suppressed by FindBugsExcludeFilter
//return itemsSV; //SpotBugs EI_EXPOSE_REP, Suppressed by FindBugsExcludeFilter
}

@Override
Expand All @@ -255,7 +258,7 @@ public final void merge(final KllSketch other) {
final KllItemsSketch<T> othItmSk = (KllItemsSketch<T>)other;
if (othItmSk.isEmpty()) { return; }
KllItemsHelper.mergeItemImpl(this, othItmSk, comparator);
kllItemsSV = null;
itemsSV = null;
}

@Override
Expand All @@ -270,7 +273,7 @@ public void reset() {
setMinItem(null);
setMaxItem(null);
setItemsArray(new Object[k]);
kllItemsSV = null;
itemsSV = null;
}

public byte[] toByteArray() {
Expand All @@ -293,7 +296,7 @@ public void update(final T item) {
if (item == null) { return; } //ignore
if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); }
KllItemsHelper.updateItem(this, item);
kllItemsSV = null;
itemsSV = null;
}

/**
Expand All @@ -307,7 +310,7 @@ public void update(final T item, final long weight) {
if (weight < 1L) { throw new SketchesArgumentException("Weight is less than one."); }
if (weight == 1L) { KllItemsHelper.updateItem(this, item); }
else { KllItemsHelper.updateItem(this, item, weight); }
kllItemsSV = null;
itemsSV = null;
}

//restricted
Expand Down Expand Up @@ -394,19 +397,20 @@ void updateMinMax(final T item) {
}
}

private final KllItemsSketchSortedView<T> refreshSortedView() {
if (kllItemsSV == null) {
private final ItemsSketchSortedView<T> refreshSortedView() {
if (itemsSV == null) {
final CreateSortedView csv = new CreateSortedView();
kllItemsSV = csv.getSV();
itemsSV = csv.getSV();
}
return kllItemsSV;
return itemsSV;
}

@SuppressWarnings({"rawtypes"})
private final class CreateSortedView {
T[] quantiles;
long[] cumWeights;

KllItemsSketchSortedView<T> getSV() {
ItemsSketchSortedView<T> getSV() {
if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); }
if (getN() == 0) { throw new SketchesArgumentException(EMPTY_MSG); }
final T[] srcQuantiles = getTotalItemsArray();
Expand All @@ -417,11 +421,13 @@ KllItemsSketchSortedView<T> getSV() {
Arrays.sort(srcQuantiles, srcLevels[0], srcLevels[1], comparator);
if (!hasMemory()) { setLevelZeroSorted(true); }
}
final int numQuantiles = srcLevels[srcNumLevels] - srcLevels[0]; //remove free space
final int numQuantiles = getNumRetained();
quantiles = (T[]) Array.newInstance(serDe.getClassOfT(), numQuantiles);
cumWeights = new long[numQuantiles];
populateFromSketch(srcQuantiles, srcLevels, srcNumLevels, numQuantiles);
return new KllItemsSketchSortedView<>(quantiles, cumWeights, getN(), comparator, getMaxItem(), getMinItem());
final double normRankErr = getNormalizedRankError(getK(), true);
return new ItemsSketchSortedView(
quantiles, cumWeights, getN(), comparator, getMaxItem(), getMinItem(), normRankErr);
}

private void populateFromSketch(final Object[] srcQuantiles, final int[] srcLevels,
Expand Down
Loading

0 comments on commit c0c20ac

Please sign in to comment.