diff --git a/.github/workflows/cron-job-its.yml b/.github/workflows/cron-job-its.yml index 65471ad81b05..0a5c2f45ce24 100644 --- a/.github/workflows/cron-job-its.yml +++ b/.github/workflows/cron-job-its.yml @@ -34,8 +34,11 @@ jobs: - name: Checkout branch uses: actions/checkout@v3 - - name: Setup java - run: export JAVA_HOME=$JAVA_HOME_8_X64 + - name: setup java + uses: actions/setup-java@v3 + with: + java-version: '8' + distribution: 'zulu' - name: Cache Maven m2 repository id: maven diff --git a/.github/workflows/reusable-revised-its.yml b/.github/workflows/reusable-revised-its.yml index 4558c3434371..60b5261b3038 100644 --- a/.github/workflows/reusable-revised-its.yml +++ b/.github/workflows/reusable-revised-its.yml @@ -77,9 +77,11 @@ jobs: - name: Checkout branch uses: actions/checkout@v3 - - name: Setup java - run: | - echo "JAVA_HOME=$JAVA_HOME_${{ inputs.build_jdk }}_X64" >> $GITHUB_ENV + - name: setup java + uses: actions/setup-java@v3 + with: + java-version: ${{ inputs.build_jdk }} + distribution: 'zulu' - name: Restore Maven repository id: maven-restore @@ -158,7 +160,7 @@ jobs: - name: Collect service logs on failure if: ${{ failure() && steps.run-it.conclusion == 'failure' }} run: | - tar cvzf ./service-logs.tgz ~/shared/logs + tar cvzf ./service-logs.tgz integration-tests-ex/cases/target/${{ inputs.it }}/logs - name: Upload Druid service logs to GitHub if: ${{ failure() && steps.run-it.conclusion == 'failure' }} diff --git a/.github/workflows/reusable-standard-its.yml b/.github/workflows/reusable-standard-its.yml index d0adace22fbe..821ecd625863 100644 --- a/.github/workflows/reusable-standard-its.yml +++ b/.github/workflows/reusable-standard-its.yml @@ -62,9 +62,11 @@ jobs: - name: Checkout branch uses: actions/checkout@v3 - - name: Setup java - run: | - echo "JAVA_HOME=$JAVA_HOME_${{ inputs.runtime_jdk }}_X64" >> $GITHUB_ENV + - name: setup java + uses: actions/setup-java@v3 + with: + java-version: ${{ inputs.runtime_jdk }} + distribution: 'zulu' - name: Restore Maven repository id: maven-restore diff --git a/.github/workflows/standard-its.yml b/.github/workflows/standard-its.yml index ae78a1f2a836..79c6ae0f0742 100644 --- a/.github/workflows/standard-its.yml +++ b/.github/workflows/standard-its.yml @@ -93,7 +93,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [8, 17] + jdk: [8, 17, 21] uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: @@ -150,8 +150,11 @@ jobs: - name: Checkout branch uses: actions/checkout@v3 - - name: Setup java - run: export JAVA_HOME=$JAVA_HOME_8_X64 + - name: setup java + uses: actions/setup-java@v3 + with: + java-version: '8' + distribution: 'zulu' # the build step produces SNAPSHOT artifacts into the local maven repository, # we include github.sha in the cache key to make it specific to that build/jdk diff --git a/examples/bin/run-java b/examples/bin/run-java index ce8baaa5274c..80190d0a793c 100755 --- a/examples/bin/run-java +++ b/examples/bin/run-java @@ -24,7 +24,7 @@ if [ -z "$JAVA_BIN" ]; then exit 1 fi -JAVA_MAJOR="$("$JAVA_BIN" -version 2>&1 | sed -n -E 's/.* version "([^."]*).*/\1/p')" +JAVA_MAJOR="$("$JAVA_BIN" -version 2>&1 | sed -n -E 's/.* version "([^."-]*).*/\1/p')" if [ "$JAVA_MAJOR" != "" ] && [ "$JAVA_MAJOR" -ge "11" ] then diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesAndWorkerTaskRunner.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesAndWorkerTaskRunner.java index 243f6626c664..8c41772aea69 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesAndWorkerTaskRunner.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesAndWorkerTaskRunner.java @@ -283,10 +283,4 @@ public void updateStatus(Task task, TaskStatus status) { kubernetesTaskRunner.updateStatus(task, status); } - - @Override - public void updateLocation(Task task, TaskLocation location) - { - kubernetesTaskRunner.updateLocation(task, location); - } } diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycle.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycle.java index 5c6c7c6b3ebe..ea3c2a19d1c2 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycle.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycle.java @@ -31,6 +31,9 @@ import org.apache.druid.indexer.TaskLocation; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.overlord.TaskRunnerListener; +import org.apache.druid.indexing.overlord.TaskRunnerUtils; +import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.k8s.overlord.common.DruidK8sConstants; @@ -47,6 +50,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -89,6 +94,8 @@ protected enum State private final KubernetesPeonClient kubernetesClient; private final ObjectMapper mapper; private final TaskStateListener stateListener; + private final List> listeners; + @MonotonicNonNull private LogWatch logWatch; @@ -99,7 +106,8 @@ protected KubernetesPeonLifecycle( KubernetesPeonClient kubernetesClient, TaskLogs taskLogs, ObjectMapper mapper, - TaskStateListener stateListener + TaskStateListener stateListener, + List> listeners ) { this.taskId = new K8sTaskId(task); @@ -108,6 +116,7 @@ protected KubernetesPeonLifecycle( this.taskLogs = taskLogs; this.mapper = mapper; this.stateListener = stateListener; + this.listeners = listeners; } /** @@ -178,7 +187,11 @@ protected synchronized TaskStatus join(long timeout) throws IllegalStateExceptio { try { updateState(new State[]{State.NOT_STARTED, State.PENDING}, State.RUNNING); - + TaskRunnerUtils.notifyLocationChanged( + listeners, + task.getId(), + getTaskLocation() + ); JobResponse jobResponse = kubernetesClient.waitForPeonJobCompletion( taskId, timeout, @@ -190,12 +203,14 @@ protected synchronized TaskStatus join(long timeout) throws IllegalStateExceptio finally { try { saveLogs(); + shutdown(); } catch (Exception e) { - log.warn(e, "Log processing failed for task [%s]", taskId); + log.warn(e, "Cleanup failed for task [%s]", taskId); + } + finally { + stopTask(); } - - stopTask(); } } diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleFactory.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleFactory.java index bf4e3a712577..2998f3fc9212 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleFactory.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleFactory.java @@ -21,9 +21,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.overlord.TaskRunnerListener; +import org.apache.druid.java.util.common.Pair; import org.apache.druid.k8s.overlord.common.KubernetesPeonClient; import org.apache.druid.tasklogs.TaskLogs; +import java.util.List; +import java.util.concurrent.Executor; + public class KubernetesPeonLifecycleFactory implements PeonLifecycleFactory { private final KubernetesPeonClient client; @@ -42,14 +47,15 @@ public KubernetesPeonLifecycleFactory( } @Override - public KubernetesPeonLifecycle build(Task task, KubernetesPeonLifecycle.TaskStateListener stateListener) + public KubernetesPeonLifecycle build(Task task, KubernetesPeonLifecycle.TaskStateListener stateListener, List> listeners) { return new KubernetesPeonLifecycle( task, client, taskLogs, mapper, - stateListener + stateListener, + listeners ); } } diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunner.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunner.java index a0a29dcbbb92..56de37c37980 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunner.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunner.java @@ -146,16 +146,20 @@ public Optional streamTaskLog(String taskid, long offset) public ListenableFuture run(Task task) { synchronized (tasks) { - return tasks.computeIfAbsent(task.getId(), k -> new KubernetesWorkItem(task, exec.submit(() -> runTask(task)))) - .getResult(); + return tasks.computeIfAbsent(task.getId(), k -> { + ListenableFuture unused = exec.submit(() -> runTask(task)); + return new KubernetesWorkItem(task); + }).getResult(); } } protected ListenableFuture joinAsync(Task task) { synchronized (tasks) { - return tasks.computeIfAbsent(task.getId(), k -> new KubernetesWorkItem(task, exec.submit(() -> joinTask(task)))) - .getResult(); + return tasks.computeIfAbsent(task.getId(), k -> { + ListenableFuture unused = exec.submit(() -> joinTask(task)); + return new KubernetesWorkItem(task); + }).getResult(); } } @@ -172,10 +176,12 @@ private TaskStatus joinTask(Task task) @VisibleForTesting protected TaskStatus doTask(Task task, boolean run) { + TaskStatus taskStatus = TaskStatus.failure(task.getId(), "Task execution never started"); try { KubernetesPeonLifecycle peonLifecycle = peonLifecycleFactory.build( task, - this::emitTaskStateMetrics + this::emitTaskStateMetrics, + listeners ); synchronized (tasks) { @@ -188,7 +194,6 @@ protected TaskStatus doTask(Task task, boolean run) workItem.setKubernetesPeonLifecycle(peonLifecycle); } - TaskStatus taskStatus; if (run) { taskStatus = peonLifecycle.run( adapter.fromTask(task), @@ -201,15 +206,17 @@ protected TaskStatus doTask(Task task, boolean run) config.getTaskTimeout().toStandardDuration().getMillis() ); } - - updateStatus(task, taskStatus); - return taskStatus; } catch (Exception e) { log.error(e, "Task [%s] execution caught an exception", task.getId()); + taskStatus = TaskStatus.failure(task.getId(), "Could not start task execution"); throw new RuntimeException(e); } + finally { + updateStatus(task, taskStatus); + TaskRunnerUtils.notifyLocationChanged(listeners, task.getId(), TaskLocation.unknown()); + } } @VisibleForTesting @@ -242,13 +249,13 @@ protected void emitTaskStateMetrics(KubernetesPeonLifecycle.State state, String @Override public void updateStatus(Task task, TaskStatus status) { - TaskRunnerUtils.notifyStatusChanged(listeners, task.getId(), status); - } + KubernetesWorkItem workItem = tasks.get(task.getId()); + if (workItem != null && !workItem.getResult().isDone() && status.isComplete()) { + workItem.setResult(status); + } - @Override - public void updateLocation(Task task, TaskLocation location) - { - TaskRunnerUtils.notifyLocationChanged(listeners, task.getId(), location); + // Notify listeners even if the result is set to handle the shutdown case. + TaskRunnerUtils.notifyStatusChanged(listeners, task.getId(), status); } @Override @@ -417,6 +424,16 @@ public void registerListener(TaskRunnerListener listener, Executor executor) final Pair listenerPair = Pair.of(listener, executor); log.debug("Registered listener [%s]", listener.getListenerId()); listeners.add(listenerPair); + + for (Map.Entry entry : tasks.entrySet()) { + if (entry.getValue().isRunning()) { + TaskRunnerUtils.notifyLocationChanged( + ImmutableList.of(listenerPair), + entry.getKey(), + entry.getValue().getLocation() + ); + } + } } @Override diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerConfig.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerConfig.java index 0d67c55b30aa..5fdcd9cfbb71 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerConfig.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerConfig.java @@ -78,7 +78,7 @@ public class KubernetesTaskRunnerConfig @JsonProperty @NotNull // how long to wait for the jobs to be cleaned up. - private Period taskCleanupDelay = new Period("P2D"); + private Period taskCleanupDelay = new Period("PT1H"); @JsonProperty @NotNull diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesWorkItem.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesWorkItem.java index 94d4bbb67f63..b089b4dd2db0 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesWorkItem.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesWorkItem.java @@ -19,9 +19,10 @@ package org.apache.druid.k8s.overlord; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; import org.apache.druid.indexer.RunnerTaskState; import org.apache.druid.indexer.TaskLocation; import org.apache.druid.indexer.TaskStatus; @@ -36,9 +37,18 @@ public class KubernetesWorkItem extends TaskRunnerWorkItem private final Task task; private KubernetesPeonLifecycle kubernetesPeonLifecycle = null; - public KubernetesWorkItem(Task task, ListenableFuture statusFuture) + private final SettableFuture result; + + public KubernetesWorkItem(Task task) + { + this(task, SettableFuture.create()); + } + + @VisibleForTesting + public KubernetesWorkItem(Task task, SettableFuture result) { - super(task.getId(), statusFuture); + super(task.getId(), result); + this.result = result; this.task = task; } @@ -51,7 +61,7 @@ protected synchronized void setKubernetesPeonLifecycle(KubernetesPeonLifecycle k protected synchronized void shutdown() { - if (this.kubernetesPeonLifecycle != null) { + if (this.kubernetesPeonLifecycle != null && !result.isDone()) { this.kubernetesPeonLifecycle.startWatchingLogs(); this.kubernetesPeonLifecycle.shutdown(); } @@ -119,4 +129,9 @@ public Task getTask() { return task; } + + public void setResult(TaskStatus status) + { + result.set(status); + } } diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/PeonLifecycleFactory.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/PeonLifecycleFactory.java index 2a234ebc5786..2b180fb9dac0 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/PeonLifecycleFactory.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/PeonLifecycleFactory.java @@ -20,8 +20,13 @@ package org.apache.druid.k8s.overlord; import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.overlord.TaskRunnerListener; +import org.apache.druid.java.util.common.Pair; + +import java.util.List; +import java.util.concurrent.Executor; public interface PeonLifecycleFactory { - KubernetesPeonLifecycle build(Task task, KubernetesPeonLifecycle.TaskStateListener stateListener); + KubernetesPeonLifecycle build(Task task, KubernetesPeonLifecycle.TaskStateListener stateListener, List> listeners); } diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/JobResponse.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/JobResponse.java index a7a8156468f6..26fe5b98f0bd 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/JobResponse.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/JobResponse.java @@ -32,12 +32,10 @@ public class JobResponse private static final EmittingLogger LOGGER = new EmittingLogger(JobResponse.class); private final Job job; - private final PeonPhase phase; - public JobResponse(@Nullable Job job, PeonPhase phase) + public JobResponse(@Nullable Job job) { this.job = job; - this.phase = phase; } public Job getJob() @@ -45,11 +43,6 @@ public Job getJob() return job; } - public PeonPhase getPhase() - { - return phase; - } - public long getJobDuration() { long duration = -1L; diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/KubernetesPeonClient.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/KubernetesPeonClient.java index 9fdc25fa6455..147ea732d0ca 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/KubernetesPeonClient.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/KubernetesPeonClient.java @@ -101,13 +101,13 @@ public JobResponse waitForPeonJobCompletion(K8sTaskId taskId, long howLong, Time ); if (job == null) { log.info("K8s job for the task [%s] was not found. It can happen if the task was canceled", taskId); - return new JobResponse(null, PeonPhase.FAILED); + return new JobResponse(null); } if (job.getStatus().getSucceeded() != null) { - return new JobResponse(job, PeonPhase.SUCCEEDED); + return new JobResponse(job); } log.warn("Task %s failed with status %s", taskId, job.getStatus()); - return new JobResponse(job, PeonPhase.FAILED); + return new JobResponse(job); }); } diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/PeonPhase.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/PeonPhase.java deleted file mode 100644 index 6efcd34872b8..000000000000 --- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/PeonPhase.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.k8s.overlord.common; - -import io.fabric8.kubernetes.api.model.Pod; - -import java.util.Arrays; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - -public enum PeonPhase -{ - PENDING("Pending"), - SUCCEEDED("Succeeded"), - FAILED("Failed"), - UNKNOWN("Unknown"), - RUNNING("Running"); - - private static final Map PHASE_MAP = Arrays.stream(PeonPhase.values()) - .collect(Collectors.toMap( - PeonPhase::getPhase, - Function.identity() - )); - private final String phase; - - PeonPhase(String phase) - { - this.phase = phase; - } - - public String getPhase() - { - return phase; - } - - public static PeonPhase getPhaseFor(Pod pod) - { - if (pod == null) { - return UNKNOWN; - } - return PHASE_MAP.get(pod.getStatus().getPhase()); - } - -} diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesAndWorkerTaskRunnerTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesAndWorkerTaskRunnerTest.java index af5a6c39bb0b..80010b9e5396 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesAndWorkerTaskRunnerTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesAndWorkerTaskRunnerTest.java @@ -332,13 +332,4 @@ public void test_updateStatus() runner.updateStatus(task, TaskStatus.running(ID)); verifyAll(); } - - @Test - public void test_updateLocation() - { - kubernetesTaskRunner.updateLocation(task, TaskLocation.unknown()); - replayAll(); - runner.updateLocation(task, TaskLocation.unknown()); - verifyAll(); - } } diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleTest.java index 1c6e429a3dc3..e984e449b282 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesPeonLifecycleTest.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodBuilder; import io.fabric8.kubernetes.api.model.batch.v1.Job; @@ -31,11 +32,12 @@ import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.TestUtils; import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.overlord.TaskRunnerListener; +import org.apache.druid.java.util.common.Pair; import org.apache.druid.k8s.overlord.common.JobResponse; import org.apache.druid.k8s.overlord.common.K8sTaskId; import org.apache.druid.k8s.overlord.common.K8sTestUtils; import org.apache.druid.k8s.overlord.common.KubernetesPeonClient; -import org.apache.druid.k8s.overlord.common.PeonPhase; import org.apache.druid.tasklogs.TaskLogs; import org.easymock.EasyMock; import org.easymock.EasyMockRunner; @@ -50,6 +52,8 @@ import java.io.IOException; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -65,6 +69,8 @@ public class KubernetesPeonLifecycleTest extends EasyMockSupport @Mock LogWatch logWatch; @Mock KubernetesPeonLifecycle.TaskStateListener stateListener; + List> listeners = ImmutableList.of(); + private ObjectMapper mapper; private Task task; private K8sTaskId k8sTaskId; @@ -86,7 +92,8 @@ public void test_run() throws IOException kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ) { @Override @@ -131,7 +138,8 @@ public void test_run_useTaskManager() throws IOException kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ) { @Override @@ -175,7 +183,8 @@ public void test_run_whenCalledMultipleTimes_raisesIllegalStateException() throw kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ) { @Override @@ -224,7 +233,8 @@ public void test_run_whenExceptionRaised_setsRunnerTaskStateToNone() kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ) { @Override @@ -268,15 +278,19 @@ public void test_join_withoutJob_returnsFailedTaskStatus() throws IOException kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); EasyMock.expect(kubernetesClient.waitForPeonJobCompletion( EasyMock.eq(k8sTaskId), EasyMock.anyLong(), EasyMock.eq(TimeUnit.MILLISECONDS) - )).andReturn(new JobResponse(null, PeonPhase.FAILED)); + )).andReturn(new JobResponse(null)); EasyMock.expect(kubernetesClient.getPeonLogWatcher(k8sTaskId)).andReturn(Optional.of(logWatch)); + EasyMock.expect(kubernetesClient.getPeonPod(k8sTaskId.getK8sJobName())).andReturn(Optional.absent()); + EasyMock.expect(kubernetesClient.deletePeonJob(k8sTaskId)).andReturn(true); + EasyMock.expect(taskLogs.streamTaskStatus(ID)).andReturn(Optional.absent()); taskLogs.pushTaskLog(EasyMock.eq(ID), EasyMock.anyObject(File.class)); EasyMock.expectLastCall(); @@ -302,12 +316,15 @@ public void test_join_withoutJob_returnsFailedTaskStatus() throws IOException @Test public void test_join() throws IOException { + Executor executor = EasyMock.mock(Executor.class); + TaskRunnerListener taskRunnerListener = EasyMock.mock(TaskRunnerListener.class); KubernetesPeonLifecycle peonLifecycle = new KubernetesPeonLifecycle( task, kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + ImmutableList.of(Pair.of(taskRunnerListener, executor)) ); Job job = new JobBuilder() @@ -325,8 +342,12 @@ public void test_join() throws IOException EasyMock.eq(k8sTaskId), EasyMock.anyLong(), EasyMock.eq(TimeUnit.MILLISECONDS) - )).andReturn(new JobResponse(job, PeonPhase.SUCCEEDED)); + )).andReturn(new JobResponse(job)); EasyMock.expect(kubernetesClient.getPeonLogWatcher(k8sTaskId)).andReturn(Optional.of(logWatch)); + EasyMock.expect(kubernetesClient.getPeonPod(k8sTaskId.getK8sJobName())).andReturn( + Optional.of(new PodBuilder().withNewMetadata().withName("job-pod").endMetadata().withNewStatus().withPodIP("ip").endStatus().build()) + ); + EasyMock.expect(kubernetesClient.deletePeonJob(k8sTaskId)).andReturn(true); EasyMock.expect(taskLogs.streamTaskStatus(ID)).andReturn(Optional.of( IOUtils.toInputStream(mapper.writeValueAsString(SUCCESS), StandardCharsets.UTF_8) )); @@ -338,6 +359,10 @@ public void test_join() throws IOException EasyMock.expectLastCall().once(); logWatch.close(); EasyMock.expectLastCall(); + executor.execute(EasyMock.anyObject()); + EasyMock.expectLastCall(); + taskRunnerListener.locationChanged(EasyMock.anyObject(), EasyMock.anyObject()); + EasyMock.expectLastCall(); Assert.assertEquals(KubernetesPeonLifecycle.State.NOT_STARTED, peonLifecycle.getState()); @@ -359,7 +384,8 @@ public void test_join_whenCalledMultipleTimes_raisesIllegalStateException() thro kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); Job job = new JobBuilder() @@ -375,8 +401,15 @@ public void test_join_whenCalledMultipleTimes_raisesIllegalStateException() thro EasyMock.eq(k8sTaskId), EasyMock.anyLong(), EasyMock.eq(TimeUnit.MILLISECONDS) - )).andReturn(new JobResponse(job, PeonPhase.SUCCEEDED)); + )).andReturn(new JobResponse(job)); EasyMock.expect(kubernetesClient.getPeonLogWatcher(k8sTaskId)).andReturn(Optional.of(logWatch)); + + // Only update the location the first time, second call doesn't reach this point in the logic + EasyMock.expect(kubernetesClient.getPeonPod(k8sTaskId.getK8sJobName())).andReturn( + Optional.of(new PodBuilder().withNewMetadata().withName("job-pod").endMetadata().withNewStatus().withPodIP("ip").endStatus().build()) + ); + // Always try to delete the job + EasyMock.expect(kubernetesClient.deletePeonJob(k8sTaskId)).andReturn(true).times(2); EasyMock.expect(taskLogs.streamTaskStatus(ID)).andReturn( Optional.of(IOUtils.toInputStream(mapper.writeValueAsString(SUCCESS), StandardCharsets.UTF_8)) ); @@ -419,7 +452,8 @@ public void test_join_withoutTaskStatus_returnsFailedTaskStatus() throws IOExcep kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); Job job = new JobBuilder() @@ -435,8 +469,12 @@ public void test_join_withoutTaskStatus_returnsFailedTaskStatus() throws IOExcep EasyMock.eq(k8sTaskId), EasyMock.anyLong(), EasyMock.eq(TimeUnit.MILLISECONDS) - )).andReturn(new JobResponse(job, PeonPhase.SUCCEEDED)); + )).andReturn(new JobResponse(job)); EasyMock.expect(kubernetesClient.getPeonLogWatcher(k8sTaskId)).andReturn(Optional.of(logWatch)); + EasyMock.expect(kubernetesClient.getPeonPod(k8sTaskId.getK8sJobName())).andReturn( + Optional.of(new PodBuilder().withNewMetadata().withName("job-pod").endMetadata().withNewStatus().withPodIP("ip").endStatus().build()) + ); + EasyMock.expect(kubernetesClient.deletePeonJob(k8sTaskId)).andReturn(true); EasyMock.expect(taskLogs.streamTaskStatus(ID)).andReturn(Optional.absent()); taskLogs.pushTaskLog(EasyMock.eq(ID), EasyMock.anyObject(File.class)); EasyMock.expectLastCall(); @@ -469,7 +507,8 @@ public void test_join_whenIOExceptionThrownWhileStreamingTaskStatus_returnsFaile kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); Job job = new JobBuilder() @@ -485,8 +524,12 @@ public void test_join_whenIOExceptionThrownWhileStreamingTaskStatus_returnsFaile EasyMock.eq(k8sTaskId), EasyMock.anyLong(), EasyMock.eq(TimeUnit.MILLISECONDS) - )).andReturn(new JobResponse(job, PeonPhase.SUCCEEDED)); + )).andReturn(new JobResponse(job)); EasyMock.expect(kubernetesClient.getPeonLogWatcher(k8sTaskId)).andReturn(Optional.of(logWatch)); + EasyMock.expect(kubernetesClient.getPeonPod(k8sTaskId.getK8sJobName())).andReturn( + Optional.of(new PodBuilder().withNewMetadata().withName("job-pod").endMetadata().withNewStatus().withPodIP("ip").endStatus().build()) + ); + EasyMock.expect(kubernetesClient.deletePeonJob(k8sTaskId)).andReturn(true); EasyMock.expect(taskLogs.streamTaskStatus(ID)).andThrow(new IOException()); taskLogs.pushTaskLog(EasyMock.eq(ID), EasyMock.anyObject(File.class)); EasyMock.expectLastCall(); @@ -519,7 +562,8 @@ public void test_join_whenIOExceptionThrownWhileStreamingTaskLogs_isIgnored() th kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); Job job = new JobBuilder() @@ -535,8 +579,11 @@ public void test_join_whenIOExceptionThrownWhileStreamingTaskLogs_isIgnored() th EasyMock.eq(k8sTaskId), EasyMock.anyLong(), EasyMock.eq(TimeUnit.MILLISECONDS) - )).andReturn(new JobResponse(job, PeonPhase.SUCCEEDED)); + )).andReturn(new JobResponse(job)); EasyMock.expect(kubernetesClient.getPeonLogWatcher(k8sTaskId)).andReturn(Optional.of(logWatch)); + EasyMock.expect(kubernetesClient.getPeonPod(k8sTaskId.getK8sJobName())).andReturn( + Optional.of(new PodBuilder().withNewMetadata().withName("job-pod").endMetadata().withNewStatus().withPodIP("ip").endStatus().build()) + ); EasyMock.expect(taskLogs.streamTaskStatus(ID)).andReturn( Optional.of(IOUtils.toInputStream(mapper.writeValueAsString(SUCCESS), StandardCharsets.UTF_8)) ); @@ -549,6 +596,9 @@ public void test_join_whenIOExceptionThrownWhileStreamingTaskLogs_isIgnored() th logWatch.close(); EasyMock.expectLastCall(); + // We should still try to cleanup the Job after + EasyMock.expect(kubernetesClient.deletePeonJob(k8sTaskId)).andReturn(true); + Assert.assertEquals(KubernetesPeonLifecycle.State.NOT_STARTED, peonLifecycle.getState()); replayAll(); @@ -569,7 +619,8 @@ public void test_join_whenRuntimeExceptionThrownWhileWaitingForKubernetesJob_thr kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); EasyMock.expect(kubernetesClient.waitForPeonJobCompletion( @@ -578,6 +629,9 @@ public void test_join_whenRuntimeExceptionThrownWhileWaitingForKubernetesJob_thr EasyMock.eq(TimeUnit.MILLISECONDS) )).andThrow(new RuntimeException()); + EasyMock.expect(kubernetesClient.getPeonPod(k8sTaskId.getK8sJobName())).andReturn( + Optional.of(new PodBuilder().withNewMetadata().withName("job-pod").endMetadata().withNewStatus().withPodIP("ip").endStatus().build()) + ); // We should still try to push logs EasyMock.expect(kubernetesClient.getPeonLogWatcher(k8sTaskId)).andReturn(Optional.of(logWatch)); taskLogs.pushTaskLog(EasyMock.eq(ID), EasyMock.anyObject(File.class)); @@ -588,7 +642,7 @@ public void test_join_whenRuntimeExceptionThrownWhileWaitingForKubernetesJob_thr EasyMock.expectLastCall().once(); logWatch.close(); EasyMock.expectLastCall(); - + EasyMock.expect(kubernetesClient.deletePeonJob(k8sTaskId)).andReturn(true); Assert.assertEquals(KubernetesPeonLifecycle.State.NOT_STARTED, peonLifecycle.getState()); replayAll(); @@ -608,7 +662,8 @@ public void test_shutdown_withNotStartedTaskState() kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); peonLifecycle.shutdown(); } @@ -621,7 +676,8 @@ public void test_shutdown_withPendingTaskState() throws NoSuchFieldException, Il kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); setPeonLifecycleState(peonLifecycle, KubernetesPeonLifecycle.State.PENDING); @@ -642,7 +698,8 @@ public void test_shutdown_withRunningTaskState() throws NoSuchFieldException, Il kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); setPeonLifecycleState(peonLifecycle, KubernetesPeonLifecycle.State.RUNNING); @@ -663,7 +720,8 @@ public void test_shutdown_withStoppedTaskState() throws NoSuchFieldException, Il kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); setPeonLifecycleState(peonLifecycle, KubernetesPeonLifecycle.State.STOPPED); @@ -678,7 +736,8 @@ public void test_streamLogs_withNotStartedTaskState() throws NoSuchFieldExceptio kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); setPeonLifecycleState(peonLifecycle, KubernetesPeonLifecycle.State.NOT_STARTED); @@ -693,7 +752,8 @@ public void test_streamLogs_withPendingTaskState() throws NoSuchFieldException, kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); setPeonLifecycleState(peonLifecycle, KubernetesPeonLifecycle.State.PENDING); @@ -708,7 +768,8 @@ public void test_streamLogs_withRunningTaskState() throws NoSuchFieldException, kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); setPeonLifecycleState(peonLifecycle, KubernetesPeonLifecycle.State.RUNNING); @@ -731,7 +792,8 @@ public void test_streamLogs_withStoppedTaskState() throws NoSuchFieldException, kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); setPeonLifecycleState(peonLifecycle, KubernetesPeonLifecycle.State.STOPPED); @@ -747,7 +809,8 @@ public void test_getTaskLocation_withNotStartedTaskState_returnsUnknown() kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); setPeonLifecycleState(peonLifecycle, KubernetesPeonLifecycle.State.NOT_STARTED); @@ -763,7 +826,8 @@ public void test_getTaskLocation_withPendingTaskState_returnsUnknown() kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); setPeonLifecycleState(peonLifecycle, KubernetesPeonLifecycle.State.PENDING); @@ -779,7 +843,8 @@ public void test_getTaskLocation_withRunningTaskState_withoutPeonPod_returnsUnkn kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); setPeonLifecycleState(peonLifecycle, KubernetesPeonLifecycle.State.RUNNING); @@ -801,7 +866,8 @@ public void test_getTaskLocation_withRunningTaskState_withPeonPodWithoutStatus_r kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); setPeonLifecycleState(peonLifecycle, KubernetesPeonLifecycle.State.RUNNING); @@ -829,7 +895,8 @@ public void test_getTaskLocation_withRunningTaskState_withPeonPodWithStatus_retu kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); setPeonLifecycleState(peonLifecycle, KubernetesPeonLifecycle.State.RUNNING); @@ -865,7 +932,8 @@ public void test_getTaskLocation_saveTaskLocation() kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); setPeonLifecycleState(peonLifecycle, KubernetesPeonLifecycle.State.RUNNING); @@ -901,7 +969,8 @@ public void test_getTaskLocation_withRunningTaskState_withPeonPodWithStatusWithT kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); setPeonLifecycleState(peonLifecycle, KubernetesPeonLifecycle.State.RUNNING); @@ -938,7 +1007,8 @@ public void test_getTaskLocation_withStoppedTaskState_returnsUnknown() kubernetesClient, taskLogs, mapper, - stateListener + stateListener, + listeners ); setPeonLifecycleState(peonLifecycle, KubernetesPeonLifecycle.State.STOPPED); EasyMock.expect(kubernetesClient.getPeonPod(k8sTaskId.getK8sJobName())).andReturn(Optional.absent()).once(); diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerConfigTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerConfigTest.java index 1f4a7281f649..579b7539d818 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerConfigTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerConfigTest.java @@ -47,7 +47,7 @@ public void test_deserializable() throws IOException Assert.assertNull(config.getGraceTerminationPeriodSeconds()); Assert.assertTrue(config.isDisableClientProxy()); Assert.assertEquals(new Period("PT4H"), config.getTaskTimeout()); - Assert.assertEquals(new Period("P2D"), config.getTaskCleanupDelay()); + Assert.assertEquals(new Period("PT1H"), config.getTaskCleanupDelay()); Assert.assertEquals(new Period("PT10m"), config.getTaskCleanupInterval()); Assert.assertEquals(new Period("PT1H"), config.getTaskLaunchTimeout()); Assert.assertEquals(ImmutableList.of(), config.getPeonMonitors()); @@ -72,7 +72,7 @@ public void test_builder_preservesDefaults() Assert.assertNull(config.getGraceTerminationPeriodSeconds()); Assert.assertTrue(config.isDisableClientProxy()); Assert.assertEquals(new Period("PT4H"), config.getTaskTimeout()); - Assert.assertEquals(new Period("P2D"), config.getTaskCleanupDelay()); + Assert.assertEquals(new Period("PT1H"), config.getTaskCleanupDelay()); Assert.assertEquals(new Period("PT10m"), config.getTaskCleanupInterval()); Assert.assertEquals(new Period("PT1H"), config.getTaskLaunchTimeout()); Assert.assertEquals(ImmutableList.of(), config.getPeonMonitors()); diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerTest.java index 36a7b4cfcd9c..e04ef6300362 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerTest.java @@ -28,8 +28,10 @@ import org.apache.commons.io.IOUtils; import org.apache.druid.indexer.RunnerTaskState; import org.apache.druid.indexer.TaskLocation; +import org.apache.druid.indexer.TaskState; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.overlord.TaskRunnerListener; import org.apache.druid.indexing.overlord.TaskRunnerWorkItem; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.java.util.emitter.service.ServiceEventBuilder; @@ -76,6 +78,9 @@ public class KubernetesTaskRunnerTest extends EasyMockSupport @Mock private KubernetesPeonLifecycle kubernetesPeonLifecycle; @Mock private ServiceEmitter emitter; + @Mock private Executor executor; + @Mock private TaskRunnerListener taskRunnerListener; + private KubernetesTaskRunnerConfig config; private KubernetesTaskRunner runner; private Task task; @@ -116,11 +121,7 @@ protected ListenableFuture joinAsync(Task task) { return tasks.computeIfAbsent( task.getId(), - k -> new KubernetesWorkItem( - task, - Futures.immediateFuture(TaskStatus.success(task.getId())) - ) - ).getResult(); + k -> new KubernetesWorkItem(task)).getResult(); } }; @@ -249,7 +250,7 @@ public void test_run_withExistingTask_returnsExistingWorkItem() } @Test - public void test_run_whenExceptionThrown_throwsRuntimeException() throws IOException + public void test_run_whenExceptionThrown_throwsRuntimeException() throws Exception { Job job = new JobBuilder() .withNewMetadata() @@ -269,11 +270,89 @@ public void test_run_whenExceptionThrown_throwsRuntimeException() throws IOExcep replayAll(); ListenableFuture future = runner.run(task); + TaskStatus taskStatus = future.get(); + Assert.assertEquals(TaskState.FAILED, taskStatus.getStatusCode()); + Assert.assertEquals("Could not start task execution", taskStatus.getErrorMsg()); + verifyAll(); + } - Exception e = Assert.assertThrows(ExecutionException.class, future::get); - Assert.assertTrue(e.getCause() instanceof RuntimeException); + @Test + public void test_run_updateStatus() throws ExecutionException, InterruptedException + { + KubernetesTaskRunner runner = new KubernetesTaskRunner( + taskAdapter, + config, + peonClient, + httpClient, + new TestPeonLifecycleFactory(kubernetesPeonLifecycle), + emitter + ); + KubernetesWorkItem workItem = new KubernetesWorkItem(task); + runner.tasks.put(task.getId(), workItem); + TaskStatus completeTaskStatus = TaskStatus.success(task.getId()); + + replayAll(); + runner.updateStatus(task, completeTaskStatus); verifyAll(); + + assertTrue(workItem.getResult().isDone()); + assertEquals(completeTaskStatus, workItem.getResult().get()); + } + + @Test + public void test_run_updateStatus_running() + { + KubernetesTaskRunner runner = new KubernetesTaskRunner( + taskAdapter, + config, + peonClient, + httpClient, + new TestPeonLifecycleFactory(kubernetesPeonLifecycle), + emitter + ); + KubernetesWorkItem workItem = new KubernetesWorkItem(task); + runner.tasks.put(task.getId(), workItem); + TaskStatus runningTaskStatus = TaskStatus.running(task.getId()); + + replayAll(); + runner.updateStatus(task, runningTaskStatus); + verifyAll(); + + assertFalse(workItem.getResult().isDone()); + } + + @Test + public void test_registerListener_runningTask() + { + KubernetesTaskRunner runner = new KubernetesTaskRunner( + taskAdapter, + config, + peonClient, + httpClient, + new TestPeonLifecycleFactory(kubernetesPeonLifecycle), + emitter + ); + + KubernetesPeonLifecycle runningKubernetesPeonLifecycle = EasyMock.mock(KubernetesPeonLifecycle.class); + EasyMock.expect(runningKubernetesPeonLifecycle.getState()).andReturn(KubernetesPeonLifecycle.State.RUNNING); + EasyMock.expect(runningKubernetesPeonLifecycle.getTaskLocation()).andReturn(TaskLocation.unknown()); + KubernetesWorkItem workItem = new KubernetesWorkItem(task); + workItem.setKubernetesPeonLifecycle(runningKubernetesPeonLifecycle); + runner.tasks.put(task.getId(), workItem); + + Executor executor = EasyMock.mock(Executor.class); + TaskRunnerListener taskRunnerListener = EasyMock.mock(TaskRunnerListener.class); + executor.execute(EasyMock.anyObject()); + EasyMock.expectLastCall(); + taskRunnerListener.locationChanged(EasyMock.anyObject(), EasyMock.anyObject()); + EasyMock.expectLastCall(); + + replayAll(); + EasyMock.replay(runningKubernetesPeonLifecycle); + runner.registerListener(taskRunnerListener, executor); + verifyAll(); + EasyMock.verify(runningKubernetesPeonLifecycle); } @Test @@ -303,16 +382,15 @@ public void test_join_withExistingTask_returnsExistingWorkItem() } @Test - public void test_join_whenExceptionThrown_throwsRuntimeException() + public void test_join_whenExceptionThrown_throwsRuntimeException() throws ExecutionException, InterruptedException { EasyMock.expect(kubernetesPeonLifecycle.join(EasyMock.anyLong())).andThrow(new IllegalStateException()); replayAll(); ListenableFuture future = runner.joinAsync(task); - - Exception e = Assert.assertThrows(ExecutionException.class, future::get); - Assert.assertTrue(e.getCause() instanceof RuntimeException); + TaskStatus taskStatus = future.get(); + Assert.assertEquals(TaskState.FAILED, taskStatus.getStatusCode()); verifyAll(); } diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesWorkItemTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesWorkItemTest.java index 7d17193b1714..f2f398658e05 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesWorkItemTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesWorkItemTest.java @@ -56,6 +56,7 @@ public void test_setKubernetesPeonLifecycleTwice_throwsIllegalStateException() null, null, null, + null, null )); @@ -66,6 +67,7 @@ public void test_setKubernetesPeonLifecycleTwice_throwsIllegalStateException() null, null, null, + null, null )) ); @@ -80,6 +82,8 @@ public void test_shutdown_withoutKubernetesPeonLifecycle() @Test public void test_shutdown_withKubernetesPeonLifecycle() { + KubernetesWorkItem workItem = new KubernetesWorkItem(task); + kubernetesPeonLifecycle.shutdown(); EasyMock.expectLastCall(); kubernetesPeonLifecycle.startWatchingLogs(); @@ -87,7 +91,6 @@ public void test_shutdown_withKubernetesPeonLifecycle() replayAll(); workItem.setKubernetesPeonLifecycle(kubernetesPeonLifecycle); - workItem.shutdown(); verifyAll(); } @@ -158,6 +161,7 @@ public void test_getRunnerTaskState_withKubernetesPeonLifecycle_returnsPending() null, null, null, + null, null )); @@ -172,6 +176,7 @@ public void test_getRunnerTaskState_withKubernetesPeonLifecycle_inPendingState_r null, null, null, + null, null ) { @Override @@ -194,6 +199,7 @@ public void test_getRunnerTaskState_withKubernetesPeonLifecycle_inRunningState_r null, null, null, + null, null ) { @Override @@ -216,6 +222,7 @@ public void test_getRunnerTaskState_withKubernetesPeonLifecycle_inStoppedState_r null, null, null, + null, null ) { @Override @@ -244,6 +251,7 @@ public void test_streamTaskLogs_withKubernetesPeonLifecycle() null, null, null, + null, null )); Assert.assertFalse(workItem.streamTaskLogs().isPresent()); @@ -263,6 +271,7 @@ public void test_getLocation_withKubernetesPeonLifecycle() null, null, null, + null, null )); diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/TestPeonLifecycleFactory.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/TestPeonLifecycleFactory.java index 8b8c43c0d71c..fa0f79bc1d20 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/TestPeonLifecycleFactory.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/TestPeonLifecycleFactory.java @@ -20,6 +20,11 @@ package org.apache.druid.k8s.overlord; import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.overlord.TaskRunnerListener; +import org.apache.druid.java.util.common.Pair; + +import java.util.List; +import java.util.concurrent.Executor; public class TestPeonLifecycleFactory implements PeonLifecycleFactory { @@ -31,7 +36,7 @@ public TestPeonLifecycleFactory(KubernetesPeonLifecycle kubernetesPeonLifecycle) } @Override - public KubernetesPeonLifecycle build(Task task, KubernetesPeonLifecycle.TaskStateListener stateListener) + public KubernetesPeonLifecycle build(Task task, KubernetesPeonLifecycle.TaskStateListener stateListener, List> listeners) { return kubernetesPeonLifecycle; } diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/JobResponseTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/JobResponseTest.java index 2e2043578aa1..cf9f345fd7a5 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/JobResponseTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/JobResponseTest.java @@ -39,7 +39,7 @@ void testCompletionTime() .endStatus() .build(); - JobResponse response = new JobResponse(job, PeonPhase.SUCCEEDED); + JobResponse response = new JobResponse(job); Assertions.assertEquals(58000L, response.getJobDuration()); } @@ -56,7 +56,7 @@ void testNoDuration() .endStatus() .build(); - JobResponse response = new JobResponse(job, PeonPhase.SUCCEEDED); + JobResponse response = new JobResponse(job); Assertions.assertEquals(-1, response.getJobDuration()); } @@ -70,7 +70,7 @@ void testMakingCodeCoverageHappy() .endMetadata() .build(); - JobResponse response = new JobResponse(job, PeonPhase.SUCCEEDED); + JobResponse response = new JobResponse(job); Assertions.assertEquals(-1, response.getJobDuration()); } @@ -78,7 +78,7 @@ void testMakingCodeCoverageHappy() @Test void testNullJob() { - JobResponse response = new JobResponse(null, PeonPhase.SUCCEEDED); + JobResponse response = new JobResponse(null); long duration = response.getJobDuration(); Assertions.assertEquals(-1, duration); } diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/KubernetesPeonClientTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/KubernetesPeonClientTest.java index f6096b675d6c..cde7faa473bc 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/KubernetesPeonClientTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/KubernetesPeonClientTest.java @@ -153,7 +153,6 @@ void test_waitForPeonJobCompletion_withSuccessfulJob_returnsJobResponseWithJobAn TimeUnit.SECONDS ); - Assertions.assertEquals(PeonPhase.SUCCEEDED, jobResponse.getPhase()); Assertions.assertNotNull(jobResponse.getJob()); } @@ -178,7 +177,6 @@ void test_waitForPeonJobCompletion_withFailedJob_returnsJobResponseWithJobAndFai TimeUnit.SECONDS ); - Assertions.assertEquals(PeonPhase.FAILED, jobResponse.getPhase()); Assertions.assertNotNull(jobResponse.getJob()); } @@ -191,7 +189,6 @@ void test_waitforPeonJobCompletion_withoutRunningJob_returnsJobResponseWithEmpty TimeUnit.SECONDS ); - Assertions.assertEquals(PeonPhase.FAILED, jobResponse.getPhase()); Assertions.assertNull(jobResponse.getJob()); } diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/DruidPeonClientIntegrationTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/DruidPeonClientIntegrationTest.java index 098161685883..2cc5bf15c65a 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/DruidPeonClientIntegrationTest.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/DruidPeonClientIntegrationTest.java @@ -38,13 +38,11 @@ import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.k8s.overlord.KubernetesTaskRunnerConfig; import org.apache.druid.k8s.overlord.common.DruidKubernetesClient; -import org.apache.druid.k8s.overlord.common.JobResponse; import org.apache.druid.k8s.overlord.common.K8sTaskId; import org.apache.druid.k8s.overlord.common.K8sTestUtils; import org.apache.druid.k8s.overlord.common.KubernetesClientApi; import org.apache.druid.k8s.overlord.common.KubernetesPeonClient; import org.apache.druid.k8s.overlord.common.PeonCommandContext; -import org.apache.druid.k8s.overlord.common.PeonPhase; import org.apache.druid.server.DruidNode; import org.apache.druid.server.log.StartupLoggingConfig; import org.junit.jupiter.api.BeforeEach; @@ -184,9 +182,8 @@ public void testDeployingSomethingToKind(@TempDir Path tempDir) throws Exception assertEquals(task, taskFromPod); - JobResponse jobStatusResult = peonClient.waitForPeonJobCompletion(taskId, 2, TimeUnit.MINUTES); + peonClient.waitForPeonJobCompletion(taskId, 2, TimeUnit.MINUTES); thread.join(); - assertEquals(PeonPhase.SUCCEEDED, jobStatusResult.getPhase()); // as long as there were no exceptions we are good! assertEquals(expectedLogs, actualLogs); // cleanup my job diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedEphemeralOutput.yaml b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedEphemeralOutput.yaml index 30960cdbc668..741a032eb6c9 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedEphemeralOutput.yaml +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedEphemeralOutput.yaml @@ -62,4 +62,4 @@ spec: ephemeral-storage: 1Gi hostname: "id-3e70afe5cd823dfc7dd308eea616426b" restartPolicy: "Never" - ttlSecondsAfterFinished: 172800 \ No newline at end of file + ttlSecondsAfterFinished: 3600 \ No newline at end of file diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedMultiContainerOutput.yaml b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedMultiContainerOutput.yaml index 70b8b7c1d242..73f31ddfb508 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedMultiContainerOutput.yaml +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedMultiContainerOutput.yaml @@ -105,4 +105,4 @@ spec: name: "graveyard" - emptyDir: {} name: "kubexit" - ttlSecondsAfterFinished: 172800 + ttlSecondsAfterFinished: 3600 diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedMultiContainerOutputOrder.yaml b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedMultiContainerOutputOrder.yaml index 70b8b7c1d242..73f31ddfb508 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedMultiContainerOutputOrder.yaml +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedMultiContainerOutputOrder.yaml @@ -105,4 +105,4 @@ spec: name: "graveyard" - emptyDir: {} name: "kubexit" - ttlSecondsAfterFinished: 172800 + ttlSecondsAfterFinished: 3600 diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJob.yaml b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJob.yaml index 2cef837f3972..004fed9585af 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJob.yaml +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJob.yaml @@ -16,7 +16,7 @@ metadata: spec: activeDeadlineSeconds: 14400 backoffLimit: 0 - ttlSecondsAfterFinished: 172800 + ttlSecondsAfterFinished: 3600 template: metadata: labels: diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobLongIds.yaml b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobLongIds.yaml index cf16c49c5db1..b6ca8a2cefe6 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobLongIds.yaml +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobLongIds.yaml @@ -16,7 +16,7 @@ metadata: spec: activeDeadlineSeconds: 14400 backoffLimit: 0 - ttlSecondsAfterFinished: 172800 + ttlSecondsAfterFinished: 3600 template: metadata: labels: diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobNoTaskJson.yaml b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobNoTaskJson.yaml index d72d0ef37b03..8ecdaf50b012 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobNoTaskJson.yaml +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobNoTaskJson.yaml @@ -16,7 +16,7 @@ metadata: spec: activeDeadlineSeconds: 14400 backoffLimit: 0 - ttlSecondsAfterFinished: 172800 + ttlSecondsAfterFinished: 3600 template: metadata: labels: diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobTlsEnabled.yaml b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobTlsEnabled.yaml index a230ac913a60..547887e90847 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobTlsEnabled.yaml +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobTlsEnabled.yaml @@ -16,7 +16,7 @@ metadata: spec: activeDeadlineSeconds: 14400 backoffLimit: 0 - ttlSecondsAfterFinished: 172800 + ttlSecondsAfterFinished: 3600 template: metadata: labels: diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedPodSpec.yaml b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedPodSpec.yaml index e46de1337883..ecd9416c563a 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedPodSpec.yaml +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedPodSpec.yaml @@ -104,4 +104,4 @@ spec: name: "graveyard" - emptyDir: {} name: "kubexit" - ttlSecondsAfterFinished: 172800 + ttlSecondsAfterFinished: 3600 diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedSingleContainerOutput.yaml b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedSingleContainerOutput.yaml index f270368fb552..7afc393c56af 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedSingleContainerOutput.yaml +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedSingleContainerOutput.yaml @@ -57,4 +57,4 @@ spec: cpu: "1000m" memory: "2400000000" restartPolicy: "Never" - ttlSecondsAfterFinished: 172800 \ No newline at end of file + ttlSecondsAfterFinished: 3600 \ No newline at end of file diff --git a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java index ac2738534da9..d0a035be17c2 100644 --- a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java +++ b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java @@ -40,6 +40,7 @@ import org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.JodaUtils; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.StringUtils; @@ -364,7 +365,7 @@ Pair, Map>> checkSegment // Pair max(created_date), interval -> list> Pair, Map>> baseSegmentsSnapshot = getMaxCreateDateAndBaseSegments( - metadataStorageCoordinator.retrieveUsedSegmentsAndCreatedDates(spec.getBaseDataSource()) + metadataStorageCoordinator.retrieveUsedSegmentsAndCreatedDates(spec.getBaseDataSource(), Intervals.ETERNITY) ); // baseSegments are used to create HadoopIndexTask Map> baseSegments = baseSegmentsSnapshot.rhs; diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java index c108c7d679e9..f2260b055a96 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java @@ -47,6 +47,7 @@ import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.discovery.BrokerClient; +import org.apache.druid.error.DruidException; import org.apache.druid.frame.allocation.ArenaMemoryAllocator; import org.apache.druid.frame.channel.FrameChannelSequence; import org.apache.druid.frame.key.ClusterBy; @@ -69,7 +70,10 @@ import org.apache.druid.indexing.common.actions.LockReleaseAction; import org.apache.druid.indexing.common.actions.MarkSegmentsAsUnusedAction; import org.apache.druid.indexing.common.actions.SegmentAllocateAction; +import org.apache.druid.indexing.common.actions.SegmentTransactionalAppendAction; import org.apache.druid.indexing.common.actions.SegmentTransactionalInsertAction; +import org.apache.druid.indexing.common.actions.SegmentTransactionalReplaceAction; +import org.apache.druid.indexing.common.actions.TaskAction; import org.apache.druid.indexing.common.actions.TaskActionClient; import org.apache.druid.indexing.common.task.batch.parallel.TombstoneHelper; import org.apache.druid.indexing.overlord.SegmentPublishResult; @@ -962,7 +966,11 @@ private List generateSegmentIdsWithShardSpecs( ); } else { final RowKeyReader keyReader = clusterBy.keyReader(signature); - return generateSegmentIdsWithShardSpecsForAppend(destination, partitionBoundaries, keyReader); + return generateSegmentIdsWithShardSpecsForAppend( + destination, + partitionBoundaries, + keyReader, + MultiStageQueryContext.validateAndGetTaskLockType(QueryContext.of(task.getQuerySpec().getQuery().getContext()), false)); } } @@ -972,7 +980,8 @@ private List generateSegmentIdsWithShardSpecs( private List generateSegmentIdsWithShardSpecsForAppend( final DataSourceMSQDestination destination, final ClusterByPartitions partitionBoundaries, - final RowKeyReader keyReader + final RowKeyReader keyReader, + final TaskLockType taskLockType ) throws IOException { final Granularity segmentGranularity = destination.getSegmentGranularity(); @@ -998,7 +1007,7 @@ private List generateSegmentIdsWithShardSpecsForAppend( false, NumberedPartialShardSpec.instance(), LockGranularity.TIME_CHUNK, - TaskLockType.SHARED + taskLockType ) ); } @@ -1399,6 +1408,10 @@ private void publishAllSegments(final Set segments) throws IOExcept (DataSourceMSQDestination) task.getQuerySpec().getDestination(); final Set segmentsWithTombstones = new HashSet<>(segments); int numTombstones = 0; + final TaskLockType taskLockType = MultiStageQueryContext.validateAndGetTaskLockType( + QueryContext.of(task.getQuerySpec().getQuery().getContext()), + destination.isReplaceTimeChunks() + ); if (destination.isReplaceTimeChunks()) { final List intervalsToDrop = findIntervalsToDrop(Preconditions.checkNotNull(segments, "segments")); @@ -1441,7 +1454,7 @@ private void publishAllSegments(final Set segments) throws IOExcept } performSegmentPublish( context.taskActionClient(), - SegmentTransactionalInsertAction.overwriteAction(null, segmentsWithTombstones) + createOverwriteAction(taskLockType, segmentsWithTombstones) ); } } else if (!segments.isEmpty()) { @@ -1458,7 +1471,7 @@ private void publishAllSegments(final Set segments) throws IOExcept // Append mode. performSegmentPublish( context.taskActionClient(), - SegmentTransactionalInsertAction.appendAction(segments, null, null) + createAppendAction(segments, taskLockType) ); } @@ -1467,6 +1480,34 @@ private void publishAllSegments(final Set segments) throws IOExcept task.emitMetric(context.emitter(), "ingest/segments/count", segmentsWithTombstones.size()); } + private static TaskAction createAppendAction( + Set segments, + TaskLockType taskLockType + ) + { + if (taskLockType.equals(TaskLockType.APPEND)) { + return SegmentTransactionalAppendAction.forSegments(segments); + } else if (taskLockType.equals(TaskLockType.SHARED)) { + return SegmentTransactionalInsertAction.appendAction(segments, null, null); + } else { + throw DruidException.defensive("Invalid lock type [%s] received for append action", taskLockType); + } + } + + private TaskAction createOverwriteAction( + TaskLockType taskLockType, + Set segmentsWithTombstones + ) + { + if (taskLockType.equals(TaskLockType.REPLACE)) { + return SegmentTransactionalReplaceAction.create(segmentsWithTombstones); + } else if (taskLockType.equals(TaskLockType.EXCLUSIVE)) { + return SegmentTransactionalInsertAction.overwriteAction(null, segmentsWithTombstones); + } else { + throw DruidException.defensive("Invalid lock type [%s] received for overwrite action", taskLockType); + } + } + /** * When doing an ingestion with {@link DataSourceMSQDestination#isReplaceTimeChunks()}, finds intervals * containing data that should be dropped. @@ -2282,7 +2323,7 @@ private static Map copyOfStageRuntimesEndingAtCurrentTime( */ static void performSegmentPublish( final TaskActionClient client, - final SegmentTransactionalInsertAction action + final TaskAction action ) throws IOException { try { diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQControllerTask.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQControllerTask.java index 43967e7d748a..3cdf706ba163 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQControllerTask.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQControllerTask.java @@ -51,6 +51,8 @@ import org.apache.druid.msq.indexing.destination.DataSourceMSQDestination; import org.apache.druid.msq.indexing.destination.DurableStorageMSQDestination; import org.apache.druid.msq.indexing.destination.MSQDestination; +import org.apache.druid.msq.util.MultiStageQueryContext; +import org.apache.druid.query.QueryContext; import org.apache.druid.rpc.ServiceClientFactory; import org.apache.druid.rpc.StandardRetryPolicy; import org.apache.druid.rpc.indexing.OverlordClient; @@ -204,12 +206,19 @@ public boolean isReady(TaskActionClient taskActionClient) throws Exception { // If we're in replace mode, acquire locks for all intervals before declaring the task ready. if (isIngestion(querySpec) && ((DataSourceMSQDestination) querySpec.getDestination()).isReplaceTimeChunks()) { + final TaskLockType taskLockType = + MultiStageQueryContext.validateAndGetTaskLockType(QueryContext.of(querySpec.getQuery().getContext()), true); final List intervals = ((DataSourceMSQDestination) querySpec.getDestination()).getReplaceTimeChunks(); - log.debug("Task[%s] trying to acquire[%s] locks for intervals[%s] to become ready", getId(), TaskLockType.EXCLUSIVE, intervals); + log.debug( + "Task[%s] trying to acquire[%s] locks for intervals[%s] to become ready", + getId(), + taskLockType, + intervals + ); for (final Interval interval : intervals) { final TaskLock taskLock = - taskActionClient.submit(new TimeChunkLockTryAcquireAction(TaskLockType.EXCLUSIVE, interval)); + taskActionClient.submit(new TimeChunkLockTryAcquireAction(taskLockType, interval)); if (taskLock == null) { return false; diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/input/external/ExternalSegment.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/input/external/ExternalSegment.java index f53c2cf7c540..93f24cbdff6d 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/input/external/ExternalSegment.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/input/external/ExternalSegment.java @@ -45,6 +45,7 @@ public class ExternalSegment extends RowBasedSegment { private final InputSource inputSource; + private final RowSignature signature; public static final String SEGMENT_ID = "__external"; /** @@ -145,6 +146,7 @@ public void cleanup(CloseableIterator iterFromMake) signature ); this.inputSource = inputSource; + this.signature = signature; } /** @@ -154,4 +156,12 @@ public InputSource externalInputSource() { return inputSource; } + + /** + * Returns the signature of the external input source + */ + public RowSignature signature() + { + return signature; + } } diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ExternalColumnSelectorFactory.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ExternalColumnSelectorFactory.java index fbe82c240dbf..fc9f59ad32c6 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ExternalColumnSelectorFactory.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ExternalColumnSelectorFactory.java @@ -21,6 +21,8 @@ import org.apache.druid.data.input.InputSource; import org.apache.druid.java.util.common.parsers.ParseException; +import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.math.expr.ExpressionType; import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.query.filter.ValueMatcher; @@ -32,6 +34,7 @@ import org.apache.druid.segment.RowIdSupplier; import org.apache.druid.segment.SimpleSettableOffset; import org.apache.druid.segment.column.ColumnCapabilities; +import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.data.IndexedInts; import javax.annotation.Nullable; @@ -48,16 +51,19 @@ public class ExternalColumnSelectorFactory implements ColumnSelectorFactory private final ColumnSelectorFactory delegate; private final InputSource inputSource; + private final RowSignature rowSignature; private final SimpleSettableOffset offset; public ExternalColumnSelectorFactory( final ColumnSelectorFactory delgate, final InputSource inputSource, + final RowSignature rowSignature, final SimpleSettableOffset offset ) { this.delegate = delgate; this.inputSource = inputSource; + this.rowSignature = rowSignature; this.offset = offset; } @@ -67,6 +73,7 @@ public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec) return new DimensionSelector() { final DimensionSelector delegateDimensionSelector = delegate.makeDimensionSelector(dimensionSpec); + final ExpressionType expressionType = ExpressionType.fromColumnType(dimensionSpec.getOutputType()); @Override public IndexedInts getRow() @@ -97,7 +104,10 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) public Object getObject() { try { - return delegateDimensionSelector.getObject(); + if (expressionType == null) { + return delegateDimensionSelector.getObject(); + } + return ExprEval.ofType(expressionType, delegateDimensionSelector.getObject()).value(); } catch (Exception e) { throw createException(e, dimensionSpec.getDimension(), inputSource, offset); @@ -144,6 +154,9 @@ public ColumnValueSelector makeColumnValueSelector(String columnName) return new ColumnValueSelector() { final ColumnValueSelector delegateColumnValueSelector = delegate.makeColumnValueSelector(columnName); + final ExpressionType expressionType = ExpressionType.fromColumnType( + rowSignature.getColumnType(columnName).orElse(null) + ); @Override public double getDouble() @@ -195,7 +208,10 @@ public boolean isNull() public Object getObject() { try { - return delegateColumnValueSelector.getObject(); + if (expressionType == null) { + return delegateColumnValueSelector.getObject(); + } + return ExprEval.ofType(expressionType, delegateColumnValueSelector.getObject()).value(); } catch (Exception e) { throw createException(e, columnName, inputSource, offset); diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ScanQueryFrameProcessor.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ScanQueryFrameProcessor.java index 1541d314f215..278a9c251dea 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ScanQueryFrameProcessor.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/scan/ScanQueryFrameProcessor.java @@ -392,6 +392,7 @@ private ColumnSelectorFactory wrapColumnSelectorFactoryIfNeeded(final ColumnSele return new ExternalColumnSelectorFactory( baseColumnSelectorFactory, ((ExternalSegment) segment).externalInputSource(), + ((ExternalSegment) segment).signature(), cursorOffset ); } diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java index de48387db200..d38fa1a8dc64 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/MSQTaskQueryMaker.java @@ -226,12 +226,15 @@ public QueryResponse runQuery(final DruidQuery druidQuery) fieldMapping.stream().map(f -> f.right).collect(Collectors.toList()) ); - destination = new DataSourceMSQDestination( + final DataSourceMSQDestination dataSourceMSQDestination = new DataSourceMSQDestination( targetDataSource, segmentGranularityObject, segmentSortOrder, replaceTimeChunks ); + MultiStageQueryContext.validateAndGetTaskLockType(sqlQueryContext, + dataSourceMSQDestination.isReplaceTimeChunks()); + destination = dataSourceMSQDestination; } else { final MSQSelectDestination msqSelectDestination = MultiStageQueryContext.getSelectDestination(sqlQueryContext); if (msqSelectDestination.equals(MSQSelectDestination.TASKREPORT)) { diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java index 77b11a287687..f6caa6da0590 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java @@ -25,6 +25,10 @@ import com.google.common.annotations.VisibleForTesting; import com.opencsv.RFC4180Parser; import com.opencsv.RFC4180ParserBuilder; +import org.apache.druid.error.DruidException; +import org.apache.druid.indexing.common.TaskLockType; +import org.apache.druid.indexing.common.task.Tasks; +import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.msq.exec.ClusterStatisticsMergeMode; import org.apache.druid.msq.exec.Limits; import org.apache.druid.msq.exec.SegmentSource; @@ -78,6 +82,11 @@ * ingested via MSQ. If set to 'none', arrays are not allowed to be ingested in MSQ. If set to 'array', array types * can be ingested as expected. If set to 'mvd', numeric arrays can not be ingested, and string arrays will be * ingested as MVDs (this is kept for legacy purpose). + * + *
  • taskLockType: Temporary flag to allow MSQ to use experimental lock types. Valid values are present in + * {@link TaskLockType}. If the flag is not set, msq uses {@link TaskLockType#EXCLUSIVE} for replace queries and + * {@link TaskLockType#SHARED} for insert queries. + * * **/ public class MultiStageQueryContext @@ -350,4 +359,53 @@ static IndexSpec decodeIndexSpec(@Nullable final Object indexSpecObject, final O throw QueryContexts.badValueException(CTX_INDEX_SPEC, "an indexSpec", indexSpecObject); } } + + /** + * This method is used to validate and get the taskLockType from the queryContext. + * If the queryContext does not contain the taskLockType, then {@link TaskLockType#EXCLUSIVE} is used for replace queries and + * {@link TaskLockType#SHARED} is used for insert queries. + * If the queryContext contains the taskLockType, then it is validated and returned. + */ + public static TaskLockType validateAndGetTaskLockType(QueryContext queryContext, boolean isReplaceQuery) + { + final TaskLockType taskLockType = QueryContexts.getAsEnum( + Tasks.TASK_LOCK_TYPE, + queryContext.getString(Tasks.TASK_LOCK_TYPE, null), + TaskLockType.class + ); + if (taskLockType == null) { + if (isReplaceQuery) { + return TaskLockType.EXCLUSIVE; + } else { + return TaskLockType.SHARED; + } + } + final String appendErrorMessage = StringUtils.format( + " Please use [%s] key in the context parameter and use one of the TaskLock types as mentioned earlier or " + + "remove this key for automatic lock type selection", Tasks.TASK_LOCK_TYPE); + + if (isReplaceQuery && !(taskLockType.equals(TaskLockType.EXCLUSIVE) || taskLockType.equals(TaskLockType.REPLACE))) { + throw DruidException.forPersona(DruidException.Persona.USER) + .ofCategory(DruidException.Category.INVALID_INPUT) + .build( + "TaskLock must be of type [%s] or [%s] for a REPLACE query. Found invalid type [%s] set." + + appendErrorMessage, + TaskLockType.EXCLUSIVE, + TaskLockType.REPLACE, + taskLockType + ); + } + if (!isReplaceQuery && !(taskLockType.equals(TaskLockType.SHARED) || taskLockType.equals(TaskLockType.APPEND))) { + throw DruidException.forPersona(DruidException.Persona.USER) + .ofCategory(DruidException.Category.INVALID_INPUT) + .build( + "TaskLock must be of type [%s] or [%s] for an INSERT query. Found invalid type [%s] set." + + appendErrorMessage, + TaskLockType.SHARED, + TaskLockType.APPEND, + taskLockType + ); + } + return taskLockType; + } } diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQArraysTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQArraysTest.java index 2b152cfbe1c4..282dacc115c2 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQArraysTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQArraysTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import org.apache.druid.data.input.impl.InlineInputSource; import org.apache.druid.data.input.impl.JsonInputFormat; import org.apache.druid.data.input.impl.LocalInputSource; import org.apache.druid.java.util.common.ISE; @@ -857,6 +858,100 @@ public void testScanWithOrderByOnDoubleArray() .verifyResults(); } + @Test + public void testScanExternBooleanArray() + { + final List expectedRows = Collections.singletonList( + new Object[]{Arrays.asList(1L, 0L, null)} + ); + + RowSignature scanSignature = RowSignature.builder() + .add("a_bool", ColumnType.LONG_ARRAY) + .build(); + + Query expectedQuery = newScanQueryBuilder() + .dataSource( + new ExternalDataSource( + new InlineInputSource("{\"a_bool\":[true,false,null]}"), + new JsonInputFormat(null, null, null, null, null), + scanSignature + ) + ) + .intervals(querySegmentSpec(Filtration.eternity())) + .columns("a_bool") + .context(defaultScanQueryContext(context, scanSignature)) + .build(); + + testSelectQuery().setSql("SELECT a_bool FROM TABLE(\n" + + " EXTERN(\n" + + " '{\"type\": \"inline\", \"data\":\"{\\\"a_bool\\\":[true,false,null]}\"}',\n" + + " '{\"type\": \"json\"}',\n" + + " '[{\"name\": \"a_bool\", \"type\": \"ARRAY\"}]'\n" + + " )\n" + + ")") + .setQueryContext(context) + .setExpectedMSQSpec(MSQSpec + .builder() + .query(expectedQuery) + .columnMappings(new ColumnMappings(ImmutableList.of( + new ColumnMapping("a_bool", "a_bool") + ))) + .tuningConfig(MSQTuningConfig.defaultConfig()) + .destination(TaskReportMSQDestination.INSTANCE) + .build() + ) + .setExpectedRowSignature(scanSignature) + .setExpectedResultRows(expectedRows) + .verifyResults(); + } + + @Test + public void testScanExternArrayWithNonConvertibleType() + { + final List expectedRows = Collections.singletonList( + new Object[]{Arrays.asList(null, null)} + ); + + RowSignature scanSignature = RowSignature.builder() + .add("a_bool", ColumnType.LONG_ARRAY) + .build(); + + Query expectedQuery = newScanQueryBuilder() + .dataSource( + new ExternalDataSource( + new InlineInputSource("{\"a_bool\":[\"Test\",\"Test2\"]}"), + new JsonInputFormat(null, null, null, null, null), + scanSignature + ) + ) + .intervals(querySegmentSpec(Filtration.eternity())) + .columns("a_bool") + .context(defaultScanQueryContext(context, scanSignature)) + .build(); + + testSelectQuery().setSql("SELECT a_bool FROM TABLE(\n" + + " EXTERN(\n" + + " '{\"type\": \"inline\", \"data\":\"{\\\"a_bool\\\":[\\\"Test\\\",\\\"Test2\\\"]}\"}',\n" + + " '{\"type\": \"json\"}',\n" + + " '[{\"name\": \"a_bool\", \"type\": \"ARRAY\"}]'\n" + + " )\n" + + ")") + .setQueryContext(context) + .setExpectedMSQSpec(MSQSpec + .builder() + .query(expectedQuery) + .columnMappings(new ColumnMappings(ImmutableList.of( + new ColumnMapping("a_bool", "a_bool") + ))) + .tuningConfig(MSQTuningConfig.defaultConfig()) + .destination(TaskReportMSQDestination.INSTANCE) + .build() + ) + .setExpectedRowSignature(scanSignature) + .setExpectedResultRows(expectedRows) + .verifyResults(); + } + private List expectedMultiValueFooRowsToArray() { List expectedRows = new ArrayList<>(); diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQFaultsTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQFaultsTest.java index 4b77dd78b339..612dee3bbd83 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQFaultsTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQFaultsTest.java @@ -22,7 +22,9 @@ import com.google.common.collect.ImmutableMap; import org.apache.druid.error.DruidException; import org.apache.druid.error.DruidExceptionMatcher; +import org.apache.druid.indexing.common.TaskLockType; import org.apache.druid.indexing.common.actions.SegmentAllocateAction; +import org.apache.druid.indexing.common.task.Tasks; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.msq.indexing.error.InsertCannotAllocateSegmentFault; @@ -45,6 +47,7 @@ import java.io.File; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -175,10 +178,10 @@ public void testInsertTimeNullFault() .build(); final String sql = "INSERT INTO foo1\n" - + "SELECT TIME_PARSE(dim1) AS __time, dim1 as cnt\n" - + "FROM foo\n" - + "PARTITIONED BY DAY\n" - + "CLUSTERED BY dim1"; + + "SELECT TIME_PARSE(dim1) AS __time, dim1 as cnt\n" + + "FROM foo\n" + + "PARTITIONED BY DAY\n" + + "CLUSTERED BY dim1"; testIngestQuery() .setSql(sql) @@ -349,8 +352,9 @@ public void testUnionAllWithDifferentColumnNames() DruidException.Persona.ADMIN, DruidException.Category.INVALID_INPUT, "general" - ).expectMessageContains("SQL requires union between two tables and column names queried for each table are different " - + "Left: [dim2, dim1, m1], Right: [dim1, dim2, m1].")) + ).expectMessageContains( + "SQL requires union between two tables and column names queried for each table are different " + + "Left: [dim2, dim1, m1], Right: [dim1, dim2, m1].")) .verifyPlanningErrors(); } @@ -374,4 +378,47 @@ public void testTopLevelUnionAllWithJoins() "SQL requires union between inputs that are not simple table scans and involve a filter or aliasing")) .verifyPlanningErrors(); } + + @Test + public void testInsertWithReplaceAndExcludeLocks() + { + for (TaskLockType taskLockType : new TaskLockType[]{TaskLockType.EXCLUSIVE, TaskLockType.REPLACE}) { + testLockTypes( + taskLockType, + "INSERT INTO foo1 select * from foo partitioned by day", + "TaskLock must be of type [SHARED] or [APPEND] for an INSERT query" + ); + } + } + + @Test + public void testReplaceWithAppendAndSharedLocks() + { + for (TaskLockType taskLockType : new TaskLockType[]{TaskLockType.APPEND, TaskLockType.SHARED}) { + testLockTypes( + taskLockType, + "REPLACE INTO foo1 overwrite ALL select * from foo partitioned by day", + "TaskLock must be of type [EXCLUSIVE] or [REPLACE] for a REPLACE query" + ); + } + } + + private void testLockTypes(TaskLockType contextTaskLockType, String sql, String errorMessage) + { + Map context = new HashMap<>(DEFAULT_MSQ_CONTEXT); + context.put(Tasks.TASK_LOCK_TYPE, contextTaskLockType.name()); + testIngestQuery() + .setSql( + sql + ) + .setQueryContext(context) + .setExpectedValidationErrorMatcher( + new DruidExceptionMatcher( + DruidException.Persona.USER, + DruidException.Category.INVALID_INPUT, + "general" + ).expectMessageContains( + errorMessage)) + .verifyPlanningErrors(); + } } diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java index b43dd72e88c8..2314c10d7e6e 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java @@ -28,6 +28,8 @@ import org.apache.druid.error.DruidException; import org.apache.druid.error.DruidExceptionMatcher; import org.apache.druid.hll.HyperLogLogCollector; +import org.apache.druid.indexing.common.TaskLockType; +import org.apache.druid.indexing.common.task.Tasks; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; @@ -59,6 +61,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -67,6 +70,16 @@ @RunWith(Parameterized.class) public class MSQInsertTest extends MSQTestBase { + + private static final String WITH_APPEND_LOCK = "WITH_APPEND_LOCK"; + private static final Map QUERY_CONTEXT_WITH_APPEND_LOCK = + ImmutableMap.builder() + .putAll(DEFAULT_MSQ_CONTEXT) + .put( + Tasks.TASK_LOCK_TYPE, + TaskLockType.APPEND.name().toLowerCase(Locale.ENGLISH) + ) + .build(); private final HashFunction fn = Hashing.murmur3_128(); @Parameterized.Parameters(name = "{index}:with context {0}") @@ -76,7 +89,8 @@ public static Collection data() {DEFAULT, DEFAULT_MSQ_CONTEXT}, {DURABLE_STORAGE, DURABLE_STORAGE_MSQ_CONTEXT}, {FAULT_TOLERANCE, FAULT_TOLERANCE_MSQ_CONTEXT}, - {PARALLEL_MERGE, PARALLEL_MERGE_MSQ_CONTEXT} + {PARALLEL_MERGE, PARALLEL_MERGE_MSQ_CONTEXT}, + {WITH_APPEND_LOCK, QUERY_CONTEXT_WITH_APPEND_LOCK} }; return Arrays.asList(data); } diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java index 0a43fdaea721..144e74b084e9 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQReplaceTest.java @@ -20,10 +20,14 @@ package org.apache.druid.msq.exec; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.indexing.common.TaskLockType; import org.apache.druid.indexing.common.actions.RetrieveUsedSegmentsAction; +import org.apache.druid.indexing.common.task.Tasks; import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.msq.test.CounterSnapshotMatcher; import org.apache.druid.msq.test.MSQTestBase; import org.apache.druid.msq.test.MSQTestFileUtils; @@ -54,6 +58,17 @@ @RunWith(Parameterized.class) public class MSQReplaceTest extends MSQTestBase { + + private static final String WITH_REPLACE_LOCK = "WITH_REPLACE_LOCK"; + private static final Map QUERY_CONTEXT_WITH_REPLACE_LOCK = + ImmutableMap.builder() + .putAll(DEFAULT_MSQ_CONTEXT) + .put( + Tasks.TASK_LOCK_TYPE, + StringUtils.toLowerCase(TaskLockType.REPLACE.name()) + ) + .build(); + @Parameterized.Parameters(name = "{index}:with context {0}") public static Collection data() { @@ -61,7 +76,8 @@ public static Collection data() {DEFAULT, DEFAULT_MSQ_CONTEXT}, {DURABLE_STORAGE, DURABLE_STORAGE_MSQ_CONTEXT}, {FAULT_TOLERANCE, FAULT_TOLERANCE_MSQ_CONTEXT}, - {PARALLEL_MERGE, PARALLEL_MERGE_MSQ_CONTEXT} + {PARALLEL_MERGE, PARALLEL_MERGE_MSQ_CONTEXT}, + {WITH_REPLACE_LOCK, QUERY_CONTEXT_WITH_REPLACE_LOCK} }; return Arrays.asList(data); } diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java index 219af6a31883..b63ee479e202 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java @@ -97,7 +97,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; @RunWith(Parameterized.class) @@ -114,7 +113,7 @@ public class MSQSelectTest extends MSQTestBase .put(MultiStageQueryContext.CTX_ROWS_PER_PAGE, 2) .put( MultiStageQueryContext.CTX_SELECT_DESTINATION, - MSQSelectDestination.DURABLESTORAGE.getName().toLowerCase(Locale.ENGLISH) + StringUtils.toLowerCase(MSQSelectDestination.DURABLESTORAGE.getName()) ) .build(); @@ -124,7 +123,7 @@ public class MSQSelectTest extends MSQTestBase .putAll(DEFAULT_MSQ_CONTEXT) .put( MultiStageQueryContext.CTX_SELECT_DESTINATION, - MSQSelectDestination.DURABLESTORAGE.getName().toLowerCase(Locale.ENGLISH) + StringUtils.toLowerCase(MSQSelectDestination.DURABLESTORAGE.getName()) ) .build(); diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestTaskActionClient.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestTaskActionClient.java index 897e57c93bc8..31b3272b74ff 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestTaskActionClient.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestTaskActionClient.java @@ -28,7 +28,9 @@ import org.apache.druid.indexing.common.actions.LockListAction; import org.apache.druid.indexing.common.actions.RetrieveUsedSegmentsAction; import org.apache.druid.indexing.common.actions.SegmentAllocateAction; +import org.apache.druid.indexing.common.actions.SegmentTransactionalAppendAction; import org.apache.druid.indexing.common.actions.SegmentTransactionalInsertAction; +import org.apache.druid.indexing.common.actions.SegmentTransactionalReplaceAction; import org.apache.druid.indexing.common.actions.TaskAction; import org.apache.druid.indexing.common.actions.TaskActionClient; import org.apache.druid.indexing.overlord.SegmentPublishResult; @@ -121,10 +123,17 @@ public RetType submit(TaskAction taskAction) ).collect(Collectors.toSet()); } } else if (taskAction instanceof SegmentTransactionalInsertAction) { - // Always OK. final Set segments = ((SegmentTransactionalInsertAction) taskAction).getSegments(); publishedSegments.addAll(segments); return (RetType) SegmentPublishResult.ok(segments); + } else if (taskAction instanceof SegmentTransactionalReplaceAction) { + final Set segments = ((SegmentTransactionalReplaceAction) taskAction).getSegments(); + publishedSegments.addAll(segments); + return (RetType) SegmentPublishResult.ok(segments); + } else if (taskAction instanceof SegmentTransactionalAppendAction) { + final Set segments = ((SegmentTransactionalAppendAction) taskAction).getSegments(); + publishedSegments.addAll(segments); + return (RetType) SegmentPublishResult.ok(segments); } else { return null; } diff --git a/extensions-core/parquet-extensions/pom.xml b/extensions-core/parquet-extensions/pom.xml index 4951c1c49e8d..b72afc7d8902 100644 --- a/extensions-core/parquet-extensions/pom.xml +++ b/extensions-core/parquet-extensions/pom.xml @@ -33,7 +33,7 @@ 4.0.0 - 1.12.0 + 1.13.1 @@ -162,6 +162,11 @@ avro ${avro.version} + + com.github.rvesse + airline + provided + junit junit diff --git a/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/ParquetToJson.java b/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/ParquetToJson.java index 5cc40df45c2c..e75b17960ac4 100644 --- a/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/ParquetToJson.java +++ b/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/ParquetToJson.java @@ -21,6 +21,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SequenceWriter; +import com.github.rvesse.airline.Cli; +import com.github.rvesse.airline.annotations.Arguments; +import com.github.rvesse.airline.annotations.Command; +import com.github.rvesse.airline.annotations.Option; +import com.github.rvesse.airline.builder.CliBuilder; import org.apache.druid.data.input.parquet.simple.ParquetGroupConverter; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.IAE; @@ -29,27 +34,59 @@ import org.apache.parquet.hadoop.example.GroupReadSupport; import java.io.File; +import java.util.List; +import java.util.concurrent.Callable; /** - * Converts parquet files into new-deliminated JSON object files. Takes a single argument (an input directory) - * and processes all files that end with a ".parquet" extension. Writes out a new file in the same directory named - * by appending ".json" to the old file name. Will overwrite any output file that already exists. + * Converts parquet files into new-deliminated JSON object files. Takes a single + * argument (an input directory) and processes all files that end with a + * ".parquet" extension. Writes out a new file in the same directory named by + * appending ".json" to the old file name. Will overwrite any output file that + * already exists. */ -public class ParquetToJson +@Command(name = "ParquetToJson") +public class ParquetToJson implements Callable { + @Option(name = "--convert-corrupt-dates") + public boolean convertCorruptDates = false; + + @Arguments(description = "directory") + public List directories; + + public static void main(String[] args) throws Exception { - if (args.length != 1) { - throw new IAE("Usage: directory"); + CliBuilder builder = Cli.builder("ParquetToJson"); + builder.withDefaultCommand(ParquetToJson.class); + builder.build().parse(args).call(); + } + + private File[] getInputFiles() + { + if (directories == null || directories.size() != 1) { + throw new IAE("Only one directory argument is supported!"); } - ParquetGroupConverter converter = new ParquetGroupConverter(true); + File dir = new File(directories.get(0)); + if (!dir.isDirectory()) { + throw new IAE("Not a directory [%s]", dir); + } + File[] inputFiles = dir.listFiles( + pathname -> pathname.getName().endsWith(".parquet")); + if (inputFiles == null || inputFiles.length == 0) { + throw new IAE("No parquet files in directory [%s]", dir); + } + return inputFiles; + } + + @Override + public Void call() throws Exception + { ObjectMapper mapper = new DefaultObjectMapper(); - File[] inputFiles = new File(args[0]).listFiles( - pathname -> pathname.getName().endsWith(".parquet") - ); + File[] inputFiles = getInputFiles(); + for (File inputFile : inputFiles) { File outputFile = new File(inputFile.getAbsolutePath() + ".json"); @@ -57,13 +94,14 @@ public static void main(String[] args) throws Exception final org.apache.parquet.hadoop.ParquetReader reader = org.apache.parquet.hadoop.ParquetReader .builder(new GroupReadSupport(), new Path(inputFile.toURI())) .build(); - final SequenceWriter writer = mapper.writer().withRootValueSeparator("\n").writeValues(outputFile) - ) { + final SequenceWriter writer = mapper.writer().withRootValueSeparator("\n").writeValues(outputFile)) { + ParquetGroupConverter converter = new ParquetGroupConverter(true, convertCorruptDates); Group group; while ((group = reader.read()) != null) { writer.write(converter.convertGroup(group)); } } } + return null; } } diff --git a/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/simple/ParquetGroupConverter.java b/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/simple/ParquetGroupConverter.java index d2bd643304a3..4571da3d7246 100644 --- a/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/simple/ParquetGroupConverter.java +++ b/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/simple/ParquetGroupConverter.java @@ -52,10 +52,55 @@ public class ParquetGroupConverter private static final long NANOS_PER_MILLISECOND = TimeUnit.MILLISECONDS.toNanos(1); /** - * See {@link ParquetGroupConverter#convertField(Group, String)} + * https://github.com/apache/drill/blob/2ab46a9411a52f12a0f9acb1144a318059439bc4/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetReaderUtility.java#L89 + */ + public static final long CORRECT_CORRUPT_DATE_SHIFT = 2 * JULIAN_EPOCH_OFFSET_DAYS; + + private final boolean binaryAsString; + private final boolean convertCorruptDates; + + public ParquetGroupConverter(boolean binaryAsString, boolean convertCorruptDates) + { + this.binaryAsString = binaryAsString; + this.convertCorruptDates = convertCorruptDates; + } + + /** + * Recursively converts a group into native Java Map + * + * @param g the group + * @return the native Java object + */ + public Object convertGroup(Group g) + { + Map retVal = new LinkedHashMap<>(); + + for (Type field : g.getType().getFields()) { + final String fieldName = field.getName(); + retVal.put(fieldName, convertField(g, fieldName)); + } + + return retVal; + } + + Object unwrapListElement(Object o) + { + if (o instanceof Group) { + Group g = (Group) o; + return convertListElement(g); + } + return o; + } + + /** + * Convert a parquet group field as though it were a map. Logical types of 'list' and 'map' will be transformed + * into java lists and maps respectively ({@link ParquetGroupConverter#convertLogicalList} and + * {@link ParquetGroupConverter#convertLogicalMap}), repeated fields will also be translated to lists, and + * primitive types will be extracted into an ingestion friendly state (e.g. 'int' and 'long'). Finally, + * if a field is not present, this method will return null. */ @Nullable - private static Object convertField(Group g, String fieldName, boolean binaryAsString) + Object convertField(Group g, String fieldName) { if (!g.getType().containsField(fieldName)) { return null; @@ -76,22 +121,22 @@ private static Object convertField(Group g, String fieldName, boolean binaryAsSt int repeated = g.getFieldRepetitionCount(fieldIndex); List vals = new ArrayList<>(); for (int i = 0; i < repeated; i++) { - vals.add(convertPrimitiveField(g, fieldIndex, i, binaryAsString)); + vals.add(convertPrimitiveField(g, fieldIndex, i)); } return vals; } - return convertPrimitiveField(g, fieldIndex, binaryAsString); + return convertPrimitiveField(g, fieldIndex); } else { if (fieldType.isRepetition(Type.Repetition.REPEATED)) { - return convertRepeatedFieldToList(g, fieldIndex, binaryAsString); + return convertRepeatedFieldToList(g, fieldIndex); } if (isLogicalMapType(fieldType)) { - return convertLogicalMap(g.getGroup(fieldIndex, 0), binaryAsString); + return convertLogicalMap(g.getGroup(fieldIndex, 0)); } if (isLogicalListType(fieldType)) { - return convertLogicalList(g.getGroup(fieldIndex, 0), binaryAsString); + return convertLogicalList(g.getGroup(fieldIndex, 0)); } // not a list, but not a primitive, return the nested group type @@ -102,7 +147,7 @@ private static Object convertField(Group g, String fieldName, boolean binaryAsSt /** * convert a repeated field into a list of primitives or groups */ - private static List convertRepeatedFieldToList(Group g, int fieldIndex, boolean binaryAsString) + private List convertRepeatedFieldToList(Group g, int fieldIndex) { Type t = g.getType().getFields().get(fieldIndex); @@ -111,7 +156,7 @@ private static List convertRepeatedFieldToList(Group g, int fieldIndex, List vals = new ArrayList<>(); for (int i = 0; i < repeated; i++) { if (t.isPrimitive()) { - vals.add(convertPrimitiveField(g, fieldIndex, i, binaryAsString)); + vals.add(convertPrimitiveField(g, fieldIndex, i)); } else { vals.add(g.getGroup(fieldIndex, i)); } @@ -134,7 +179,7 @@ private static boolean isLogicalListType(Type listType) /** * convert a parquet 'list' logical type {@link Group} to a java list of primitives or groups */ - private static List convertLogicalList(Group g, boolean binaryAsString) + private List convertLogicalList(Group g) { /* // List (nullable list, non-null elements) @@ -181,16 +226,16 @@ optional group my_list (LIST) { for (int i = 0; i < repeated; i++) { if (isListItemPrimitive) { - vals.add(convertPrimitiveField(g, 0, i, binaryAsString)); + vals.add(convertPrimitiveField(g, 0, i)); } else { Group listItem = g.getGroup(0, i); - vals.add(convertListElement(listItem, binaryAsString)); + vals.add(convertListElement(listItem)); } } return vals; } - private static Object convertListElement(Group listItem, boolean binaryAsString) + private Object convertListElement(Group listItem) { if ( listItem.getType().isRepetition(Type.Repetition.REPEATED) && @@ -199,7 +244,7 @@ private static Object convertListElement(Group listItem, boolean binaryAsString) listItem.getType().getFields().get(0).isPrimitive() ) { // nullable primitive list elements can have a repeating wrapper element, peel it off - return convertPrimitiveField(listItem, 0, binaryAsString); + return convertPrimitiveField(listItem, 0); } else if ( listItem.getType().isRepetition(Type.Repetition.REPEATED) && listItem.getType().getFieldCount() == 1 && @@ -244,7 +289,7 @@ private static boolean isLogicalMapType(Type groupType) /** * Convert a parquet 'map' logical type {@link Group} to a java map of string keys to groups/lists/primitive values */ - private static Map convertLogicalMap(Group g, boolean binaryAsString) + private Map convertLogicalMap(Group g) { /* // Map (nullable map, non-null values) @@ -268,8 +313,8 @@ optional group my_map (MAP_KEY_VALUE) {( Map converted = new HashMap<>(); for (int i = 0; i < mapEntries; i++) { Group mapEntry = g.getGroup(0, i); - String key = convertPrimitiveField(mapEntry, 0, binaryAsString).toString(); - Object value = convertField(mapEntry, "value", binaryAsString); + String key = convertPrimitiveField(mapEntry, 0).toString(); + Object value = convertField(mapEntry, "value"); converted.put(key, value); } return converted; @@ -281,17 +326,17 @@ optional group my_map (MAP_KEY_VALUE) {( * @return "ingestion ready" java object, or null */ @Nullable - private static Object convertPrimitiveField(Group g, int fieldIndex, boolean binaryAsString) + private Object convertPrimitiveField(Group g, int fieldIndex) { PrimitiveType pt = (PrimitiveType) g.getType().getFields().get(fieldIndex); if (pt.isRepetition(Type.Repetition.REPEATED) && g.getFieldRepetitionCount(fieldIndex) > 1) { List vals = new ArrayList<>(); for (int i = 0; i < g.getFieldRepetitionCount(fieldIndex); i++) { - vals.add(convertPrimitiveField(g, fieldIndex, i, binaryAsString)); + vals.add(convertPrimitiveField(g, fieldIndex, i)); } return vals; } - return convertPrimitiveField(g, fieldIndex, 0, binaryAsString); + return convertPrimitiveField(g, fieldIndex, 0); } /** @@ -300,7 +345,7 @@ private static Object convertPrimitiveField(Group g, int fieldIndex, boolean bin * @return "ingestion ready" java object, or null */ @Nullable - private static Object convertPrimitiveField(Group g, int fieldIndex, int index, boolean binaryAsString) + private Object convertPrimitiveField(Group g, int fieldIndex, int index) { PrimitiveType pt = (PrimitiveType) g.getType().getFields().get(fieldIndex); OriginalType ot = pt.getOriginalType(); @@ -310,7 +355,7 @@ private static Object convertPrimitiveField(Group g, int fieldIndex, int index, // convert logical types switch (ot) { case DATE: - long ts = g.getInteger(fieldIndex, index) * MILLIS_IN_DAY; + long ts = convertDateToMillis(g.getInteger(fieldIndex, index)); return ts; case TIME_MICROS: return g.getLong(fieldIndex, index); @@ -443,6 +488,14 @@ private static Object convertPrimitiveField(Group g, int fieldIndex, int index, } } + private long convertDateToMillis(int value) + { + if (convertCorruptDates) { + value -= CORRECT_CORRUPT_DATE_SHIFT; + } + return value * MILLIS_IN_DAY; + } + /** * convert deprecated parquet int96 nanosecond timestamp to a long, based on * https://github.com/prestodb/presto/blob/master/presto-parquet/src/main/java/com/facebook/presto/parquet/ParquetTimestampUtils.java#L44 @@ -490,51 +543,4 @@ private static BigDecimal convertBinaryToDecimal(Binary value, int precision, in return new BigDecimal(new BigInteger(value.getBytes()), scale); } } - - private final boolean binaryAsString; - - public ParquetGroupConverter(boolean binaryAsString) - { - this.binaryAsString = binaryAsString; - } - - /** - * Recursively converts a group into native Java Map - * - * @param g the group - * @return the native Java object - */ - public Object convertGroup(Group g) - { - Map retVal = new LinkedHashMap<>(); - - for (Type field : g.getType().getFields()) { - final String fieldName = field.getName(); - retVal.put(fieldName, convertField(g, fieldName)); - } - - return retVal; - } - - /** - * Convert a parquet group field as though it were a map. Logical types of 'list' and 'map' will be transformed - * into java lists and maps respectively ({@link ParquetGroupConverter#convertLogicalList} and - * {@link ParquetGroupConverter#convertLogicalMap}), repeated fields will also be translated to lists, and - * primitive types will be extracted into an ingestion friendly state (e.g. 'int' and 'long'). Finally, - * if a field is not present, this method will return null. - */ - @Nullable - Object convertField(Group g, String fieldName) - { - return convertField(g, fieldName, binaryAsString); - } - - Object unwrapListElement(Object o) - { - if (o instanceof Group) { - Group g = (Group) o; - return convertListElement(g, binaryAsString); - } - return o; - } } diff --git a/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/simple/ParquetGroupFlattenerMaker.java b/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/simple/ParquetGroupFlattenerMaker.java index a243107cc238..abf6f6ed1aea 100644 --- a/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/simple/ParquetGroupFlattenerMaker.java +++ b/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/simple/ParquetGroupFlattenerMaker.java @@ -45,7 +45,7 @@ public class ParquetGroupFlattenerMaker implements ObjectFlatteners.FlattenerMak public ParquetGroupFlattenerMaker(boolean binaryAsString, boolean discoverNestedFields) { - this.converter = new ParquetGroupConverter(binaryAsString); + this.converter = new ParquetGroupConverter(binaryAsString, false); this.parquetJsonProvider = new ParquetGroupJsonProvider(converter); this.jsonPathConfiguration = Configuration.builder() .jsonProvider(parquetJsonProvider) diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/ParquetToJsonTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/ParquetToJsonTest.java index dd191e36dd14..ed0c5b3a9181 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/ParquetToJsonTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/ParquetToJsonTest.java @@ -33,6 +33,9 @@ import java.nio.file.Files; import java.util.List; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + @SuppressWarnings("ALL") public class ParquetToJsonTest { @@ -70,10 +73,65 @@ public void testSanity() throws Exception ); } + @Test + public void testConvertedDates() throws Exception + { + final File tmpDir = tmp.newFolder(); + try (InputStream in = new BufferedInputStream(ClassLoader.getSystemResourceAsStream("smlTbl.parquet"))) { + Files.copy(in, tmpDir.toPath().resolve("smlTbl.parquet")); + } + + ParquetToJson.main(new String[]{"--convert-corrupt-dates", tmpDir.toString()}); + + DefaultObjectMapper mapper = DefaultObjectMapper.INSTANCE; + List objs = mapper.readerFor(Object.class).readValues(new File(tmpDir, "smlTbl.parquet.json")).readAll(); + + Assert.assertEquals(56, objs.size()); + Assert.assertEquals( + ImmutableMap + .builder() + .put("col_int", 8122) + .put("col_bgint", 817200) + .put("col_char_2", "IN") + .put("col_vchar_52", "AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB") + .put("col_tmstmp", 1409617682418L) + .put("col_dt", 984009600000L) + .put("col_booln", false) + .put("col_dbl", 12900.48) + .put("col_tm", 33109170) + .build(), + objs.get(0) + ); + } + + @Test public void testInputValidation() { Assert.assertThrows(IAE.class, () -> ParquetToJson.main(new String[]{})); Assert.assertThrows(IAE.class, () -> ParquetToJson.main(new String[]{"a", "b"})); } + + @Test + public void testEmptyDir() throws Exception + { + final File tmpDir = tmp.newFolder(); + Assert.assertThrows(IAE.class, () -> ParquetToJson.main(new String[] {tmpDir.getAbsolutePath()})); + } + + @Test + public void testSomeFile() throws Exception + { + final File file = tmp.newFile(); + assertTrue(file.exists()); + Assert.assertThrows(IAE.class, () -> ParquetToJson.main(new String[] {file.getAbsolutePath()})); + } + + @Test + public void testNonExistentFile() throws Exception + { + final File file = new File(tmp.getRoot(), "nonExistent"); + assertFalse(file.exists()); + Assert.assertThrows(IAE.class, () -> ParquetToJson.main(new String[] {file.getAbsolutePath()})); + } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/RetrieveSegmentsToReplaceAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/RetrieveSegmentsToReplaceAction.java new file mode 100644 index 000000000000..78e6ada5c1e2 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/RetrieveSegmentsToReplaceAction.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.common.actions; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.type.TypeReference; +import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.common.task.batch.parallel.AbstractBatchSubtask; +import org.apache.druid.indexing.overlord.Segments; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.metadata.ReplaceTaskLock; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.Partitions; +import org.apache.druid.timeline.SegmentTimeline; +import org.joda.time.Interval; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * This action exists in addition to retrieveUsedSegmentsAction because that action suffers + * from a race condition described by the following sequence of events: + * + * -Segments S1, S2, S3 exist + * -Compact acquires a replace lock + * -A concurrent appending job publishes a segment S4 which needs to be upgraded to the replace lock's version + * -Compact task processes S1-S4 to create new segments + * -Compact task publishes new segments and carries S4 forward to the new version + * + * This can lead to the data in S4 being duplicated + * + * This TaskAction returns a collection of segments which have data within the specified interval and are marked as + * used, and have been created before a REPLACE lock, if any, was acquired. + * This ensures that a consistent set of segments is returned each time this action is called + */ +public class RetrieveSegmentsToReplaceAction implements TaskAction> +{ + private static final Logger log = new Logger(RetrieveSegmentsToReplaceAction.class); + + @JsonIgnore + private final String dataSource; + + @JsonIgnore + private final Interval interval; + + @JsonCreator + public RetrieveSegmentsToReplaceAction( + @JsonProperty("dataSource") String dataSource, + @JsonProperty("interval") Interval interval + ) + { + this.dataSource = dataSource; + this.interval = interval; + } + + @JsonProperty + public String getDataSource() + { + return dataSource; + } + + @JsonProperty + public Interval getInterval() + { + return interval; + } + + @Override + public TypeReference> getReturnTypeReference() + { + return new TypeReference>() {}; + } + + @Override + public Collection perform(Task task, TaskActionToolbox toolbox) + { + // The DruidInputSource can be used to read from one datasource and write to another. + // In such a case, the race condition described in the class-level docs cannot occur, + // and the action can simply fetch all visible segments for the datasource and interval + if (!task.getDataSource().equals(dataSource)) { + return retrieveAllVisibleSegments(toolbox); + } + + final String supervisorId; + if (task instanceof AbstractBatchSubtask) { + supervisorId = ((AbstractBatchSubtask) task).getSupervisorTaskId(); + } else { + supervisorId = task.getId(); + } + + final Set replaceLocksForTask = toolbox + .getTaskLockbox() + .getAllReplaceLocksForDatasource(task.getDataSource()) + .stream() + .filter(lock -> supervisorId.equals(lock.getSupervisorTaskId())) + .collect(Collectors.toSet()); + + // If there are no replace locks for the task, simply fetch all visible segments for the interval + if (replaceLocksForTask.isEmpty()) { + return retrieveAllVisibleSegments(toolbox); + } + + Map>> intervalToCreatedToSegments = new HashMap<>(); + for (Pair segmentAndCreatedDate : + toolbox.getIndexerMetadataStorageCoordinator().retrieveUsedSegmentsAndCreatedDates(dataSource, interval)) { + final DataSegment segment = segmentAndCreatedDate.lhs; + final String created = segmentAndCreatedDate.rhs; + intervalToCreatedToSegments.computeIfAbsent(segment.getInterval(), s -> new HashMap<>()) + .computeIfAbsent(created, c -> new HashSet<>()) + .add(segment); + } + + Set allSegmentsToBeReplaced = new HashSet<>(); + for (final Map.Entry>> entry : intervalToCreatedToSegments.entrySet()) { + final Interval segmentInterval = entry.getKey(); + String lockVersion = null; + for (ReplaceTaskLock replaceLock : replaceLocksForTask) { + if (replaceLock.getInterval().contains(segmentInterval)) { + lockVersion = replaceLock.getVersion(); + } + } + final Map> createdToSegmentsMap = entry.getValue(); + for (Map.Entry> createdAndSegments : createdToSegmentsMap.entrySet()) { + if (lockVersion == null || lockVersion.compareTo(createdAndSegments.getKey()) > 0) { + allSegmentsToBeReplaced.addAll(createdAndSegments.getValue()); + } else { + for (DataSegment segment : createdAndSegments.getValue()) { + log.info("Ignoring segment[%s] as it has created_date[%s] greater than the REPLACE lock version[%s]", + segment.getId(), createdAndSegments.getKey(), lockVersion); + } + } + } + } + + return SegmentTimeline.forSegments(allSegmentsToBeReplaced) + .findNonOvershadowedObjectsInInterval(Intervals.ETERNITY, Partitions.ONLY_COMPLETE); + } + + private Collection retrieveAllVisibleSegments(TaskActionToolbox toolbox) + { + return toolbox.getIndexerMetadataStorageCoordinator() + .retrieveUsedSegmentsForInterval(dataSource, interval, Segments.ONLY_VISIBLE); + } + + @Override + public boolean isAudited() + { + return false; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + RetrieveSegmentsToReplaceAction that = (RetrieveSegmentsToReplaceAction) o; + + if (!dataSource.equals(that.dataSource)) { + return false; + } + return interval.equals(that.interval); + } + + @Override + public int hashCode() + { + return Objects.hash(dataSource, interval); + } + + @Override + public String toString() + { + return getClass().getSimpleName() + "{" + + "dataSource='" + dataSource + '\'' + + ", interval=" + interval + + '}'; + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TaskAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TaskAction.java index 171d53b9cdd6..e251626f8690 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TaskAction.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/TaskAction.java @@ -38,6 +38,7 @@ @JsonSubTypes.Type(name = "segmentTransactionalInsert", value = SegmentTransactionalInsertAction.class), @JsonSubTypes.Type(name = "segmentTransactionalAppend", value = SegmentTransactionalAppendAction.class), @JsonSubTypes.Type(name = "segmentTransactionalReplace", value = SegmentTransactionalReplaceAction.class), + @JsonSubTypes.Type(name = "retrieveSegmentsToReplace", value = RetrieveSegmentsToReplaceAction.class), // Type name doesn't correspond to the name of the class for backward compatibility. @JsonSubTypes.Type(name = "segmentListUsed", value = RetrieveUsedSegmentsAction.class), // Type name doesn't correspond to the name of the class for backward compatibility. diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/UpdateLocationAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/UpdateLocationAction.java index f4926864dcbe..088169d3a537 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/UpdateLocationAction.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/UpdateLocationAction.java @@ -23,11 +23,15 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.base.Optional; import org.apache.druid.indexer.TaskLocation; import org.apache.druid.indexing.common.task.Task; -import org.apache.druid.indexing.overlord.TaskRunner; +/* This class was added for mm-less ingestion in order to let the peon manage its own location lifecycle by submitting +actions to the overlord. https://github.com/apache/druid/pull/15133 moved this location logic to the overlord itself +so this Action is no longer needed. For backwards compatibility with old peons, this class was left in but can be deprecated +for a later druid release. +*/ +@Deprecated public class UpdateLocationAction implements TaskAction { @JsonIgnore @@ -58,10 +62,6 @@ public TypeReference getReturnTypeReference() @Override public Void perform(Task task, TaskActionToolbox toolbox) { - Optional taskRunner = toolbox.getTaskRunner(); - if (taskRunner.isPresent()) { - taskRunner.get().updateLocation(task, taskLocation); - } return null; } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java index fe19b35391e9..cbc469b61333 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java @@ -62,6 +62,7 @@ import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.druid.query.DruidMetrics; +import org.apache.druid.query.QueryContexts; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.segment.handoff.SegmentHandoffNotifier; import org.apache.druid.segment.incremental.ParseExceptionHandler; @@ -483,13 +484,18 @@ private TaskLockType determineLockType(LockGranularity lockGranularity) return TaskLockType.EXCLUSIVE; } - final String contextLockType = getContextValue(Tasks.TASK_LOCK_TYPE); + final TaskLockType contextTaskLockType = QueryContexts.getAsEnum( + Tasks.TASK_LOCK_TYPE, + getContextValue(Tasks.TASK_LOCK_TYPE), + TaskLockType.class + ); + final TaskLockType lockType; - if (contextLockType == null) { + if (contextTaskLockType == null) { lockType = getContextValue(Tasks.USE_SHARED_LOCK, false) ? TaskLockType.SHARED : TaskLockType.EXCLUSIVE; } else { - lockType = TaskLockType.valueOf(contextLockType); + lockType = contextTaskLockType; } final IngestionMode ingestionMode = getIngestionMode(); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractTask.java index e5b6ab1b7312..ea0cb566b2ce 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractTask.java @@ -25,13 +25,11 @@ import com.google.common.base.Objects; import com.google.common.base.Preconditions; import org.apache.druid.common.utils.IdUtils; -import org.apache.druid.indexer.TaskLocation; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.TaskLock; import org.apache.druid.indexing.common.TaskToolbox; import org.apache.druid.indexing.common.actions.LockListAction; import org.apache.druid.indexing.common.actions.TaskActionClient; -import org.apache.druid.indexing.common.actions.UpdateLocationAction; import org.apache.druid.indexing.common.actions.UpdateStatusAction; import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.IAE; @@ -42,14 +40,12 @@ import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunner; import org.apache.druid.segment.indexing.BatchIOConfig; -import org.apache.druid.server.DruidNode; import org.joda.time.Interval; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.File; import java.io.IOException; -import java.net.InetAddress; import java.nio.file.Files; import java.nio.file.Paths; import java.util.HashMap; @@ -155,11 +151,6 @@ public String setup(TaskToolbox toolbox) throws Exception FileUtils.mkdirp(attemptDir); reportsFile = new File(attemptDir, "report.json"); statusFile = new File(attemptDir, "status.json"); - InetAddress hostName = InetAddress.getLocalHost(); - DruidNode node = toolbox.getTaskExecutorNode(); - toolbox.getTaskActionClient().submit(new UpdateLocationAction(TaskLocation.create( - hostName.getHostAddress(), node.getPlaintextPort(), node.getTlsPort(), node.isEnablePlaintextPort() - ))); } log.debug("Task setup complete"); return null; @@ -211,7 +202,6 @@ public void cleanUp(TaskToolbox toolbox, TaskStatus taskStatus) throws Exception status = new UpdateStatusAction("failure"); } toolbox.getTaskActionClient().submit(status); - toolbox.getTaskActionClient().submit(new UpdateLocationAction(TaskLocation.unknown())); if (reportsFile != null && reportsFile.exists()) { toolbox.getTaskLogPusher().pushTaskReports(id, reportsFile); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractBatchSubtask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractBatchSubtask.java index 37b70c53ed55..e6de0cf25949 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractBatchSubtask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractBatchSubtask.java @@ -19,6 +19,7 @@ package org.apache.druid.indexing.common.task.batch.parallel; +import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.indexing.common.task.AbstractBatchIndexTask; import org.apache.druid.indexing.common.task.TaskResource; @@ -29,16 +30,20 @@ public abstract class AbstractBatchSubtask extends AbstractBatchIndexTask { + private final String supervisorTaskId; + protected AbstractBatchSubtask( String id, @Nullable String groupId, @Nullable TaskResource taskResource, String dataSource, @Nullable Map context, - @Nonnull IngestionMode ingestionMode + @Nonnull IngestionMode ingestionMode, + @Nonnull String supervisorTaskId ) { super(id, groupId, taskResource, dataSource, context, -1, ingestionMode); + this.supervisorTaskId = supervisorTaskId; } /** @@ -46,4 +51,13 @@ protected AbstractBatchSubtask( * This ID is used to identify duplicate work of retry tasks for the same spec. */ public abstract String getSubtaskSpecId(); + + /** + * @return Task ID of the {@code ParallelIndexSupervisorTask} which launched this sub-task. + */ + @JsonProperty + public String getSupervisorTaskId() + { + return supervisorTaskId; + } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionCardinalityTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionCardinalityTask.java index 625b20517836..698f8c9d0e95 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionCardinalityTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionCardinalityTask.java @@ -69,7 +69,6 @@ public class PartialDimensionCardinalityTask extends PerfectRollupWorkerTask private final int numAttempts; private final ParallelIndexIngestionSpec ingestionSchema; - private final String supervisorTaskId; private final String subtaskSpecId; private final ObjectMapper jsonMapper; @@ -95,7 +94,8 @@ public class PartialDimensionCardinalityTask extends PerfectRollupWorkerTask taskResource, ingestionSchema.getDataSchema(), ingestionSchema.getTuningConfig(), - context + context, + supervisorTaskId ); Preconditions.checkArgument( @@ -107,7 +107,6 @@ public class PartialDimensionCardinalityTask extends PerfectRollupWorkerTask this.subtaskSpecId = subtaskSpecId; this.numAttempts = numAttempts; this.ingestionSchema = ingestionSchema; - this.supervisorTaskId = supervisorTaskId; this.jsonMapper = jsonMapper; } @@ -123,12 +122,6 @@ private ParallelIndexIngestionSpec getIngestionSchema() return ingestionSchema; } - @JsonProperty - private String getSupervisorTaskId() - { - return supervisorTaskId; - } - @JsonProperty @Override public String getSubtaskSpecId() @@ -163,7 +156,7 @@ public boolean isReady(TaskActionClient taskActionClient) throws Exception { if (!getIngestionSchema().getDataSchema().getGranularitySpec().inputIntervals().isEmpty()) { return tryTimeChunkLock( - new SurrogateTaskActionClient(supervisorTaskId, taskActionClient), + new SurrogateTaskActionClient(getSupervisorTaskId(), taskActionClient), getIngestionSchema().getDataSchema().getGranularitySpec().inputIntervals() ); } else { @@ -274,7 +267,7 @@ private void sendReport(TaskToolbox toolbox, DimensionCardinalityReport report) { final ParallelIndexSupervisorTaskClient taskClient = toolbox.getSupervisorTaskClientProvider().build( - supervisorTaskId, + getSupervisorTaskId(), ingestionSchema.getTuningConfig().getChatHandlerTimeout(), ingestionSchema.getTuningConfig().getChatHandlerNumRetries() ); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionDistributionTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionDistributionTask.java index b2ecd3dc2695..8f03c3bfa55e 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionDistributionTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionDistributionTask.java @@ -82,7 +82,6 @@ public class PartialDimensionDistributionTask extends PerfectRollupWorkerTask private final int numAttempts; private final ParallelIndexIngestionSpec ingestionSchema; - private final String supervisorTaskId; private final String subtaskSpecId; // For testing @@ -136,7 +135,8 @@ public class PartialDimensionDistributionTask extends PerfectRollupWorkerTask taskResource, ingestionSchema.getDataSchema(), ingestionSchema.getTuningConfig(), - context + context, + supervisorTaskId ); Preconditions.checkArgument( @@ -148,7 +148,6 @@ public class PartialDimensionDistributionTask extends PerfectRollupWorkerTask this.subtaskSpecId = subtaskSpecId; this.numAttempts = numAttempts; this.ingestionSchema = ingestionSchema; - this.supervisorTaskId = supervisorTaskId; this.dedupInputRowFilterSupplier = dedupRowDimValueFilterSupplier; } @@ -164,12 +163,6 @@ private ParallelIndexIngestionSpec getIngestionSchema() return ingestionSchema; } - @JsonProperty - private String getSupervisorTaskId() - { - return supervisorTaskId; - } - @JsonProperty @Override public String getSubtaskSpecId() @@ -204,7 +197,7 @@ public boolean isReady(TaskActionClient taskActionClient) throws Exception { if (!getIngestionSchema().getDataSchema().getGranularitySpec().inputIntervals().isEmpty()) { return tryTimeChunkLock( - new SurrogateTaskActionClient(supervisorTaskId, taskActionClient), + new SurrogateTaskActionClient(getSupervisorTaskId(), taskActionClient), getIngestionSchema().getDataSchema().getGranularitySpec().inputIntervals() ); } else { @@ -326,7 +319,7 @@ private Map determineDistribution( private void sendReport(TaskToolbox toolbox, DimensionDistributionReport report) { final ParallelIndexSupervisorTaskClient taskClient = toolbox.getSupervisorTaskClientProvider().build( - supervisorTaskId, + getSupervisorTaskId(), ingestionSchema.getTuningConfig().getChatHandlerTimeout(), ingestionSchema.getTuningConfig().getChatHandlerNumRetries() ); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialHashSegmentGenerateTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialHashSegmentGenerateTask.java index b91a6ce3a821..49e3591ff18a 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialHashSegmentGenerateTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialHashSegmentGenerateTask.java @@ -63,7 +63,6 @@ public class PartialHashSegmentGenerateTask extends PartialSegmentGenerateTask intervalToNumShardsOverride; @@ -96,7 +95,6 @@ public PartialHashSegmentGenerateTask( this.subtaskSpecId = subtaskSpecId; this.numAttempts = numAttempts; this.ingestionSchema = ingestionSchema; - this.supervisorTaskId = supervisorTaskId; this.intervalToNumShardsOverride = intervalToNumShardsOverride; } @@ -112,12 +110,6 @@ public ParallelIndexIngestionSpec getIngestionSchema() return ingestionSchema; } - @JsonProperty - public String getSupervisorTaskId() - { - return supervisorTaskId; - } - @JsonProperty @Override public String getSubtaskSpecId() @@ -158,7 +150,7 @@ public Set getInputSourceResources() public boolean isReady(TaskActionClient taskActionClient) throws Exception { return tryTimeChunkLock( - new SurrogateTaskActionClient(supervisorTaskId, taskActionClient), + new SurrogateTaskActionClient(getSupervisorTaskId(), taskActionClient), getIngestionSchema().getDataSchema().getGranularitySpec().inputIntervals() ); } @@ -175,7 +167,7 @@ SegmentAllocatorForBatch createSegmentAllocator(TaskToolbox toolbox, ParallelInd getDataSource(), getSubtaskSpecId(), granularitySpec, - new SupervisorTaskAccess(supervisorTaskId, taskClient), + new SupervisorTaskAccess(getSupervisorTaskId(), taskClient), createHashPartitionAnalysisFromPartitionsSpec( granularitySpec, partitionsSpec, diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialRangeSegmentGenerateTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialRangeSegmentGenerateTask.java index 147a1fbf1212..27604eb7e770 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialRangeSegmentGenerateTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialRangeSegmentGenerateTask.java @@ -64,7 +64,6 @@ public class PartialRangeSegmentGenerateTask extends PartialSegmentGenerateTask< private static final String PROP_SPEC = "spec"; private static final boolean SKIP_NULL = true; - private final String supervisorTaskId; private final String subtaskSpecId; private final int numAttempts; private final ParallelIndexIngestionSpec ingestionSchema; @@ -98,7 +97,6 @@ public PartialRangeSegmentGenerateTask( this.subtaskSpecId = subtaskSpecId; this.numAttempts = numAttempts; this.ingestionSchema = ingestionSchema; - this.supervisorTaskId = supervisorTaskId; this.intervalToPartitions = intervalToPartitions; } @@ -131,12 +129,6 @@ public ParallelIndexIngestionSpec getIngestionSchema() return ingestionSchema; } - @JsonProperty - public String getSupervisorTaskId() - { - return supervisorTaskId; - } - @JsonProperty @Override public String getSubtaskSpecId() @@ -176,7 +168,7 @@ public Set getInputSourceResources() public boolean isReady(TaskActionClient taskActionClient) throws IOException { return tryTimeChunkLock( - new SurrogateTaskActionClient(supervisorTaskId, taskActionClient), + new SurrogateTaskActionClient(getSupervisorTaskId(), taskActionClient), getIngestionSchema().getDataSchema().getGranularitySpec().inputIntervals() ); } @@ -194,7 +186,7 @@ SegmentAllocatorForBatch createSegmentAllocator(TaskToolbox toolbox, ParallelInd getDataSource(), getSubtaskSpecId(), ingestionSchema.getDataSchema().getGranularitySpec(), - new SupervisorTaskAccess(supervisorTaskId, taskClient), + new SupervisorTaskAccess(getSupervisorTaskId(), taskClient), partitionAnalysis ); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialSegmentGenerateTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialSegmentGenerateTask.java index dc2a7ef5bf92..e20c7bdbe352 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialSegmentGenerateTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialSegmentGenerateTask.java @@ -94,7 +94,8 @@ abstract class PartialSegmentGenerateTask e taskResource, ingestionSchema.getDataSchema(), ingestionSchema.getTuningConfig(), - context + context, + supervisorTaskId ); Preconditions.checkArgument( diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialSegmentMergeTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialSegmentMergeTask.java index bb933169d4b7..b59ec65716c2 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialSegmentMergeTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialSegmentMergeTask.java @@ -78,7 +78,6 @@ abstract class PartialSegmentMergeTask extends PerfectRollu private final PartialSegmentMergeIOConfig ioConfig; private final int numAttempts; - private final String supervisorTaskId; private final String subtaskSpecId; PartialSegmentMergeTask( @@ -101,7 +100,8 @@ abstract class PartialSegmentMergeTask extends PerfectRollu taskResource, dataSchema, tuningConfig, - context + context, + supervisorTaskId ); Preconditions.checkArgument( @@ -111,7 +111,6 @@ abstract class PartialSegmentMergeTask extends PerfectRollu this.subtaskSpecId = subtaskSpecId; this.ioConfig = ioConfig; this.numAttempts = numAttempts; - this.supervisorTaskId = supervisorTaskId; } @JsonProperty @@ -120,12 +119,6 @@ public int getNumAttempts() return numAttempts; } - @JsonProperty - public String getSupervisorTaskId() - { - return supervisorTaskId; - } - @JsonProperty @Override public String getSubtaskSpecId() @@ -151,7 +144,7 @@ public TaskStatus runTask(TaskToolbox toolbox) throws Exception } final List locks = toolbox.getTaskActionClient().submit( - new SurrogateAction<>(supervisorTaskId, new LockListAction()) + new SurrogateAction<>(getSupervisorTaskId(), new LockListAction()) ); final Map intervalToVersion = Maps.newHashMapWithExpectedSize(locks.size()); locks.forEach(lock -> { @@ -179,7 +172,7 @@ public TaskStatus runTask(TaskToolbox toolbox) throws Exception LOG.info("Fetch took [%s] seconds", fetchTime); final ParallelIndexSupervisorTaskClient taskClient = toolbox.getSupervisorTaskClientProvider().build( - supervisorTaskId, + getSupervisorTaskId(), getTuningConfig().getChatHandlerTimeout(), getTuningConfig().getChatHandlerNumRetries() ); @@ -225,7 +218,7 @@ private Map>> fetchSegmentFiles( ); FileUtils.mkdirp(partitionDir); for (PartitionLocation location : entryPerBucketId.getValue()) { - final File unzippedDir = toolbox.getShuffleClient().fetchSegmentFile(partitionDir, supervisorTaskId, location); + final File unzippedDir = toolbox.getShuffleClient().fetchSegmentFile(partitionDir, getSupervisorTaskId(), location); intervalToUnzippedFiles.computeIfAbsent(interval, k -> new Int2ObjectOpenHashMap<>()) .computeIfAbsent(bucketId, k -> new ArrayList<>()) .add(unzippedDir); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PerfectRollupWorkerTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PerfectRollupWorkerTask.java index 4259922b43ad..3b00f0fedf67 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PerfectRollupWorkerTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PerfectRollupWorkerTask.java @@ -49,10 +49,11 @@ abstract class PerfectRollupWorkerTask extends AbstractBatchSubtask @Nullable TaskResource taskResource, DataSchema dataSchema, ParallelIndexTuningConfig tuningConfig, - @Nullable Map context + @Nullable Map context, + String supervisorTaskId ) { - super(id, groupId, taskResource, dataSchema.getDataSource(), context, IngestionMode.NONE); + super(id, groupId, taskResource, dataSchema.getDataSource(), context, IngestionMode.NONE, supervisorTaskId); Preconditions.checkArgument( tuningConfig.isForceGuaranteedRollup(), diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/SinglePhaseSubTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/SinglePhaseSubTask.java index 183cde7c66d0..a3bd47d29603 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/SinglePhaseSubTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/SinglePhaseSubTask.java @@ -118,7 +118,6 @@ public class SinglePhaseSubTask extends AbstractBatchSubtask implements ChatHand private final int numAttempts; private final ParallelIndexIngestionSpec ingestionSchema; - private final String supervisorTaskId; private final String subtaskSpecId; /** @@ -169,7 +168,8 @@ public SinglePhaseSubTask( taskResource, ingestionSchema.getDataSchema().getDataSource(), context, - AbstractTask.computeBatchIngestionMode(ingestionSchema.getIOConfig()) + AbstractTask.computeBatchIngestionMode(ingestionSchema.getIOConfig()), + supervisorTaskId ); if (ingestionSchema.getTuningConfig().isForceGuaranteedRollup()) { @@ -179,7 +179,6 @@ public SinglePhaseSubTask( this.subtaskSpecId = subtaskSpecId; this.numAttempts = numAttempts; this.ingestionSchema = ingestionSchema; - this.supervisorTaskId = supervisorTaskId; this.missingIntervalsInOverwriteMode = ingestionSchema.getIOConfig().isAppendToExisting() != true && ingestionSchema.getDataSchema() .getGranularitySpec() @@ -217,7 +216,7 @@ public Set getInputSourceResources() public boolean isReady(TaskActionClient taskActionClient) throws IOException { return determineLockGranularityAndTryLock( - new SurrogateTaskActionClient(supervisorTaskId, taskActionClient), + new SurrogateTaskActionClient(getSupervisorTaskId(), taskActionClient), ingestionSchema.getDataSchema().getGranularitySpec().inputIntervals() ); } @@ -234,12 +233,6 @@ public ParallelIndexIngestionSpec getIngestionSchema() return ingestionSchema; } - @JsonProperty - public String getSupervisorTaskId() - { - return supervisorTaskId; - } - @Override @JsonProperty public String getSubtaskSpecId() @@ -272,7 +265,7 @@ public TaskStatus runTask(final TaskToolbox toolbox) throws Exception final InputSource inputSource = ingestionSchema.getIOConfig().getNonNullInputSource(toolbox); final ParallelIndexSupervisorTaskClient taskClient = toolbox.getSupervisorTaskClientProvider().build( - supervisorTaskId, + getSupervisorTaskId(), ingestionSchema.getTuningConfig().getChatHandlerTimeout(), ingestionSchema.getTuningConfig().getChatHandlerNumRetries() ); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/input/DruidInputSource.java b/indexing-service/src/main/java/org/apache/druid/indexing/input/DruidInputSource.java index bf8b4bfb1d51..8056c69901fc 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/input/DruidInputSource.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/input/DruidInputSource.java @@ -48,10 +48,9 @@ import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.indexing.common.SegmentCacheManagerFactory; import org.apache.druid.indexing.common.TaskToolbox; -import org.apache.druid.indexing.common.actions.RetrieveUsedSegmentsAction; +import org.apache.druid.indexing.common.actions.RetrieveSegmentsToReplaceAction; import org.apache.druid.indexing.common.config.TaskConfig; import org.apache.druid.indexing.firehose.WindowedSegmentId; -import org.apache.druid.indexing.overlord.Segments; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.guava.Comparators; @@ -552,14 +551,7 @@ public static List> getTimelineForInte } else { try { usedSegments = toolbox.getTaskActionClient() - .submit( - new RetrieveUsedSegmentsAction( - dataSource, - null, - Collections.singletonList(interval), - Segments.ONLY_VISIBLE - ) - ); + .submit(new RetrieveSegmentsToReplaceAction(dataSource, interval)); } catch (IOException e) { LOG.error(e, "Error retrieving the used segments for interval[%s].", interval); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskRunner.java index ac1fd124ef55..99b0c05b8323 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskRunner.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskRunner.java @@ -141,11 +141,6 @@ default void updateStatus(Task task, TaskStatus status) // do nothing } - default void updateLocation(Task task, TaskLocation location) - { - // do nothing - } - /** * The maximum number of tasks this TaskRunner can run concurrently. * Can return -1 if this method is not implemented or capacity can't be found. diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/UpdateLocationActionTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/UpdateLocationActionTest.java deleted file mode 100644 index 83aeb382dc0a..000000000000 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/UpdateLocationActionTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.indexing.common.actions; - -import com.google.common.base.Optional; -import org.apache.druid.indexer.TaskLocation; -import org.apache.druid.indexing.common.task.NoopTask; -import org.apache.druid.indexing.common.task.Task; -import org.apache.druid.indexing.overlord.TaskRunner; -import org.junit.Test; - -import java.net.InetAddress; -import java.net.UnknownHostException; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class UpdateLocationActionTest -{ - @Test - public void testFlow() throws UnknownHostException - { - // get my task location - InetAddress hostName = InetAddress.getLocalHost(); - TaskLocation myLocation = TaskLocation.create(hostName.getHostAddress(), 1, 2); - UpdateLocationAction action = new UpdateLocationAction(myLocation); - Task task = NoopTask.create(); - TaskActionToolbox toolbox = mock(TaskActionToolbox.class); - TaskRunner runner = mock(TaskRunner.class); - when(toolbox.getTaskRunner()).thenReturn(Optional.of(runner)); - action.perform(task, toolbox); - verify(runner, times(1)).updateLocation(eq(task), eq(myLocation)); - } - - @Test - public void testWithNoTaskRunner() throws UnknownHostException - { - // get my task location - InetAddress hostName = InetAddress.getLocalHost(); - TaskLocation myLocation = TaskLocation.create(hostName.getHostAddress(), 1, 2); - UpdateLocationAction action = new UpdateLocationAction(myLocation); - Task task = NoopTask.create(); - TaskActionToolbox toolbox = mock(TaskActionToolbox.class); - TaskRunner runner = mock(TaskRunner.class); - when(toolbox.getTaskRunner()).thenReturn(Optional.absent()); - action.perform(task, toolbox); - verify(runner, never()).updateStatus(any(), any()); - } -} diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/AbstractTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/AbstractTaskTest.java index 5bcadcfb7125..39b0bdfcfc50 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/AbstractTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/AbstractTaskTest.java @@ -109,7 +109,7 @@ public String setup(TaskToolbox toolbox) throws Exception task.run(toolbox); // call it 3 times, once to update location in setup, then one for status and location in cleanup - Mockito.verify(taskActionClient, times(3)).submit(any()); + Mockito.verify(taskActionClient, times(1)).submit(any()); verify(pusher, times(1)).pushTaskReports(eq("myID"), any()); verify(pusher, times(1)).pushTaskStatus(eq("myID"), any()); } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/PerfectRollupWorkerTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/PerfectRollupWorkerTaskTest.java index 71b474e77359..b95fd53c74f7 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/PerfectRollupWorkerTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/PerfectRollupWorkerTaskTest.java @@ -111,7 +111,8 @@ PerfectRollupWorkerTask build() null, createDataSchema(granularitySpecInputIntervals), createTuningConfig(forceGuaranteedRollup, partitionsSpec), - null + null, + "supervisor-id" ); } @@ -149,10 +150,11 @@ private static class TestPerfectRollupWorkerTask extends PerfectRollupWorkerTask @Nullable TaskResource taskResource, DataSchema dataSchema, ParallelIndexTuningConfig tuningConfig, - @Nullable Map context + @Nullable Map context, + String supervisorId ) { - super(id, groupId, taskResource, dataSchema, tuningConfig, context); + super(id, groupId, taskResource, dataSchema, tuningConfig, context, supervisorId); } @Override diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/concurrent/ConcurrentReplaceAndAppendTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/concurrent/ConcurrentReplaceAndAppendTest.java index 22f21fb79b62..1c4b6809c387 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/concurrent/ConcurrentReplaceAndAppendTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/concurrent/ConcurrentReplaceAndAppendTest.java @@ -27,6 +27,7 @@ import org.apache.druid.indexing.common.TaskStorageDirTracker; import org.apache.druid.indexing.common.TaskToolbox; import org.apache.druid.indexing.common.TaskToolboxFactory; +import org.apache.druid.indexing.common.actions.RetrieveSegmentsToReplaceAction; import org.apache.druid.indexing.common.actions.RetrieveUsedSegmentsAction; import org.apache.druid.indexing.common.actions.TaskActionClient; import org.apache.druid.indexing.common.actions.TaskActionClientFactory; @@ -845,6 +846,54 @@ public void testSegmentIsAllocatedAtLatestVersion() verifyIntervalHasVisibleSegments(YEAR_23, segmentV20, segmentV21, segmentV22, segmentV23); } + @Test + public void testSegmentsToReplace() + { + final SegmentIdWithShardSpec pendingSegmentV01 + = appendTask.allocateSegmentForTimestamp(JAN_23.getStart(), Granularities.MONTH); + Assert.assertEquals(SEGMENT_V0, pendingSegmentV01.getVersion()); + Assert.assertEquals(JAN_23, pendingSegmentV01.getInterval()); + final DataSegment segment1 = asSegment(pendingSegmentV01); + appendTask.commitAppendSegments(segment1); + + final SegmentIdWithShardSpec pendingSegmentV02 + = appendTask.allocateSegmentForTimestamp(JAN_23.getStart(), Granularities.MONTH); + Assert.assertNotEquals(pendingSegmentV01.asSegmentId(), pendingSegmentV02.asSegmentId()); + Assert.assertEquals(SEGMENT_V0, pendingSegmentV02.getVersion()); + Assert.assertEquals(JAN_23, pendingSegmentV02.getInterval()); + + verifyInputSegments(replaceTask, JAN_23, segment1); + + replaceTask.acquireReplaceLockOn(JAN_23); + + final DataSegment segment2 = asSegment(pendingSegmentV02); + appendTask.commitAppendSegments(segment2); + + // Despite segment2 existing, it is not chosen to be replaced because it was created after the tasklock was acquired + verifyInputSegments(replaceTask, JAN_23, segment1); + + replaceTask.releaseLock(JAN_23); + + final SegmentIdWithShardSpec pendingSegmentV03 + = appendTask.allocateSegmentForTimestamp(JAN_23.getStart(), Granularities.MONTH); + Assert.assertNotEquals(pendingSegmentV01.asSegmentId(), pendingSegmentV03.asSegmentId()); + Assert.assertNotEquals(pendingSegmentV02.asSegmentId(), pendingSegmentV03.asSegmentId()); + Assert.assertEquals(SEGMENT_V0, pendingSegmentV03.getVersion()); + Assert.assertEquals(JAN_23, pendingSegmentV03.getInterval()); + final DataSegment segment3 = asSegment(pendingSegmentV03); + appendTask.commitAppendSegments(segment3); + appendTask.releaseLock(JAN_23); + + replaceTask.acquireReplaceLockOn(FIRST_OF_JAN_23); + // The new lock was acquired before segment3 was created but it doesn't contain the month's interval + // So, all three segments are chosen + verifyInputSegments(replaceTask, JAN_23, segment1, segment2, segment3); + + replaceTask.releaseLock(FIRST_OF_JAN_23); + // All the segments are chosen when there's no lock + verifyInputSegments(replaceTask, JAN_23, segment1, segment2, segment3); + } + @Nullable private DataSegment findSegmentWith(String version, Map loadSpec, Set segments) { @@ -901,6 +950,23 @@ private void verifySegments(Interval interval, Segments visibility, DataSegment. } } + private void verifyInputSegments(Task task, Interval interval, DataSegment... expectedSegments) + { + try { + final TaskActionClient taskActionClient = taskActionClientFactory.create(task); + Collection allUsedSegments = taskActionClient.submit( + new RetrieveSegmentsToReplaceAction( + WIKI, + interval + ) + ); + Assert.assertEquals(Sets.newHashSet(expectedSegments), Sets.newHashSet(allUsedSegments)); + } + catch (IOException e) { + throw new ISE(e, "Error while fetching segments to replace in interval[%s]", interval); + } + } + private TaskToolboxFactory createToolboxFactory( TaskConfig taskConfig, TaskActionClientFactory taskActionClientFactory diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/test/TestIndexerMetadataStorageCoordinator.java b/indexing-service/src/test/java/org/apache/druid/indexing/test/TestIndexerMetadataStorageCoordinator.java index d1c72485011f..108833422c88 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/test/TestIndexerMetadataStorageCoordinator.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/test/TestIndexerMetadataStorageCoordinator.java @@ -89,7 +89,7 @@ public List retrieveAllUsedSegments(String dataSource, Segments vis } @Override - public List> retrieveUsedSegmentsAndCreatedDates(String dataSource) + public List> retrieveUsedSegmentsAndCreatedDates(String dataSource, Interval interval) { return ImmutableList.of(); } diff --git a/integration-tests/script/docker_build_containers.sh b/integration-tests/script/docker_build_containers.sh index 95e32f3ec027..ba2dd7dcd822 100755 --- a/integration-tests/script/docker_build_containers.sh +++ b/integration-tests/script/docker_build_containers.sh @@ -26,21 +26,18 @@ then else echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is set with value ${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" case "${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" in - 8) - echo "Build druid-cluster with Java 8" - docker build -t druid/cluster --build-arg JDK_VERSION=8-slim-buster --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg MARIA_VERSION --build-arg MYSQL_DRIVER_CLASSNAME --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker - ;; - 11) - echo "Build druid-cluster with Java 11" - docker build -t druid/cluster --build-arg JDK_VERSION=11-slim-buster --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg MARIA_VERSION --build-arg MYSQL_DRIVER_CLASSNAME --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker - ;; - 15) - echo "Build druid-cluster with Java 15" - docker build -t druid/cluster --build-arg JDK_VERSION=15-slim-buster --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg MARIA_VERSION --build-arg MYSQL_DRIVER_CLASSNAME --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker - ;; - 17) - echo "Build druid-cluster with Java 17" - docker build -t druid/cluster --build-arg JDK_VERSION=17-slim-buster --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg MARIA_VERSION --build-arg MYSQL_DRIVER_CLASSNAME --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker + 8 | 11 | 17 | 21) + echo "Build druid-cluster with Java $DRUID_INTEGRATION_TEST_JVM_RUNTIME" + docker build -t druid/cluster \ + --build-arg JDK_VERSION=$DRUID_INTEGRATION_TEST_JVM_RUNTIME-slim-buster \ + --build-arg ZK_VERSION \ + --build-arg KAFKA_VERSION \ + --build-arg CONFLUENT_VERSION \ + --build-arg MYSQL_VERSION \ + --build-arg MARIA_VERSION \ + --build-arg MYSQL_DRIVER_CLASSNAME \ + --build-arg APACHE_ARCHIVE_MIRROR_HOST \ + $SHARED_DIR/docker ;; *) echo "Invalid JVM Runtime given. Stopping" diff --git a/licenses.yaml b/licenses.yaml index f2c01acaffde..a207fb1b0acf 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -3527,7 +3527,7 @@ libraries: --- name: Apache Kafka -version: 3.5.1 +version: 3.6.0 license_category: binary module: extensions/druid-kafka-indexing-service license_name: Apache License version 2.0 @@ -4464,7 +4464,7 @@ name: Apache Kafka license_category: binary module: extensions/kafka-extraction-namespace license_name: Apache License version 2.0 -version: 3.5.1 +version: 3.6.0 libraries: - org.apache.kafka: kafka-clients notices: diff --git a/pom.xml b/pom.xml index e0617ffbcc2c..26f353534369 100644 --- a/pom.xml +++ b/pom.xml @@ -76,7 +76,7 @@ UTF-8 0.9.0.M2 5.5.0 - 3.5.1 + 3.6.0 2.0.0 2.2.4 2.13.11 diff --git a/processing/src/main/java/org/apache/druid/data/input/Rows.java b/processing/src/main/java/org/apache/druid/data/input/Rows.java index c10460c86c8a..add6781b81b1 100644 --- a/processing/src/main/java/org/apache/druid/data/input/Rows.java +++ b/processing/src/main/java/org/apache/druid/data/input/Rows.java @@ -25,7 +25,6 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.parsers.ParseException; -import org.apache.druid.math.expr.Evals; import javax.annotation.Nullable; import java.util.Arrays; @@ -60,7 +59,11 @@ public static List toGroupKey(long timeStamp, InputRow inputRow) } /** - * Convert an object to a list of strings. + * Convert an object to a list of strings. This function translates single value nulls into an empty list, and any + * nulls inside of a list or array into the string "null". Do not use this method if you don't want this behavior, + * but note that many implementations of {@link InputRow#getDimension(String)} do use this method, so it is + * recommended to use {@link InputRow#getRaw(String)} if you want the actual value without this coercion. For legacy + * reasons, some stuff counts on this incorrect behavior, (such as {@link Rows#toGroupKey(long, InputRow)}). */ public static List objectToStrings(final Object inputValue) { @@ -73,7 +76,7 @@ public static List objectToStrings(final Object inputValue) // convert byte[] to base64 encoded string return Collections.singletonList(StringUtils.encodeBase64String((byte[]) inputValue)); } else if (inputValue instanceof Object[]) { - return Arrays.stream((Object[]) inputValue).map(Evals::asString).collect(Collectors.toList()); + return Arrays.stream((Object[]) inputValue).map(String::valueOf).collect(Collectors.toList()); } else { return Collections.singletonList(String.valueOf(inputValue)); } diff --git a/processing/src/main/java/org/apache/druid/frame/field/DoubleArrayFieldReader.java b/processing/src/main/java/org/apache/druid/frame/field/DoubleArrayFieldReader.java index 48f2a5117ee8..ec7de095e12c 100644 --- a/processing/src/main/java/org/apache/druid/frame/field/DoubleArrayFieldReader.java +++ b/processing/src/main/java/org/apache/druid/frame/field/DoubleArrayFieldReader.java @@ -37,6 +37,7 @@ public ColumnValueSelector makeColumnValueSelector( { return new NumericArrayFieldSelector(memory, fieldPointer) { + private static final int FIELD_SIZE = Byte.BYTES + Double.BYTES; final SettableFieldPointer fieldPointer = new SettableFieldPointer(); final ColumnValueSelector columnValueSelector = DoubleFieldReader.forArray().makeColumnValueSelector(memory, fieldPointer); @@ -45,7 +46,7 @@ public ColumnValueSelector makeColumnValueSelector( @Override public Double getIndividualValueAtMemory(long position) { - fieldPointer.setPosition(position); + fieldPointer.setPositionAndLength(position, FIELD_SIZE); if (columnValueSelector.isNull()) { return null; } @@ -55,7 +56,7 @@ public Double getIndividualValueAtMemory(long position) @Override public int getIndividualFieldSize() { - return Byte.BYTES + Double.BYTES; + return FIELD_SIZE; } }; } diff --git a/processing/src/main/java/org/apache/druid/frame/field/FloatArrayFieldReader.java b/processing/src/main/java/org/apache/druid/frame/field/FloatArrayFieldReader.java index fcbe407bdb26..e97af071824e 100644 --- a/processing/src/main/java/org/apache/druid/frame/field/FloatArrayFieldReader.java +++ b/processing/src/main/java/org/apache/druid/frame/field/FloatArrayFieldReader.java @@ -37,6 +37,7 @@ public ColumnValueSelector makeColumnValueSelector( { return new NumericArrayFieldSelector(memory, fieldPointer) { + private static final int FIELD_SIZE = Byte.BYTES + Float.BYTES; final SettableFieldPointer fieldPointer = new SettableFieldPointer(); final ColumnValueSelector columnValueSelector = FloatFieldReader.forArray().makeColumnValueSelector(memory, fieldPointer); @@ -45,7 +46,7 @@ public ColumnValueSelector makeColumnValueSelector( @Override public Float getIndividualValueAtMemory(long position) { - fieldPointer.setPosition(position); + fieldPointer.setPositionAndLength(position, FIELD_SIZE); if (columnValueSelector.isNull()) { return null; } @@ -55,7 +56,7 @@ public Float getIndividualValueAtMemory(long position) @Override public int getIndividualFieldSize() { - return Byte.BYTES + Float.BYTES; + return FIELD_SIZE; } }; } diff --git a/processing/src/main/java/org/apache/druid/frame/field/LongArrayFieldReader.java b/processing/src/main/java/org/apache/druid/frame/field/LongArrayFieldReader.java index b52b39d13c47..8f7578c07d38 100644 --- a/processing/src/main/java/org/apache/druid/frame/field/LongArrayFieldReader.java +++ b/processing/src/main/java/org/apache/druid/frame/field/LongArrayFieldReader.java @@ -37,6 +37,7 @@ public ColumnValueSelector makeColumnValueSelector( { return new NumericArrayFieldSelector(memory, fieldPointer) { + private static final int FIELD_SIZE = Byte.BYTES + Long.BYTES; final SettableFieldPointer fieldPointer = new SettableFieldPointer(); final ColumnValueSelector columnValueSelector = LongFieldReader.forArray().makeColumnValueSelector(memory, fieldPointer); @@ -45,7 +46,7 @@ public ColumnValueSelector makeColumnValueSelector( @Override public Long getIndividualValueAtMemory(long position) { - fieldPointer.setPosition(position); + fieldPointer.setPositionAndLength(position, FIELD_SIZE); if (columnValueSelector.isNull()) { return null; } @@ -55,7 +56,7 @@ public Long getIndividualValueAtMemory(long position) @Override public int getIndividualFieldSize() { - return Byte.BYTES + Long.BYTES; + return FIELD_SIZE; } }; } diff --git a/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldSelector.java b/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldSelector.java index 1871aef06e08..f15361d47ea1 100644 --- a/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldSelector.java +++ b/processing/src/main/java/org/apache/druid/frame/field/NumericArrayFieldSelector.java @@ -25,8 +25,6 @@ import org.apache.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.List; /** * Base implementation of the column value selector that the concrete numeric field reader implementations inherit from. @@ -66,12 +64,8 @@ public abstract class NumericArrayFieldSelector impl /** * Value of the row at the location beginning at {@link #currentFieldPosition} */ - private final List currentRow = new ArrayList<>(); - - /** - * Nullity of the row at the location beginning at {@link #currentFieldPosition} - */ - private boolean currentRowIsNull; + @Nullable + private Number[] currentRow = null; public NumericArrayFieldSelector(final Memory memory, final ReadableFieldPointer fieldPointer) { @@ -89,13 +83,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public Object getObject() { - final List currentArray = computeCurrentArray(); - - if (currentArray == null) { - return null; - } - - return currentArray.toArray(); + return computeCurrentArray(); } @Override @@ -143,34 +131,29 @@ public boolean isNull() public abstract int getIndividualFieldSize(); @Nullable - private List computeCurrentArray() + private Number[] computeCurrentArray() { final long fieldPosition = fieldPointer.position(); + final long fieldLength = fieldPointer.length(); if (fieldPosition != currentFieldPosition) { - updateCurrentArray(fieldPosition); + updateCurrentArray(fieldPosition, fieldLength); } this.currentFieldPosition = fieldPosition; - - if (currentRowIsNull) { - return null; - } return currentRow; - } - private void updateCurrentArray(final long fieldPosition) + private void updateCurrentArray(final long fieldPosition, final long fieldLength) { - currentRow.clear(); - currentRowIsNull = false; + currentRow = null; long position = fieldPosition; long limit = memory.getCapacity(); // Check the first byte, and if it is null, update the current value to null and return if (isNull()) { - currentRowIsNull = true; + // Already set the currentRow to null return; } @@ -179,9 +162,13 @@ private void updateCurrentArray(final long fieldPosition) position++; } + int numElements = numElements(fieldLength); + currentRow = new Number[numElements]; + // Sanity check, to make sure that we see the rowTerminator at the end boolean rowTerminatorSeen = false; + int curElement = 0; while (position < limit) { final byte kind = memory.getByte(position); @@ -193,12 +180,26 @@ private void updateCurrentArray(final long fieldPosition) // If terminator not seen, then read the field at that location, and increment the position by the element's field // size to read the next element. - currentRow.add(getIndividualValueAtMemory(position)); + currentRow[curElement] = getIndividualValueAtMemory(position); position += getIndividualFieldSize(); + curElement++; } - if (!rowTerminatorSeen) { + if (!rowTerminatorSeen || curElement != numElements) { throw DruidException.defensive("Unexpected end of field"); } } + + int numElements(long fieldSize) + { + if (fieldSize <= 1) { + throw DruidException.defensive("fieldSize should be greater than 1 for non null array elements"); + } + // Remove one byte for the nullity byte, and one for the array terminator + long cumulativeFieldSize = fieldSize - Byte.BYTES - Byte.BYTES; + if (cumulativeFieldSize % getIndividualFieldSize() != 0) { + throw DruidException.defensive("cumulativeFieldSize should be a multiple of the individual fieldSize"); + } + return Math.toIntExact(cumulativeFieldSize / getIndividualFieldSize()); + } } diff --git a/processing/src/main/java/org/apache/druid/frame/field/ReadableFieldPointer.java b/processing/src/main/java/org/apache/druid/frame/field/ReadableFieldPointer.java index 29412b507af8..09de051a691e 100644 --- a/processing/src/main/java/org/apache/druid/frame/field/ReadableFieldPointer.java +++ b/processing/src/main/java/org/apache/druid/frame/field/ReadableFieldPointer.java @@ -31,4 +31,9 @@ public interface ReadableFieldPointer * Starting position of the field. */ long position(); + + /** + * Length of the field. + */ + long length(); } diff --git a/processing/src/main/java/org/apache/druid/frame/field/RowMemoryFieldPointer.java b/processing/src/main/java/org/apache/druid/frame/field/RowMemoryFieldPointer.java index 881c9adb04f9..d90eb3eaa22a 100644 --- a/processing/src/main/java/org/apache/druid/frame/field/RowMemoryFieldPointer.java +++ b/processing/src/main/java/org/apache/druid/frame/field/RowMemoryFieldPointer.java @@ -24,6 +24,10 @@ /** * A {@link ReadableFieldPointer} that is derived from a row-based frame. + * + * Returns the position and the length of a field at a particular position for the row that the rowPointer is pointing + * to at the time. It caches the values of the position and the length based on position of the rowPointer. + * This method is not thread-safe */ public class RowMemoryFieldPointer implements ReadableFieldPointer { @@ -32,6 +36,16 @@ public class RowMemoryFieldPointer implements ReadableFieldPointer private final int fieldNumber; private final int fieldCount; + // Caching of position() calls + private long rowWithCachedPosition = -1L; + private long cachedPosition = -1L; + + // Caching of length() calls + // We cache the length() calls separately, because not all field types call length(), therefore it's wasteful to + // compute and cache length() if we are not reading it + private long rowWithCachedLength = -1L; + private long cachedLength = -1L; + public RowMemoryFieldPointer( final Memory memory, final ReadableFrameRowPointer rowPointer, @@ -47,6 +61,63 @@ public RowMemoryFieldPointer( @Override public long position() + { + updatePosition(); + return cachedPosition; + } + + @Override + public long length() + { + updatePositionAndLength(); + return cachedLength; + } + + private void updatePosition() + { + long rowPointerPosition = rowPointer.position(); + if (rowPointerPosition == rowWithCachedPosition) { + return; + } + // Update the cached position for position() + rowWithCachedPosition = rowPointerPosition; + + // Update the start position + cachedPosition = startPosition(fieldNumber); + } + + // Not all field types call length(), and therefore there's no need to cache the length of the field. This method + // updates both the position and the length. + private void updatePositionAndLength() + { + updatePosition(); + + // rowPointerPosition = rowPointer.position() = rowWithCachedPosition, since that was updated in the call to update + // position above + long rowPointerPosition = rowWithCachedPosition; + + if (rowPointerPosition == rowWithCachedLength) { + return; + } + // Update the cached position for length() + rowWithCachedLength = rowPointerPosition; + + if (fieldNumber == fieldCount - 1) { + // If the field is the last field in the row, then the length of the field would be the end of the row minus the + // start position of the field. End of the row is the start of the row plus the length of the row. + cachedLength = (rowPointerPosition + rowPointer.length()) - cachedPosition; + } else { + // Else the length of the field would be the difference between the start position of the given field and + // the subsequent field + cachedLength = startPosition(fieldNumber + 1) - cachedPosition; + } + } + + /** + * Given a fieldNumber, computes the start position of the field. Requires a memory access to read the start position, + * therefore callers should cache the value for better efficiency. + */ + private long startPosition(int fieldNumber) { if (fieldNumber == 0) { // First field starts after the field end pointers -- one integer per field. diff --git a/processing/src/main/java/org/apache/druid/frame/field/SettableFieldPointer.java b/processing/src/main/java/org/apache/druid/frame/field/SettableFieldPointer.java index d26f84f251de..0242b412ca08 100644 --- a/processing/src/main/java/org/apache/druid/frame/field/SettableFieldPointer.java +++ b/processing/src/main/java/org/apache/druid/frame/field/SettableFieldPointer.java @@ -20,16 +20,20 @@ package org.apache.druid.frame.field; /** - * A simple {@link ReadableFieldPointer} that returns the position that was set on its object. + * A simple {@link ReadableFieldPointer} that returns the position and the length that was set on its object. */ public class SettableFieldPointer implements ReadableFieldPointer { - long position = 0; + long length = -1; - public void setPosition(long position) + /** + * Sets the position and the length to be returned when interface's methods are called. + */ + public void setPositionAndLength(long position, long length) { this.position = position; + this.length = length; } @Override @@ -37,4 +41,10 @@ public long position() { return position; } + + @Override + public long length() + { + return length; + } } diff --git a/processing/src/main/java/org/apache/druid/frame/segment/row/ReadableFrameRowPointer.java b/processing/src/main/java/org/apache/druid/frame/segment/row/ReadableFrameRowPointer.java index 714195396c8e..8276e26b28c3 100644 --- a/processing/src/main/java/org/apache/druid/frame/segment/row/ReadableFrameRowPointer.java +++ b/processing/src/main/java/org/apache/druid/frame/segment/row/ReadableFrameRowPointer.java @@ -30,7 +30,13 @@ */ public interface ReadableFrameRowPointer { + /** + * Position of the start of the row relative to the start of the Frame + */ long position(); + /** + * Length of the row (in bytes) + */ long length(); } diff --git a/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java index 4e4a3b10ec94..95c7b78862b9 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java @@ -51,7 +51,9 @@ @JsonSubTypes.Type(name = "false", value = FalseDimFilter.class), @JsonSubTypes.Type(name = "null", value = NullFilter.class), @JsonSubTypes.Type(name = "equals", value = EqualityFilter.class), - @JsonSubTypes.Type(name = "range", value = RangeFilter.class) + @JsonSubTypes.Type(name = "range", value = RangeFilter.class), + @JsonSubTypes.Type(name = "isfalse", value = IsFalseDimFilter.class), + @JsonSubTypes.Type(name = "istrue", value = IsTrueDimFilter.class) }) public interface DimFilter extends Cacheable { diff --git a/processing/src/main/java/org/apache/druid/query/filter/DimFilterUtils.java b/processing/src/main/java/org/apache/druid/query/filter/DimFilterUtils.java index 27a0581d4752..ed03efac38a6 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/DimFilterUtils.java +++ b/processing/src/main/java/org/apache/druid/query/filter/DimFilterUtils.java @@ -58,6 +58,8 @@ public class DimFilterUtils static final byte EQUALS_CACHE_ID = 0x13; static final byte RANGE_CACHE_ID = 0x14; + static final byte IS_FILTER_BOOLEAN_FILTER_CACHE_ID = 0x15; + public static final byte STRING_SEPARATOR = (byte) 0xFF; diff --git a/processing/src/main/java/org/apache/druid/query/filter/IsBooleanDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/IsBooleanDimFilter.java new file mode 100644 index 000000000000..4b766de3af92 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/filter/IsBooleanDimFilter.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.filter; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.RangeSet; +import org.apache.druid.error.DruidException; +import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.filter.IsBooleanFilter; + +import java.util.Objects; +import java.util.Set; + +/** + * Abstract SQL three-value logic wrapper for some child {@link DimFilter} to implement '{filter} IS TRUE' and + * '{filter} IS FALSE'. + * + * @see IsTrueDimFilter - IS TRUE + * @see IsFalseDimFilter - IS FALSE + * @see IsBooleanFilter - actual filtering logic + */ +public abstract class IsBooleanDimFilter extends AbstractOptimizableDimFilter +{ + private final DimFilter field; + private final boolean isTrue; + + public IsBooleanDimFilter( + DimFilter field, + boolean isTrue + ) + { + if (field == null) { + throw DruidException.forPersona(DruidException.Persona.USER) + .ofCategory(DruidException.Category.INVALID_INPUT) + .build("IS %s operator requires a non-null filter for field", isTrue ? "TRUE" : "FALSE"); + } + this.field = field; + this.isTrue = isTrue; + } + + @JsonProperty("field") + public DimFilter getField() + { + return field; + } + + @Override + public byte[] getCacheKey() + { + return new CacheKeyBuilder(DimFilterUtils.IS_FILTER_BOOLEAN_FILTER_CACHE_ID).appendBoolean(isTrue) + .appendCacheable(field) + .build(); + } + + @Override + public Filter toFilter() + { + return new IsBooleanFilter(field.toFilter(), isTrue); + } + + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + return null; + } + + @Override + public Set getRequiredColumns() + { + return field.getRequiredColumns(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + IsBooleanDimFilter that = (IsBooleanDimFilter) o; + + if (field != null ? !field.equals(that.field) : that.field != null) { + return false; + } + + return isTrue == that.isTrue; + } + + @Override + public int hashCode() + { + return Objects.hash(field, isTrue); + } + + @Override + public String toString() + { + return "(" + field + ") IS " + (isTrue ? "TRUE" : "FALSE"); + } +} diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/PeonPhaseTest.java b/processing/src/main/java/org/apache/druid/query/filter/IsFalseDimFilter.java similarity index 53% rename from extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/PeonPhaseTest.java rename to processing/src/main/java/org/apache/druid/query/filter/IsFalseDimFilter.java index 3f6bd71312be..7e674869a883 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/PeonPhaseTest.java +++ b/processing/src/main/java/org/apache/druid/query/filter/IsFalseDimFilter.java @@ -17,27 +17,29 @@ * under the License. */ -package org.apache.druid.k8s.overlord.common; +package org.apache.druid.query.filter; -import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.api.model.PodStatus; -import org.junit.jupiter.api.Test; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class PeonPhaseTest +public class IsFalseDimFilter extends IsBooleanDimFilter { + public static IsFalseDimFilter of(DimFilter field) + { + return new IsFalseDimFilter(field); + } + + @JsonCreator + public IsFalseDimFilter( + @JsonProperty("field") DimFilter field + ) + { + super(field, false); + } - @Test - void testGetPhaseForToMakeCoverageHappy() + @Override + public DimFilter optimize() { - Pod pod = mock(Pod.class); - PodStatus status = mock(PodStatus.class); - when(status.getPhase()).thenReturn("Succeeded"); - when(pod.getStatus()).thenReturn(status); - assertEquals(PeonPhase.UNKNOWN, PeonPhase.getPhaseFor(null)); - assertEquals(PeonPhase.SUCCEEDED, PeonPhase.getPhaseFor(pod)); + return new IsFalseDimFilter(getField().optimize()); } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/IsTrueDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/IsTrueDimFilter.java new file mode 100644 index 000000000000..a61897f10f0f --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/filter/IsTrueDimFilter.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.filter; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class IsTrueDimFilter extends IsBooleanDimFilter +{ + public static IsTrueDimFilter of(DimFilter field) + { + return new IsTrueDimFilter(field); + } + + @JsonCreator + public IsTrueDimFilter( + @JsonProperty("field") DimFilter field + ) + { + super(field, true); + } + + @Override + public DimFilter optimize() + { + return new IsTrueDimFilter(getField().optimize()); + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/UnnestDimensionCursor.java b/processing/src/main/java/org/apache/druid/segment/UnnestDimensionCursor.java index 5db2dd414e03..a9880c705f9c 100644 --- a/processing/src/main/java/org/apache/druid/segment/UnnestDimensionCursor.java +++ b/processing/src/main/java/org/apache/druid/segment/UnnestDimensionCursor.java @@ -126,13 +126,21 @@ public ValueMatcher makeValueMatcher(@Nullable String value) @Override public boolean matches(boolean includeUnknown) { - return includeUnknown; + // don't match anything, except for null values when includeUnknown is true + if (includeUnknown) { + if (indexedIntsForCurrentRow == null || indexedIntsForCurrentRow.size() <= 0) { + return true; + } + final int rowId = indexedIntsForCurrentRow.get(index); + return lookupName(rowId) == null; + } + return false; } @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { - + inspector.visit("selector", dimSelector); } }; } @@ -155,7 +163,7 @@ public boolean matches(boolean includeUnknown) @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { - dimSelector.inspectRuntimeShape(inspector); + inspector.visit("selector", dimSelector); } }; } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java index 6d2bfab4b6a3..1208865d4ce1 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java @@ -186,7 +186,7 @@ public boolean matches(boolean includeUnknown) return Arrays.stream(dResult).filter(Objects::nonNull).anyMatch(o -> Evals.asBoolean((double) o)); } } - return (includeUnknown && eval.value() == null) || eval.asBoolean(); + return eval.asBoolean(); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/IsBooleanFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/IsBooleanFilter.java new file mode 100644 index 000000000000..ddf3972ccff8 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/filter/IsBooleanFilter.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.filter; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.query.BitmapResultFactory; +import org.apache.druid.query.filter.ColumnIndexSelector; +import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.ValueMatcher; +import org.apache.druid.query.filter.vector.BaseVectorValueMatcher; +import org.apache.druid.query.filter.vector.ReadableVectorMatch; +import org.apache.druid.query.filter.vector.VectorMatch; +import org.apache.druid.query.filter.vector.VectorValueMatcher; +import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import org.apache.druid.segment.ColumnInspector; +import org.apache.druid.segment.ColumnSelector; +import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ColumnIndexCapabilities; +import org.apache.druid.segment.index.BitmapColumnIndex; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; + +import javax.annotation.Nullable; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * SQL three-value logic wrapper for some child {@link Filter} to implement '{filter} IS TRUE' and + * '{filter} IS FALSE'. Primarily useful when living beneath a {@link NotFilter} because this filter purposely ignores + * the value of {@code includeUnknown} and so always correctly only returns values that definitely match or do not match + * the filter to produce correct results for '{filter} IS NOT TRUE' and '{filter} IS NOT FALSE'. This filter is a + * relatively thin wrapper, so should be relatively harmless if used without a 'NOT' filter. + * + * @see org.apache.druid.query.filter.IsBooleanDimFilter + * @see org.apache.druid.query.filter.IsTrueDimFilter + * @see org.apache.druid.query.filter.IsFalseDimFilter + */ +public class IsBooleanFilter implements Filter +{ + private final Filter baseFilter; + private final boolean isTrue; + + public IsBooleanFilter(Filter baseFilter, boolean isTrue) + { + this.baseFilter = baseFilter; + this.isTrue = isTrue; + } + + @Nullable + @Override + public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector) + { + final BitmapColumnIndex baseIndex = baseFilter.getBitmapColumnIndex(selector); + if (baseIndex != null && (isTrue || baseIndex.getIndexCapabilities().isInvertible())) { + return new BitmapColumnIndex() + { + private final boolean useThreeValueLogic = NullHandling.useThreeValueLogic(); + @Override + public ColumnIndexCapabilities getIndexCapabilities() + { + return baseIndex.getIndexCapabilities(); + } + + @Override + public double estimateSelectivity(int totalRows) + { + return 1. - baseFilter.estimateSelectivity(selector); + } + + @Override + public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) + { + if (isTrue) { + return baseIndex.computeBitmapResult(bitmapResultFactory, false); + } + return bitmapResultFactory.complement( + baseIndex.computeBitmapResult(bitmapResultFactory, useThreeValueLogic), + selector.getNumRows() + ); + } + }; + } + return null; + } + + @Override + public ValueMatcher makeMatcher(ColumnSelectorFactory factory) + { + final ValueMatcher baseMatcher = baseFilter.makeMatcher(factory); + + return new ValueMatcher() + { + private final boolean useThreeValueLogic = NullHandling.useThreeValueLogic(); + @Override + public boolean matches(boolean includeUnknown) + { + if (isTrue) { + return baseMatcher.matches(false); + } + return !baseMatcher.matches(useThreeValueLogic); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("baseMatcher", baseMatcher); + } + }; + } + + @Override + public VectorValueMatcher makeVectorMatcher(final VectorColumnSelectorFactory factory) + { + final VectorValueMatcher baseMatcher = baseFilter.makeVectorMatcher(factory); + + return new BaseVectorValueMatcher(baseMatcher) + { + private final VectorMatch scratch = VectorMatch.wrap(new int[factory.getMaxVectorSize()]); + private final boolean useThreeValueLogic = NullHandling.useThreeValueLogic(); + + @Override + public ReadableVectorMatch match(final ReadableVectorMatch mask, boolean includeUnknown) + { + if (isTrue) { + return baseMatcher.match(mask, false); + } + final ReadableVectorMatch baseMatch = baseMatcher.match(mask, useThreeValueLogic); + + scratch.copyFrom(mask); + scratch.removeAll(baseMatch); + assert scratch.isValid(mask); + return scratch; + } + }; + } + + @Override + public boolean canVectorizeMatcher(ColumnInspector inspector) + { + return baseFilter.canVectorizeMatcher(inspector); + } + + @Override + public Set getRequiredColumns() + { + return baseFilter.getRequiredColumns(); + } + + @Override + public boolean supportsRequiredColumnRewrite() + { + return baseFilter.supportsRequiredColumnRewrite(); + } + + @Override + public Filter rewriteRequiredColumns(Map columnRewrites) + { + return new IsBooleanFilter(baseFilter.rewriteRequiredColumns(columnRewrites), isTrue); + } + + @Override + public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, ColumnIndexSelector indexSelector) + { + return baseFilter.supportsSelectivityEstimation(columnSelector, indexSelector); + } + + @Override + public String toString() + { + return StringUtils.format("(%s) IS %s", baseFilter, isTrue ? "TRUE" : "FALSE"); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + IsBooleanFilter isFilter = (IsBooleanFilter) o; + return Objects.equals(baseFilter, isFilter.baseFilter); + } + + @Override + public int hashCode() + { + // to return a different hash from baseFilter + return Objects.hash(1, baseFilter, isTrue); + } +} diff --git a/processing/src/test/java/org/apache/druid/frame/field/ComplexFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/ComplexFieldReaderTest.java index 598e7d28edc1..d4970d9bd0cd 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/ComplexFieldReaderTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/ComplexFieldReaderTest.java @@ -123,7 +123,7 @@ public void test_makeColumnValueSelector_null() writeToMemory(null); final ColumnValueSelector readSelector = - new ComplexFieldReader(SERDE).makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new ComplexFieldReader(SERDE).makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertNull(readSelector.getObject()); } @@ -134,7 +134,7 @@ public void test_makeColumnValueSelector_aValue() writeToMemory("foo"); final ColumnValueSelector readSelector = - new ComplexFieldReader(SERDE).makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new ComplexFieldReader(SERDE).makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals("foo", readSelector.getObject()); } diff --git a/processing/src/main/java/org/apache/druid/frame/field/ConstantFieldPointer.java b/processing/src/test/java/org/apache/druid/frame/field/ConstantFieldPointer.java similarity index 85% rename from processing/src/main/java/org/apache/druid/frame/field/ConstantFieldPointer.java rename to processing/src/test/java/org/apache/druid/frame/field/ConstantFieldPointer.java index 17ff1058794b..0079246c2623 100644 --- a/processing/src/main/java/org/apache/druid/frame/field/ConstantFieldPointer.java +++ b/processing/src/test/java/org/apache/druid/frame/field/ConstantFieldPointer.java @@ -22,10 +22,12 @@ public class ConstantFieldPointer implements ReadableFieldPointer { private final long position; + private final long length; - public ConstantFieldPointer(long position) + public ConstantFieldPointer(long position, long length) { this.position = position; + this.length = length; } @Override @@ -33,4 +35,10 @@ public long position() { return position; } + + @Override + public long length() + { + return length; + } } diff --git a/processing/src/test/java/org/apache/druid/frame/field/DoubleArrayFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/DoubleArrayFieldReaderTest.java index 6381138f62db..c931b7aac904 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/DoubleArrayFieldReaderTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/DoubleArrayFieldReaderTest.java @@ -149,10 +149,13 @@ public void test_isNull_arrayWithSingleNullElement() @Test public void test_makeColumnValueSelector_null() { - writeToMemory(null, MEMORY_POSITION); + long sz = writeToMemory(null, MEMORY_POSITION); final ColumnValueSelector readSelector = - new DoubleArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new DoubleArrayFieldReader().makeColumnValueSelector( + memory, + new ConstantFieldPointer(MEMORY_POSITION, sz) + ); Assert.assertTrue(readSelector.isNull()); } @@ -160,10 +163,14 @@ public void test_makeColumnValueSelector_null() @Test public void test_makeColumnValueSelector_aValue() { - writeToMemory(DOUBLES_ARRAY_1, MEMORY_POSITION); + long sz = writeToMemory(DOUBLES_ARRAY_1, MEMORY_POSITION); final ColumnValueSelector readSelector = - new DoubleArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new DoubleArrayFieldReader() + .makeColumnValueSelector( + memory, + new ConstantFieldPointer(MEMORY_POSITION, sz) + ); assertResults(DOUBLES_LIST_1, readSelector.getObject()); } @@ -172,15 +179,15 @@ public void test_makeColumnValueSelector_aValue() public void test_makeColumnValueSelector_multipleValues() { long sz = writeToMemory(DOUBLES_ARRAY_1, MEMORY_POSITION); - writeToMemory(DOUBLES_ARRAY_2, MEMORY_POSITION + sz); - IndexArrayFieldPointer pointer = new IndexArrayFieldPointer(ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz)); - + long sz2 = writeToMemory(DOUBLES_ARRAY_2, MEMORY_POSITION + sz); + IndexArrayFieldPointer pointer = new IndexArrayFieldPointer( + ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz), + ImmutableList.of(sz, sz2) + ); final ColumnValueSelector readSelector = new DoubleArrayFieldReader().makeColumnValueSelector(memory, pointer); - pointer.setPointer(0); assertResults(DOUBLES_LIST_1, readSelector.getObject()); - pointer.setPointer(1); assertResults(DOUBLES_LIST_2, readSelector.getObject()); } @@ -188,10 +195,11 @@ public void test_makeColumnValueSelector_multipleValues() @Test public void test_makeColumnValueSelector_emptyArray() { - writeToMemory(new Object[]{}, MEMORY_POSITION); + long sz = writeToMemory(new Object[]{}, MEMORY_POSITION); final ColumnValueSelector readSelector = - new DoubleArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new DoubleArrayFieldReader() + .makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); assertResults(Collections.emptyList(), readSelector.getObject()); } @@ -199,10 +207,13 @@ public void test_makeColumnValueSelector_emptyArray() @Test public void test_makeColumnValueSelector_arrayWithSingleNullElement() { - writeToMemory(new Object[]{null}, MEMORY_POSITION); + long sz = writeToMemory(new Object[]{null}, MEMORY_POSITION); final ColumnValueSelector readSelector = - new DoubleArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new DoubleArrayFieldReader().makeColumnValueSelector( + memory, + new ConstantFieldPointer(MEMORY_POSITION, sz) + ); assertResults(Collections.singletonList(null), readSelector.getObject()); } diff --git a/processing/src/test/java/org/apache/druid/frame/field/DoubleFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/DoubleFieldReaderTest.java index a18b7cc70ec9..c4a300b5bc3c 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/DoubleFieldReaderTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/DoubleFieldReaderTest.java @@ -89,7 +89,7 @@ public void test_makeColumnValueSelector_defaultOrNull() writeToMemory(NullHandling.defaultDoubleValue()); final ColumnValueSelector readSelector = - DoubleFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + DoubleFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals(!NullHandling.replaceWithDefault(), readSelector.isNull()); @@ -104,7 +104,7 @@ public void test_makeColumnValueSelector_aValue() writeToMemory(5.1d); final ColumnValueSelector readSelector = - DoubleFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + DoubleFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals(5.1d, readSelector.getObject()); } @@ -115,7 +115,8 @@ public void test_makeDimensionSelector_defaultOrNull() writeToMemory(NullHandling.defaultDoubleValue()); final DimensionSelector readSelector = - DoubleFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null); + DoubleFieldReader.forPrimitive() + .makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null); // Data retrieval tests. final IndexedInts row = readSelector.getRow(); @@ -149,7 +150,8 @@ public void test_makeDimensionSelector_aValue() writeToMemory(5.1d); final DimensionSelector readSelector = - DoubleFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null); + DoubleFieldReader.forPrimitive() + .makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null); // Data retrieval tests. final IndexedInts row = readSelector.getRow(); @@ -178,7 +180,7 @@ public void test_makeDimensionSelector_aValue_extractionFn() final DimensionSelector readSelector = DoubleFieldReader.forPrimitive().makeDimensionSelector( memory, - new ConstantFieldPointer(MEMORY_POSITION), + new ConstantFieldPointer(MEMORY_POSITION, -1), new SubstringDimExtractionFn(1, null) ); diff --git a/processing/src/test/java/org/apache/druid/frame/field/FloatArrayFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/FloatArrayFieldReaderTest.java index e61e40db1cb1..3c404ed28dd9 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/FloatArrayFieldReaderTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/FloatArrayFieldReaderTest.java @@ -150,10 +150,10 @@ public void test_isNull_arrayWithSingleNullElement() @Test public void test_makeColumnValueSelector_null() { - writeToMemory(null, MEMORY_POSITION); + long sz = writeToMemory(null, MEMORY_POSITION); final ColumnValueSelector readSelector = - new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); Assert.assertTrue(readSelector.isNull()); } @@ -161,10 +161,10 @@ public void test_makeColumnValueSelector_null() @Test public void test_makeColumnValueSelector_aValue() { - writeToMemory(FLOATS_ARRAY_1, MEMORY_POSITION); + long sz = writeToMemory(FLOATS_ARRAY_1, MEMORY_POSITION); final ColumnValueSelector readSelector = - new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); assertResults(FLOATS_LIST_1, readSelector.getObject()); } @@ -173,15 +173,15 @@ public void test_makeColumnValueSelector_aValue() public void test_makeColumnValueSelector_multipleValues() { long sz = writeToMemory(FLOATS_ARRAY_1, MEMORY_POSITION); - writeToMemory(FLOATS_ARRAY_2, MEMORY_POSITION + sz); - IndexArrayFieldPointer pointer = new IndexArrayFieldPointer(ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz)); - + long sz2 = writeToMemory(FLOATS_ARRAY_2, MEMORY_POSITION + sz); + IndexArrayFieldPointer pointer = new IndexArrayFieldPointer( + ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz), + ImmutableList.of(sz, sz2) + ); final ColumnValueSelector readSelector = new FloatArrayFieldReader().makeColumnValueSelector(memory, pointer); - pointer.setPointer(0); assertResults(FLOATS_LIST_1, readSelector.getObject()); - pointer.setPointer(1); assertResults(FLOATS_LIST_2, readSelector.getObject()); } @@ -189,10 +189,10 @@ public void test_makeColumnValueSelector_multipleValues() @Test public void test_makeColumnValueSelector_emptyArray() { - writeToMemory(new Object[]{}, MEMORY_POSITION); + long sz = writeToMemory(new Object[]{}, MEMORY_POSITION); final ColumnValueSelector readSelector = - new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); assertResults(Collections.emptyList(), readSelector.getObject()); } @@ -200,10 +200,10 @@ public void test_makeColumnValueSelector_emptyArray() @Test public void test_makeColumnValueSelector_arrayWithSingleNullElement() { - writeToMemory(new Object[]{null}, MEMORY_POSITION); + long sz = writeToMemory(new Object[]{null}, MEMORY_POSITION); final ColumnValueSelector readSelector = - new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new FloatArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); assertResults(Collections.singletonList(null), readSelector.getObject()); } diff --git a/processing/src/test/java/org/apache/druid/frame/field/FloatFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/FloatFieldReaderTest.java index 6bae52f1c50d..f25845f9f4ed 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/FloatFieldReaderTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/FloatFieldReaderTest.java @@ -89,7 +89,7 @@ public void test_makeColumnValueSelector_defaultOrNull() writeToMemory(NullHandling.defaultFloatValue()); final ColumnValueSelector readSelector = - FloatFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + FloatFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals(!NullHandling.replaceWithDefault(), readSelector.isNull()); @@ -104,7 +104,7 @@ public void test_makeColumnValueSelector_aValue() writeToMemory(5.1f); final ColumnValueSelector readSelector = - FloatFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + FloatFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals(5.1f, readSelector.getObject()); } @@ -115,7 +115,8 @@ public void test_makeDimensionSelector_defaultOrNull() writeToMemory(NullHandling.defaultFloatValue()); final DimensionSelector readSelector = - FloatFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null); + FloatFieldReader.forPrimitive() + .makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null); // Data retrieval tests. final IndexedInts row = readSelector.getRow(); @@ -149,7 +150,8 @@ public void test_makeDimensionSelector_aValue() writeToMemory(5.1f); final DimensionSelector readSelector = - FloatFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null); + FloatFieldReader.forPrimitive() + .makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null); // Data retrieval tests. final IndexedInts row = readSelector.getRow(); @@ -178,7 +180,7 @@ public void test_makeDimensionSelector_aValue_extractionFn() final DimensionSelector readSelector = FloatFieldReader.forPrimitive().makeDimensionSelector( memory, - new ConstantFieldPointer(MEMORY_POSITION), + new ConstantFieldPointer(MEMORY_POSITION, -1), new SubstringDimExtractionFn(1, null) ); diff --git a/processing/src/test/java/org/apache/druid/frame/field/IndexArrayFieldPointer.java b/processing/src/test/java/org/apache/druid/frame/field/IndexArrayFieldPointer.java index 1e115f48e3c5..22fdbacd1b71 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/IndexArrayFieldPointer.java +++ b/processing/src/test/java/org/apache/druid/frame/field/IndexArrayFieldPointer.java @@ -30,11 +30,13 @@ public class IndexArrayFieldPointer implements ReadableFieldPointer { private final LongArrayList indices; + private final LongArrayList lengths; private int pointer = 0; - public IndexArrayFieldPointer(final List indices) + public IndexArrayFieldPointer(final List indices, final List lengths) { this.indices = new LongArrayList(indices); + this.lengths = new LongArrayList(lengths); } private int numIndices() @@ -53,4 +55,10 @@ public long position() { return indices.getLong(pointer); } + + @Override + public long length() + { + return lengths.getLong(pointer); + } } diff --git a/processing/src/test/java/org/apache/druid/frame/field/LongArrayFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/LongArrayFieldReaderTest.java index aa34cd6afaf3..f679bb2b3ed9 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/LongArrayFieldReaderTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/LongArrayFieldReaderTest.java @@ -126,10 +126,10 @@ public void test_isNull_arrayWithSingleNullElement() @Test public void test_makeColumnValueSelector_null() { - writeToMemory(null, MEMORY_POSITION); + long sz = writeToMemory(null, MEMORY_POSITION); final ColumnValueSelector readSelector = - new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); Assert.assertTrue(readSelector.isNull()); } @@ -137,10 +137,10 @@ public void test_makeColumnValueSelector_null() @Test public void test_makeColumnValueSelector_aValue() { - writeToMemory(LONGS_ARRAY_1, MEMORY_POSITION); + long sz = writeToMemory(LONGS_ARRAY_1, MEMORY_POSITION); final ColumnValueSelector readSelector = - new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); assertResults(LONGS_LIST_1, readSelector.getObject()); } @@ -149,15 +149,15 @@ public void test_makeColumnValueSelector_aValue() public void test_makeColumnValueSelector_multipleValues() { long sz = writeToMemory(LONGS_ARRAY_1, MEMORY_POSITION); - writeToMemory(LONGS_ARRAY_2, MEMORY_POSITION + sz); - IndexArrayFieldPointer pointer = new IndexArrayFieldPointer(ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz)); - + long sz2 = writeToMemory(LONGS_ARRAY_2, MEMORY_POSITION + sz); + IndexArrayFieldPointer pointer = new IndexArrayFieldPointer( + ImmutableList.of(MEMORY_POSITION, MEMORY_POSITION + sz), + ImmutableList.of(sz, sz2) + ); final ColumnValueSelector readSelector = new LongArrayFieldReader().makeColumnValueSelector(memory, pointer); - pointer.setPointer(0); assertResults(LONGS_LIST_1, readSelector.getObject()); - pointer.setPointer(1); assertResults(LONGS_LIST_2, readSelector.getObject()); } @@ -165,10 +165,10 @@ public void test_makeColumnValueSelector_multipleValues() @Test public void test_makeColumnValueSelector_emptyArray() { - writeToMemory(new Object[]{}, MEMORY_POSITION); + long sz = writeToMemory(new Object[]{}, MEMORY_POSITION); final ColumnValueSelector readSelector = - new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); assertResults(Collections.emptyList(), readSelector.getObject()); } @@ -176,10 +176,10 @@ public void test_makeColumnValueSelector_emptyArray() @Test public void test_makeColumnValueSelector_arrayWithSingleNullElement() { - writeToMemory(new Object[]{null}, MEMORY_POSITION); + long sz = writeToMemory(new Object[]{null}, MEMORY_POSITION); final ColumnValueSelector readSelector = - new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new LongArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, sz)); assertResults(Collections.singletonList(null), readSelector.getObject()); } diff --git a/processing/src/test/java/org/apache/druid/frame/field/LongFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/LongFieldReaderTest.java index 643846ee0e3a..8a201394083d 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/LongFieldReaderTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/LongFieldReaderTest.java @@ -89,7 +89,7 @@ public void test_makeColumnValueSelector_defaultOrNull() writeToMemory(NullHandling.defaultLongValue()); final ColumnValueSelector readSelector = - LongFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + LongFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals(!NullHandling.replaceWithDefault(), readSelector.isNull()); @@ -104,7 +104,7 @@ public void test_makeColumnValueSelector_aValue() writeToMemory(5L); final ColumnValueSelector readSelector = - LongFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + LongFieldReader.forPrimitive().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals(5L, readSelector.getObject()); } @@ -115,7 +115,8 @@ public void test_makeDimensionSelector_defaultOrNull() writeToMemory(NullHandling.defaultLongValue()); final DimensionSelector readSelector = - LongFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null); + LongFieldReader.forPrimitive() + .makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null); // Data retrieval tests. final IndexedInts row = readSelector.getRow(); @@ -149,7 +150,8 @@ public void test_makeDimensionSelector_aValue() writeToMemory(5L); final DimensionSelector readSelector = - LongFieldReader.forPrimitive().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null); + LongFieldReader.forPrimitive() + .makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null); // Data retrieval tests. final IndexedInts row = readSelector.getRow(); @@ -178,7 +180,7 @@ public void test_makeDimensionSelector_aValue_extractionFn() final DimensionSelector readSelector = LongFieldReader.forPrimitive().makeDimensionSelector( memory, - new ConstantFieldPointer(MEMORY_POSITION), + new ConstantFieldPointer(MEMORY_POSITION, -1), new SubstringDimExtractionFn(1, null) ); diff --git a/processing/src/test/java/org/apache/druid/frame/field/StringArrayFieldWriterTest.java b/processing/src/test/java/org/apache/druid/frame/field/StringArrayFieldWriterTest.java index 63dd03d48bf0..02d4d44cbfcc 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/StringArrayFieldWriterTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/StringArrayFieldWriterTest.java @@ -140,7 +140,7 @@ private List readFromMemory(final long written) final FieldReader fieldReader = FieldReaders.create("columnNameDoesntMatterHere", ColumnType.STRING_ARRAY); final ColumnValueSelector selector = - fieldReader.makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + fieldReader.makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); final Object o = selector.getObject(); //noinspection rawtypes,unchecked diff --git a/processing/src/test/java/org/apache/druid/frame/field/StringFieldReaderTest.java b/processing/src/test/java/org/apache/druid/frame/field/StringFieldReaderTest.java index a682e658ca76..04296cb78c3c 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/StringFieldReaderTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/StringFieldReaderTest.java @@ -143,9 +143,9 @@ public void test_makeColumnValueSelector_singleString_notArray() writeToMemory(Collections.singletonList("foo")); final ColumnValueSelector readSelector = - new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); final ColumnValueSelector readSelectorAsArray = - new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals("foo", readSelector.getObject()); Assert.assertArrayEquals(new Object[]{"foo"}, (Object[]) readSelectorAsArray.getObject()); @@ -157,9 +157,9 @@ public void test_makeColumnValueSelector_multiString() writeToMemory(ImmutableList.of("foo", "bar")); final ColumnValueSelector readSelector = - new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); final ColumnValueSelector readSelectorAsArray = - new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertEquals(ImmutableList.of("foo", "bar"), readSelector.getObject()); Assert.assertArrayEquals(new Object[]{"foo", "bar"}, (Object[]) readSelectorAsArray.getObject()); @@ -171,9 +171,9 @@ public void test_makeColumnValueSelector_null() writeToMemory(Collections.singletonList(null)); final ColumnValueSelector readSelector = - new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); final ColumnValueSelector readSelectorAsArray = - new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertNull(readSelector.getObject()); Assert.assertArrayEquals(new Object[]{null}, (Object[]) readSelectorAsArray.getObject()); @@ -185,9 +185,9 @@ public void test_makeColumnValueSelector_empty() writeToMemory(Collections.emptyList()); final ColumnValueSelector readSelector = - new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new StringFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); final ColumnValueSelector readSelectorAsArray = - new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + new StringArrayFieldReader().makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); Assert.assertNull(readSelector.getObject()); Assert.assertArrayEquals(ObjectArrays.EMPTY_ARRAY, (Object[]) readSelectorAsArray.getObject()); @@ -200,7 +200,11 @@ public void test_makeDimensionSelector_multiString_asArray() final IllegalStateException e = Assert.assertThrows( IllegalStateException.class, - () -> new StringArrayFieldReader().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null) + () -> new StringArrayFieldReader().makeDimensionSelector( + memory, + new ConstantFieldPointer(MEMORY_POSITION, -1), + null + ) ); MatcherAssert.assertThat( @@ -215,7 +219,7 @@ public void test_makeDimensionSelector_multiString() writeToMemory(ImmutableList.of("foo", "bar")); final DimensionSelector readSelector = - new StringFieldReader().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION), null); + new StringFieldReader().makeDimensionSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1), null); // Data retrieval tests. final IndexedInts row = readSelector.getRow(); @@ -247,7 +251,7 @@ public void test_makeDimensionSelector_multiString_withExtractionFn() final DimensionSelector readSelector = new StringFieldReader().makeDimensionSelector( memory, - new ConstantFieldPointer(MEMORY_POSITION), + new ConstantFieldPointer(MEMORY_POSITION, -1), new SubstringDimExtractionFn(1, null) ); diff --git a/processing/src/test/java/org/apache/druid/frame/field/StringFieldWriterTest.java b/processing/src/test/java/org/apache/druid/frame/field/StringFieldWriterTest.java index 12bbf8238bf2..f44b69e2810f 100644 --- a/processing/src/test/java/org/apache/druid/frame/field/StringFieldWriterTest.java +++ b/processing/src/test/java/org/apache/druid/frame/field/StringFieldWriterTest.java @@ -184,7 +184,7 @@ private Object[] readFromMemory(final long written) final FieldReader fieldReader = FieldReaders.create("columnNameDoesntMatterHere", ColumnType.STRING_ARRAY); final ColumnValueSelector selector = - fieldReader.makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION)); + fieldReader.makeColumnValueSelector(memory, new ConstantFieldPointer(MEMORY_POSITION, -1)); return (Object[]) selector.getObject(); } diff --git a/processing/src/test/java/org/apache/druid/query/filter/IsBooleanDimFilterTest.java b/processing/src/test/java/org/apache/druid/query/filter/IsBooleanDimFilterTest.java new file mode 100644 index 000000000000..7c4be474efef --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/filter/IsBooleanDimFilterTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.filter; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.apache.druid.error.DruidException; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.testing.InitializedNullHandlingTest; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; + +public class IsBooleanDimFilterTest extends InitializedNullHandlingTest +{ + @Test + public void testSerde() throws JsonProcessingException + { + ObjectMapper mapper = new DefaultObjectMapper(); + EqualityFilter baseFilter = new EqualityFilter("x", ColumnType.STRING, "hello", null); + + IsTrueDimFilter trueFilter = IsTrueDimFilter.of(baseFilter); + String s = mapper.writeValueAsString(trueFilter); + Assert.assertEquals(trueFilter, mapper.readValue(s, IsTrueDimFilter.class)); + + IsFalseDimFilter falseFilter = IsFalseDimFilter.of(baseFilter); + s = mapper.writeValueAsString(falseFilter); + Assert.assertEquals(falseFilter, mapper.readValue(s, IsFalseDimFilter.class)); + + } + + @Test + public void testGetCacheKey() + { + EqualityFilter f1 = new EqualityFilter("x", ColumnType.STRING, "hello", null); + EqualityFilter f1_2 = new EqualityFilter("x", ColumnType.STRING, "hello", null); + EqualityFilter f2 = new EqualityFilter("x", ColumnType.STRING, "world", null); + EqualityFilter f3 = new EqualityFilter("x", ColumnType.STRING, "hello", new FilterTuning(true, null, null)); + Assert.assertArrayEquals(f1.getCacheKey(), f1_2.getCacheKey()); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); + Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); + + } + + @Test + public void testInvalidParameters() + { + Throwable t = Assert.assertThrows( + DruidException.class, + () -> new IsTrueDimFilter(null) + ); + Assert.assertEquals("IS TRUE operator requires a non-null filter for field", t.getMessage()); + t = Assert.assertThrows( + DruidException.class, + () -> new IsFalseDimFilter(null) + ); + Assert.assertEquals("IS FALSE operator requires a non-null filter for field", t.getMessage()); + } + + @Test + public void test_equals() + { + EqualsVerifier.forClass(IsTrueDimFilter.class).usingGetClass() + .withNonnullFields("field") + .withIgnoredFields("cachedOptimizedFilter") + .verify(); + + EqualsVerifier.forClass(IsFalseDimFilter.class).usingGetClass() + .withNonnullFields("field") + .withIgnoredFields("cachedOptimizedFilter") + .verify(); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/groupby/UnnestGroupByQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/groupby/UnnestGroupByQueryRunnerTest.java index d0a7d94c3967..62caef2493a2 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/UnnestGroupByQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/UnnestGroupByQueryRunnerTest.java @@ -40,6 +40,8 @@ import org.apache.druid.query.dimension.ExtractionDimensionSpec; import org.apache.druid.query.expression.TestExprMacroTable; import org.apache.druid.query.extraction.StringFormatExtractionFn; +import org.apache.druid.query.filter.EqualityFilter; +import org.apache.druid.query.filter.NotDimFilter; import org.apache.druid.query.groupby.orderby.OrderByColumnSpec; import org.apache.druid.segment.IncrementalIndexSegment; import org.apache.druid.segment.TestHelper; @@ -704,6 +706,228 @@ private Iterable runQuery(final GroupByQuery query, final Incremental return GroupByQueryRunnerTestHelper.runQuery(factory, queryRunner, query); } + @Test + public void testGroupByOnUnnestedFilterMatch() + { + // testGroupByOnUnnestedColumn but with filter to match single value + cannotVectorize(); + + final DataSource unnestDataSource = UnnestDataSource.create( + new TableDataSource(QueryRunnerTestHelper.DATA_SOURCE), + new ExpressionVirtualColumn( + QueryRunnerTestHelper.PLACEMENTISH_DIMENSION_UNNEST, + "\"" + QueryRunnerTestHelper.PLACEMENTISH_DIMENSION + "\"", + null, + ExprMacroTable.nil() + ), + null + ); + + GroupByQuery query = makeQueryBuilder() + .setDataSource(unnestDataSource) + .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD) + .setDimensions( + new DefaultDimensionSpec(QueryRunnerTestHelper.PLACEMENTISH_DIMENSION_UNNEST, "alias0") + ) + .setDimFilter( + new EqualityFilter(QueryRunnerTestHelper.PLACEMENTISH_DIMENSION_UNNEST, ColumnType.STRING, "a", null) + ) + .setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT) + .setGranularity(QueryRunnerTestHelper.ALL_GRAN) + .addOrderByColumn("alias0", OrderByColumnSpec.Direction.ASCENDING) + .build(); + + List expectedResults = Collections.singletonList( + makeRow( + query, + "2011-04-01", + "alias0", "a", + "rows", 2L + ) + ); + + Iterable results = runQuery(query, TestIndex.getIncrementalTestIndex()); + TestHelper.assertExpectedObjects(expectedResults, results, "groupBy-on-unnested-virtual-column"); + } + + @Test + public void testGroupByOnUnnestedNotFilterMatch() + { + // testGroupByOnUnnestedColumn but with negated filter to match everything except 1 value + cannotVectorize(); + + final DataSource unnestDataSource = UnnestDataSource.create( + new TableDataSource(QueryRunnerTestHelper.DATA_SOURCE), + new ExpressionVirtualColumn( + QueryRunnerTestHelper.PLACEMENTISH_DIMENSION_UNNEST, + "\"" + QueryRunnerTestHelper.PLACEMENTISH_DIMENSION + "\"", + null, + ExprMacroTable.nil() + ), + null + ); + + GroupByQuery query = makeQueryBuilder() + .setDataSource(unnestDataSource) + .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD) + .setDimensions( + new DefaultDimensionSpec(QueryRunnerTestHelper.PLACEMENTISH_DIMENSION_UNNEST, "alias0") + ) + .setDimFilter( + NotDimFilter.of(new EqualityFilter(QueryRunnerTestHelper.PLACEMENTISH_DIMENSION_UNNEST, ColumnType.STRING, "a", null)) + ) + .setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT) + .setGranularity(QueryRunnerTestHelper.ALL_GRAN) + .addOrderByColumn("alias0", OrderByColumnSpec.Direction.ASCENDING) + .build(); + + List expectedResults = Arrays.asList( + makeRow( + query, + "2011-04-01", + "alias0", "b", + "rows", 2L + ), + makeRow( + query, + "2011-04-01", + "alias0", "e", + "rows", 2L + ), + makeRow( + query, + "2011-04-01", + "alias0", "h", + "rows", 2L + ), + makeRow( + query, + "2011-04-01", + "alias0", "m", + "rows", 6L + ), + makeRow( + query, + "2011-04-01", + "alias0", "n", + "rows", 2L + ), + makeRow( + query, + "2011-04-01", + "alias0", "p", + "rows", 6L + ), + makeRow( + query, + "2011-04-01", + "alias0", "preferred", + "rows", 26L + ), + makeRow( + query, + "2011-04-01", + "alias0", "t", + "rows", 4L + ) + ); + + Iterable results = runQuery(query, TestIndex.getIncrementalTestIndex()); + TestHelper.assertExpectedObjects(expectedResults, results, "groupBy-on-unnested-virtual-column"); + } + + @Test + public void testGroupByOnUnnestedNotFilterMatchNonexistentValue() + { + // testGroupByOnUnnestedColumn but with negated filter on nonexistent value to still match everything + cannotVectorize(); + + final DataSource unnestDataSource = UnnestDataSource.create( + new TableDataSource(QueryRunnerTestHelper.DATA_SOURCE), + new ExpressionVirtualColumn( + QueryRunnerTestHelper.PLACEMENTISH_DIMENSION_UNNEST, + "\"" + QueryRunnerTestHelper.PLACEMENTISH_DIMENSION + "\"", + null, + ExprMacroTable.nil() + ), + null + ); + + GroupByQuery query = makeQueryBuilder() + .setDataSource(unnestDataSource) + .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD) + .setDimensions( + new DefaultDimensionSpec(QueryRunnerTestHelper.PLACEMENTISH_DIMENSION_UNNEST, "alias0") + ) + .setDimFilter( + NotDimFilter.of(new EqualityFilter(QueryRunnerTestHelper.PLACEMENTISH_DIMENSION_UNNEST, ColumnType.STRING, "noexist", null)) + ) + .setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT) + .setGranularity(QueryRunnerTestHelper.ALL_GRAN) + .addOrderByColumn("alias0", OrderByColumnSpec.Direction.ASCENDING) + .build(); + + List expectedResults = Arrays.asList( + makeRow( + query, + "2011-04-01", + "alias0", "a", + "rows", 2L + ), + makeRow( + query, + "2011-04-01", + "alias0", "b", + "rows", 2L + ), + makeRow( + query, + "2011-04-01", + "alias0", "e", + "rows", 2L + ), + makeRow( + query, + "2011-04-01", + "alias0", "h", + "rows", 2L + ), + makeRow( + query, + "2011-04-01", + "alias0", "m", + "rows", 6L + ), + makeRow( + query, + "2011-04-01", + "alias0", "n", + "rows", 2L + ), + makeRow( + query, + "2011-04-01", + "alias0", "p", + "rows", 6L + ), + makeRow( + query, + "2011-04-01", + "alias0", "preferred", + "rows", 26L + ), + makeRow( + query, + "2011-04-01", + "alias0", "t", + "rows", 4L + ) + ); + + Iterable results = runQuery(query, TestIndex.getIncrementalTestIndex()); + TestHelper.assertExpectedObjects(expectedResults, results, "groupBy-on-unnested-virtual-column"); + } + private Map makeContext() { return ImmutableMap.builder() diff --git a/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java index 286a636e89a3..6df1766bf984 100644 --- a/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java @@ -35,6 +35,7 @@ import org.apache.druid.query.filter.EqualityFilter; import org.apache.druid.query.filter.Filter; import org.apache.druid.query.filter.SelectorDimFilter; +import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.ValueType; @@ -300,12 +301,6 @@ public void test_two_levels_of_unnest_adapters() }); } - private static void assertColumnReadsIdentifier(final VirtualColumn column, final String identifier) - { - MatcherAssert.assertThat(column, CoreMatchers.instanceOf(ExpressionVirtualColumn.class)); - Assert.assertEquals("\"" + identifier + "\"", ((ExpressionVirtualColumn) column).getExpression()); - } - @Test public void test_pushdown_or_filters_unnested_and_original_dimension_with_unnest_adapters() { @@ -350,6 +345,7 @@ public void test_pushdown_or_filters_unnested_and_original_dimension_with_unnest return null; }); } + @Test public void test_nested_filters_unnested_and_original_dimension_with_unnest_adapters() { @@ -483,7 +479,6 @@ public void test_nested_filters_unnested_and_topLevelORAnd3filtersInOR() "(unnested-multi-string1 = 3 || (newcol = 2 && multi-string1 = 2 && unnested-multi-string1 = 1))" ); } - @Test public void test_nested_filters_unnested_and_topLevelAND3filtersInORWithNestedOrs() { @@ -734,6 +729,62 @@ public void test_pushdown_filters_unnested_dimension_outside() }); } + @Test + public void testUnnestValueMatcherValueDoesntExist() + { + final String inputColumn = "multi-string5"; + final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("expression-testbench"); + + final DataSegment dataSegment = DataSegment.builder() + .dataSource("foo") + .interval(schemaInfo.getDataInterval()) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(); + final SegmentGenerator segmentGenerator = CLOSER.register(new SegmentGenerator()); + + IncrementalIndex index = CLOSER.register( + segmentGenerator.generateIncrementalIndex(dataSegment, schemaInfo, Granularities.HOUR, 100) + ); + IncrementalIndexStorageAdapter adapter = new IncrementalIndexStorageAdapter(index); + UnnestStorageAdapter withNullsStorageAdapter = new UnnestStorageAdapter( + adapter, + new ExpressionVirtualColumn(OUTPUT_COLUMN_NAME, "\"" + inputColumn + "\"", null, ExprMacroTable.nil()), + null + ); + Sequence cursorSequence = withNullsStorageAdapter.makeCursors( + null, + withNullsStorageAdapter.getInterval(), + VirtualColumns.EMPTY, + Granularities.ALL, + false, + null + ); + + cursorSequence.accumulate(null, (accumulated, cursor) -> { + ColumnSelectorFactory factory = cursor.getColumnSelectorFactory(); + + DimensionSelector dimSelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(OUTPUT_COLUMN_NAME)); + // wont match anything + ValueMatcher matcher = dimSelector.makeValueMatcher("x"); + int count = 0; + while (!cursor.isDone()) { + Object dimSelectorVal = dimSelector.getObject(); + if (dimSelectorVal == null) { + Assert.assertNull(dimSelectorVal); + Assert.assertTrue(matcher.matches(true)); + } + Assert.assertFalse(matcher.matches(false)); + cursor.advance(); + count++; + } + Assert.assertEquals(count, 618); + return null; + }); + + } + public void testComputeBaseAndPostUnnestFilters( Filter testQueryFilter, String expectedBasePushDown, @@ -777,6 +828,12 @@ public void testComputeBaseAndPostUnnestFilters( actualPostUnnestFilter == null ? "" : actualPostUnnestFilter.toString() ); } + + private static void assertColumnReadsIdentifier(final VirtualColumn column, final String identifier) + { + MatcherAssert.assertThat(column, CoreMatchers.instanceOf(ExpressionVirtualColumn.class)); + Assert.assertEquals("\"" + identifier + "\"", ((ExpressionVirtualColumn) column).getExpression()); + } } /** diff --git a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java index 5403ac7aca6c..06ca5ab3fd48 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java @@ -37,6 +37,8 @@ import org.apache.druid.math.expr.ExpressionType; import org.apache.druid.query.filter.EqualityFilter; import org.apache.druid.query.filter.FilterTuning; +import org.apache.druid.query.filter.IsFalseDimFilter; +import org.apache.druid.query.filter.IsTrueDimFilter; import org.apache.druid.query.filter.NotDimFilter; import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.StorageAdapter; @@ -255,6 +257,26 @@ public void testSingleValueStringColumnWithNulls() NotDimFilter.of(new EqualityFilter("s0", ColumnType.STRING, "a", null)), ImmutableList.of("0", "2", "4") ); + // "(s0 = 'a') is not true", same rows as "s0 <> 'a'", but also with null rows + assertFilterMatches( + NotDimFilter.of(IsTrueDimFilter.of(new EqualityFilter("s0", ColumnType.STRING, "a", null))), + ImmutableList.of("0", "2", "3", "4") + ); + // "(s0 = 'a') is true", equivalent to "s0 = 'a'" + assertFilterMatches( + IsTrueDimFilter.of(new EqualityFilter("s0", ColumnType.STRING, "a", null)), + ImmutableList.of("1", "5") + ); + // "(s0 = 'a') is false", equivalent results to "s0 <> 'a'" + assertFilterMatches( + IsFalseDimFilter.of(new EqualityFilter("s0", ColumnType.STRING, "a", null)), + ImmutableList.of("0", "2", "4") + ); + // "(s0 = 'a') is not false", same rows as "s0 = 'a'", but also with null rows + assertFilterMatches( + NotDimFilter.of(IsFalseDimFilter.of(new EqualityFilter("s0", ColumnType.STRING, "a", null))), + ImmutableList.of("1", "3", "5") + ); try { // make sure if 3vl is disabled with behave with 2vl @@ -288,6 +310,28 @@ public void testSingleValueStringColumnWithNulls() NotDimFilter.of(new EqualityFilter("s0", ColumnType.STRING, "noexist", null)), ImmutableList.of("0", "1", "2", "3", "4", "5") ); + + // in default value mode, is true/is false are basically pointless since they have the same behavior as = and <> + // "(s0 = 'a') is not true" equivalent to "s0 <> 'a'" + assertFilterMatches( + NotDimFilter.of(IsTrueDimFilter.of(new EqualityFilter("s0", ColumnType.STRING, "a", null))), + ImmutableList.of("0", "2", "3", "4") + ); + // "(s0 = 'a') is true", equivalent to "s0 = 'a'" + assertFilterMatches( + IsTrueDimFilter.of(new EqualityFilter("s0", ColumnType.STRING, "a", null)), + ImmutableList.of("1", "5") + ); + // "(s0 = 'a') is false" equivalent to "s0 <> 'a'" + assertFilterMatches( + IsFalseDimFilter.of(new EqualityFilter("s0", ColumnType.STRING, "a", null)), + ImmutableList.of("0", "2", "3", "4") + ); + // "(s0 = 'a') is not false", equivalent to "s0 = 'a'" + assertFilterMatches( + NotDimFilter.of(IsFalseDimFilter.of(new EqualityFilter("s0", ColumnType.STRING, "a", null))), + ImmutableList.of("1", "5") + ); } } diff --git a/processing/src/test/java/org/apache/druid/segment/transform/TransformerTest.java b/processing/src/test/java/org/apache/druid/segment/transform/TransformerTest.java index c1aa1d0c6e88..0c33b3c2d1db 100644 --- a/processing/src/test/java/org/apache/druid/segment/transform/TransformerTest.java +++ b/processing/src/test/java/org/apache/druid/segment/transform/TransformerTest.java @@ -389,7 +389,7 @@ public void testTransformWithArrayLongInputs() Assert.assertNotNull(actual); Assert.assertEquals(ImmutableList.of("dim"), actual.getDimensions()); Assert.assertArrayEquals(new Object[]{1L, 2L, null, 3L}, (Object[]) actual.getRaw("dim")); - Assert.assertArrayEquals(new String[]{"1", "2", null, "3"}, actual.getDimension("dim").toArray()); + Assert.assertEquals(ImmutableList.of("1", "2", "null", "3"), actual.getDimension("dim")); Assert.assertEquals(row.getTimestamp(), actual.getTimestamp()); } @@ -416,9 +416,9 @@ public void testTransformWithArrayFloatInputs() Assert.assertEquals(2.3, (Double) raw[1], 0.00001); Assert.assertNull(raw[2]); Assert.assertEquals(3.4, (Double) raw[3], 0.00001); - Assert.assertArrayEquals( - new String[]{"1.2000000476837158", "2.299999952316284", null, "3.4000000953674316"}, - actual.getDimension("dim").toArray() + Assert.assertEquals( + ImmutableList.of("1.2000000476837158", "2.299999952316284", "null", "3.4000000953674316"), + actual.getDimension("dim") ); Assert.assertEquals(row.getTimestamp(), actual.getTimestamp()); } @@ -445,12 +445,12 @@ public void testTransformWithArrayDoubleInputs() Assert.assertEquals(2.3, (Double) raw[1], 0.0); Assert.assertNull(raw[2]); Assert.assertEquals(3.4, (Double) raw[3], 0.0); - Assert.assertArrayEquals(new String[]{"1.2", "2.3", null, "3.4"}, actual.getDimension("dim").toArray()); + Assert.assertEquals(ImmutableList.of("1.2", "2.3", "null", "3.4"), actual.getDimension("dim")); Assert.assertEquals(row.getTimestamp(), actual.getTimestamp()); } @Test - public void testTransformWithExpr() + public void testTransformWithArrayExpr() { final Transformer transformer = new Transformer( new TransformSpec( @@ -517,6 +517,6 @@ public int compareTo(Row o) }); Assert.assertEquals(actualTranformedRow.getDimension("dim"), dimList.subList(0, 5)); Assert.assertArrayEquals(dimList.subList(0, 5).toArray(), (Object[]) actualTranformedRow.getRaw("dim")); - Assert.assertArrayEquals(new Object[]{"a"}, actualTranformedRow.getDimension("dim1").toArray()); + Assert.assertEquals(ImmutableList.of("a"), actualTranformedRow.getDimension("dim1")); } } diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageCoordinator.java b/server/src/main/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageCoordinator.java index 2c2a6bc0f77b..7c6710048a1a 100644 --- a/server/src/main/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageCoordinator.java +++ b/server/src/main/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageCoordinator.java @@ -82,8 +82,9 @@ default Collection retrieveUsedSegmentsForInterval( Collection retrieveAllUsedSegments(String dataSource, Segments visibility); /** + * * Retrieve all published segments which are marked as used and the created_date of these segments belonging to the - * given data source from the metadata store. + * given data source and interval from the metadata store. * * Unlike other similar methods in this interface, this method doesn't accept a {@link Segments} "visibility" * parameter. The returned collection may include overshadowed segments and their created_dates, as if {@link @@ -91,10 +92,11 @@ default Collection retrieveUsedSegmentsForInterval( * if needed. * * @param dataSource The data source to query + * @param interval The interval to query * * @return The DataSegments and the related created_date of segments */ - Collection> retrieveUsedSegmentsAndCreatedDates(String dataSource); + Collection> retrieveUsedSegmentsAndCreatedDates(String dataSource, Interval interval); /** * Retrieve all published segments which may include any data in the given intervals and are marked as used from the diff --git a/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java b/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java index 226663c32330..c654d5e229b7 100644 --- a/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java +++ b/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java @@ -174,15 +174,34 @@ private Collection doRetrieveUsedSegments( } @Override - public List> retrieveUsedSegmentsAndCreatedDates(String dataSource) + public List> retrieveUsedSegmentsAndCreatedDates(String dataSource, Interval interval) { - String rawQueryString = "SELECT created_date, payload FROM %1$s WHERE dataSource = :dataSource AND used = true"; - final String queryString = StringUtils.format(rawQueryString, dbTables.getSegmentsTable()); + StringBuilder queryBuilder = new StringBuilder( + "SELECT created_date, payload FROM %1$s WHERE dataSource = :dataSource AND used = true" + ); + + final List intervals = new ArrayList<>(); + // Do not need an interval condition if the interval is ETERNITY + if (!Intervals.isEternity(interval)) { + intervals.add(interval); + } + + SqlSegmentsMetadataQuery.appendConditionForIntervalsAndMatchMode( + queryBuilder, + intervals, + SqlSegmentsMetadataQuery.IntervalMode.OVERLAPS, + connector + ); + + final String queryString = StringUtils.format(queryBuilder.toString(), dbTables.getSegmentsTable()); return connector.retryWithHandle( handle -> { Query> query = handle .createQuery(queryString) .bind("dataSource", dataSource); + + SqlSegmentsMetadataQuery.bindQueryIntervals(query, intervals); + return query .map((int index, ResultSet r, StatementContext ctx) -> new Pair<>( diff --git a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java index 20b176c50914..76e4f9745762 100644 --- a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java +++ b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java @@ -261,6 +261,82 @@ public DataSegment retrieveSegmentForId(String id) return null; } + /** + * Append the condition for the interval and match mode to the given string builder with a partial query + * @param sb - StringBuilder containing the paritial query with SELECT clause and WHERE condition for used, datasource + * @param intervals - intervals to fetch the segments for + * @param matchMode - Interval match mode - overlaps or contains + * @param connector - SQL connector + */ + public static void appendConditionForIntervalsAndMatchMode( + final StringBuilder sb, + final Collection intervals, + final IntervalMode matchMode, + final SQLMetadataConnector connector + ) + { + if (intervals.isEmpty()) { + return; + } + + sb.append(" AND ("); + for (int i = 0; i < intervals.size(); i++) { + sb.append( + matchMode.makeSqlCondition( + connector.getQuoteString(), + StringUtils.format(":start%d", i), + StringUtils.format(":end%d", i) + ) + ); + + // Add a special check for a segment which have one end at eternity and the other at some finite value. Since + // we are using string comparison, a segment with this start or end will not be returned otherwise. + if (matchMode.equals(IntervalMode.OVERLAPS)) { + sb.append(StringUtils.format( + " OR (start = '%s' AND \"end\" != '%s' AND \"end\" > :start%d)", + Intervals.ETERNITY.getStart(), Intervals.ETERNITY.getEnd(), i + )); + sb.append(StringUtils.format( + " OR (start != '%s' AND \"end\" = '%s' AND start < :end%d)", + Intervals.ETERNITY.getStart(), Intervals.ETERNITY.getEnd(), i + )); + } + + if (i != intervals.size() - 1) { + sb.append(" OR "); + } + } + + // Add a special check for a single segment with eternity. Since we are using string comparison, a segment with + // this start and end will not be returned otherwise. + // Known Issue: https://github.com/apache/druid/issues/12860 + if (matchMode.equals(IntervalMode.OVERLAPS)) { + sb.append(StringUtils.format( + " OR (start = '%s' AND \"end\" = '%s')", Intervals.ETERNITY.getStart(), Intervals.ETERNITY.getEnd() + )); + } + sb.append(")"); + } + + /** + * Given a Query object bind the input intervals to it + * @param query Query to fetch segments + * @param intervals Intervals to fetch segments for + */ + public static void bindQueryIntervals(final Query> query, final Collection intervals) + { + if (intervals.isEmpty()) { + return; + } + + final Iterator iterator = intervals.iterator(); + for (int i = 0; iterator.hasNext(); i++) { + Interval interval = iterator.next(); + query.bind(StringUtils.format("start%d", i), interval.getStart().toString()) + .bind(StringUtils.format("end%d", i), interval.getEnd().toString()); + } + } + private CloseableIterator retrieveSegments( final String dataSource, final Collection intervals, @@ -275,36 +351,8 @@ private CloseableIterator retrieveSegments( final StringBuilder sb = new StringBuilder(); sb.append("SELECT payload FROM %s WHERE used = :used AND dataSource = :dataSource"); - if (compareAsString && !intervals.isEmpty()) { - sb.append(" AND ("); - for (int i = 0; i < intervals.size(); i++) { - sb.append( - matchMode.makeSqlCondition( - connector.getQuoteString(), - StringUtils.format(":start%d", i), - StringUtils.format(":end%d", i) - ) - ); - - // Add a special check for a segment which have one end at eternity and the other at some finite value. Since - // we are using string comparison, a segment with this start or end will not be returned otherwise. - if (matchMode.equals(IntervalMode.OVERLAPS)) { - sb.append(StringUtils.format(" OR (start = '%s' AND \"end\" != '%s' AND \"end\" > :start%d)", Intervals.ETERNITY.getStart(), Intervals.ETERNITY.getEnd(), i)); - sb.append(StringUtils.format(" OR (start != '%s' AND \"end\" = '%s' AND start < :end%d)", Intervals.ETERNITY.getStart(), Intervals.ETERNITY.getEnd(), i)); - } - - if (i != intervals.size() - 1) { - sb.append(" OR "); - } - } - - // Add a special check for a single segment with eternity. Since we are using string comparison, a segment with - // this start and end will not be returned otherwise. - // Known Issue: https://github.com/apache/druid/issues/12860 - if (matchMode.equals(IntervalMode.OVERLAPS)) { - sb.append(StringUtils.format(" OR (start = '%s' AND \"end\" = '%s')", Intervals.ETERNITY.getStart(), Intervals.ETERNITY.getEnd())); - } - sb.append(")"); + if (compareAsString) { + appendConditionForIntervalsAndMatchMode(sb, intervals, matchMode, connector); } final Query> sql = handle @@ -317,12 +365,7 @@ private CloseableIterator retrieveSegments( } if (compareAsString) { - final Iterator iterator = intervals.iterator(); - for (int i = 0; iterator.hasNext(); i++) { - Interval interval = iterator.next(); - sql.bind(StringUtils.format("start%d", i), interval.getStart().toString()) - .bind(StringUtils.format("end%d", i), interval.getEnd().toString()); - } + bindQueryIntervals(sql, intervals); } final ResultIterator resultIterator = diff --git a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java index b1b6f3aa16ea..0512357ffc10 100644 --- a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java @@ -2554,6 +2554,52 @@ public void testMarkSegmentsAsUnusedWithinIntervalTwoYears() throws IOException ); } + @Test + public void testRetrieveUsedSegmentsAndCreatedDates() + { + insertUsedSegments(ImmutableSet.of(defaultSegment)); + + List> resultForIntervalOnTheLeft = + coordinator.retrieveUsedSegmentsAndCreatedDates(defaultSegment.getDataSource(), Intervals.of("2000/2001")); + Assert.assertTrue(resultForIntervalOnTheLeft.isEmpty()); + + List> resultForIntervalOnTheRight = + coordinator.retrieveUsedSegmentsAndCreatedDates(defaultSegment.getDataSource(), Intervals.of("3000/3001")); + Assert.assertTrue(resultForIntervalOnTheRight.isEmpty()); + + List> resultForExactInterval = + coordinator.retrieveUsedSegmentsAndCreatedDates(defaultSegment.getDataSource(), defaultSegment.getInterval()); + Assert.assertEquals(1, resultForExactInterval.size()); + Assert.assertEquals(defaultSegment, resultForExactInterval.get(0).lhs); + + List> resultForIntervalWithLeftOverlap = + coordinator.retrieveUsedSegmentsAndCreatedDates(defaultSegment.getDataSource(), Intervals.of("2000/2015-01-02")); + Assert.assertEquals(resultForExactInterval, resultForIntervalWithLeftOverlap); + + List> resultForIntervalWithRightOverlap = + coordinator.retrieveUsedSegmentsAndCreatedDates(defaultSegment.getDataSource(), Intervals.of("2015-01-01/3000")); + Assert.assertEquals(resultForExactInterval, resultForIntervalWithRightOverlap); + + List> resultForEternity = + coordinator.retrieveUsedSegmentsAndCreatedDates(defaultSegment.getDataSource(), Intervals.ETERNITY); + Assert.assertEquals(resultForExactInterval, resultForEternity); + } + + @Test + public void testRetrieveUsedSegmentsAndCreatedDatesFetchesEternityForAnyInterval() + { + + insertUsedSegments(ImmutableSet.of(eternitySegment, firstHalfEternityRangeSegment, secondHalfEternityRangeSegment)); + + List> resultForRandomInterval = + coordinator.retrieveUsedSegmentsAndCreatedDates(defaultSegment.getDataSource(), defaultSegment.getInterval()); + Assert.assertEquals(3, resultForRandomInterval.size()); + + List> resultForEternity = + coordinator.retrieveUsedSegmentsAndCreatedDates(defaultSegment.getDataSource(), eternitySegment.getInterval()); + Assert.assertEquals(3, resultForEternity.size()); + } + private static class DS { static final String WIKI = "wiki"; diff --git a/services/src/main/java/org/apache/druid/server/AsyncQueryForwardingServlet.java b/services/src/main/java/org/apache/druid/server/AsyncQueryForwardingServlet.java index 75b13a39f1ff..5134a8109b8a 100644 --- a/services/src/main/java/org/apache/druid/server/AsyncQueryForwardingServlet.java +++ b/services/src/main/java/org/apache/druid/server/AsyncQueryForwardingServlet.java @@ -56,6 +56,7 @@ import org.apache.druid.server.security.AuthenticationResult; import org.apache.druid.server.security.Authenticator; import org.apache.druid.server.security.AuthenticatorMapper; +import org.apache.druid.server.security.AuthorizationUtils; import org.apache.druid.sql.http.SqlQuery; import org.apache.druid.sql.http.SqlResource; import org.eclipse.jetty.client.HttpClient; @@ -303,6 +304,7 @@ protected void service(HttpServletRequest request, HttpServletResponse response) /** * Rebuilds the {@link SqlQuery} object with sqlQueryId and queryId context parameters if not present + * * @param sqlQuery the original SqlQuery * @return an updated sqlQuery object with sqlQueryId and queryId context parameters */ @@ -367,13 +369,16 @@ void handleQueryParseException( // Log the error message final String errorMessage = exceptionToReport.getMessage() == null ? "no error message" : exceptionToReport.getMessage(); + + AuthenticationResult authenticationResult = AuthorizationUtils.authenticationResultFromRequest(request); + if (isNativeQuery) { requestLogger.logNativeQuery( RequestLogLine.forNative( null, DateTimes.nowUtc(), request.getRemoteAddr(), - new QueryStats(ImmutableMap.of("success", false, "exception", errorMessage)) + new QueryStats(ImmutableMap.of("success", false, "exception", errorMessage, "identity", authenticationResult.getIdentity())) ) ); } else { @@ -383,7 +388,7 @@ void handleQueryParseException( null, DateTimes.nowUtc(), request.getRemoteAddr(), - new QueryStats(ImmutableMap.of("success", false, "exception", errorMessage)) + new QueryStats(ImmutableMap.of("success", false, "exception", errorMessage, "identity", authenticationResult.getIdentity())) ) ); } @@ -744,6 +749,8 @@ public void onComplete(Result result) } emitQueryTime(requestTimeNs, success, sqlQueryId, queryId); + AuthenticationResult authenticationResult = AuthorizationUtils.authenticationResultFromRequest(req); + //noinspection VariableNotUsedInsideIf if (sqlQueryId != null) { // SQL query doesn't have a native query translation in router. Hence, not logging the native query. @@ -761,7 +768,9 @@ public void onComplete(Result result) TimeUnit.NANOSECONDS.toMillis(requestTimeNs), "success", success - && result.getResponse().getStatus() == Status.OK.getStatusCode() + && result.getResponse().getStatus() == Status.OK.getStatusCode(), + "identity", + authenticationResult.getIdentity() ) ) ) @@ -787,7 +796,9 @@ public void onComplete(Result result) TimeUnit.NANOSECONDS.toMillis(requestTimeNs), "success", success - && result.getResponse().getStatus() == Status.OK.getStatusCode() + && result.getResponse().getStatus() == Status.OK.getStatusCode(), + "identity", + authenticationResult.getIdentity() ) ) ) @@ -824,6 +835,7 @@ public void onFailure(Response response, Throwable failure) failedQueryCount.incrementAndGet(); emitQueryTime(requestTimeNs, false, sqlQueryId, queryId); + AuthenticationResult authenticationResult = AuthorizationUtils.authenticationResultFromRequest(req); //noinspection VariableNotUsedInsideIf if (sqlQueryId != null) { @@ -841,7 +853,9 @@ public void onFailure(Response response, Throwable failure) "success", false, "exception", - errorMessage == null ? "no message" : errorMessage + errorMessage == null ? "no message" : errorMessage, + "identity", + authenticationResult.getIdentity() ) ) ) @@ -871,7 +885,9 @@ public void onFailure(Response response, Throwable failure) "success", false, "exception", - errorMessage == null ? "no message" : errorMessage + errorMessage == null ? "no message" : errorMessage, + "identity", + authenticationResult.getIdentity() ) ) ) @@ -890,7 +906,12 @@ public void onFailure(Response response, Throwable failure) super.onFailure(response, failure); } - private void emitQueryTime(long requestTimeNs, boolean success, @Nullable String sqlQueryId, @Nullable String queryId) + private void emitQueryTime( + long requestTimeNs, + boolean success, + @Nullable String sqlQueryId, + @Nullable String queryId + ) { QueryMetrics queryMetrics; if (sqlQueryId != null) { diff --git a/services/src/test/java/org/apache/druid/server/AsyncQueryForwardingServletTest.java b/services/src/test/java/org/apache/druid/server/AsyncQueryForwardingServletTest.java index 6facaa547780..54238fe8cce6 100644 --- a/services/src/test/java/org/apache/druid/server/AsyncQueryForwardingServletTest.java +++ b/services/src/test/java/org/apache/druid/server/AsyncQueryForwardingServletTest.java @@ -65,6 +65,8 @@ import org.apache.druid.server.router.QueryHostFinder; import org.apache.druid.server.router.RendezvousHashAvaticaConnectionBalancer; import org.apache.druid.server.security.AllowAllAuthorizer; +import org.apache.druid.server.security.AuthConfig; +import org.apache.druid.server.security.AuthenticationResult; import org.apache.druid.server.security.AuthenticatorMapper; import org.apache.druid.server.security.Authorizer; import org.apache.druid.server.security.AuthorizerMapper; @@ -227,7 +229,7 @@ public void testSqlQueryProxy() throws Exception Properties properties = new Properties(); properties.setProperty("druid.router.sql.enable", "true"); - verifyServletCallsForQuery(query, true, false, hostFinder, properties); + verifyServletCallsForQuery(query, true, false, hostFinder, properties, false); } @Test @@ -244,7 +246,7 @@ public void testQueryProxy() throws Exception EasyMock.expect(hostFinder.pickServer(query)).andReturn(new TestServer("http", "1.2.3.4", 9999)).once(); EasyMock.replay(hostFinder); - verifyServletCallsForQuery(query, false, false, hostFinder, new Properties()); + verifyServletCallsForQuery(query, false, false, hostFinder, new Properties(), false); } @Test @@ -258,7 +260,7 @@ public void testJDBCSqlProxy() throws Exception .once(); EasyMock.replay(hostFinder); - verifyServletCallsForQuery(jdbcRequest, false, true, hostFinder, new Properties()); + verifyServletCallsForQuery(jdbcRequest, false, true, hostFinder, new Properties(), false); } @Test @@ -408,6 +410,7 @@ public void testHandleQueryParseExceptionWithFilterDisabled() throws Exception new Properties(), new ServerConfig() ); + Mockito.when(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).thenReturn(new AuthenticationResult("userA", "basic", "basic", null)); IOException testException = new IOException(errorMessage); servlet.handleQueryParseException(request, response, mockMapper, testException, false); ArgumentCaptor captor = ArgumentCaptor.forClass(Exception.class); @@ -454,6 +457,7 @@ public ErrorResponseTransformStrategy getErrorResponseTransformStrategy() } } ); + Mockito.when(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).thenReturn(new AuthenticationResult("userA", "basic", "basic", null)); IOException testException = new IOException(errorMessage); servlet.handleQueryParseException(request, response, mockMapper, testException, false); ArgumentCaptor captor = ArgumentCaptor.forClass(Exception.class); @@ -501,6 +505,7 @@ public ErrorResponseTransformStrategy getErrorResponseTransformStrategy() } } ); + Mockito.when(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).thenReturn(new AuthenticationResult("userA", "basic", "basic", null)); IOException testException = new IOException(errorMessage); servlet.handleQueryParseException(request, response, mockMapper, testException, false); ArgumentCaptor captor = ArgumentCaptor.forClass(Exception.class); @@ -512,6 +517,46 @@ public ErrorResponseTransformStrategy getErrorResponseTransformStrategy() Assert.assertNull(((QueryException) captor.getValue()).getHost()); } + @Test + public void testNativeQueryProxyFailure() throws Exception + { + final TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() + .dataSource("foo") + .intervals("2000/P1D") + .granularity(Granularities.ALL) + .context(ImmutableMap.of("queryId", "dummy")) + .build(); + + final QueryHostFinder hostFinder = EasyMock.createMock(QueryHostFinder.class); + EasyMock.expect(hostFinder.pickServer(query)).andReturn(new TestServer("http", "1.2.3.4", 9999)).once(); + EasyMock.replay(hostFinder); + + verifyServletCallsForQuery(query, false, false, hostFinder, new Properties(), true); + } + + @Test + public void testSqlQueryProxyFailure() throws Exception + { + final SqlQuery query = new SqlQuery( + "SELECT * FROM foo", + ResultFormat.ARRAY, + false, + false, + false, + ImmutableMap.of("sqlQueryId", "dummy"), + null + ); + final QueryHostFinder hostFinder = EasyMock.createMock(QueryHostFinder.class); + EasyMock.expect(hostFinder.findServerSql( + query.withOverridenContext(ImmutableMap.of("sqlQueryId", "dummy", "queryId", "dummy"))) + ).andReturn(new TestServer("http", "1.2.3.4", 9999)).once(); + EasyMock.replay(hostFinder); + + Properties properties = new Properties(); + properties.setProperty("druid.router.sql.enable", "true"); + verifyServletCallsForQuery(query, true, false, hostFinder, properties, true); + } + /** * Verifies that the Servlet calls the right methods the right number of times. */ @@ -520,7 +565,8 @@ private void verifyServletCallsForQuery( boolean isNativeSql, boolean isJDBCSql, QueryHostFinder hostFinder, - Properties properties + Properties properties, + boolean isFailure ) throws Exception { final ObjectMapper jsonMapper = TestHelper.makeJsonMapper(); @@ -587,27 +633,30 @@ public int read() EasyMock.expectLastCall(); requestMock.setAttribute("org.apache.druid.proxy.to.host.scheme", "http"); EasyMock.expectLastCall(); + EasyMock.expect(requestMock.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(new AuthenticationResult("userA", "basic", "basic", null)); + if (isFailure) { + EasyMock.expect(requestMock.getRemoteAddr()).andReturn("0.0.0.0:0"); + } + EasyMock.replay(requestMock); final AtomicLong didService = new AtomicLong(); final Request proxyRequestMock = Mockito.spy(Request.class); - final Result result = new Result( - proxyRequestMock, - new HttpResponse(proxyRequestMock, ImmutableList.of()) - { - @Override - public HttpFields getHeaders() - { - HttpFields httpFields = new HttpFields(); - if (isJDBCSql) { - httpFields.add(new HttpField("X-Druid-SQL-Query-Id", "jdbcDummy")); - } else if (isNativeSql) { - httpFields.add(new HttpField("X-Druid-SQL-Query-Id", "dummy")); - } - return httpFields; - } + HttpResponse response = new HttpResponse(proxyRequestMock, ImmutableList.of()) + { + @Override + public HttpFields getHeaders() + { + HttpFields httpFields = new HttpFields(); + if (isJDBCSql) { + httpFields.add(new HttpField("X-Druid-SQL-Query-Id", "jdbcDummy")); + } else if (isNativeSql) { + httpFields.add(new HttpField("X-Druid-SQL-Query-Id", "dummy")); } - ); + return httpFields; + } + }; + final Result result = new Result(proxyRequestMock, response); final StubServiceEmitter stubServiceEmitter = new StubServiceEmitter("", ""); final AsyncQueryForwardingServlet servlet = new AsyncQueryForwardingServlet( new MapQueryToolChestWarehouse(ImmutableMap.of()), @@ -640,7 +689,11 @@ protected void doService( // partial state of the servlet. Hence, only catching the exact exception to avoid possible errors. // Further, the metric assertions are also done to ensure that the metrics have emitted. try { - servlet.newProxyResponseListener(requestMock, null).onComplete(result); + if (isFailure) { + servlet.newProxyResponseListener(requestMock, null).onFailure(response, new Throwable("Proxy failed")); + } else { + servlet.newProxyResponseListener(requestMock, null).onComplete(result); + } } catch (NullPointerException ignored) { } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java index 438c666227e6..7a8786e21ef5 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java @@ -45,6 +45,8 @@ import org.apache.druid.query.filter.AndDimFilter; import org.apache.druid.query.filter.DimFilter; import org.apache.druid.query.filter.ExpressionDimFilter; +import org.apache.druid.query.filter.IsFalseDimFilter; +import org.apache.druid.query.filter.IsTrueDimFilter; import org.apache.druid.query.filter.NotDimFilter; import org.apache.druid.query.filter.NullFilter; import org.apache.druid.query.filter.OrDimFilter; @@ -379,22 +381,47 @@ public static DimFilter toFilter( { final SqlKind kind = expression.getKind(); - if (kind == SqlKind.IS_TRUE || kind == SqlKind.IS_NOT_FALSE) { - return toFilter( - plannerContext, - rowSignature, - virtualColumnRegistry, - Iterables.getOnlyElement(((RexCall) expression).getOperands()) - ); - } else if (kind == SqlKind.IS_FALSE || kind == SqlKind.IS_NOT_TRUE) { - return new NotDimFilter( - toFilter( + if (kind == SqlKind.IS_TRUE + || kind == SqlKind.IS_NOT_TRUE + || kind == SqlKind.IS_FALSE + || kind == SqlKind.IS_NOT_FALSE) { + if (NullHandling.useThreeValueLogic()) { + final DimFilter baseFilter = toFilter( + plannerContext, + rowSignature, + virtualColumnRegistry, + Iterables.getOnlyElement(((RexCall) expression).getOperands()) + ); + + if (kind == SqlKind.IS_TRUE) { + return IsTrueDimFilter.of(baseFilter); + } else if (kind == SqlKind.IS_NOT_TRUE) { + return NotDimFilter.of(IsTrueDimFilter.of(baseFilter)); + } else if (kind == SqlKind.IS_FALSE) { + return IsFalseDimFilter.of(baseFilter); + } else { // SqlKind.IS_NOT_FALSE + return NotDimFilter.of(IsFalseDimFilter.of(baseFilter)); + } + } else { + // legacy behavior + if (kind == SqlKind.IS_TRUE || kind == SqlKind.IS_NOT_FALSE) { + return toFilter( plannerContext, rowSignature, virtualColumnRegistry, Iterables.getOnlyElement(((RexCall) expression).getOperands()) - ) - ); + ); + } else { // SqlKind.IS_FALSE || SqlKind.IS_NOT_TRUE + return new NotDimFilter( + toFilter( + plannerContext, + rowSignature, + virtualColumnRegistry, + Iterables.getOnlyElement(((RexCall) expression).getOperands()) + ) + ); + } + } } else if (kind == SqlKind.CAST && expression.getType().getSqlTypeName() == SqlTypeName.BOOLEAN) { // Calcite sometimes leaves errant, useless cast-to-booleans inside filters. Strip them and continue. return toFilter( @@ -403,9 +430,7 @@ public static DimFilter toFilter( virtualColumnRegistry, Iterables.getOnlyElement(((RexCall) expression).getOperands()) ); - } else if (kind == SqlKind.AND - || kind == SqlKind.OR - || kind == SqlKind.NOT) { + } else if (kind == SqlKind.AND || kind == SqlKind.OR || kind == SqlKind.NOT) { final List filters = new ArrayList<>(); for (final RexNode rexNode : ((RexCall) expression).getOperands()) { final DimFilter nextFilter = toFilter( @@ -424,8 +449,7 @@ public static DimFilter toFilter( return new AndDimFilter(filters); } else if (kind == SqlKind.OR) { return new OrDimFilter(filters); - } else { - assert kind == SqlKind.NOT; + } else { // SqlKind.NOT return new NotDimFilter(Iterables.getOnlyElement(filters)); } } else { @@ -488,6 +512,11 @@ private static DimFilter toSimpleLeafFilter( final SqlKind kind = rexNode.getKind(); if (kind == SqlKind.IS_TRUE || kind == SqlKind.IS_NOT_FALSE) { + if (NullHandling.useThreeValueLogic()) { + // use expression filter to get istrue or notfalse expressions for correct 3vl behavior + return toExpressionLeafFilter(plannerContext, rowSignature, rexNode); + } + // legacy behavior return toSimpleLeafFilter( plannerContext, rowSignature, @@ -495,6 +524,11 @@ private static DimFilter toSimpleLeafFilter( Iterables.getOnlyElement(((RexCall) rexNode).getOperands()) ); } else if (kind == SqlKind.IS_FALSE || kind == SqlKind.IS_NOT_TRUE) { + if (NullHandling.useThreeValueLogic()) { + // use expression filter to get isfalse or nottrue expressions for correct 3vl behavior + return toExpressionLeafFilter(plannerContext, rowSignature, rexNode); + } + // legacy behavior return new NotDimFilter( toSimpleLeafFilter( plannerContext, diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/BottomUpTransform.java b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/BottomUpTransform.java index f594878b2947..a0d28f372e31 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/BottomUpTransform.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/BottomUpTransform.java @@ -23,6 +23,8 @@ import com.google.common.base.Preconditions; import org.apache.druid.query.filter.AndDimFilter; import org.apache.druid.query.filter.DimFilter; +import org.apache.druid.query.filter.IsFalseDimFilter; +import org.apache.druid.query.filter.IsTrueDimFilter; import org.apache.druid.query.filter.NotDimFilter; import org.apache.druid.query.filter.OrDimFilter; @@ -89,6 +91,22 @@ private DimFilter apply0(final DimFilter filter) } else { return checkedProcess(filter); } + } else if (filter instanceof IsTrueDimFilter) { + final DimFilter oldFilter = ((IsTrueDimFilter) filter).getField(); + final DimFilter newFilter = apply0(oldFilter); + if (!oldFilter.equals(newFilter)) { + return checkedProcess(new IsTrueDimFilter(newFilter)); + } else { + return checkedProcess(filter); + } + } else if (filter instanceof IsFalseDimFilter) { + final DimFilter oldFilter = ((IsFalseDimFilter) filter).getField(); + final DimFilter newFilter = apply0(oldFilter); + if (!oldFilter.equals(newFilter)) { + return checkedProcess(new IsFalseDimFilter(newFilter)); + } else { + return checkedProcess(filter); + } } else { return checkedProcess(filter); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Filtration.java b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Filtration.java index df03ff9662be..95466635a114 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Filtration.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Filtration.java @@ -114,6 +114,7 @@ public Filtration optimize(final RowSignature rowSignature) MoveTimeFiltersToIntervals.instance(), ConvertBoundsToSelectors.create(rowSignature), ConvertSelectorsToIns.create(rowSignature), + RemoveRedundantIsTrue.instance(), MoveMarkerFiltersToIntervals.instance(), ValidateNoMarkerFiltersRemain.instance() ) @@ -136,7 +137,8 @@ public Filtration optimizeFilterOnly(final RowSignature rowSignature) ImmutableList.of( CombineAndSimplifyBounds.instance(), ConvertBoundsToSelectors.create(rowSignature), - ConvertSelectorsToIns.create(rowSignature) + ConvertSelectorsToIns.create(rowSignature), + RemoveRedundantIsTrue.instance() ) ); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/RemoveRedundantIsTrue.java b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/RemoveRedundantIsTrue.java new file mode 100644 index 000000000000..d21b22ef4f96 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/RemoveRedundantIsTrue.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.filtration; + +import com.google.common.base.Function; +import org.apache.druid.query.filter.AndDimFilter; +import org.apache.druid.query.filter.DimFilter; +import org.apache.druid.query.filter.IsTrueDimFilter; +import org.apache.druid.query.filter.OrDimFilter; + +import java.util.ArrayList; +import java.util.List; + +/** + * Similar to {@link BottomUpTransform} except only removes redundant IS TRUE filters that are not inside of a NOT + * filter. The planner leaves behind stuff like `(x == y) IS TRUE` which is a pointless delegate when not living inside + * of a not filter to enforce proper three-value logic + */ +public class RemoveRedundantIsTrue implements Function +{ + private static final RemoveRedundantIsTrue INSTANCE = new RemoveRedundantIsTrue(); + + public static RemoveRedundantIsTrue instance() + { + return INSTANCE; + } + + @Override + public Filtration apply(Filtration filtration) + { + if (filtration.getDimFilter() != null) { + final Filtration retVal = Filtration.create(apply0(filtration.getDimFilter()), filtration.getIntervals()); + return filtration.equals(retVal) ? retVal : apply(retVal); + } else { + return filtration; + } + } + + private DimFilter apply0(final DimFilter filter) + { + // check for AND, OR to process their children, and unwrap any IS TRUE not living under a NOT, anything else we + // leave alone + if (filter instanceof AndDimFilter) { + final List oldFilters = ((AndDimFilter) filter).getFields(); + final List newFilters = new ArrayList<>(); + for (DimFilter oldFilter : oldFilters) { + final DimFilter newFilter = apply0(oldFilter); + if (newFilter != null) { + newFilters.add(newFilter); + } + } + if (!newFilters.equals(oldFilters)) { + return new AndDimFilter(newFilters); + } else { + return filter; + } + } else if (filter instanceof OrDimFilter) { + final List oldFilters = ((OrDimFilter) filter).getFields(); + final List newFilters = new ArrayList<>(); + for (DimFilter oldFilter : oldFilters) { + final DimFilter newFilter = apply0(oldFilter); + if (newFilter != null) { + newFilters.add(newFilter); + } + } + if (!newFilters.equals(oldFilters)) { + return new OrDimFilter(newFilters); + } else { + return filter; + } + } else if (filter instanceof IsTrueDimFilter) { + final DimFilter oldFilter = ((IsTrueDimFilter) filter).getField(); + final DimFilter newFilter = apply0(oldFilter); + if (!oldFilter.equals(newFilter)) { + return newFilter; + } else { + return oldFilter; + } + } else { + return filter; + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidSqlValidator.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidSqlValidator.java index 5a901c72296e..bbc4f99a1316 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidSqlValidator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidSqlValidator.java @@ -22,7 +22,14 @@ import org.apache.calcite.adapter.java.JavaTypeFactory; import org.apache.calcite.prepare.BaseDruidSqlValidator; import org.apache.calcite.prepare.CalciteCatalogReader; +import org.apache.calcite.runtime.CalciteContextException; +import org.apache.calcite.runtime.CalciteException; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOperatorTable; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.validate.SqlValidatorScope; /** * Druid extended SQL validator. (At present, it doesn't actually @@ -39,4 +46,33 @@ protected DruidSqlValidator( { super(opTab, catalogReader, typeFactory, validatorConfig); } + + @Override + public void validateCall(SqlCall call, SqlValidatorScope scope) + { + if (call.getKind() == SqlKind.NULLS_FIRST) { + SqlNode op0 = call.getOperandList().get(0); + if (op0.getKind() == SqlKind.DESCENDING) { + throw buildCalciteContextException("DESCENDING ordering with NULLS FIRST is not supported!", call); + } + } + if (call.getKind() == SqlKind.NULLS_LAST) { + SqlNode op0 = call.getOperandList().get(0); + if (op0.getKind() != SqlKind.DESCENDING) { + throw buildCalciteContextException("ASCENDING ordering with NULLS LAST is not supported!", call); + } + } + super.validateCall(call, scope); + } + + private CalciteContextException buildCalciteContextException(String message, SqlCall call) + { + SqlParserPos pos = call.getParserPosition(); + return new CalciteContextException(message, + new CalciteException(message, null), + pos.getLineNum(), + pos.getColumnNum(), + pos.getEndLineNum(), + pos.getEndColumnNum()); + } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java index 9141c4db090f..29f22d46c573 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java @@ -80,7 +80,7 @@ public class PlannerContext /** * Undocumented context key, used to enable window functions. */ - public static final String CTX_ENABLE_WINDOW_FNS = "windowsAreForClosers"; + public static final String CTX_ENABLE_WINDOW_FNS = "enableWindowing"; public static final String CTX_SQL_USE_BOUNDS_AND_SELECTORS = "sqlUseBoundAndSelectors"; public static final boolean DEFAULT_SQL_USE_BOUNDS_AND_SELECTORS = NullHandling.replaceWithDefault(); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/convertlet/DruidConvertletTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/convertlet/DruidConvertletTable.java index 59ad4ef24f05..eadd2e48268d 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/convertlet/DruidConvertletTable.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/convertlet/DruidConvertletTable.java @@ -67,6 +67,9 @@ public class DruidConvertletTable implements SqlRexConvertletTable .add(SqlStdOperatorTable.NULLIF) .add(SqlStdOperatorTable.COALESCE) .add(SqlLibraryOperators.NVL) + .add(SqlStdOperatorTable.DESC) + .add(SqlStdOperatorTable.NULLS_FIRST) + .add(SqlStdOperatorTable.NULLS_LAST) .build(); private final Map table; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index 9bfbfb8e39bd..6b2d12c6cce0 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -59,6 +59,7 @@ import org.apache.druid.query.filter.EqualityFilter; import org.apache.druid.query.filter.ExpressionDimFilter; import org.apache.druid.query.filter.InDimFilter; +import org.apache.druid.query.filter.IsTrueDimFilter; import org.apache.druid.query.filter.NotDimFilter; import org.apache.druid.query.filter.NullFilter; import org.apache.druid.query.filter.OrDimFilter; @@ -365,6 +366,11 @@ public static NotDimFilter not(DimFilter filter) return new NotDimFilter(filter); } + public static IsTrueDimFilter istrue(DimFilter filter) + { + return new IsTrueDimFilter(filter); + } + public static InDimFilter in(String dimension, Collection values, ExtractionFn extractionFn) { return new InDimFilter(dimension, values, extractionFn); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index 086633d7e59c..3a5da7d325ff 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -6470,7 +6470,9 @@ public void testUnnestVirtualWithColumnsAndNullIf() expressionFilter("(\"j0.unnest\" == \"m2\")"), and( isNull("j0.unnest"), - not(expressionFilter("(\"j0.unnest\" == \"m2\")")) + NullHandling.sqlCompatible() + ? not(istrue(expressionFilter("(\"j0.unnest\" == \"m2\")"))) + : not(expressionFilter("(\"j0.unnest\" == \"m2\")")) ) )) .legacy(false) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java index 908ccae687b2..0882f3c9cb12 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java @@ -5607,7 +5607,14 @@ public void testRegressionFilteredAggregatorsSubqueryJoins(Map q or( isNull("__j0.a0"), not( - or( + NullHandling.sqlCompatible() + ? istrue( + or( + not(expressionFilter("\"__j0.d0\"")), + notNull("__j0.d0") + ) + ) + : or( not(expressionFilter("\"__j0.d0\"")), notNull("__j0.d0") ) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 08fdbc3bfaa0..811227162ac7 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -137,6 +137,9 @@ import java.util.Map; import java.util.stream.Collectors; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertThrows; + public class CalciteQueryTest extends BaseCalciteQueryTest { @Test @@ -3101,7 +3104,9 @@ public void testNullEmptyStringEquality() equality("dim2", "a", ColumnType.STRING), and( isNull("dim2"), - not(equality("dim2", "a", ColumnType.STRING)) + NullHandling.sqlCompatible() + ? not(istrue(equality("dim2", "a", ColumnType.STRING))) + : not(selector("dim2", "a")) ) ) ) @@ -3109,12 +3114,12 @@ public void testNullEmptyStringEquality() .context(QUERY_CONTEXT_DEFAULT) .build() ), - ImmutableList.of( NullHandling.replaceWithDefault() // Matches everything but "abc" ? new Object[]{5L} - : new Object[]{2L} + // match only null values + : new Object[]{4L} ) ); } @@ -4847,13 +4852,15 @@ public void testFilteredAggregations() ), new FilteredAggregatorFactory( new LongSumAggregatorFactory("a1", "cnt"), - not(equality("dim1", "abc", ColumnType.STRING)) + NullHandling.sqlCompatible() + ? not(istrue(equality("dim1", "abc", ColumnType.STRING))) + : not(selector("dim1", "abc")) ), new FilteredAggregatorFactory( new LongSumAggregatorFactory("a2", "cnt"), - NullHandling.replaceWithDefault() - ? selector("dim1", "a", new SubstringDimExtractionFn(0, 1)) - : expressionFilter("(substring(\"dim1\", 0, 1) == 'a')") + NullHandling.sqlCompatible() + ? expressionFilter("(substring(\"dim1\", 0, 1) == 'a')") + : selector("dim1", "a", new SubstringDimExtractionFn(0, 1)) ), new FilteredAggregatorFactory( @@ -14247,4 +14254,25 @@ public void testLatestByOnStringColumnWithoutMaxBytesSpecified() new Object[] {"abc", defaultString, "def", defaultString, "def", defaultString} )); } + + @Test + public void testUnSupportedNullsFirst() + { + DruidException e = assertThrows(DruidException.class, () -> testBuilder() + .queryContext(ImmutableMap.of(PlannerContext.CTX_ENABLE_WINDOW_FNS, true)) + .sql("SELECT dim1,ROW_NUMBER() OVER (ORDER BY dim1 DESC NULLS FIRST) from druid.foo") + .run()); + + assertThat(e, invalidSqlIs("DESCENDING ordering with NULLS FIRST is not supported! (line [1], column [41])")); + } + + @Test + public void testUnSupportedNullsLast() + { + DruidException e = assertThrows(DruidException.class, () -> testBuilder() + .queryContext(ImmutableMap.of(PlannerContext.CTX_ENABLE_WINDOW_FNS, true)) + .sql("SELECT dim1,ROW_NUMBER() OVER (ORDER BY dim1 NULLS LAST) from druid.foo") + .run()); + assertThat(e, invalidSqlIs("ASCENDING ordering with NULLS LAST is not supported! (line [1], column [41])")); + } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java index 27620315ed47..7a2b9b70f1a6 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java @@ -450,7 +450,8 @@ private static Object parseLongValue(final String val) } try { LocalTime v = LocalTime.parse(val); - return v.getMillisOfDay(); + Long l = (long) v.getMillisOfDay(); + return l; } catch (Exception e) { } @@ -477,7 +478,6 @@ public void windowQueryTest() .skipVectorize(true) .queryContext(ImmutableMap.of( PlannerContext.CTX_ENABLE_WINDOW_FNS, true, - "windowsAllTheWayDown", true, QueryContexts.ENABLE_DEBUG, true) ) .sql(testCase.getQueryString()) @@ -4385,7 +4385,7 @@ public void test_nestedAggs_nstdWinView01() windowQueryTest(); } - @NotYetSupported(Modes.MISSING_DESC) + @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("aggregates/winFnQry_63") @Test public void test_aggregates_winFnQry_63() @@ -4393,7 +4393,7 @@ public void test_aggregates_winFnQry_63() windowQueryTest(); } - @NotYetSupported(Modes.MISSING_DESC) + @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/winFnQry_83") @Test public void test_aggregates_winFnQry_83() @@ -4401,7 +4401,7 @@ public void test_aggregates_winFnQry_83() windowQueryTest(); } - @NotYetSupported(Modes.MISSING_DESC) + @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("frameclause/multipl_wnwds/mulwind_01") @Test public void test_frameclause_multipl_wnwds_mulwind_01() @@ -4409,7 +4409,7 @@ public void test_frameclause_multipl_wnwds_mulwind_01() windowQueryTest(); } - @NotYetSupported(Modes.MISSING_DESC) + @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("frameclause/multipl_wnwds/mulwind_06") @Test public void test_frameclause_multipl_wnwds_mulwind_06() @@ -4417,7 +4417,7 @@ public void test_frameclause_multipl_wnwds_mulwind_06() windowQueryTest(); } - @NotYetSupported(Modes.MISSING_DESC) + @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("frameclause/multipl_wnwds/mulwind_07") @Test public void test_frameclause_multipl_wnwds_mulwind_07() @@ -4425,7 +4425,6 @@ public void test_frameclause_multipl_wnwds_mulwind_07() windowQueryTest(); } - @NotYetSupported(Modes.MISSING_DESC) @DrillTest("lag_func/lag_Fn_108") @Test public void test_lag_func_lag_Fn_108() @@ -4433,7 +4432,6 @@ public void test_lag_func_lag_Fn_108() windowQueryTest(); } - @NotYetSupported(Modes.MISSING_DESC) @DrillTest("lag_func/lag_Fn_109") @Test public void test_lag_func_lag_Fn_109() @@ -4441,7 +4439,6 @@ public void test_lag_func_lag_Fn_109() windowQueryTest(); } - @NotYetSupported(Modes.MISSING_DESC) @DrillTest("lag_func/lag_Fn_69") @Test public void test_lag_func_lag_Fn_69() @@ -4449,7 +4446,6 @@ public void test_lag_func_lag_Fn_69() windowQueryTest(); } - @NotYetSupported(Modes.MISSING_DESC) @DrillTest("lead_func/lead_Fn_103") @Test public void test_lead_func_lead_Fn_103() @@ -4457,7 +4453,6 @@ public void test_lead_func_lead_Fn_103() windowQueryTest(); } - @NotYetSupported(Modes.MISSING_DESC) @DrillTest("lead_func/lead_Fn_104") @Test public void test_lead_func_lead_Fn_104() @@ -4465,7 +4460,6 @@ public void test_lead_func_lead_Fn_104() windowQueryTest(); } - @NotYetSupported(Modes.MISSING_DESC) @DrillTest("lead_func/lead_Fn_69") @Test public void test_lead_func_lead_Fn_69() @@ -4473,7 +4467,7 @@ public void test_lead_func_lead_Fn_69() windowQueryTest(); } - @NotYetSupported(Modes.MISSING_DESC) + @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("nestedAggs/multiWin_7") @Test public void test_nestedAggs_multiWin_7() @@ -4993,7 +4987,7 @@ public void test_aggregates_winFnQry_7() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_COUNT_MISMATCH) @DrillTest("aggregates/testW_Nulls_10") @Test public void test_aggregates_testW_Nulls_10() @@ -5001,7 +4995,7 @@ public void test_aggregates_testW_Nulls_10() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/testW_Nulls_11") @Test public void test_aggregates_testW_Nulls_11() @@ -5153,7 +5147,7 @@ public void test_aggregates_testW_Nulls_29() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/testW_Nulls_2") @Test public void test_aggregates_testW_Nulls_2() @@ -5241,7 +5235,7 @@ public void test_aggregates_testW_Nulls_39() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_COUNT_MISMATCH) @DrillTest("aggregates/testW_Nulls_3") @Test public void test_aggregates_testW_Nulls_3() @@ -5249,7 +5243,7 @@ public void test_aggregates_testW_Nulls_3() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/testW_Nulls_4") @Test public void test_aggregates_testW_Nulls_4() @@ -5257,7 +5251,7 @@ public void test_aggregates_testW_Nulls_4() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.COLUMN_NOT_FOUND) @DrillTest("aggregates/testW_Nulls_5") @Test public void test_aggregates_testW_Nulls_5() @@ -5265,7 +5259,7 @@ public void test_aggregates_testW_Nulls_5() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.COLUMN_NOT_FOUND) @DrillTest("aggregates/testW_Nulls_6") @Test public void test_aggregates_testW_Nulls_6() @@ -5273,7 +5267,7 @@ public void test_aggregates_testW_Nulls_6() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/testW_Nulls_7") @Test public void test_aggregates_testW_Nulls_7() @@ -5281,7 +5275,6 @@ public void test_aggregates_testW_Nulls_7() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("aggregates/testW_Nulls_8") @Test public void test_aggregates_testW_Nulls_8() @@ -5289,7 +5282,7 @@ public void test_aggregates_testW_Nulls_8() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/testW_Nulls_9") @Test public void test_aggregates_testW_Nulls_9() @@ -5297,7 +5290,7 @@ public void test_aggregates_testW_Nulls_9() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/winFnQry_61") @Test public void test_aggregates_winFnQry_61() @@ -5305,7 +5298,7 @@ public void test_aggregates_winFnQry_61() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/winFnQry_62") @Test public void test_aggregates_winFnQry_62() @@ -5313,7 +5306,7 @@ public void test_aggregates_winFnQry_62() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/winFnQry_64") @Test public void test_aggregates_winFnQry_64() @@ -5321,7 +5314,7 @@ public void test_aggregates_winFnQry_64() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/winFnQry_65") @Test public void test_aggregates_winFnQry_65() @@ -5409,7 +5402,7 @@ public void test_aggregates_winFnQry_75() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/winFnQry_76") @Test public void test_aggregates_winFnQry_76() @@ -5417,7 +5410,7 @@ public void test_aggregates_winFnQry_76() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/winFnQry_77") @Test public void test_aggregates_winFnQry_77() @@ -5425,7 +5418,7 @@ public void test_aggregates_winFnQry_77() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/winFnQry_78") @Test public void test_aggregates_winFnQry_78() @@ -5433,7 +5426,7 @@ public void test_aggregates_winFnQry_78() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_COUNT_MISMATCH) @DrillTest("aggregates/winFnQry_79") @Test public void test_aggregates_winFnQry_79() @@ -5441,7 +5434,7 @@ public void test_aggregates_winFnQry_79() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_COUNT_MISMATCH) @DrillTest("aggregates/winFnQry_80") @Test public void test_aggregates_winFnQry_80() @@ -5449,7 +5442,7 @@ public void test_aggregates_winFnQry_80() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_COUNT_MISMATCH) @DrillTest("aggregates/winFnQry_81") @Test public void test_aggregates_winFnQry_81() @@ -5457,7 +5450,7 @@ public void test_aggregates_winFnQry_81() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/winFnQry_82") @Test public void test_aggregates_winFnQry_82() @@ -5465,7 +5458,6 @@ public void test_aggregates_winFnQry_82() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lag_func/lag_Fn_10") @Test public void test_lag_func_lag_Fn_10() @@ -5473,7 +5465,6 @@ public void test_lag_func_lag_Fn_10() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lag_func/lag_Fn_11") @Test public void test_lag_func_lag_Fn_11() @@ -5481,7 +5472,6 @@ public void test_lag_func_lag_Fn_11() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lag_func/lag_Fn_12") @Test public void test_lag_func_lag_Fn_12() @@ -5489,7 +5479,6 @@ public void test_lag_func_lag_Fn_12() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lag_func/lag_Fn_13") @Test public void test_lag_func_lag_Fn_13() @@ -5497,7 +5486,6 @@ public void test_lag_func_lag_Fn_13() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lag_func/lag_Fn_14") @Test public void test_lag_func_lag_Fn_14() @@ -5505,7 +5493,6 @@ public void test_lag_func_lag_Fn_14() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lag_func/lag_Fn_15") @Test public void test_lag_func_lag_Fn_15() @@ -5513,7 +5500,6 @@ public void test_lag_func_lag_Fn_15() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lag_func/lag_Fn_16") @Test public void test_lag_func_lag_Fn_16() @@ -5521,7 +5507,6 @@ public void test_lag_func_lag_Fn_16() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lag_func/lag_Fn_17") @Test public void test_lag_func_lag_Fn_17() @@ -5529,7 +5514,6 @@ public void test_lag_func_lag_Fn_17() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lag_func/lag_Fn_18") @Test public void test_lag_func_lag_Fn_18() @@ -5537,7 +5521,7 @@ public void test_lag_func_lag_Fn_18() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.UNSUPPORTED_NULL_ORDERING) @DrillTest("lag_func/lag_Fn_19") @Test public void test_lag_func_lag_Fn_19() @@ -5545,7 +5529,7 @@ public void test_lag_func_lag_Fn_19() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.UNSUPPORTED_NULL_ORDERING) @DrillTest("lag_func/lag_Fn_20") @Test public void test_lag_func_lag_Fn_20() @@ -5553,7 +5537,7 @@ public void test_lag_func_lag_Fn_20() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.UNSUPPORTED_NULL_ORDERING) @DrillTest("lag_func/lag_Fn_21") @Test public void test_lag_func_lag_Fn_21() @@ -5561,7 +5545,7 @@ public void test_lag_func_lag_Fn_21() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.UNSUPPORTED_NULL_ORDERING) @DrillTest("lag_func/lag_Fn_22") @Test public void test_lag_func_lag_Fn_22() @@ -5577,7 +5561,7 @@ public void test_lag_func_lag_Fn_23() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.UNSUPPORTED_NULL_ORDERING) @DrillTest("lag_func/lag_Fn_24") @Test public void test_lag_func_lag_Fn_24() @@ -5585,7 +5569,7 @@ public void test_lag_func_lag_Fn_24() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.UNSUPPORTED_NULL_ORDERING) @DrillTest("lag_func/lag_Fn_25") @Test public void test_lag_func_lag_Fn_25() @@ -5593,7 +5577,7 @@ public void test_lag_func_lag_Fn_25() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.UNSUPPORTED_NULL_ORDERING) @DrillTest("lag_func/lag_Fn_26") @Test public void test_lag_func_lag_Fn_26() @@ -5601,7 +5585,7 @@ public void test_lag_func_lag_Fn_26() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.UNSUPPORTED_NULL_ORDERING) @DrillTest("lag_func/lag_Fn_54") @Test public void test_lag_func_lag_Fn_54() @@ -5609,7 +5593,6 @@ public void test_lag_func_lag_Fn_54() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lag_func/lag_Fn_64") @Test public void test_lag_func_lag_Fn_64() @@ -5617,7 +5600,6 @@ public void test_lag_func_lag_Fn_64() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lag_func/lag_Fn_65") @Test public void test_lag_func_lag_Fn_65() @@ -5625,7 +5607,6 @@ public void test_lag_func_lag_Fn_65() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lag_func/lag_Fn_66") @Test public void test_lag_func_lag_Fn_66() @@ -5633,7 +5614,6 @@ public void test_lag_func_lag_Fn_66() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lag_func/lag_Fn_67") @Test public void test_lag_func_lag_Fn_67() @@ -5641,7 +5621,7 @@ public void test_lag_func_lag_Fn_67() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.UNSUPPORTED_NULL_ORDERING) @DrillTest("lag_func/lag_Fn_68") @Test public void test_lag_func_lag_Fn_68() @@ -5649,7 +5629,6 @@ public void test_lag_func_lag_Fn_68() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lag_func/lag_Fn_71") @Test public void test_lag_func_lag_Fn_71() @@ -5657,7 +5636,7 @@ public void test_lag_func_lag_Fn_71() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.UNSUPPORTED_NULL_ORDERING) @DrillTest("lag_func/lag_Fn_72") @Test public void test_lag_func_lag_Fn_72() @@ -5665,7 +5644,6 @@ public void test_lag_func_lag_Fn_72() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lead_func/lead_Fn_10") @Test public void test_lead_func_lead_Fn_10() @@ -5673,7 +5651,6 @@ public void test_lead_func_lead_Fn_10() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lead_func/lead_Fn_11") @Test public void test_lead_func_lead_Fn_11() @@ -5681,7 +5658,6 @@ public void test_lead_func_lead_Fn_11() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lead_func/lead_Fn_12") @Test public void test_lead_func_lead_Fn_12() @@ -5689,7 +5665,6 @@ public void test_lead_func_lead_Fn_12() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lead_func/lead_Fn_13") @Test public void test_lead_func_lead_Fn_13() @@ -5697,7 +5672,6 @@ public void test_lead_func_lead_Fn_13() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lead_func/lead_Fn_14") @Test public void test_lead_func_lead_Fn_14() @@ -5705,7 +5679,6 @@ public void test_lead_func_lead_Fn_14() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lead_func/lead_Fn_15") @Test public void test_lead_func_lead_Fn_15() @@ -5713,7 +5686,6 @@ public void test_lead_func_lead_Fn_15() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lead_func/lead_Fn_16") @Test public void test_lead_func_lead_Fn_16() @@ -5721,7 +5693,6 @@ public void test_lead_func_lead_Fn_16() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lead_func/lead_Fn_17") @Test public void test_lead_func_lead_Fn_17() @@ -5729,7 +5700,6 @@ public void test_lead_func_lead_Fn_17() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lead_func/lead_Fn_18") @Test public void test_lead_func_lead_Fn_18() @@ -5737,7 +5707,7 @@ public void test_lead_func_lead_Fn_18() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.UNSUPPORTED_NULL_ORDERING) @DrillTest("lead_func/lead_Fn_19") @Test public void test_lead_func_lead_Fn_19() @@ -5753,7 +5723,7 @@ public void test_lead_func_lead_Fn_20() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.UNSUPPORTED_NULL_ORDERING) @DrillTest("lead_func/lead_Fn_21") @Test public void test_lead_func_lead_Fn_21() @@ -5761,7 +5731,7 @@ public void test_lead_func_lead_Fn_21() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.UNSUPPORTED_NULL_ORDERING) @DrillTest("lead_func/lead_Fn_22") @Test public void test_lead_func_lead_Fn_22() @@ -5793,7 +5763,6 @@ public void test_lead_func_lead_Fn_25() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lead_func/lead_Fn_64") @Test public void test_lead_func_lead_Fn_64() @@ -5801,7 +5770,6 @@ public void test_lead_func_lead_Fn_64() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lead_func/lead_Fn_65") @Test public void test_lead_func_lead_Fn_65() @@ -5809,7 +5777,6 @@ public void test_lead_func_lead_Fn_65() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lead_func/lead_Fn_66") @Test public void test_lead_func_lead_Fn_66() @@ -5817,7 +5784,6 @@ public void test_lead_func_lead_Fn_66() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lead_func/lead_Fn_67") @Test public void test_lead_func_lead_Fn_67() @@ -5825,7 +5791,7 @@ public void test_lead_func_lead_Fn_67() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.UNSUPPORTED_NULL_ORDERING) @DrillTest("lead_func/lead_Fn_68") @Test public void test_lead_func_lead_Fn_68() @@ -5833,7 +5799,6 @@ public void test_lead_func_lead_Fn_68() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) @DrillTest("lead_func/lead_Fn_71") @Test public void test_lead_func_lead_Fn_71() @@ -5841,7 +5806,7 @@ public void test_lead_func_lead_Fn_71() windowQueryTest(); } - @NotYetSupported(Modes.NULLS_FIRST_LAST) + @NotYetSupported(Modes.UNSUPPORTED_NULL_ORDERING) @DrillTest("lead_func/lead_Fn_72") @Test public void test_lead_func_lead_Fn_72() @@ -6463,7 +6428,6 @@ public void test_aggregates_winFnQry_8() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/wo_OrdrBy_17") @Test public void test_aggregates_wo_OrdrBy_17() @@ -6471,7 +6435,6 @@ public void test_aggregates_wo_OrdrBy_17() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/wo_OrdrBy_18") @Test public void test_aggregates_wo_OrdrBy_18() @@ -6479,7 +6442,6 @@ public void test_aggregates_wo_OrdrBy_18() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/wo_OrdrBy_19") @Test public void test_aggregates_wo_OrdrBy_19() @@ -6487,7 +6449,6 @@ public void test_aggregates_wo_OrdrBy_19() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/wo_OrdrBy_20") @Test public void test_aggregates_wo_OrdrBy_20() @@ -6495,7 +6456,6 @@ public void test_aggregates_wo_OrdrBy_20() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/wo_OrdrBy_21") @Test public void test_aggregates_wo_OrdrBy_21() @@ -6503,7 +6463,6 @@ public void test_aggregates_wo_OrdrBy_21() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/wo_OrdrBy_22") @Test public void test_aggregates_wo_OrdrBy_22() @@ -6511,7 +6470,6 @@ public void test_aggregates_wo_OrdrBy_22() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/wo_OrdrBy_23") @Test public void test_aggregates_wo_OrdrBy_23() @@ -6519,7 +6477,6 @@ public void test_aggregates_wo_OrdrBy_23() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/wo_OrdrBy_24") @Test public void test_aggregates_wo_OrdrBy_24() @@ -6527,7 +6484,6 @@ public void test_aggregates_wo_OrdrBy_24() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/wo_OrdrBy_25") @Test public void test_aggregates_wo_OrdrBy_25() @@ -6535,7 +6491,6 @@ public void test_aggregates_wo_OrdrBy_25() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("aggregates/wo_OrdrBy_26") @Test public void test_aggregates_wo_OrdrBy_26() @@ -6679,7 +6634,6 @@ public void test_aggregates_woPrtnBy_9() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("first_val/firstValFn_17") @Test public void test_first_val_firstValFn_17() @@ -6687,7 +6641,6 @@ public void test_first_val_firstValFn_17() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("first_val/firstValFn_23") @Test public void test_first_val_firstValFn_23() @@ -6813,7 +6766,6 @@ public void test_frameclause_defaultFrame_RBUPACR_dt_2() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("frameclause/defaultFrame/RBUPACR_dt_4") @Test public void test_frameclause_defaultFrame_RBUPACR_dt_4() @@ -6996,7 +6948,6 @@ public void test_frameclause_RBCRACR_RBCRACR_dbl_7() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("frameclause/RBCRACR/RBCRACR_dt_1") @Test public void test_frameclause_RBCRACR_RBCRACR_dt_1() @@ -7004,7 +6955,6 @@ public void test_frameclause_RBCRACR_RBCRACR_dt_1() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("frameclause/RBCRACR/RBCRACR_dt_2") @Test public void test_frameclause_RBCRACR_RBCRACR_dt_2() @@ -7262,7 +7212,6 @@ public void test_frameclause_RBUPAUF_RBUPAUF_dbl_7() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("frameclause/RBUPAUF/RBUPAUF_dt_1") @Test public void test_frameclause_RBUPAUF_RBUPAUF_dt_1() @@ -7270,7 +7219,6 @@ public void test_frameclause_RBUPAUF_RBUPAUF_dt_1() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("frameclause/RBUPAUF/RBUPAUF_dt_2") @Test public void test_frameclause_RBUPAUF_RBUPAUF_dt_2() @@ -7278,7 +7226,6 @@ public void test_frameclause_RBUPAUF_RBUPAUF_dt_2() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("frameclause/RBUPAUF/RBUPAUF_dt_4") @Test public void test_frameclause_RBUPAUF_RBUPAUF_dt_4() @@ -7286,7 +7233,6 @@ public void test_frameclause_RBUPAUF_RBUPAUF_dt_4() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("frameclause/RBUPAUF/RBUPAUF_dt_5") @Test public void test_frameclause_RBUPAUF_RBUPAUF_dt_5() @@ -7349,7 +7295,6 @@ public void test_frameclause_subQueries_frmInSubQry_58() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("frameclause/subQueries/frmInSubQry_59") @Test public void test_frameclause_subQueries_frmInSubQry_59() @@ -7357,7 +7302,6 @@ public void test_frameclause_subQueries_frmInSubQry_59() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("frameclause/subQueries/frmInSubQry_60") @Test public void test_frameclause_subQueries_frmInSubQry_60() @@ -7373,7 +7317,6 @@ public void test_frameclause_subQueries_frmInSubQry_61() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("frameclause/subQueries/frmInSubQry_62") @Test public void test_frameclause_subQueries_frmInSubQry_62() @@ -7397,7 +7340,6 @@ public void test_frameclause_subQueries_frmInSubQry_64() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("lag_func/lag_Fn_101") @Test public void test_lag_func_lag_Fn_101() @@ -7405,7 +7347,6 @@ public void test_lag_func_lag_Fn_101() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("lag_func/lag_Fn_6") @Test public void test_lag_func_lag_Fn_6() @@ -7573,7 +7514,6 @@ public void test_last_val_lastValFn_9() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("lead_func/lead_Fn_33") @Test public void test_lead_func_lead_Fn_33() @@ -7581,7 +7521,6 @@ public void test_lead_func_lead_Fn_33() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("lead_func/lead_Fn_42") @Test public void test_lead_func_lead_Fn_42() @@ -7589,7 +7528,6 @@ public void test_lead_func_lead_Fn_42() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("lead_func/lead_Fn_51") @Test public void test_lead_func_lead_Fn_51() @@ -7597,7 +7535,6 @@ public void test_lead_func_lead_Fn_51() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("lead_func/lead_Fn_54") @Test public void test_lead_func_lead_Fn_54() @@ -7605,7 +7542,6 @@ public void test_lead_func_lead_Fn_54() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("lead_func/lead_Fn_60") @Test public void test_lead_func_lead_Fn_60() @@ -7613,7 +7549,6 @@ public void test_lead_func_lead_Fn_60() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("lead_func/lead_Fn_63") @Test public void test_lead_func_lead_Fn_63() @@ -7621,7 +7556,6 @@ public void test_lead_func_lead_Fn_63() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("lead_func/lead_Fn_6") @Test public void test_lead_func_lead_Fn_6() @@ -7629,7 +7563,6 @@ public void test_lead_func_lead_Fn_6() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("lead_func/lead_Fn_77") @Test public void test_lead_func_lead_Fn_77() @@ -7637,7 +7570,6 @@ public void test_lead_func_lead_Fn_77() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("lead_func/lead_Fn_90") @Test public void test_lead_func_lead_Fn_90() @@ -7645,7 +7577,6 @@ public void test_lead_func_lead_Fn_90() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("lead_func/lead_Fn_96") @Test public void test_lead_func_lead_Fn_96() @@ -7653,7 +7584,6 @@ public void test_lead_func_lead_Fn_96() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("lead_func/lead_Fn_9") @Test public void test_lead_func_lead_Fn_9() @@ -7669,7 +7599,6 @@ public void test_nestedAggs_basic_3() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("nestedAggs/basic_5") @Test public void test_nestedAggs_basic_5() @@ -7677,7 +7606,6 @@ public void test_nestedAggs_basic_5() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("nestedAggs/basic_7") @Test public void test_nestedAggs_basic_7() @@ -7756,7 +7684,6 @@ public void test_nestedAggs_woutPrtnBy_5() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("ntile_func/ntileFn_15") @Test public void test_ntile_func_ntileFn_15() @@ -7764,7 +7691,6 @@ public void test_ntile_func_ntileFn_15() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("ntile_func/ntileFn_17") @Test public void test_ntile_func_ntileFn_17() @@ -7772,7 +7698,6 @@ public void test_ntile_func_ntileFn_17() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("ntile_func/ntileFn_27") @Test public void test_ntile_func_ntileFn_27() @@ -7796,7 +7721,6 @@ public void test_ntile_func_ntileFn_59() windowQueryTest(); } - @NotYetSupported(Modes.RESULT_MISMATCH) @DrillTest("ntile_func/ntileFn_7") @Test public void test_ntile_func_ntileFn_7() @@ -7868,7 +7792,6 @@ public void test_frameclause_RBUPACR_RBUPACR_dt_2() windowQueryTest(); } - @NotYetSupported(Modes.T_ALLTYPES_ISSUES) @DrillTest("frameclause/RBUPACR/RBUPACR_dt_4") @Test public void test_frameclause_RBUPACR_RBUPACR_dt_4() @@ -7908,7 +7831,6 @@ public void test_frameclause_RBUPACR_RBUPACR_int9() windowQueryTest(); } - @NotYetSupported(Modes.T_ALLTYPES_ISSUES) @DrillTest("lag_func/lag_Fn_33") @Test public void test_lag_func_lag_Fn_33() @@ -7916,7 +7838,6 @@ public void test_lag_func_lag_Fn_33() windowQueryTest(); } - @NotYetSupported(Modes.T_ALLTYPES_ISSUES) @DrillTest("lag_func/lag_Fn_51") @Test public void test_lag_func_lag_Fn_51() @@ -7924,7 +7845,6 @@ public void test_lag_func_lag_Fn_51() windowQueryTest(); } - @NotYetSupported(Modes.T_ALLTYPES_ISSUES) @DrillTest("lag_func/lag_Fn_60") @Test public void test_lag_func_lag_Fn_60() @@ -7932,7 +7852,6 @@ public void test_lag_func_lag_Fn_60() windowQueryTest(); } - @NotYetSupported(Modes.T_ALLTYPES_ISSUES) @DrillTest("lag_func/lag_Fn_77") @Test public void test_lag_func_lag_Fn_77() @@ -7940,7 +7859,6 @@ public void test_lag_func_lag_Fn_77() windowQueryTest(); } - @NotYetSupported(Modes.T_ALLTYPES_ISSUES) @DrillTest("lag_func/lag_Fn_95") @Test public void test_lag_func_lag_Fn_95() @@ -7948,7 +7866,6 @@ public void test_lag_func_lag_Fn_95() windowQueryTest(); } - @NotYetSupported(Modes.T_ALLTYPES_ISSUES) @DrillTest("nestedAggs/frmclause03") @Test public void test_nestedAggs_frmclause03() diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java b/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java index a8634ba5db94..86e5c41d6de8 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java @@ -88,7 +88,8 @@ enum Modes INCORRECT_SYNTAX(DruidException.class, "Incorrect syntax near the keyword"), // at least c7 is represented oddly in the parquet file T_ALLTYPES_ISSUES(AssertionError.class, "(t_alltype|allTypsUniq|fewRowsAllData).parquet.*Verifier.verify"), - RESULT_MISMATCH(AssertionError.class, "assertResultsEquals"); + RESULT_MISMATCH(AssertionError.class, "assertResultsEquals"), + UNSUPPORTED_NULL_ORDERING(DruidException.class, "(A|DE)SCENDING ordering with NULLS (LAST|FIRST)"); public Class throwableClass; public String regex; diff --git a/sql/src/test/resources/calcite/tests/window/orderByDescNulls.sqlTest b/sql/src/test/resources/calcite/tests/window/orderByDescNulls.sqlTest new file mode 100644 index 000000000000..320b448c7488 --- /dev/null +++ b/sql/src/test/resources/calcite/tests/window/orderByDescNulls.sqlTest @@ -0,0 +1,45 @@ +type: "operatorValidation" + +sql: | + SELECT + cityName, + __time, + ROW_NUMBER() OVER (ORDER BY cityName desc nulls last, __time ) windowedDelta, + ROW_NUMBER() OVER (ORDER BY cityName nulls first, __time ) windowedDelta + FROM wikipedia + where page < '0' and channel like '#en%' + +expectedOperators: + - type: "naiveSort" + columns: + - column: "cityName" + direction: "DESC" + - column: "__time" + direction: "ASC" + - type: "naivePartition" + partitionColumns: [ ] + - type: "window" + processor: + type: "rowNumber" + outputColumn: "w0" + - type: "naiveSort" + columns: + - column: "cityName" + direction: "ASC" + - column: "__time" + direction: "ASC" + - type: "naivePartition" + partitionColumns: [ ] + - type: "window" + processor: + type: "rowNumber" + outputColumn: "w1" +expectedResults: + - [null,1442019358364,3,1] + - [null,1442021099146,4,2] + - [null,1442033539153,5,3] + - [null,1442095704125,6,4] + - [null,1442096110867,7,5] + - [null,1442100368226,8,6] + - ["Crescent City",1442035449448,2,7] + - ["Vinnytsya",1442100940306,1,8] diff --git a/sql/src/test/resources/drill/window/datasources/allTypsUniq.parquet.json b/sql/src/test/resources/drill/window/datasources/allTypsUniq.parquet.json index abfffddc411e..f675bb0721d9 100644 --- a/sql/src/test/resources/drill/window/datasources/allTypsUniq.parquet.json +++ b/sql/src/test/resources/drill/window/datasources/allTypsUniq.parquet.json @@ -1,22 +1,22 @@ -{"col0":1,"col1":65534,"col2":256.0,"col3":1234.9,"col4":73578580,"col5":1393720082338,"col6":421185052800000,"col7":false,"col8":"CA","col9":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXZ"} -{"col0":2,"col1":10000000,"col2":-256.0,"col3":11.0,"col4":39598119,"col5":1388622482228,"col6":422086982400000,"col7":true,"col8":"WI","col9":"BXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXD"} -{"col0":3,"col1":-1,"col2":255.9993,"col3":0.0,"col4":82189300,"col5":1409617682616,"col6":422680464000000,"col7":false,"col8":"NY","col9":"CXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} -{"col0":4,"col1":0,"col2":128.9,"col3":1.0,"col4":77989310,"col5":1422836882404,"col6":421019078400000,"col7":true,"col8":"AZ","col9":"DXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} -{"col0":5,"col1":1,"col2":-128.0,"col3":-1.0,"col4":74969310,"col5":1404260882309,"col6":422434137600000,"col7":false,"col8":"TX","col9":"HXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXZ"} -{"col0":6,"col1":13,"col2":32.9,"col3":256.9,"col4":77761240,"col5":481249682638,"col6":422546803200000,"col7":true,"col8":"CO","col9":"IXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXD"} -{"col0":7,"col1":17,"col2":-33.0,"col3":33.9,"col4":47593130,"col5":1146529682748,"col6":422457724800000,"col7":false,"col8":"IA","col9":"UXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} -{"col0":8,"col1":23,"col2":13.79,"col3":-1.1,"col4":43932120,"col5":1117672082321,"col6":421673299200000,"col7":true,"col8":"SD","col9":"YXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} -{"col0":9,"col1":1000,"col2":29.13,"col3":0.9,"col4":54613101,"col5":-612747117889,"col6":422199820800000,"col7":false,"col8":"RI","col9":"TXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXD"} -{"col0":10,"col1":9999999,"col2":100.09,"col3":10000.0,"col4":85535120,"col5":-710119917582,"col6":420884294400000,"col7":true,"col8":"FL","col9":"EXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXZ"} -{"col0":2147483647,"col1":30,"col2":123.129,"col3":512.999,"col4":79322101,"col5":107828882418,"col6":422596656000000,"col7":false,"col8":"IN","col9":"WXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} -{"col0":65535,"col1":25,"col2":255.99,"col3":11111.11,"col4":84210222,"col5":707444882418,"col6":421353014400000,"col7":true,"col8":"MN","col9":"FXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} -{"col0":1073741823,"col1":1001,"col2":128.978,"col3":131313.19,"col4":59745100,"col5":770516882418,"col6":423167155200000,"col7":false,"col8":"MA","col9":"MXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} -{"col0":-1,"col1":-65535,"col2":13.9,"col3":10000.09,"col4":69885200,"col5":959905682418,"col6":423153676800000,"col7":true,"col8":"VT","col9":"KXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} -{"col0":-65535,"col1":5000,"col2":127.0,"col3":127.99,"col4":73220300,"col5":1022977682418,"col6":423132854400000,"col7":false,"col8":"NJ","col9":"OXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} -{"col0":10000000,"col1":3000,"col2":127.9,"col3":65534.99,"col4":55230230,"col5":1054513682418,"col6":423090777600000,"col7":true,"col8":"OR","col9":"QXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXZ"} -{"col0":13,"col1":200,"col2":1.0,"col3":-65534.0,"col4":69630500,"col5":1086136082418,"col6":421804713600000,"col7":false,"col8":"GA","col9":"PXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXD"} -{"col0":19,"col1":197,"col2":0.0,"col3":12345.987,"col4":62060100,"col5":1275438482118,"col6":422335987200000,"col7":true,"col8":"WY","col9":"LXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} -{"col0":29,"col1":4611686018427387903,"col2":-1.0,"col3":100.9,"col4":58576160,"col5":1306974482218,"col6":421473196800000,"col7":false,"col8":"NC","col9":"NXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} -{"col0":23,"col1":9223372036854775806,"col2":99.9,"col3":303.12,"col4":70230300,"col5":1338596882418,"col6":422924803200000,"col7":true,"col8":"SC","col9":"SXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} -{"col0":0,"col1":9223372036854775807,"col2":63.99,"col3":9898.68,"col4":80611100,"col5":1370132882318,"col6":422613849600000,"col7":false,"col8":"KS","col9":"ZXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXZ"} -{"col0":109,"col1":92233720385475807,"col2":69.89,"col3":9798.68,"col4":81210180,"col5":1438547282318,"col6":422648928000000,"col7":true,"col8":"NE","col9":"VXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXZ"} \ No newline at end of file +{"col0":1,"col1":65534,"col2":256.0,"col3":1234.9,"col4":73578580,"col5":1393720082338,"col6":-548553600000,"col7":false,"col8":"CA","col9":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXZ"} +{"col0":2,"col1":10000000,"col2":-256.0,"col3":11.0,"col4":39598119,"col5":1388622482228,"col6":353376000000,"col7":true,"col8":"WI","col9":"BXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXD"} +{"col0":3,"col1":-1,"col2":255.9993,"col3":0.0,"col4":82189300,"col5":1409617682616,"col6":946857600000,"col7":false,"col8":"NY","col9":"CXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} +{"col0":4,"col1":0,"col2":128.9,"col3":1.0,"col4":77989310,"col5":1422836882404,"col6":-714528000000,"col7":true,"col8":"AZ","col9":"DXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} +{"col0":5,"col1":1,"col2":-128.0,"col3":-1.0,"col4":74969310,"col5":1404260882309,"col6":700531200000,"col7":false,"col8":"TX","col9":"HXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXZ"} +{"col0":6,"col1":13,"col2":32.9,"col3":256.9,"col4":77761240,"col5":481249682638,"col6":813196800000,"col7":true,"col8":"CO","col9":"IXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXD"} +{"col0":7,"col1":17,"col2":-33.0,"col3":33.9,"col4":47593130,"col5":1146529682748,"col6":724118400000,"col7":false,"col8":"IA","col9":"UXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} +{"col0":8,"col1":23,"col2":13.79,"col3":-1.1,"col4":43932120,"col5":1117672082321,"col6":-60307200000,"col7":true,"col8":"SD","col9":"YXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} +{"col0":9,"col1":1000,"col2":29.13,"col3":0.9,"col4":54613101,"col5":-612747117889,"col6":466214400000,"col7":false,"col8":"RI","col9":"TXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXD"} +{"col0":10,"col1":9999999,"col2":100.09,"col3":10000.0,"col4":85535120,"col5":-710119917582,"col6":-849312000000,"col7":true,"col8":"FL","col9":"EXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXZ"} +{"col0":2147483647,"col1":30,"col2":123.129,"col3":512.999,"col4":79322101,"col5":107828882418,"col6":863049600000,"col7":false,"col8":"IN","col9":"WXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} +{"col0":65535,"col1":25,"col2":255.99,"col3":11111.11,"col4":84210222,"col5":707444882418,"col6":-380592000000,"col7":true,"col8":"MN","col9":"FXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} +{"col0":1073741823,"col1":1001,"col2":128.978,"col3":131313.19,"col4":59745100,"col5":770516882418,"col6":1433548800000,"col7":false,"col8":"MA","col9":"MXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} +{"col0":-1,"col1":-65535,"col2":13.9,"col3":10000.09,"col4":69885200,"col5":959905682418,"col6":1420070400000,"col7":true,"col8":"VT","col9":"KXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} +{"col0":-65535,"col1":5000,"col2":127.0,"col3":127.99,"col4":73220300,"col5":1022977682418,"col6":1399248000000,"col7":false,"col8":"NJ","col9":"OXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} +{"col0":10000000,"col1":3000,"col2":127.9,"col3":65534.99,"col4":55230230,"col5":1054513682418,"col6":1357171200000,"col7":true,"col8":"OR","col9":"QXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXZ"} +{"col0":13,"col1":200,"col2":1.0,"col3":-65534.0,"col4":69630500,"col5":1086136082418,"col6":71107200000,"col7":false,"col8":"GA","col9":"PXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXD"} +{"col0":19,"col1":197,"col2":0.0,"col3":12345.987,"col4":62060100,"col5":1275438482118,"col6":602380800000,"col7":true,"col8":"WY","col9":"LXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} +{"col0":29,"col1":4611686018427387903,"col2":-1.0,"col3":100.9,"col4":58576160,"col5":1306974482218,"col6":-260409600000,"col7":false,"col8":"NC","col9":"NXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} +{"col0":23,"col1":9223372036854775806,"col2":99.9,"col3":303.12,"col4":70230300,"col5":1338596882418,"col6":1191196800000,"col7":true,"col8":"SC","col9":"SXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB"} +{"col0":0,"col1":9223372036854775807,"col2":63.99,"col3":9898.68,"col4":80611100,"col5":1370132882318,"col6":880243200000,"col7":false,"col8":"KS","col9":"ZXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXZ"} +{"col0":109,"col1":92233720385475807,"col2":69.89,"col3":9798.68,"col4":81210180,"col5":1438547282318,"col6":915321600000,"col7":true,"col8":"NE","col9":"VXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXZ"} \ No newline at end of file diff --git a/sql/src/test/resources/drill/window/datasources/fewRowsAllData.parquet.json b/sql/src/test/resources/drill/window/datasources/fewRowsAllData.parquet.json index 2dffec37526c..79a2193e36fe 100644 --- a/sql/src/test/resources/drill/window/datasources/fewRowsAllData.parquet.json +++ b/sql/src/test/resources/drill/window/datasources/fewRowsAllData.parquet.json @@ -1,78 +1,78 @@ -{"col0":12024,"col1":307168,"col2":"VT","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1338596882419,"col5":422705433600000,"col6":true,"col7":3.95110006277E8,"col8":67465430} -{"col0":-1704,"col1":48600128,"col2":"ND","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882420,"col5":422407699200000,"col6":true,"col7":1.29358204137E9,"col8":75128560} -{"col0":-19521,"col1":3640288,"col2":"SD","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1370132882420,"col5":421580246400000,"col6":false,"col7":9.83657842924E8,"col8":71170420} -{"col0":-4709,"col1":3783896,"col2":"MN","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1306974482420,"col5":422796153600000,"col6":false,"col7":4.66674349327E8,"col8":55294390} -{"col0":1600,"col1":8107680,"col2":"MA","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882421,"col5":422410982400000,"col6":false,"col7":4.7471347117E8,"col8":37915240} -{"col0":-1950,"col1":40547891,"col2":"IN","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882421,"col5":421379884800000,"col6":false,"col7":8.89911941945E8,"col8":5867310} -{"col0":-1968,"col1":6212808,"col2":"SD","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882421,"col5":422410032000000,"col6":false,"col7":4.72880577743E8,"col8":22332400} -{"col0":-788,"col1":7171344,"col2":"CO","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882422,"col5":421916601600000,"col6":false,"col7":1.04871325474E9,"col8":34731100} -{"col0":-1846,"col1":5077360,"col2":"WI","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882422,"col5":422439753600000,"col6":false,"col7":1.4793288844E8,"col8":72125200} -{"col0":1001,"col1":7819200,"col2":"MD","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1370132882422,"col5":421684012800000,"col6":true,"col7":9.36437541786E8,"col8":75052800} -{"col0":19860,"col1":133164,"col2":"VT","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882422,"col5":421253049600000,"col6":false,"col7":1.09109659183E9,"col8":73602340} -{"col0":-1,"col1":-1,"col2":"VT","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882422,"col5":421253049600000,"col6":false,"col7":1.09109659183E9,"col8":73602340} -{"col0":1140,"col1":108878,"col2":"HI","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1338596882422,"col5":421429392000000,"col6":false,"col7":3.71555243082E8,"col8":7378300} -{"col0":-8984,"col1":677944,"col2":"GA","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1306974482422,"col5":422895340800000,"col6":true,"col7":1.42680533942E9,"col8":66670320} -{"col0":-14211,"col1":7985264,"col2":"MN","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1338596882423,"col5":422065296000000,"col6":false,"col7":3.90513711473E8,"col8":16118230} -{"col0":-19479,"col1":240352,"col2":"GA","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1338596882423,"col5":421930252800000,"col6":false,"col7":9.72747639559E8,"col8":63177700} -{"col0":4242,"col1":502144,"col2":"WY","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882423,"col5":421398806400000,"col6":true,"col7":1.13430261855E9,"col8":9238280} -{"col0":-3576,"col1":5392352,"col2":"MI","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1370132882423,"col5":421822339200000,"col6":false,"col7":4.70197757462E7,"col8":43115460} -{"col0":-1816,"col1":29832,"col2":"MA","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1338596882423,"col5":423068745600000,"col6":true,"col7":2.68051680205E8,"col8":8566390} -{"col0":362,"col1":24064,"col2":"OR","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882423,"col5":423099072000000,"col6":false,"col7":3.84895861757E8,"col8":69763320} -{"col0":-1645,"col1":908720,"col2":"MA","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882424,"col5":421244323200000,"col6":true,"col7":2.73055641297E8,"col8":71068900} -{"col0":-1159,"col1":50752,"col2":"RI","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882424,"col5":421906147200000,"col6":true,"col7":4.01229124414E8,"col8":5649490} -{"col0":-202,"col1":5700,"col2":"MO","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882424,"col5":422671564800000,"col6":false,"col7":1.84965459559E8,"col8":29932570} -{"col0":-3399,"col1":74400,"col2":"RI","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882424,"col5":421853702400000,"col6":true,"col7":1.24109748587E9,"col8":40411560} -{"col0":564,"col1":537456,"col2":"AZ","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882425,"col5":421991683200000,"col6":true,"col7":1.09531044142E9,"col8":39161450} -{"col0":739,"col1":103690,"col2":"OR","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882427,"col5":422474054400000,"col6":true,"col7":8.68835049003E8,"col8":17273570} -{"col0":14587,"col1":617504,"col2":"WY","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1306974482427,"col5":422182627200000,"col6":false,"col7":5.3037680439E8,"col8":81757180} -{"col0":-61299,"col1":383096,"col2":"CO","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882427,"col5":421627161600000,"col6":false,"col7":2.69607476481E7,"col8":27971300} -{"col0":11345,"col1":8788,"col2":"MO","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882428,"col5":422137872000000,"col6":true,"col7":9.11002550279E8,"col8":20510140} -{"col0":95690,"col1":189392,"col2":"VT","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1306974482428,"col5":422953401600000,"col6":false,"col7":1.25727344595E8,"col8":29546560} -{"col0":-199048,"col1":357520,"col2":"SC","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1370132882429,"col5":421109020800000,"col6":false,"col7":1.27909910034E9,"col8":51408700} -{"col0":216553,"col1":86744,"col2":"SC","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1338596882429,"col5":421599945600000,"col6":false,"col7":1.17889933749E9,"col8":53136170} -{"col0":-126232,"col1":38160,"col2":"WY","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882429,"col5":422958240000000,"col6":true,"col7":4.20766450283E8,"col8":62477170} -{"col0":-14479922,"col1":18402,"col2":"NE","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882429,"col5":423112896000000,"col6":false,"col7":1.1120849455E9,"col8":25806550} -{"col0":-185276,"col1":1150,"col2":"SD","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882430,"col5":421212009600000,"col6":true,"col7":9.3620425577E8,"col8":37363300} -{"col0":-14479,"col1":51192,"col2":"IN","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1338596882430,"col5":422682019200000,"col6":false,"col7":1.17748065647E9,"col8":12482490} -{"col0":155858,"col1":46640,"col2":"RI","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882430,"col5":422123356800000,"col6":true,"col7":1.02765112757E9,"col8":40761560} -{"col0":1,"col1":0,"col2":"RI","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882430,"col5":422123356800000,"col6":true,"col7":1.02765112757E9,"col8":40761560} -{"col0":-823,"col1":6352,"col2":"MN","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882430,"col5":421587244800000,"col6":true,"col7":1.02635426051E8,"col8":17111380} -{"col0":13899,"col1":29548,"col2":"UT","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882431,"col5":421273872000000,"col6":true,"col7":1.12572986097E9,"col8":8261230} -{"col0":-20449,"col1":5872,"col2":"NY","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1370132882431,"col5":421681680000000,"col6":false,"col7":4.22305438068E8,"col8":18255480} -{"col0":-20449,"col1":5872,"col2":"NY","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1370132882431,"col5":421681680000000,"col6":false,"col7":1.0,"col8":18255480} -{"col0":-86821,"col1":115648,"col2":"LA","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882431,"col5":422381779200000,"col6":true,"col7":1.26712293161E9,"col8":82189300} -{"col0":5562,"col1":58144,"col2":"AK","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882431,"col5":421205184000000,"col6":true,"col7":8.30455866482E8,"col8":42999360} -{"col0":3894,"col1":247308,"col2":"OH","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882431,"col5":422359920000000,"col6":true,"col7":7.63454104971E8,"col8":50267140} -{"col0":10770,"col1":212492,"col2":"PA","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1370132882432,"col5":422397417600000,"col6":true,"col7":7.2797834549E8,"col8":13490300} -{"col0":14016,"col1":7755464,"col2":"IN","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882432,"col5":422252870400000,"col6":true,"col7":8.28874169301E8,"col8":55007110} -{"col0":2532,"col1":338648,"col2":"DE","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882432,"col5":421450560000000,"col6":true,"col7":3.99985456569E8,"col8":73722390} -{"col0":17069,"col1":648376,"col2":"MN","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1370132882432,"col5":422475696000000,"col6":true,"col7":1.42275732068E9,"col8":13465170} -{"col0":20257,"col1":195470,"col2":"IA","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1306974482432,"col5":421379452800000,"col6":true,"col7":1.23097632842E8,"col8":27742430} -{"col0":-71899,"col1":104126,"col2":"KS","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882432,"col5":421108329600000,"col6":false,"col7":0.303,"col8":77761240} -{"col0":72309,"col1":752768,"col2":"WI","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1306974482433,"col5":422809632000000,"col6":true,"col7":6.16602756464E7,"col8":8470130} -{"col0":11281,"col1":45224,"col2":"GA","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1338596882433,"col5":421905283200000,"col6":true,"col7":1.40238980845E9,"col8":5527400} -{"col0":19294,"col1":831584,"col2":"AZ","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882433,"col5":422289676800000,"col6":true,"col7":1.41978080866E9,"col8":49582500} -{"col0":8966,"col1":80944,"col2":"MD","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882434,"col5":423039974400000,"col6":true,"col7":5.6890579253E8,"col8":35118460} -{"col0":-17427,"col1":85048,"col2":"MO","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882434,"col5":421333920000000,"col6":true,"col7":4.6673697742E8,"col8":22981110} -{"col0":-110,"col1":2632,"col2":"GA","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882434,"col5":421494451200000,"col6":false,"col7":1.44804901984E8,"col8":59898370} -{"col0":3418,"col1":493976,"col2":"WY","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882434,"col5":422340566400000,"col6":true,"col7":8.56574532391E8,"col8":37636440} -{"col0":-446,"col1":25236,"col2":"OH","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882434,"col5":421380316800000,"col6":true,"col7":7.35715255558E8,"col8":51842560} -{"col0":5530,"col1":199400,"col2":"NH","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1370132882435,"col5":421374960000000,"col6":true,"col7":8.91015086626E8,"col8":71064450} -{"col0":0,"col1":1,"col2":"NH","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1370132882435,"col5":421374960000000,"col6":true,"col7":8.91015086626E8,"col8":71064450} -{"col0":-977,"col1":458168,"col2":"ME","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882435,"col5":421534108800000,"col6":true,"col7":6.677148957E7,"col8":63195250} -{"col0":-1727,"col1":41872,"col2":"IA","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1338596882435,"col5":421150492800000,"col6":false,"col7":1.37327670868E9,"col8":58690170} -{"col0":-8117,"col1":84976,"col2":"MN","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1306974482435,"col5":421977081600000,"col6":false,"col7":5.0893542491E8,"col8":52269240} -{"col0":-14062,"col1":74776,"col2":"IA","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882436,"col5":422806694400000,"col6":true,"col7":1.38237309946E9,"col8":66897160} -{"col0":1919,"col1":50312,"col2":"UT","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882436,"col5":421410297600000,"col6":false,"col7":8.02591009467E7,"col8":10039350} -{"col0":15935,"col1":806060,"col2":"IA","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882436,"col5":421357766400000,"col6":true,"col7":-1.067,"col8":29033340} -{"col0":374,"col1":43592,"col2":"NC","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882436,"col5":421799529600000,"col6":false,"col7":3.06514832189E7,"col8":9995490} -{"col0":1868,"col1":290820,"col2":"NY","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882436,"col5":421818796800000,"col6":false,"col7":1.02289267124E9,"col8":11248450} -{"col0":11107,"col1":186364,"col2":"MN","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882436,"col5":423020620800000,"col6":false,"col7":7.91697316619E8,"col8":53224800} -{"col0":63389,"col1":38640,"col2":"UT","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1433204882437,"col5":421578691200000,"col6":true,"col7":1.00485076217E9,"col8":22871350} -{"col0":15426,"col1":32192,"col2":"WV","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882437,"col5":422003520000000,"col6":true,"col7":1.0252414664E9,"col8":56382320} -{"col0":-292190,"col1":385288,"col2":"CO","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1370132882437,"col5":422413142400000,"col6":false,"col7":8.0256170003E8,"col8":66881100} -{"col0":0,"col1":0,"col2":"ND","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1306974482437,"col5":421739395200000,"col6":false,"col7":3.30283711488E8,"col8":73737330} -{"col0":-1732385,"col1":56312,"col2":"IN","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1370132882437,"col5":422905363200000,"col6":false,"col7":2.18199000393E8,"col8":79702400} -{"col0":-955329,"col1":778984,"col2":"CA","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1275438482437,"col5":421725744000000,"col6":true,"col7":1.38262054833E9,"col8":29513360} -{"col0":1722833,"col1":5376880,"col2":"ME","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668892193,"col5":421672608000000,"col6":true,"col7":1.19950218188E9,"col8":36795330} -{"col0":-5186159,"col1":31933,"col2":"NH","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1275438492194,"col5":421295817600000,"col6":false,"col7":2.30933430625E8,"col8":10938520} \ No newline at end of file +{"col0":12024,"col1":307168,"col2":"VT","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1338596882419,"col5":971827200000,"col6":true,"col7":3.95110006277E8,"col8":67465430} +{"col0":-1704,"col1":48600128,"col2":"ND","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882420,"col5":674092800000,"col6":true,"col7":1.29358204137E9,"col8":75128560} +{"col0":-19521,"col1":3640288,"col2":"SD","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1370132882420,"col5":-153360000000,"col6":false,"col7":9.83657842924E8,"col8":71170420} +{"col0":-4709,"col1":3783896,"col2":"MN","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1306974482420,"col5":1062547200000,"col6":false,"col7":4.66674349327E8,"col8":55294390} +{"col0":1600,"col1":8107680,"col2":"MA","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882421,"col5":677376000000,"col6":false,"col7":4.7471347117E8,"col8":37915240} +{"col0":-1950,"col1":40547891,"col2":"IN","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882421,"col5":-353721600000,"col6":false,"col7":8.89911941945E8,"col8":5867310} +{"col0":-1968,"col1":6212808,"col2":"SD","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882421,"col5":676425600000,"col6":false,"col7":4.72880577743E8,"col8":22332400} +{"col0":-788,"col1":7171344,"col2":"CO","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882422,"col5":182995200000,"col6":false,"col7":1.04871325474E9,"col8":34731100} +{"col0":-1846,"col1":5077360,"col2":"WI","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882422,"col5":706147200000,"col6":false,"col7":1.4793288844E8,"col8":72125200} +{"col0":1001,"col1":7819200,"col2":"MD","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1370132882422,"col5":-49593600000,"col6":true,"col7":9.36437541786E8,"col8":75052800} +{"col0":19860,"col1":133164,"col2":"VT","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882422,"col5":-480556800000,"col6":false,"col7":1.09109659183E9,"col8":73602340} +{"col0":-1,"col1":-1,"col2":"VT","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882422,"col5":-480556800000,"col6":false,"col7":1.09109659183E9,"col8":73602340} +{"col0":1140,"col1":108878,"col2":"HI","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1338596882422,"col5":-304214400000,"col6":false,"col7":3.71555243082E8,"col8":7378300} +{"col0":-8984,"col1":677944,"col2":"GA","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1306974482422,"col5":1161734400000,"col6":true,"col7":1.42680533942E9,"col8":66670320} +{"col0":-14211,"col1":7985264,"col2":"MN","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1338596882423,"col5":331689600000,"col6":false,"col7":3.90513711473E8,"col8":16118230} +{"col0":-19479,"col1":240352,"col2":"GA","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1338596882423,"col5":196646400000,"col6":false,"col7":9.72747639559E8,"col8":63177700} +{"col0":4242,"col1":502144,"col2":"WY","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882423,"col5":-334800000000,"col6":true,"col7":1.13430261855E9,"col8":9238280} +{"col0":-3576,"col1":5392352,"col2":"MI","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1370132882423,"col5":88732800000,"col6":false,"col7":4.70197757462E7,"col8":43115460} +{"col0":-1816,"col1":29832,"col2":"MA","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1338596882423,"col5":1335139200000,"col6":true,"col7":2.68051680205E8,"col8":8566390} +{"col0":362,"col1":24064,"col2":"OR","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882423,"col5":1365465600000,"col6":false,"col7":3.84895861757E8,"col8":69763320} +{"col0":-1645,"col1":908720,"col2":"MA","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882424,"col5":-489283200000,"col6":true,"col7":2.73055641297E8,"col8":71068900} +{"col0":-1159,"col1":50752,"col2":"RI","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882424,"col5":172540800000,"col6":true,"col7":4.01229124414E8,"col8":5649490} +{"col0":-202,"col1":5700,"col2":"MO","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882424,"col5":937958400000,"col6":false,"col7":1.84965459559E8,"col8":29932570} +{"col0":-3399,"col1":74400,"col2":"RI","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882424,"col5":120096000000,"col6":true,"col7":1.24109748587E9,"col8":40411560} +{"col0":564,"col1":537456,"col2":"AZ","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882425,"col5":258076800000,"col6":true,"col7":1.09531044142E9,"col8":39161450} +{"col0":739,"col1":103690,"col2":"OR","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882427,"col5":740448000000,"col6":true,"col7":8.68835049003E8,"col8":17273570} +{"col0":14587,"col1":617504,"col2":"WY","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1306974482427,"col5":449020800000,"col6":false,"col7":5.3037680439E8,"col8":81757180} +{"col0":-61299,"col1":383096,"col2":"CO","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882427,"col5":-106444800000,"col6":false,"col7":2.69607476481E7,"col8":27971300} +{"col0":11345,"col1":8788,"col2":"MO","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882428,"col5":404265600000,"col6":true,"col7":9.11002550279E8,"col8":20510140} +{"col0":95690,"col1":189392,"col2":"VT","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1306974482428,"col5":1219795200000,"col6":false,"col7":1.25727344595E8,"col8":29546560} +{"col0":-199048,"col1":357520,"col2":"SC","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1370132882429,"col5":-624585600000,"col6":false,"col7":1.27909910034E9,"col8":51408700} +{"col0":216553,"col1":86744,"col2":"SC","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1338596882429,"col5":-133660800000,"col6":false,"col7":1.17889933749E9,"col8":53136170} +{"col0":-126232,"col1":38160,"col2":"WY","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882429,"col5":1224633600000,"col6":true,"col7":4.20766450283E8,"col8":62477170} +{"col0":-14479922,"col1":18402,"col2":"NE","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882429,"col5":1379289600000,"col6":false,"col7":1.1120849455E9,"col8":25806550} +{"col0":-185276,"col1":1150,"col2":"SD","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882430,"col5":-521596800000,"col6":true,"col7":9.3620425577E8,"col8":37363300} +{"col0":-14479,"col1":51192,"col2":"IN","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1338596882430,"col5":948412800000,"col6":false,"col7":1.17748065647E9,"col8":12482490} +{"col0":155858,"col1":46640,"col2":"RI","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882430,"col5":389750400000,"col6":true,"col7":1.02765112757E9,"col8":40761560} +{"col0":1,"col1":0,"col2":"RI","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882430,"col5":389750400000,"col6":true,"col7":1.02765112757E9,"col8":40761560} +{"col0":-823,"col1":6352,"col2":"MN","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882430,"col5":-146361600000,"col6":true,"col7":1.02635426051E8,"col8":17111380} +{"col0":13899,"col1":29548,"col2":"UT","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882431,"col5":-459734400000,"col6":true,"col7":1.12572986097E9,"col8":8261230} +{"col0":-20449,"col1":5872,"col2":"NY","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1370132882431,"col5":-51926400000,"col6":false,"col7":4.22305438068E8,"col8":18255480} +{"col0":-20449,"col1":5872,"col2":"NY","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1370132882431,"col5":-51926400000,"col6":false,"col7":1.0,"col8":18255480} +{"col0":-86821,"col1":115648,"col2":"LA","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882431,"col5":648172800000,"col6":true,"col7":1.26712293161E9,"col8":82189300} +{"col0":5562,"col1":58144,"col2":"AK","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882431,"col5":-528422400000,"col6":true,"col7":8.30455866482E8,"col8":42999360} +{"col0":3894,"col1":247308,"col2":"OH","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882431,"col5":626313600000,"col6":true,"col7":7.63454104971E8,"col8":50267140} +{"col0":10770,"col1":212492,"col2":"PA","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1370132882432,"col5":663811200000,"col6":true,"col7":7.2797834549E8,"col8":13490300} +{"col0":14016,"col1":7755464,"col2":"IN","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882432,"col5":519264000000,"col6":true,"col7":8.28874169301E8,"col8":55007110} +{"col0":2532,"col1":338648,"col2":"DE","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882432,"col5":-283046400000,"col6":true,"col7":3.99985456569E8,"col8":73722390} +{"col0":17069,"col1":648376,"col2":"MN","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1370132882432,"col5":742089600000,"col6":true,"col7":1.42275732068E9,"col8":13465170} +{"col0":20257,"col1":195470,"col2":"IA","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1306974482432,"col5":-354153600000,"col6":true,"col7":1.23097632842E8,"col8":27742430} +{"col0":-71899,"col1":104126,"col2":"KS","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882432,"col5":-625276800000,"col6":false,"col7":0.303,"col8":77761240} +{"col0":72309,"col1":752768,"col2":"WI","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1306974482433,"col5":1076025600000,"col6":true,"col7":6.16602756464E7,"col8":8470130} +{"col0":11281,"col1":45224,"col2":"GA","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1338596882433,"col5":171676800000,"col6":true,"col7":1.40238980845E9,"col8":5527400} +{"col0":19294,"col1":831584,"col2":"AZ","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882433,"col5":556070400000,"col6":true,"col7":1.41978080866E9,"col8":49582500} +{"col0":8966,"col1":80944,"col2":"MD","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882434,"col5":1306368000000,"col6":true,"col7":5.6890579253E8,"col8":35118460} +{"col0":-17427,"col1":85048,"col2":"MO","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882434,"col5":-399686400000,"col6":true,"col7":4.6673697742E8,"col8":22981110} +{"col0":-110,"col1":2632,"col2":"GA","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882434,"col5":-239155200000,"col6":false,"col7":1.44804901984E8,"col8":59898370} +{"col0":3418,"col1":493976,"col2":"WY","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882434,"col5":606960000000,"col6":true,"col7":8.56574532391E8,"col8":37636440} +{"col0":-446,"col1":25236,"col2":"OH","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882434,"col5":-353289600000,"col6":true,"col7":7.35715255558E8,"col8":51842560} +{"col0":5530,"col1":199400,"col2":"NH","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1370132882435,"col5":-358646400000,"col6":true,"col7":8.91015086626E8,"col8":71064450} +{"col0":0,"col1":1,"col2":"NH","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1370132882435,"col5":-358646400000,"col6":true,"col7":8.91015086626E8,"col8":71064450} +{"col0":-977,"col1":458168,"col2":"ME","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882435,"col5":-199497600000,"col6":true,"col7":6.677148957E7,"col8":63195250} +{"col0":-1727,"col1":41872,"col2":"IA","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1338596882435,"col5":-583113600000,"col6":false,"col7":1.37327670868E9,"col8":58690170} +{"col0":-8117,"col1":84976,"col2":"MN","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1306974482435,"col5":243475200000,"col6":false,"col7":5.0893542491E8,"col8":52269240} +{"col0":-14062,"col1":74776,"col2":"IA","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882436,"col5":1073088000000,"col6":true,"col7":1.38237309946E9,"col8":66897160} +{"col0":1919,"col1":50312,"col2":"UT","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882436,"col5":-323308800000,"col6":false,"col7":8.02591009467E7,"col8":10039350} +{"col0":15935,"col1":806060,"col2":"IA","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1401668882436,"col5":-375840000000,"col6":true,"col7":-1.067,"col8":29033340} +{"col0":374,"col1":43592,"col2":"NC","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882436,"col5":65923200000,"col6":false,"col7":3.06514832189E7,"col8":9995490} +{"col0":1868,"col1":290820,"col2":"NY","col3":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col4":1401668882436,"col5":85190400000,"col6":false,"col7":1.02289267124E9,"col8":11248450} +{"col0":11107,"col1":186364,"col2":"MN","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668882436,"col5":1287014400000,"col6":false,"col7":7.91697316619E8,"col8":53224800} +{"col0":63389,"col1":38640,"col2":"UT","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1433204882437,"col5":-154915200000,"col6":true,"col7":1.00485076217E9,"col8":22871350} +{"col0":15426,"col1":32192,"col2":"WV","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1401668882437,"col5":269913600000,"col6":true,"col7":1.0252414664E9,"col8":56382320} +{"col0":-292190,"col1":385288,"col2":"CO","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1370132882437,"col5":679536000000,"col6":false,"col7":8.0256170003E8,"col8":66881100} +{"col0":0,"col1":0,"col2":"ND","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1306974482437,"col5":5788800000,"col6":false,"col7":3.30283711488E8,"col8":73737330} +{"col0":-1732385,"col1":56312,"col2":"IN","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1370132882437,"col5":1171756800000,"col6":false,"col7":2.18199000393E8,"col8":79702400} +{"col0":-955329,"col1":778984,"col2":"CA","col3":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col4":1275438482437,"col5":-7862400000,"col6":true,"col7":1.38262054833E9,"col8":29513360} +{"col0":1722833,"col1":5376880,"col2":"ME","col3":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col4":1401668892193,"col5":-60998400000,"col6":true,"col7":1.19950218188E9,"col8":36795330} +{"col0":-5186159,"col1":31933,"col2":"NH","col3":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col4":1275438492194,"col5":-437788800000,"col6":false,"col7":2.30933430625E8,"col8":10938520} \ No newline at end of file diff --git a/sql/src/test/resources/drill/window/datasources/forViewCrn.parquet.json b/sql/src/test/resources/drill/window/datasources/forViewCrn.parquet.json index 6f727e7cc8ca..f8972f68a1e4 100644 --- a/sql/src/test/resources/drill/window/datasources/forViewCrn.parquet.json +++ b/sql/src/test/resources/drill/window/datasources/forViewCrn.parquet.json @@ -1,30 +1,30 @@ -{"col_int":22,"col_bigint":7,"col_char_2":"IN","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1283387282418,"col_dt":422717616000000,"col_booln":false,"col_dbl":12900.48,"col_tm":33109170} -{"col_int":4,"col_bigint":3,"col_char_2":"VT","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":996625682419,"col_dt":422705433600000,"col_booln":true,"col_dbl":39006.277,"col_tm":67465430} -{"col_int":4,"col_bigint":4,"col_char_2":"ND","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1080865682420,"col_dt":422407699200000,"col_booln":true,"col_dbl":12041.37,"col_tm":75128560} -{"col_int":1,"col_bigint":3,"col_char_2":"SD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1425428882420,"col_dt":421580246400000,"col_booln":false,"col_dbl":97842.924,"col_tm":71170420} -{"col_int":9,"col_bigint":3,"col_char_2":"MN","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1357086482420,"col_dt":422796153600000,"col_booln":false,"col_dbl":449.327,"col_tm":55294390} -{"col_int":0,"col_bigint":8,"col_char_2":"MA","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":707963282421,"col_dt":422410982400000,"col_booln":false,"col_dbl":4771.17,"col_tm":37915240} -{"col_int":-1,"col_bigint":8,"col_char_2":"IN","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882421,"col_dt":421379884800000,"col_booln":false,"col_dbl":8841.945,"col_tm":5867310} -{"col_int":8,"col_bigint":3,"col_char_2":"FL","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1212366482421,"col_dt":421621977600000,"col_booln":false,"col_dbl":1183.32,"col_tm":6605110} -{"col_int":8,"col_bigint":6,"col_char_2":"ND","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882421,"col_dt":421788297600000,"col_booln":true,"col_dbl":436.897,"col_tm":41305100} -{"col_int":-6,"col_bigint":6,"col_char_2":"SD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1397003282421,"col_dt":422410032000000,"col_booln":false,"col_dbl":42577.743,"col_tm":22332400} -{"col_int":-7,"col_bigint":4,"col_char_2":"CO","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1409617682422,"col_dt":421916601600000,"col_booln":false,"col_dbl":10254.74,"col_tm":34731100} -{"col_int":-1,"col_bigint":0,"col_char_2":"CO","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1388968082422,"col_dt":421916601600000,"col_booln":false,"col_dbl":10254.74,"col_tm":34731100} -{"col_int":8,"col_bigint":5,"col_char_2":"WI","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1393720082422,"col_dt":422439753600000,"col_booln":false,"col_dbl":1488.44,"col_tm":72125200} -{"col_int":1,"col_bigint":70,"col_char_2":"MD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1306974482422,"col_dt":421684012800000,"col_booln":true,"col_dbl":97541.786,"col_tm":75052800} -{"col_int":10,"col_bigint":3,"col_char_2":"VT","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":946772882422,"col_dt":421253049600000,"col_booln":false,"col_dbl":10591.83,"col_tm":73602340} -{"col_int":1,"col_bigint":1,"col_char_2":"HI","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":905214482422,"col_dt":421429392000000,"col_booln":false,"col_dbl":3243.082,"col_tm":7378300} -{"col_int":8,"col_bigint":6,"col_char_2":"GA","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":444529682422,"col_dt":422895340800000,"col_booln":true,"col_dbl":14339.42,"col_tm":66670320} -{"col_int":-1,"col_bigint":8,"col_char_2":"MN","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1401668882423,"col_dt":422065296000000,"col_booln":false,"col_dbl":311.473,"col_tm":16118230} -{"col_int":-19,"col_bigint":5,"col_char_2":"GA","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882423,"col_dt":421930252800000,"col_booln":false,"col_dbl":9739.559,"col_tm":63177700} -{"col_int":2,"col_bigint":4,"col_char_2":"WY","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1406939282423,"col_dt":421398806400000,"col_booln":true,"col_dbl":1118.55,"col_tm":9238280} -{"col_int":3,"col_bigint":5,"col_char_2":"MI","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401841682423,"col_dt":421822339200000,"col_booln":false,"col_dbl":475.7462,"col_tm":43115460} -{"col_int":1,"col_bigint":3,"col_char_2":"MA","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1330907282423,"col_dt":423068745600000,"col_booln":true,"col_dbl":2680.205,"col_tm":8566390} -{"col_int":3,"col_bigint":3,"col_char_2":"OR","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1376008082423,"col_dt":423099072000000,"col_booln":false,"col_dbl":3861.757,"col_tm":69763320} -{"col_int":4,"col_bigint":2,"col_char_2":"MA","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401755282424,"col_dt":421244323200000,"col_booln":true,"col_dbl":27641.297,"col_tm":71068900} -{"col_int":5,"col_bigint":2,"col_char_2":"RI","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1396398482424,"col_dt":421906147200000,"col_booln":true,"col_dbl":4024.414,"col_tm":5649490} -{"col_int":0,"col_bigint":0,"col_char_2":"MO","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1307492882424,"col_dt":422671564800000,"col_booln":false,"col_dbl":1859.559,"col_tm":29932570} -{"col_int":9,"col_bigint":7,"col_char_2":"RI","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1391300882424,"col_dt":421853702400000,"col_booln":true,"col_dbl":185.87,"col_tm":40411560} -{"col_int":1,"col_bigint":-1,"col_char_2":"RI","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1275438482424,"col_dt":421853702400000,"col_booln":true,"col_dbl":185.87,"col_tm":40411560} -{"col_int":6,"col_bigint":5,"col_char_2":"AZ","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1396398482425,"col_dt":421991683200000,"col_booln":true,"col_dbl":10441.42,"col_tm":39161450} -{"col_int":8,"col_bigint":14,"col_char_2":"MD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1406939282426,"col_dt":422856806400000,"col_booln":false,"col_dbl":26268.761,"col_tm":28015290} \ No newline at end of file +{"col_int":22,"col_bigint":7,"col_char_2":"IN","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1283387282418,"col_dt":984009600000,"col_booln":false,"col_dbl":12900.48,"col_tm":33109170} +{"col_int":4,"col_bigint":3,"col_char_2":"VT","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":996625682419,"col_dt":971827200000,"col_booln":true,"col_dbl":39006.277,"col_tm":67465430} +{"col_int":4,"col_bigint":4,"col_char_2":"ND","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1080865682420,"col_dt":674092800000,"col_booln":true,"col_dbl":12041.37,"col_tm":75128560} +{"col_int":1,"col_bigint":3,"col_char_2":"SD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1425428882420,"col_dt":-153360000000,"col_booln":false,"col_dbl":97842.924,"col_tm":71170420} +{"col_int":9,"col_bigint":3,"col_char_2":"MN","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1357086482420,"col_dt":1062547200000,"col_booln":false,"col_dbl":449.327,"col_tm":55294390} +{"col_int":0,"col_bigint":8,"col_char_2":"MA","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":707963282421,"col_dt":677376000000,"col_booln":false,"col_dbl":4771.17,"col_tm":37915240} +{"col_int":-1,"col_bigint":8,"col_char_2":"IN","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882421,"col_dt":-353721600000,"col_booln":false,"col_dbl":8841.945,"col_tm":5867310} +{"col_int":8,"col_bigint":3,"col_char_2":"FL","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1212366482421,"col_dt":-111628800000,"col_booln":false,"col_dbl":1183.32,"col_tm":6605110} +{"col_int":8,"col_bigint":6,"col_char_2":"ND","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882421,"col_dt":54691200000,"col_booln":true,"col_dbl":436.897,"col_tm":41305100} +{"col_int":-6,"col_bigint":6,"col_char_2":"SD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1397003282421,"col_dt":676425600000,"col_booln":false,"col_dbl":42577.743,"col_tm":22332400} +{"col_int":-7,"col_bigint":4,"col_char_2":"CO","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1409617682422,"col_dt":182995200000,"col_booln":false,"col_dbl":10254.74,"col_tm":34731100} +{"col_int":-1,"col_bigint":0,"col_char_2":"CO","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1388968082422,"col_dt":182995200000,"col_booln":false,"col_dbl":10254.74,"col_tm":34731100} +{"col_int":8,"col_bigint":5,"col_char_2":"WI","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1393720082422,"col_dt":706147200000,"col_booln":false,"col_dbl":1488.44,"col_tm":72125200} +{"col_int":1,"col_bigint":70,"col_char_2":"MD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1306974482422,"col_dt":-49593600000,"col_booln":true,"col_dbl":97541.786,"col_tm":75052800} +{"col_int":10,"col_bigint":3,"col_char_2":"VT","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":946772882422,"col_dt":-480556800000,"col_booln":false,"col_dbl":10591.83,"col_tm":73602340} +{"col_int":1,"col_bigint":1,"col_char_2":"HI","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":905214482422,"col_dt":-304214400000,"col_booln":false,"col_dbl":3243.082,"col_tm":7378300} +{"col_int":8,"col_bigint":6,"col_char_2":"GA","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":444529682422,"col_dt":1161734400000,"col_booln":true,"col_dbl":14339.42,"col_tm":66670320} +{"col_int":-1,"col_bigint":8,"col_char_2":"MN","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1401668882423,"col_dt":331689600000,"col_booln":false,"col_dbl":311.473,"col_tm":16118230} +{"col_int":-19,"col_bigint":5,"col_char_2":"GA","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882423,"col_dt":196646400000,"col_booln":false,"col_dbl":9739.559,"col_tm":63177700} +{"col_int":2,"col_bigint":4,"col_char_2":"WY","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1406939282423,"col_dt":-334800000000,"col_booln":true,"col_dbl":1118.55,"col_tm":9238280} +{"col_int":3,"col_bigint":5,"col_char_2":"MI","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401841682423,"col_dt":88732800000,"col_booln":false,"col_dbl":475.7462,"col_tm":43115460} +{"col_int":1,"col_bigint":3,"col_char_2":"MA","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1330907282423,"col_dt":1335139200000,"col_booln":true,"col_dbl":2680.205,"col_tm":8566390} +{"col_int":3,"col_bigint":3,"col_char_2":"OR","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1376008082423,"col_dt":1365465600000,"col_booln":false,"col_dbl":3861.757,"col_tm":69763320} +{"col_int":4,"col_bigint":2,"col_char_2":"MA","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401755282424,"col_dt":-489283200000,"col_booln":true,"col_dbl":27641.297,"col_tm":71068900} +{"col_int":5,"col_bigint":2,"col_char_2":"RI","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1396398482424,"col_dt":172540800000,"col_booln":true,"col_dbl":4024.414,"col_tm":5649490} +{"col_int":0,"col_bigint":0,"col_char_2":"MO","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1307492882424,"col_dt":937958400000,"col_booln":false,"col_dbl":1859.559,"col_tm":29932570} +{"col_int":9,"col_bigint":7,"col_char_2":"RI","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1391300882424,"col_dt":120096000000,"col_booln":true,"col_dbl":185.87,"col_tm":40411560} +{"col_int":1,"col_bigint":-1,"col_char_2":"RI","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1275438482424,"col_dt":120096000000,"col_booln":true,"col_dbl":185.87,"col_tm":40411560} +{"col_int":6,"col_bigint":5,"col_char_2":"AZ","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1396398482425,"col_dt":258076800000,"col_booln":true,"col_dbl":10441.42,"col_tm":39161450} +{"col_int":8,"col_bigint":14,"col_char_2":"MD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1406939282426,"col_dt":1123200000000,"col_booln":false,"col_dbl":26268.761,"col_tm":28015290} \ No newline at end of file diff --git a/sql/src/test/resources/drill/window/datasources/smlTbl.parquet.json b/sql/src/test/resources/drill/window/datasources/smlTbl.parquet.json index 3b502952e30f..81fca800dd87 100644 --- a/sql/src/test/resources/drill/window/datasources/smlTbl.parquet.json +++ b/sql/src/test/resources/drill/window/datasources/smlTbl.parquet.json @@ -1,56 +1,56 @@ -{"col_int":8122,"col_bgint":817200,"col_char_2":"IN","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1409617682418,"col_dt":422717616000000,"col_booln":false,"col_dbl":12900.48,"col_tm":33109170} -{"col_int":407024,"col_bgint":37168,"col_char_2":"VT","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1312244882419,"col_dt":422705433600000,"col_booln":true,"col_dbl":39006.277,"col_tm":67465430} -{"col_int":41704,"col_bgint":470128,"col_char_2":"ND","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1396398482420,"col_dt":422407699200000,"col_booln":true,"col_dbl":12041.37,"col_tm":75128560} -{"col_int":-19521,"col_bgint":30288,"col_char_2":"SD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1425256082420,"col_dt":421580246400000,"col_booln":false,"col_dbl":97842.924,"col_tm":71170420} -{"col_int":-4709,"col_bgint":373896,"col_char_2":"MN","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1357086482420,"col_dt":422796153600000,"col_booln":false,"col_dbl":449.327,"col_tm":55294390} -{"col_int":16300,"col_bgint":81680,"col_char_2":"MA","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":707444882421,"col_dt":422410982400000,"col_booln":false,"col_dbl":4771.17,"col_tm":37915240} -{"col_int":-15950,"col_bgint":4547891,"col_char_2":"IN","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882421,"col_dt":421379884800000,"col_booln":false,"col_dbl":8841.945,"col_tm":5867310} -{"col_int":8888,"col_bgint":314696,"col_char_2":"FL","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882421,"col_dt":421621977600000,"col_booln":false,"col_dbl":1183.32,"col_tm":6605110} -{"col_int":82588,"col_bgint":6672,"col_char_2":"ND","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882421,"col_dt":421788297600000,"col_booln":true,"col_dbl":436.897,"col_tm":41305100} -{"col_int":-13968,"col_bgint":62808,"col_char_2":"SD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882421,"col_dt":422410032000000,"col_booln":false,"col_dbl":42577.743,"col_tm":22332400} -{"col_int":-74788,"col_bgint":7344,"col_char_2":"CO","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882422,"col_dt":421916601600000,"col_booln":false,"col_dbl":10254.74,"col_tm":34731100} -{"col_int":-1,"col_bgint":0,"col_char_2":"CO","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882422,"col_dt":421916601600000,"col_booln":false,"col_dbl":10254.74,"col_tm":34731100} -{"col_int":-10846,"col_bgint":57360,"col_char_2":"WI","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882422,"col_dt":422439753600000,"col_booln":false,"col_dbl":1488.44,"col_tm":72125200} -{"col_int":101701,"col_bgint":719200,"col_char_2":"MD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882422,"col_dt":421684012800000,"col_booln":true,"col_dbl":97541.786,"col_tm":75052800} -{"col_int":198860,"col_bgint":103164,"col_char_2":"VT","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882422,"col_dt":421253049600000,"col_booln":false,"col_dbl":10591.83,"col_tm":73602340} -{"col_int":11140,"col_bgint":10878,"col_char_2":"HI","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":896747282422,"col_dt":421429392000000,"col_booln":false,"col_dbl":3243.082,"col_tm":7378300} -{"col_int":-84984,"col_bgint":67944,"col_char_2":"GA","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882422,"col_dt":422895340800000,"col_booln":true,"col_dbl":14339.42,"col_tm":66670320} -{"col_int":-184211,"col_bgint":85264,"col_char_2":"MN","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1401668882423,"col_dt":422065296000000,"col_booln":false,"col_dbl":311.473,"col_tm":16118230} -{"col_int":-169479,"col_bgint":540352,"col_char_2":"GA","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882423,"col_dt":421930252800000,"col_booln":false,"col_dbl":9739.559,"col_tm":63177700} -{"col_int":42942,"col_bgint":502144,"col_char_2":"WY","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1401668882423,"col_dt":421398806400000,"col_booln":true,"col_dbl":1118.55,"col_tm":9238280} -{"col_int":-33576,"col_bgint":592352,"col_char_2":"MI","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882423,"col_dt":421822339200000,"col_booln":false,"col_dbl":475.7462,"col_tm":43115460} -{"col_int":-127816,"col_bgint":9832,"col_char_2":"MA","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1401668882423,"col_dt":423068745600000,"col_booln":true,"col_dbl":2680.205,"col_tm":8566390} -{"col_int":36582,"col_bgint":243364,"col_char_2":"OR","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1375403282423,"col_dt":423099072000000,"col_booln":false,"col_dbl":3861.757,"col_tm":69763320} -{"col_int":-149645,"col_bgint":8720,"col_char_2":"MA","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882424,"col_dt":421244323200000,"col_booln":true,"col_dbl":27641.297,"col_tm":71068900} -{"col_int":-12159,"col_bgint":500752,"col_char_2":"RI","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1401668882424,"col_dt":421906147200000,"col_booln":true,"col_dbl":4024.414,"col_tm":5649490} -{"col_int":-2402,"col_bgint":5756000,"col_char_2":"MO","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882424,"col_dt":422671564800000,"col_booln":false,"col_dbl":1859.559,"col_tm":29932570} -{"col_int":-33399,"col_bgint":7486400,"col_char_2":"RI","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882424,"col_dt":421853702400000,"col_booln":true,"col_dbl":185.87,"col_tm":40411560} -{"col_int":1,"col_bgint":-1,"col_char_2":"RI","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882424,"col_dt":421853702400000,"col_booln":true,"col_dbl":185.87,"col_tm":40411560} -{"col_int":56214,"col_bgint":5372456,"col_char_2":"AZ","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882425,"col_dt":421991683200000,"col_booln":true,"col_dbl":10441.42,"col_tm":39161450} -{"col_int":-12209,"col_bgint":7899414,"col_char_2":"MD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882426,"col_dt":422856806400000,"col_booln":false,"col_dbl":26268.761,"col_tm":28015290} -{"col_int":18763,"col_bgint":921520,"col_char_2":"NE","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882426,"col_dt":421187472000000,"col_booln":true,"col_dbl":4966.2418,"col_tm":39006130} -{"col_int":211066,"col_bgint":104832,"col_char_2":"NJ","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1404260882426,"col_dt":421838841600000,"col_booln":true,"col_dbl":1155.5,"col_tm":9018340} -{"col_int":73839,"col_bgint":103690,"col_char_2":"OR","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1401668882427,"col_dt":422474054400000,"col_booln":true,"col_dbl":8049.003,"col_tm":17273570} -{"col_int":124587,"col_bgint":617504,"col_char_2":"WY","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1401668882427,"col_dt":422182627200000,"col_booln":false,"col_dbl":5804.39,"col_tm":81757180} -{"col_int":-61299,"col_bgint":38096,"col_char_2":"CO","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1401668882427,"col_dt":421627161600000,"col_booln":false,"col_dbl":247.6481,"col_tm":27971300} -{"col_int":11345,"col_bgint":87088,"col_char_2":"MO","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1401668882428,"col_dt":422137872000000,"col_booln":true,"col_dbl":9150.279,"col_tm":20510140} -{"col_int":-92531,"col_bgint":21100,"col_char_2":"CA","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882428,"col_dt":423093542400000,"col_booln":true,"col_dbl":9240.11,"col_tm":46147130} -{"col_int":95690,"col_bgint":186392,"col_char_2":"VT","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1406939282428,"col_dt":422953401600000,"col_booln":false,"col_dbl":12344.595,"col_tm":29546560} -{"col_int":-199048,"col_bgint":3520,"col_char_2":"SC","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882429,"col_dt":421109020800000,"col_booln":false,"col_dbl":19100.34,"col_tm":51408700} -{"col_int":216553,"col_bgint":86744,"col_char_2":"SC","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1409617682429,"col_dt":421599945600000,"col_booln":false,"col_dbl":1337.49,"col_tm":53136170} -{"col_int":-126232,"col_bgint":3160,"col_char_2":"WY","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1412209682429,"col_dt":422958240000000,"col_booln":true,"col_dbl":450.283,"col_tm":62477170} -{"col_int":120079,"col_bgint":6736,"col_char_2":"CO","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882429,"col_dt":422613849600000,"col_booln":true,"col_dbl":94963.943,"col_tm":16752380} -{"col_int":-179922,"col_bgint":198402,"col_char_2":"NE","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882429,"col_dt":423112896000000,"col_booln":false,"col_dbl":1145.5,"col_tm":25806550} -{"col_int":196285,"col_bgint":84560,"col_char_2":"IA","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1401668882430,"col_dt":421431984000000,"col_booln":false,"col_dbl":95854.701,"col_tm":7694500} -{"col_int":77907,"col_bgint":412832,"col_char_2":"FL","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1401668882430,"col_dt":422734550400000,"col_booln":false,"col_dbl":135568.86,"col_tm":21999400} -{"col_int":0,"col_bgint":0,"col_char_2":"FL","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1401668882430,"col_dt":422734550400000,"col_booln":false,"col_dbl":135568.86,"col_tm":21999400} -{"col_int":-185276,"col_bgint":163150,"col_char_2":"SD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882430,"col_dt":421212009600000,"col_booln":true,"col_dbl":9255.77,"col_tm":37363300} -{"col_int":-14479,"col_bgint":5192,"col_char_2":"IN","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1401668882430,"col_dt":422682019200000,"col_booln":false,"col_dbl":1156.47,"col_tm":12482490} -{"col_int":-165881,"col_bgint":7104,"col_char_2":"SD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882430,"col_dt":422484336000000,"col_booln":false,"col_dbl":14879.08,"col_tm":21046300} -{"col_int":155858,"col_bgint":4640,"col_char_2":"RI","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":991441682430,"col_dt":422123356800000,"col_booln":true,"col_dbl":10227.57,"col_tm":40761560} -{"col_int":-8523,"col_bgint":696,"col_char_2":"MN","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1401668882430,"col_dt":421587244800000,"col_booln":true,"col_dbl":1026.051,"col_tm":17111380} -{"col_int":137899,"col_bgint":2799548,"col_char_2":"UT","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":959905682431,"col_dt":421273872000000,"col_booln":true,"col_dbl":1860.97,"col_tm":8261230} -{"col_int":1,"col_bgint":1,"col_char_2":"UT","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1306974482431,"col_dt":421273872000000,"col_booln":true,"col_dbl":1860.97,"col_tm":8261230} -{"col_int":-20449,"col_bgint":5172,"col_char_2":"NY","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1338596882431,"col_dt":421681680000000,"col_booln":false,"col_dbl":438.068,"col_tm":18255480} -{"col_int":-86821,"col_bgint":11648,"col_char_2":"LA","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882431,"col_dt":422381779200000,"col_booln":true,"col_dbl":1231.61,"col_tm":82189300} -{"col_int":65535,"col_bgint":65535,"col_char_2":"LA","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1370132882431,"col_dt":422381779200000,"col_booln":true,"col_dbl":1231.61,"col_tm":82189300} \ No newline at end of file +{"col_int":8122,"col_bgint":817200,"col_char_2":"IN","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1409617682418,"col_dt":984009600000,"col_booln":false,"col_dbl":12900.48,"col_tm":33109170} +{"col_int":407024,"col_bgint":37168,"col_char_2":"VT","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1312244882419,"col_dt":971827200000,"col_booln":true,"col_dbl":39006.277,"col_tm":67465430} +{"col_int":41704,"col_bgint":470128,"col_char_2":"ND","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1396398482420,"col_dt":674092800000,"col_booln":true,"col_dbl":12041.37,"col_tm":75128560} +{"col_int":-19521,"col_bgint":30288,"col_char_2":"SD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1425256082420,"col_dt":-153360000000,"col_booln":false,"col_dbl":97842.924,"col_tm":71170420} +{"col_int":-4709,"col_bgint":373896,"col_char_2":"MN","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1357086482420,"col_dt":1062547200000,"col_booln":false,"col_dbl":449.327,"col_tm":55294390} +{"col_int":16300,"col_bgint":81680,"col_char_2":"MA","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":707444882421,"col_dt":677376000000,"col_booln":false,"col_dbl":4771.17,"col_tm":37915240} +{"col_int":-15950,"col_bgint":4547891,"col_char_2":"IN","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882421,"col_dt":-353721600000,"col_booln":false,"col_dbl":8841.945,"col_tm":5867310} +{"col_int":8888,"col_bgint":314696,"col_char_2":"FL","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882421,"col_dt":-111628800000,"col_booln":false,"col_dbl":1183.32,"col_tm":6605110} +{"col_int":82588,"col_bgint":6672,"col_char_2":"ND","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882421,"col_dt":54691200000,"col_booln":true,"col_dbl":436.897,"col_tm":41305100} +{"col_int":-13968,"col_bgint":62808,"col_char_2":"SD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882421,"col_dt":676425600000,"col_booln":false,"col_dbl":42577.743,"col_tm":22332400} +{"col_int":-74788,"col_bgint":7344,"col_char_2":"CO","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882422,"col_dt":182995200000,"col_booln":false,"col_dbl":10254.74,"col_tm":34731100} +{"col_int":-1,"col_bgint":0,"col_char_2":"CO","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882422,"col_dt":182995200000,"col_booln":false,"col_dbl":10254.74,"col_tm":34731100} +{"col_int":-10846,"col_bgint":57360,"col_char_2":"WI","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882422,"col_dt":706147200000,"col_booln":false,"col_dbl":1488.44,"col_tm":72125200} +{"col_int":101701,"col_bgint":719200,"col_char_2":"MD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882422,"col_dt":-49593600000,"col_booln":true,"col_dbl":97541.786,"col_tm":75052800} +{"col_int":198860,"col_bgint":103164,"col_char_2":"VT","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882422,"col_dt":-480556800000,"col_booln":false,"col_dbl":10591.83,"col_tm":73602340} +{"col_int":11140,"col_bgint":10878,"col_char_2":"HI","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":896747282422,"col_dt":-304214400000,"col_booln":false,"col_dbl":3243.082,"col_tm":7378300} +{"col_int":-84984,"col_bgint":67944,"col_char_2":"GA","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882422,"col_dt":1161734400000,"col_booln":true,"col_dbl":14339.42,"col_tm":66670320} +{"col_int":-184211,"col_bgint":85264,"col_char_2":"MN","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1401668882423,"col_dt":331689600000,"col_booln":false,"col_dbl":311.473,"col_tm":16118230} +{"col_int":-169479,"col_bgint":540352,"col_char_2":"GA","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882423,"col_dt":196646400000,"col_booln":false,"col_dbl":9739.559,"col_tm":63177700} +{"col_int":42942,"col_bgint":502144,"col_char_2":"WY","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1401668882423,"col_dt":-334800000000,"col_booln":true,"col_dbl":1118.55,"col_tm":9238280} +{"col_int":-33576,"col_bgint":592352,"col_char_2":"MI","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882423,"col_dt":88732800000,"col_booln":false,"col_dbl":475.7462,"col_tm":43115460} +{"col_int":-127816,"col_bgint":9832,"col_char_2":"MA","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1401668882423,"col_dt":1335139200000,"col_booln":true,"col_dbl":2680.205,"col_tm":8566390} +{"col_int":36582,"col_bgint":243364,"col_char_2":"OR","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1375403282423,"col_dt":1365465600000,"col_booln":false,"col_dbl":3861.757,"col_tm":69763320} +{"col_int":-149645,"col_bgint":8720,"col_char_2":"MA","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882424,"col_dt":-489283200000,"col_booln":true,"col_dbl":27641.297,"col_tm":71068900} +{"col_int":-12159,"col_bgint":500752,"col_char_2":"RI","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1401668882424,"col_dt":172540800000,"col_booln":true,"col_dbl":4024.414,"col_tm":5649490} +{"col_int":-2402,"col_bgint":5756000,"col_char_2":"MO","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882424,"col_dt":937958400000,"col_booln":false,"col_dbl":1859.559,"col_tm":29932570} +{"col_int":-33399,"col_bgint":7486400,"col_char_2":"RI","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882424,"col_dt":120096000000,"col_booln":true,"col_dbl":185.87,"col_tm":40411560} +{"col_int":1,"col_bgint":-1,"col_char_2":"RI","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882424,"col_dt":120096000000,"col_booln":true,"col_dbl":185.87,"col_tm":40411560} +{"col_int":56214,"col_bgint":5372456,"col_char_2":"AZ","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882425,"col_dt":258076800000,"col_booln":true,"col_dbl":10441.42,"col_tm":39161450} +{"col_int":-12209,"col_bgint":7899414,"col_char_2":"MD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882426,"col_dt":1123200000000,"col_booln":false,"col_dbl":26268.761,"col_tm":28015290} +{"col_int":18763,"col_bgint":921520,"col_char_2":"NE","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882426,"col_dt":-546134400000,"col_booln":true,"col_dbl":4966.2418,"col_tm":39006130} +{"col_int":211066,"col_bgint":104832,"col_char_2":"NJ","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1404260882426,"col_dt":105235200000,"col_booln":true,"col_dbl":1155.5,"col_tm":9018340} +{"col_int":73839,"col_bgint":103690,"col_char_2":"OR","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1401668882427,"col_dt":740448000000,"col_booln":true,"col_dbl":8049.003,"col_tm":17273570} +{"col_int":124587,"col_bgint":617504,"col_char_2":"WY","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1401668882427,"col_dt":449020800000,"col_booln":false,"col_dbl":5804.39,"col_tm":81757180} +{"col_int":-61299,"col_bgint":38096,"col_char_2":"CO","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1401668882427,"col_dt":-106444800000,"col_booln":false,"col_dbl":247.6481,"col_tm":27971300} +{"col_int":11345,"col_bgint":87088,"col_char_2":"MO","col_vchar_52":"AXXXXXXXXXXXXXXXXXXXXXXXXXCXXXXXXXXXXXXXXXXXXXXXXXXB","col_tmstmp":1401668882428,"col_dt":404265600000,"col_booln":true,"col_dbl":9150.279,"col_tm":20510140} +{"col_int":-92531,"col_bgint":21100,"col_char_2":"CA","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882428,"col_dt":1359936000000,"col_booln":true,"col_dbl":9240.11,"col_tm":46147130} +{"col_int":95690,"col_bgint":186392,"col_char_2":"VT","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1406939282428,"col_dt":1219795200000,"col_booln":false,"col_dbl":12344.595,"col_tm":29546560} +{"col_int":-199048,"col_bgint":3520,"col_char_2":"SC","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882429,"col_dt":-624585600000,"col_booln":false,"col_dbl":19100.34,"col_tm":51408700} +{"col_int":216553,"col_bgint":86744,"col_char_2":"SC","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1409617682429,"col_dt":-133660800000,"col_booln":false,"col_dbl":1337.49,"col_tm":53136170} +{"col_int":-126232,"col_bgint":3160,"col_char_2":"WY","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1412209682429,"col_dt":1224633600000,"col_booln":true,"col_dbl":450.283,"col_tm":62477170} +{"col_int":120079,"col_bgint":6736,"col_char_2":"CO","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882429,"col_dt":880243200000,"col_booln":true,"col_dbl":94963.943,"col_tm":16752380} +{"col_int":-179922,"col_bgint":198402,"col_char_2":"NE","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882429,"col_dt":1379289600000,"col_booln":false,"col_dbl":1145.5,"col_tm":25806550} +{"col_int":196285,"col_bgint":84560,"col_char_2":"IA","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1401668882430,"col_dt":-301622400000,"col_booln":false,"col_dbl":95854.701,"col_tm":7694500} +{"col_int":77907,"col_bgint":412832,"col_char_2":"FL","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1401668882430,"col_dt":1000944000000,"col_booln":false,"col_dbl":135568.86,"col_tm":21999400} +{"col_int":0,"col_bgint":0,"col_char_2":"FL","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1401668882430,"col_dt":1000944000000,"col_booln":false,"col_dbl":135568.86,"col_tm":21999400} +{"col_int":-185276,"col_bgint":163150,"col_char_2":"SD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882430,"col_dt":-521596800000,"col_booln":true,"col_dbl":9255.77,"col_tm":37363300} +{"col_int":-14479,"col_bgint":5192,"col_char_2":"IN","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1401668882430,"col_dt":948412800000,"col_booln":false,"col_dbl":1156.47,"col_tm":12482490} +{"col_int":-165881,"col_bgint":7104,"col_char_2":"SD","col_vchar_52":"HXXXXXXXXXXXXXXXXXXXXXXXXXIXXXXXXXXXXXXXXXXXXXXXXXXJ","col_tmstmp":1401668882430,"col_dt":750729600000,"col_booln":false,"col_dbl":14879.08,"col_tm":21046300} +{"col_int":155858,"col_bgint":4640,"col_char_2":"RI","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":991441682430,"col_dt":389750400000,"col_booln":true,"col_dbl":10227.57,"col_tm":40761560} +{"col_int":-8523,"col_bgint":696,"col_char_2":"MN","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1401668882430,"col_dt":-146361600000,"col_booln":true,"col_dbl":1026.051,"col_tm":17111380} +{"col_int":137899,"col_bgint":2799548,"col_char_2":"UT","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":959905682431,"col_dt":-459734400000,"col_booln":true,"col_dbl":1860.97,"col_tm":8261230} +{"col_int":1,"col_bgint":1,"col_char_2":"UT","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1306974482431,"col_dt":-459734400000,"col_booln":true,"col_dbl":1860.97,"col_tm":8261230} +{"col_int":-20449,"col_bgint":5172,"col_char_2":"NY","col_vchar_52":"DXXXXXXXXXXXXXXXXXXXXXXXXXEXXXXXXXXXXXXXXXXXXXXXXXXF","col_tmstmp":1338596882431,"col_dt":-51926400000,"col_booln":false,"col_dbl":438.068,"col_tm":18255480} +{"col_int":-86821,"col_bgint":11648,"col_char_2":"LA","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1401668882431,"col_dt":648172800000,"col_booln":true,"col_dbl":1231.61,"col_tm":82189300} +{"col_int":65535,"col_bgint":65535,"col_char_2":"LA","col_vchar_52":"GXXXXXXXXXXXXXXXXXXXXXXXXXHXXXXXXXXXXXXXXXXXXXXXXXXI","col_tmstmp":1370132882431,"col_dt":648172800000,"col_booln":true,"col_dbl":1231.61,"col_tm":82189300} \ No newline at end of file diff --git a/sql/src/test/resources/drill/window/datasources/t_alltype.parquet.json b/sql/src/test/resources/drill/window/datasources/t_alltype.parquet.json index bef5694f7c70..1663caec1ee2 100644 --- a/sql/src/test/resources/drill/window/datasources/t_alltype.parquet.json +++ b/sql/src/test/resources/drill/window/datasources/t_alltype.parquet.json @@ -1,145 +1,145 @@ -{"c1":1,"c2":592475043,"c3":616080519999272,"c4":"ObHeWTDEcbGzssDwPwurfs","c5":"0sZxIfZ CGwTOaLWZ6nWkUNx","c6":1456290852307,"c7":421426627200000,"c8":true,"c9":0.626179100469} -{"c1":2,"c2":1194343172,"c3":3136812789494,"c4":"bSrLdMAtGNmXsaFkmNpRaZnlPuUuSljiMqxJdgzoViIy","c5":"sluPVQz3991EyMTJL86YF8NEODzE1w2BtcKwPutL5TFUS7MM0c8pEYeWz2uq5pr0f0o0JcV5HRZs6gnwj3CPAdjisUk3LM5DvZcGUMkKpIEfziNrlcCrjRR10vWYVLeG6y5hG1ZsZgoSDWubDY2K1bZOaJTEP26AMDpYwIo","c6":1456290852308,"c7":420167260800000,"c8":true,"c9":0.0794280235964} -{"c1":3,"c2":-581053021,"c3":14567676734720,"c4":"zBmUwoxbiWhazyIFwkyTsCo","c5":"b4iHRz7x3jgWYQASSBusBaOS5KjIKpdxRqEOkVvZSqgwzwS3r1 5C81WnF5RWzLNDOi8qgJK0JrMMjVxOpFi13socmJ94NVelUSvb6umYkzBrZhpzYDFHZmXtAfWzRBkXfESrfYF2ME 5lI2v79oP54eucAvtIQApA1Bak8Fw9VgRznhGwE nKzuew1UpXk6dAW91ag099LyD29bViFQxWzCyf7FFDEy","c6":1456290852309,"c7":421521667200000,"c8":true,"c9":0.256111757801} -{"c1":4,"c2":719544129,"c3":26552329517384,"c4":"VlVwoLMKaardyJCdUttoBDIYVZ","c5":"TwDutiLBRsXHs0K0KvlzY6SHzNDJTpQFOigQ32YOYD7jCEpMJrT2SrvWBk9gJ7idoJhqIjW 4tPTI8u0cxFiEsU1IUNnONTYF5E3fGo8aXdA2Y0CAWqn4nHyY QPnVt3n5FnuQtQ88ayT3a0J86FOc641y4vCrTAqlN3jXVRJ1sg","c6":1456290852309,"c7":422532201600000,"c8":true,"c9":0.367266831393} -{"c1":5,"c2":-265354187,"c3":869394999808,"c4":"gwfrW","c5":"h79IKUK0fji8POwoapUs7YCKsC5WDOI1Oge25fe7QXMAQrxIgyEIhj3upa62rTJxOVA2vCMg6U 4koGrPZzda1aNjvGiK5Qjc0qUPnGCjoGdgjE87rfON4Uu02J5 Q7NiZ15CtSu8IuY6oqYyZclq47L6FC2grvhcuBTRiw9C1zgQ6d1523I2RCONHyYiZi7RrNhziA2tSZCPmI42GYHtjZ","c6":1456290852309,"c7":421190841600000,"c8":false,"c9":0.886914049583} -{"c1":6,"c2":-739409800,"c3":458112011400,"c4":"upOMZlypwcASQHtEkgpYybPwvjCtFHpuJkfNlQBSIrWqznQJEzEV","c5":"tmZET5agjU4yZaPXZDGW8iSPwRmkuGq4dpdkZqtv7edggkdZm3Im1j9okgoqqcuqPmo8r2EDK0bPZdk5YFXutZoAVhK5ecd06gGdNj2tWkSVKO7wqCUR2E7obM9ZYpZlgDecXTjzLvLUYuZy0TV sTjyt8DCU1QnKChPKD MbcopdpZkaJ5vWd5tIM8eYOB5kjdTiLfV3TVYb2LmZGt2f4xHbih odqbeq2ZDeEuFAwaoyL3UQI7Euj8","c6":1456290852310,"c7":421723670400000,"c8":true,"c9":0.492183947524} -{"c1":7,"c2":2022947631,"c3":122811962676,"c4":"ZAFOcferhjkcl","c5":"1wnKPdZ2a6nHOkC5kuo6lpCUtShnNjjjUd4f3vBs7AdhwQDoBPA CRMC7CCBTU VOx38gwZurb9HrSQNRrSd2kDuHB6WpqffMTwNfHPvqxnwHtbh4M8GwVEE0ocFsvYPkd4mIl7 uhdOOOS3H8jZpGa6Yt OoalDERUEcgZz","c6":1456290852311,"c7":420189379200000,"c8":false,"c9":0.60032636853} +{"c1":1,"c2":592475043,"c3":616080519999272,"c4":"ObHeWTDEcbGzssDwPwurfs","c5":"0sZxIfZ CGwTOaLWZ6nWkUNx","c6":1456290852307,"c7":-306979200000,"c8":true,"c9":0.626179100469} +{"c1":2,"c2":1194343172,"c3":3136812789494,"c4":"bSrLdMAtGNmXsaFkmNpRaZnlPuUuSljiMqxJdgzoViIy","c5":"sluPVQz3991EyMTJL86YF8NEODzE1w2BtcKwPutL5TFUS7MM0c8pEYeWz2uq5pr0f0o0JcV5HRZs6gnwj3CPAdjisUk3LM5DvZcGUMkKpIEfziNrlcCrjRR10vWYVLeG6y5hG1ZsZgoSDWubDY2K1bZOaJTEP26AMDpYwIo","c6":1456290852308,"c7":-1566345600000,"c8":true,"c9":0.0794280235964} +{"c1":3,"c2":-581053021,"c3":14567676734720,"c4":"zBmUwoxbiWhazyIFwkyTsCo","c5":"b4iHRz7x3jgWYQASSBusBaOS5KjIKpdxRqEOkVvZSqgwzwS3r1 5C81WnF5RWzLNDOi8qgJK0JrMMjVxOpFi13socmJ94NVelUSvb6umYkzBrZhpzYDFHZmXtAfWzRBkXfESrfYF2ME 5lI2v79oP54eucAvtIQApA1Bak8Fw9VgRznhGwE nKzuew1UpXk6dAW91ag099LyD29bViFQxWzCyf7FFDEy","c6":1456290852309,"c7":-211939200000,"c8":true,"c9":0.256111757801} +{"c1":4,"c2":719544129,"c3":26552329517384,"c4":"VlVwoLMKaardyJCdUttoBDIYVZ","c5":"TwDutiLBRsXHs0K0KvlzY6SHzNDJTpQFOigQ32YOYD7jCEpMJrT2SrvWBk9gJ7idoJhqIjW 4tPTI8u0cxFiEsU1IUNnONTYF5E3fGo8aXdA2Y0CAWqn4nHyY QPnVt3n5FnuQtQ88ayT3a0J86FOc641y4vCrTAqlN3jXVRJ1sg","c6":1456290852309,"c7":798595200000,"c8":true,"c9":0.367266831393} +{"c1":5,"c2":-265354187,"c3":869394999808,"c4":"gwfrW","c5":"h79IKUK0fji8POwoapUs7YCKsC5WDOI1Oge25fe7QXMAQrxIgyEIhj3upa62rTJxOVA2vCMg6U 4koGrPZzda1aNjvGiK5Qjc0qUPnGCjoGdgjE87rfON4Uu02J5 Q7NiZ15CtSu8IuY6oqYyZclq47L6FC2grvhcuBTRiw9C1zgQ6d1523I2RCONHyYiZi7RrNhziA2tSZCPmI42GYHtjZ","c6":1456290852309,"c7":-542764800000,"c8":false,"c9":0.886914049583} +{"c1":6,"c2":-739409800,"c3":458112011400,"c4":"upOMZlypwcASQHtEkgpYybPwvjCtFHpuJkfNlQBSIrWqznQJEzEV","c5":"tmZET5agjU4yZaPXZDGW8iSPwRmkuGq4dpdkZqtv7edggkdZm3Im1j9okgoqqcuqPmo8r2EDK0bPZdk5YFXutZoAVhK5ecd06gGdNj2tWkSVKO7wqCUR2E7obM9ZYpZlgDecXTjzLvLUYuZy0TV sTjyt8DCU1QnKChPKD MbcopdpZkaJ5vWd5tIM8eYOB5kjdTiLfV3TVYb2LmZGt2f4xHbih odqbeq2ZDeEuFAwaoyL3UQI7Euj8","c6":1456290852310,"c7":-9936000000,"c8":true,"c9":0.492183947524} +{"c1":7,"c2":2022947631,"c3":122811962676,"c4":"ZAFOcferhjkcl","c5":"1wnKPdZ2a6nHOkC5kuo6lpCUtShnNjjjUd4f3vBs7AdhwQDoBPA CRMC7CCBTU VOx38gwZurb9HrSQNRrSd2kDuHB6WpqffMTwNfHPvqxnwHtbh4M8GwVEE0ocFsvYPkd4mIl7 uhdOOOS3H8jZpGa6Yt OoalDERUEcgZz","c6":1456290852311,"c7":-1544227200000,"c8":false,"c9":0.60032636853} {"c1":8,"c2":null,"c3":null,"c4":null,"c5":null,"c6":null,"c7":null,"c8":null,"c9":null} -{"c1":9,"c2":358750156,"c3":128037619138,"c4":"rNMsKIDnkuQFIYoJorACrHkFprLYMpp","c5":"fEwIVF31I9jk4C04qBYtw6pxswxIuThE3id0UuKTp6CgcnP9eNlVwJh82xvwsN 8CmX1RSQhtozud6Umv6kLCKSVF7BQbxRS07V78HgbrH0ZDV","c6":1456290852311,"c7":420977606400000,"c8":false,"c9":0.361570558696} -{"c1":10,"c2":-116481491,"c3":362751773072,"c4":"IcStZtjaSEfWDmrulvsnCwUQvfgn","c5":"aTISWde3pj5OJd9l7Zo7j9Kn8RRdWimrrPlc5BDFN3xpqrliH3Che9OJERMsQgwiafBwavLrTe21ORPrcml4Bh44fDbC4fBVvMMLTPVWbmgvmXm02Ct7A1WBMTM2C74LLJmIdZwPpwMXumO5dYZg8Z3KU2FnJ2bl80qqgyjJNlv9","c6":1456290852312,"c7":422991244800000,"c8":true,"c9":0.488525932471} -{"c1":11,"c2":993964305,"c3":713109032528,"c4":"TuFaJOrTexEQKCXg","c5":"9VLdcEruZzhtTbrz6Tu5vvfwwIy7jVtc7R5XEeZY","c6":1456291198301,"c7":420906067200000,"c8":true,"c9":0.957695228383} -{"c1":12,"c2":-1735090787,"c3":841324401984,"c4":"dPRmWDinhineYImsqZzRZSSMx","c5":"srBmkYpJY7TC0W","c6":1456291198302,"c7":420699571200000,"c8":false,"c9":0.707961195747} -{"c1":13,"c2":-101621245,"c3":4569922715352,"c4":"KkQGklqtAFOBwaNXDWkN","c5":"WMhbfG37BBZSwtryJVwMlS67TYe5UjqLYa5pHES13GC","c6":1456291198302,"c7":421197494400000,"c8":true,"c9":0.156305915074} -{"c1":14,"c2":202631122,"c3":6199873199432,"c4":"qV","c5":"w5Cs5Jas0T","c6":1456291198303,"c7":421541020800000,"c8":false,"c9":0.709658352873} -{"c1":15,"c2":-1800226884,"c3":2408646998880,"c4":"AtfZqknYOrYBXcFCORVnzcKmXyaEe","c5":"xfxBmCGvTcaBxEFXSvPQTaCZRCgE9Cus","c6":1456291198303,"c7":421215984000000,"c8":false,"c9":0.22155866245} -{"c1":16,"c2":2042714193,"c3":2437409008036,"c4":"BQyDpxxWldiVnQlFFfasgfwIIzpgJd","c5":"YF6qr4rAyBrHa94CosX6BWpfbwE8EXWXrupuFPoekm G1PE6vTKeliuW gxuq","c6":1456291198304,"c7":420776899200000,"c8":false,"c9":0.305618728365} -{"c1":17,"c2":-1236268475,"c3":90275297566,"c4":"OGnxbGFEaWMZhrJ","c5":"Tc0A7vQFGyzGXnoa90wXRVEMaZMmSTix2IqRw48WKqE50hCw","c6":1456291198305,"c7":423092073600000,"c8":true,"c9":0.141061129745} -{"c1":18,"c2":-1451268845,"c3":7456257725280,"c4":"HAUtIjkTOkHLzjhucQeLuH","c5":"a","c6":1456291198305,"c7":420537052800000,"c8":false,"c9":0.555072055925} -{"c1":19,"c2":-148845,"c3":3757280,"c4":"HALzjhucQeLuH","c5":null,"c6":1456291198305,"c7":419905900800000,"c8":false,"c9":0.455072055925} -{"c1":20,"c2":533611126,"c3":784427291198608,"c4":"JgPGWgjgNEjMXYYmWtC","c5":"PY250MG5pFd3NgQJGckqfC9mwY0DNazIAFmqRy1J","c6":1456291198305,"c7":420459120000000,"c8":false,"c9":0.355503179385} -{"c1":21,"c2":-642124822,"c3":43598566290864,"c4":"syewpw","c5":"iV24XUsay9jBU2wZgUoCErzIYo","c6":1456291198306,"c7":421299705600000,"c8":true,"c9":0.456846886729} -{"c1":22,"c2":735561305,"c3":6307081519934,"c4":"tMi","c5":"ig9ClFlYmzIAb99UFoGaUyjSorP9E9qA34km539T4so3qm9kRZZTZ","c6":1456291198306,"c7":422750188800000,"c8":true,"c9":0.740283135928} -{"c1":23,"c2":-350986527,"c3":69653384392040,"c4":"DkPvPKYuCdSatlPrzU","c5":"zfSgmg7tAawKVSJhkipg2nSNuprAp75eKDJry0focwPij J4hE","c6":1456291198311,"c7":420485040000000,"c8":false,"c9":0.336419256965} -{"c1":24,"c2":1847465545,"c3":38629268606556,"c4":"PV","c5":"D9lY2D4VJGmFffgSV9MoyojaAjNVxcVyBiEdHN911nwK EBRi2wwFuWwbnyLl","c6":1456291198312,"c7":420271459200000,"c8":false,"c9":0.964243969107} -{"c1":25,"c2":1442790109,"c3":61368071783056,"c4":"myUrrzUHGnxasz","c5":"wyweWUTyBPuo3jOBJLPfSWqji","c6":1456291198313,"c7":422857238400000,"c8":false,"c9":0.507695339931} -{"c1":26,"c2":-2125195491,"c3":57004017502044,"c4":"iqwqmwkUhrtTliZiPXRTstVpqgm","c5":"yCko0ouh0ueXmXK0O9 MPvLcCmlp7d7AI4ca1ru8d4jGGtbPc7mPx","c6":1456291198314,"c7":420243120000000,"c8":false,"c9":0.962960554859} -{"c1":27,"c2":-603256047,"c3":64691852740816,"c4":"lXEgAlS","c5":"7ceX9a22fDB1wb9RUb6TQWqvkbE6sXp2Sq0Lt","c6":1456291198314,"c7":420876604800000,"c8":false,"c9":0.940132365478} -{"c1":28,"c2":-1806928495,"c3":73151265354384,"c4":"yfjoc","c5":"sqxWGArGIX5jKOvvOI6BjG2l","c6":1456291198315,"c7":423006883200000,"c8":true,"c9":0.769634808418} -{"c1":29,"c2":-926147543,"c3":18430182,"c4":"kyTyhomxLi","c5":"9TR790gkKg3KMkc5k47V9QAuos2Ojig04QwTdO5nB","c6":1456291198315,"c7":421728768000000,"c8":false,"c9":0.754223352027} -{"c1":30,"c2":626932646,"c3":3285206109420,"c4":"IWZriOkCaYLDGOzMSp","c5":"oSv1RhKIOtLgOLe","c6":1456291198316,"c7":422313868800000,"c8":false,"c9":0.444467141108} -{"c1":31,"c2":-430498956,"c3":1258459028142,"c4":"V","c5":"Q1Am3KtwYAQx Vj7kk9nsawCmN BRciIQiN3hF","c6":1456291701220,"c7":420287529600000,"c8":true,"c9":0.210929121391} -{"c1":32,"c2":-938167176,"c3":6017016766800,"c4":"PStTin","c5":"dfOY 50EejHegPw9axCPz","c6":1456291701221,"c7":420197328000000,"c8":false,"c9":0.827238030237} -{"c1":33,"c2":-464056232,"c3":269431279554,"c4":"HHqSmnhL","c5":"VVRz QWFjn96AtBYO7eR001KZwKGLUrEqeMrPVcKSbQowuQ90XXFrl3iG","c6":1456291701221,"c7":422397676800000,"c8":true,"c9":0.764629411376} -{"c1":34,"c2":-1564810747,"c3":471032170712,"c4":"XJF","c5":"8D9e1bT5wwFpLD1Hr92VKaCQJS","c6":1456291701222,"c7":421392844800000,"c8":true,"c9":0.804630256964} -{"c1":35,"c2":2047004122,"c3":6190065899528,"c4":"PAobaowvKNSaPsKEdhLtCRWe","c5":"wGlZo1sVGPg","c6":1456291701222,"c7":422983382400000,"c8":false,"c9":0.119950105444} -{"c1":36,"c2":-682355573,"c3":879651728,"c4":"BANiwCtAOccuLIqNMVOQGBIQKNw","c5":"KxgxdOznNSNaltnFxkxj94qZ7C05fwQVUyBkN","c6":1456291701222,"c7":423138556800000,"c8":true,"c9":0.152121113454} -{"c1":37,"c2":1168899469,"c3":311335712,"c4":"bQoLkPBFAUXqcMpcp","c5":"6YE84Kb8G6Z8 HR9qkTp30a7cnBh3T rC1uWP4o3S7DSm","c6":1456291701222,"c7":422246476800000,"c8":false,"c9":0.674319849007} -{"c1":38,"c2":-1678398290,"c3":895040440,"c4":"CSavGyXMbAANckDiCshpuNzCmBXuHbHw","c5":"V VabLjrkDF4IJ7twnT9gqTv","c6":1456291701223,"c7":420476140800000,"c8":false,"c9":0.249230799416} -{"c1":39,"c2":-1433668371,"c3":102790112,"c4":"yytBlhCMRSmV","c5":"fanhgAi1Cla 3lwYXQTvWYdIvTF5ziIBc","c6":1456291701223,"c7":421929648000000,"c8":false,"c9":0.538207272457} -{"c1":40,"c2":1300934512,"c3":1526006148,"c4":"WYqpkjPLSeCA","c5":"4wK m9vhE XTHlpHv1U","c6":1456291701224,"c7":420625958400000,"c8":true,"c9":0.0125838026238} -{"c1":41,"c2":67513000,"c3":8652621344,"c4":"cTPjdg","c5":"KvYiWTtwEwEBTWUArtiUnPciZY V8VzRIVI03Gyt","c6":1456291701225,"c7":421659302400000,"c8":true,"c9":0.0616766496679} -{"c1":42,"c2":1071705345,"c3":774763920,"c4":"YfwqGtgVkBewvkIlQnEEnt","c5":"EtNWvXm45 Tw5CI9YjwflvI4zVqHUUpLal rAt2f8X0Wfp","c6":1456291701225,"c7":421992201600000,"c8":false,"c9":0.881191135812} -{"c1":43,"c2":-705176971,"c3":1305480928,"c4":"xLYbWdnPrlTIt","c5":"mGVBbGVhQkAhjQdAhiWAwHlCNN5f1wggwKxRjsBBbmlvodajek4E1r","c6":1456291701225,"c7":420234825600000,"c8":false,"c9":0.0963362944467} -{"c1":44,"c2":-790060808,"c3":2091106612,"c4":"chvsFysOhXbqRnniYOGeLUppugM","c5":"XZkwRGa0gxPwmnAG4VOQ1bNNgzM9cO1RzrcynKKBssqEP3BeEhibq","c6":1456291701226,"c7":420386025600000,"c8":false,"c9":0.292481581027} -{"c1":45,"c2":1221413457,"c3":8339791280,"c4":"SeQaEfXkDgumVjpOKOglGiXVe","c5":"CwZrVQVWdAfMtFcqWFJzjTcrnpWT48Rbkd7CO","c6":1456291701227,"c7":421768425600000,"c8":false,"c9":0.648820911939} -{"c1":46,"c2":627701102,"c3":33308856,"c4":"qaOEIqTfNGD","c5":"OoDOYgbcJ09XmxqPv1e","c6":1456291701227,"c7":422446579200000,"c8":true,"c9":0.40799240898} -{"c1":47,"c2":-841849915,"c3":42746339248,"c4":"YPiESbIjBms","c5":"xFfL8aYVA5EiiSWmvKkHAYRGYCiqctdxQwdkH4DOP","c6":1456291701228,"c7":420577660800000,"c8":false,"c9":0.324089746843} -{"c1":48,"c2":735186776,"c3":61478943200,"c4":"BJIPsOpsMTNRCO","c5":"80Z7O0f6A6fj1Beww7oOkwLSZdjO","c6":1456291701228,"c7":421328044800000,"c8":false,"c9":0.359520992037} -{"c1":49,"c2":56927672,"c3":84551700064,"c4":"Y","c5":"Wvu9T8ilDl8k3oaB0PFNjHirDoMdwv1YYCah6ahqiuATp1","c6":1456291701229,"c7":421813180800000,"c8":true,"c9":0.328379339497} -{"c1":50,"c2":1182933670,"c3":45692038432,"c4":"whFjvGloFQxwYmvLkshjwnADhUQlV","c5":"yt49EB4Y2RAnPQXNwTqfp7qzgDH1Jf1rjY0FnpIp2y4","c6":1456292105544,"c7":423012153600000,"c8":true,"c9":0.835195212647} -{"c1":51,"c2":980460718,"c3":5971386168,"c4":"OMJEMdywCTpQgqJUYDuKDxAPBkWdEP","c5":"akbcHSDflWlG8K6MFnIOOLBMZ4z8NV2FecgqpLO8","c6":1456292105544,"c7":420768691200000,"c8":true,"c9":0.0594494786115} -{"c1":52,"c2":-1263020365,"c3":17414114976,"c4":"vkUYqgXGGfELNjVwyYKVJMd","c5":"7zP0naU","c6":1456292105545,"c7":422403206400000,"c8":false,"c9":0.128559830476} -{"c1":53,"c2":-2078473628,"c3":23813625600,"c4":"PVAKCnVrihXStlfqJf","c5":"HKyyDE7cHRTzEQX2F","c6":1456292105545,"c7":421790371200000,"c8":true,"c9":0.497832439707} -{"c1":54,"c2":-388853860,"c3":4882213296,"c4":"NkSynIivNZiIplsAegulLrz","c5":"2AWYA664YJkqeGXGZeI8vf3fhX06wjhlUoRdzRBJnylTOnUsrz3LcyxbTyIQHr1YCMXl9jFx6LyhnaOYZorN4rOazR44zI2","c6":1456292105546,"c7":422153337600000,"c8":false,"c9":0.806218120908} -{"c1":55,"c2":147766593,"c3":3544613368,"c4":"ByZRmalvDsZD","c5":"XgulsUB2lwxhZaMz0POOTcxIAt7mPaBialPFYzth6Aa54OPBoTY9nlJpXWf89oIczO0QVg5lbkmQQUlljIMEe2Mh8rjtDoYox2uDykAXeH0BXGVtOZ12EIwe","c6":1456292105546,"c7":421184448000000,"c8":true,"c9":0.189172246469} -{"c1":56,"c2":420230528,"c3":7209802880,"c4":"ZwiFgCchxUqJTEShealG","c5":"K7RQD052LFrkPBfuL9hIihch","c6":1456292105546,"c7":422883417600000,"c8":true,"c9":0.462810537514} -{"c1":57,"c2":-1894216997,"c3":5384805744,"c4":"eBKZaQYxrdPMRRwgeqPtJjxjqOBqUH","c5":"z7cJERSftFvbtxxgVbdF31jZCHmkHBmazWaHAa3eKAp1PPEObxkcuhjA1vf6cSlGe4TT8Yektdwqcr1Jd","c6":1456292105546,"c7":420506121600000,"c8":true,"c9":0.604145500315} -{"c1":58,"c2":-263032861,"c3":56565941904,"c4":"dqXVKoNjMFKcIszKFRXCotmbejjxGU","c5":"3QDSogOdQYe6NfDPkEPIHhtECUKeT3eXsRdbwCj3omYIHz3HX","c6":1456292105547,"c7":420541459200000,"c8":false,"c9":0.472430725946} -{"c1":59,"c2":-1661674051,"c3":9595495792,"c4":"atHZ","c5":"7SWlwF1qaCNs3Jea1NollJfLsgTK95yuo","c6":1456292105547,"c7":421871846400000,"c8":true,"c9":0.0923867105652} -{"c1":60,"c2":-1129907976,"c3":6907332688,"c4":"Mc","c5":"4FN","c6":1456292105547,"c7":421541020800000,"c8":false,"c9":0.701638057307} -{"c1":61,"c2":-514254242,"c3":195633336,"c4":"kenCPo","c5":"eqVUbd9TzGrQS2W1HuV1apKCt1rkYLRTHgED2RIk1jhowGYzlRa2DVMpzTVc8RVO44rm9stOD9uvJPLNE48jVMQfhLwtNDlKXUchY8N0FOEsMW2G37X7dYnzod","c6":1456292105547,"c7":422432323200000,"c8":true,"c9":0.511369563541} +{"c1":9,"c2":358750156,"c3":128037619138,"c4":"rNMsKIDnkuQFIYoJorACrHkFprLYMpp","c5":"fEwIVF31I9jk4C04qBYtw6pxswxIuThE3id0UuKTp6CgcnP9eNlVwJh82xvwsN 8CmX1RSQhtozud6Umv6kLCKSVF7BQbxRS07V78HgbrH0ZDV","c6":1456290852311,"c7":-756000000000,"c8":false,"c9":0.361570558696} +{"c1":10,"c2":-116481491,"c3":362751773072,"c4":"IcStZtjaSEfWDmrulvsnCwUQvfgn","c5":"aTISWde3pj5OJd9l7Zo7j9Kn8RRdWimrrPlc5BDFN3xpqrliH3Che9OJERMsQgwiafBwavLrTe21ORPrcml4Bh44fDbC4fBVvMMLTPVWbmgvmXm02Ct7A1WBMTM2C74LLJmIdZwPpwMXumO5dYZg8Z3KU2FnJ2bl80qqgyjJNlv9","c6":1456290852312,"c7":1257638400000,"c8":true,"c9":0.488525932471} +{"c1":11,"c2":993964305,"c3":713109032528,"c4":"TuFaJOrTexEQKCXg","c5":"9VLdcEruZzhtTbrz6Tu5vvfwwIy7jVtc7R5XEeZY","c6":1456291198301,"c7":-827539200000,"c8":true,"c9":0.957695228383} +{"c1":12,"c2":-1735090787,"c3":841324401984,"c4":"dPRmWDinhineYImsqZzRZSSMx","c5":"srBmkYpJY7TC0W","c6":1456291198302,"c7":-1034035200000,"c8":false,"c9":0.707961195747} +{"c1":13,"c2":-101621245,"c3":4569922715352,"c4":"KkQGklqtAFOBwaNXDWkN","c5":"WMhbfG37BBZSwtryJVwMlS67TYe5UjqLYa5pHES13GC","c6":1456291198302,"c7":-536112000000,"c8":true,"c9":0.156305915074} +{"c1":14,"c2":202631122,"c3":6199873199432,"c4":"qV","c5":"w5Cs5Jas0T","c6":1456291198303,"c7":-192585600000,"c8":false,"c9":0.709658352873} +{"c1":15,"c2":-1800226884,"c3":2408646998880,"c4":"AtfZqknYOrYBXcFCORVnzcKmXyaEe","c5":"xfxBmCGvTcaBxEFXSvPQTaCZRCgE9Cus","c6":1456291198303,"c7":-517622400000,"c8":false,"c9":0.22155866245} +{"c1":16,"c2":2042714193,"c3":2437409008036,"c4":"BQyDpxxWldiVnQlFFfasgfwIIzpgJd","c5":"YF6qr4rAyBrHa94CosX6BWpfbwE8EXWXrupuFPoekm G1PE6vTKeliuW gxuq","c6":1456291198304,"c7":-956707200000,"c8":false,"c9":0.305618728365} +{"c1":17,"c2":-1236268475,"c3":90275297566,"c4":"OGnxbGFEaWMZhrJ","c5":"Tc0A7vQFGyzGXnoa90wXRVEMaZMmSTix2IqRw48WKqE50hCw","c6":1456291198305,"c7":1358467200000,"c8":true,"c9":0.141061129745} +{"c1":18,"c2":-1451268845,"c3":7456257725280,"c4":"HAUtIjkTOkHLzjhucQeLuH","c5":"a","c6":1456291198305,"c7":-1196553600000,"c8":false,"c9":0.555072055925} +{"c1":19,"c2":-148845,"c3":3757280,"c4":"HALzjhucQeLuH","c5":null,"c6":1456291198305,"c7":-1827705600000,"c8":false,"c9":0.455072055925} +{"c1":20,"c2":533611126,"c3":784427291198608,"c4":"JgPGWgjgNEjMXYYmWtC","c5":"PY250MG5pFd3NgQJGckqfC9mwY0DNazIAFmqRy1J","c6":1456291198305,"c7":-1274486400000,"c8":false,"c9":0.355503179385} +{"c1":21,"c2":-642124822,"c3":43598566290864,"c4":"syewpw","c5":"iV24XUsay9jBU2wZgUoCErzIYo","c6":1456291198306,"c7":-433900800000,"c8":true,"c9":0.456846886729} +{"c1":22,"c2":735561305,"c3":6307081519934,"c4":"tMi","c5":"ig9ClFlYmzIAb99UFoGaUyjSorP9E9qA34km539T4so3qm9kRZZTZ","c6":1456291198306,"c7":1016582400000,"c8":true,"c9":0.740283135928} +{"c1":23,"c2":-350986527,"c3":69653384392040,"c4":"DkPvPKYuCdSatlPrzU","c5":"zfSgmg7tAawKVSJhkipg2nSNuprAp75eKDJry0focwPij J4hE","c6":1456291198311,"c7":-1248566400000,"c8":false,"c9":0.336419256965} +{"c1":24,"c2":1847465545,"c3":38629268606556,"c4":"PV","c5":"D9lY2D4VJGmFffgSV9MoyojaAjNVxcVyBiEdHN911nwK EBRi2wwFuWwbnyLl","c6":1456291198312,"c7":-1462147200000,"c8":false,"c9":0.964243969107} +{"c1":25,"c2":1442790109,"c3":61368071783056,"c4":"myUrrzUHGnxasz","c5":"wyweWUTyBPuo3jOBJLPfSWqji","c6":1456291198313,"c7":1123632000000,"c8":false,"c9":0.507695339931} +{"c1":26,"c2":-2125195491,"c3":57004017502044,"c4":"iqwqmwkUhrtTliZiPXRTstVpqgm","c5":"yCko0ouh0ueXmXK0O9 MPvLcCmlp7d7AI4ca1ru8d4jGGtbPc7mPx","c6":1456291198314,"c7":-1490486400000,"c8":false,"c9":0.962960554859} +{"c1":27,"c2":-603256047,"c3":64691852740816,"c4":"lXEgAlS","c5":"7ceX9a22fDB1wb9RUb6TQWqvkbE6sXp2Sq0Lt","c6":1456291198314,"c7":-857001600000,"c8":false,"c9":0.940132365478} +{"c1":28,"c2":-1806928495,"c3":73151265354384,"c4":"yfjoc","c5":"sqxWGArGIX5jKOvvOI6BjG2l","c6":1456291198315,"c7":1273276800000,"c8":true,"c9":0.769634808418} +{"c1":29,"c2":-926147543,"c3":18430182,"c4":"kyTyhomxLi","c5":"9TR790gkKg3KMkc5k47V9QAuos2Ojig04QwTdO5nB","c6":1456291198315,"c7":-4838400000,"c8":false,"c9":0.754223352027} +{"c1":30,"c2":626932646,"c3":3285206109420,"c4":"IWZriOkCaYLDGOzMSp","c5":"oSv1RhKIOtLgOLe","c6":1456291198316,"c7":580262400000,"c8":false,"c9":0.444467141108} +{"c1":31,"c2":-430498956,"c3":1258459028142,"c4":"V","c5":"Q1Am3KtwYAQx Vj7kk9nsawCmN BRciIQiN3hF","c6":1456291701220,"c7":-1446076800000,"c8":true,"c9":0.210929121391} +{"c1":32,"c2":-938167176,"c3":6017016766800,"c4":"PStTin","c5":"dfOY 50EejHegPw9axCPz","c6":1456291701221,"c7":-1536278400000,"c8":false,"c9":0.827238030237} +{"c1":33,"c2":-464056232,"c3":269431279554,"c4":"HHqSmnhL","c5":"VVRz QWFjn96AtBYO7eR001KZwKGLUrEqeMrPVcKSbQowuQ90XXFrl3iG","c6":1456291701221,"c7":664070400000,"c8":true,"c9":0.764629411376} +{"c1":34,"c2":-1564810747,"c3":471032170712,"c4":"XJF","c5":"8D9e1bT5wwFpLD1Hr92VKaCQJS","c6":1456291701222,"c7":-340761600000,"c8":true,"c9":0.804630256964} +{"c1":35,"c2":2047004122,"c3":6190065899528,"c4":"PAobaowvKNSaPsKEdhLtCRWe","c5":"wGlZo1sVGPg","c6":1456291701222,"c7":1249776000000,"c8":false,"c9":0.119950105444} +{"c1":36,"c2":-682355573,"c3":879651728,"c4":"BANiwCtAOccuLIqNMVOQGBIQKNw","c5":"KxgxdOznNSNaltnFxkxj94qZ7C05fwQVUyBkN","c6":1456291701222,"c7":1404950400000,"c8":true,"c9":0.152121113454} +{"c1":37,"c2":1168899469,"c3":311335712,"c4":"bQoLkPBFAUXqcMpcp","c5":"6YE84Kb8G6Z8 HR9qkTp30a7cnBh3T rC1uWP4o3S7DSm","c6":1456291701222,"c7":512870400000,"c8":false,"c9":0.674319849007} +{"c1":38,"c2":-1678398290,"c3":895040440,"c4":"CSavGyXMbAANckDiCshpuNzCmBXuHbHw","c5":"V VabLjrkDF4IJ7twnT9gqTv","c6":1456291701223,"c7":-1257465600000,"c8":false,"c9":0.249230799416} +{"c1":39,"c2":-1433668371,"c3":102790112,"c4":"yytBlhCMRSmV","c5":"fanhgAi1Cla 3lwYXQTvWYdIvTF5ziIBc","c6":1456291701223,"c7":196041600000,"c8":false,"c9":0.538207272457} +{"c1":40,"c2":1300934512,"c3":1526006148,"c4":"WYqpkjPLSeCA","c5":"4wK m9vhE XTHlpHv1U","c6":1456291701224,"c7":-1107648000000,"c8":true,"c9":0.0125838026238} +{"c1":41,"c2":67513000,"c3":8652621344,"c4":"cTPjdg","c5":"KvYiWTtwEwEBTWUArtiUnPciZY V8VzRIVI03Gyt","c6":1456291701225,"c7":-74304000000,"c8":true,"c9":0.0616766496679} +{"c1":42,"c2":1071705345,"c3":774763920,"c4":"YfwqGtgVkBewvkIlQnEEnt","c5":"EtNWvXm45 Tw5CI9YjwflvI4zVqHUUpLal rAt2f8X0Wfp","c6":1456291701225,"c7":258595200000,"c8":false,"c9":0.881191135812} +{"c1":43,"c2":-705176971,"c3":1305480928,"c4":"xLYbWdnPrlTIt","c5":"mGVBbGVhQkAhjQdAhiWAwHlCNN5f1wggwKxRjsBBbmlvodajek4E1r","c6":1456291701225,"c7":-1498780800000,"c8":false,"c9":0.0963362944467} +{"c1":44,"c2":-790060808,"c3":2091106612,"c4":"chvsFysOhXbqRnniYOGeLUppugM","c5":"XZkwRGa0gxPwmnAG4VOQ1bNNgzM9cO1RzrcynKKBssqEP3BeEhibq","c6":1456291701226,"c7":-1347580800000,"c8":false,"c9":0.292481581027} +{"c1":45,"c2":1221413457,"c3":8339791280,"c4":"SeQaEfXkDgumVjpOKOglGiXVe","c5":"CwZrVQVWdAfMtFcqWFJzjTcrnpWT48Rbkd7CO","c6":1456291701227,"c7":34819200000,"c8":false,"c9":0.648820911939} +{"c1":46,"c2":627701102,"c3":33308856,"c4":"qaOEIqTfNGD","c5":"OoDOYgbcJ09XmxqPv1e","c6":1456291701227,"c7":712972800000,"c8":true,"c9":0.40799240898} +{"c1":47,"c2":-841849915,"c3":42746339248,"c4":"YPiESbIjBms","c5":"xFfL8aYVA5EiiSWmvKkHAYRGYCiqctdxQwdkH4DOP","c6":1456291701228,"c7":-1155945600000,"c8":false,"c9":0.324089746843} +{"c1":48,"c2":735186776,"c3":61478943200,"c4":"BJIPsOpsMTNRCO","c5":"80Z7O0f6A6fj1Beww7oOkwLSZdjO","c6":1456291701228,"c7":-405561600000,"c8":false,"c9":0.359520992037} +{"c1":49,"c2":56927672,"c3":84551700064,"c4":"Y","c5":"Wvu9T8ilDl8k3oaB0PFNjHirDoMdwv1YYCah6ahqiuATp1","c6":1456291701229,"c7":79574400000,"c8":true,"c9":0.328379339497} +{"c1":50,"c2":1182933670,"c3":45692038432,"c4":"whFjvGloFQxwYmvLkshjwnADhUQlV","c5":"yt49EB4Y2RAnPQXNwTqfp7qzgDH1Jf1rjY0FnpIp2y4","c6":1456292105544,"c7":1278547200000,"c8":true,"c9":0.835195212647} +{"c1":51,"c2":980460718,"c3":5971386168,"c4":"OMJEMdywCTpQgqJUYDuKDxAPBkWdEP","c5":"akbcHSDflWlG8K6MFnIOOLBMZ4z8NV2FecgqpLO8","c6":1456292105544,"c7":-964915200000,"c8":true,"c9":0.0594494786115} +{"c1":52,"c2":-1263020365,"c3":17414114976,"c4":"vkUYqgXGGfELNjVwyYKVJMd","c5":"7zP0naU","c6":1456292105545,"c7":669600000000,"c8":false,"c9":0.128559830476} +{"c1":53,"c2":-2078473628,"c3":23813625600,"c4":"PVAKCnVrihXStlfqJf","c5":"HKyyDE7cHRTzEQX2F","c6":1456292105545,"c7":56764800000,"c8":true,"c9":0.497832439707} +{"c1":54,"c2":-388853860,"c3":4882213296,"c4":"NkSynIivNZiIplsAegulLrz","c5":"2AWYA664YJkqeGXGZeI8vf3fhX06wjhlUoRdzRBJnylTOnUsrz3LcyxbTyIQHr1YCMXl9jFx6LyhnaOYZorN4rOazR44zI2","c6":1456292105546,"c7":419731200000,"c8":false,"c9":0.806218120908} +{"c1":55,"c2":147766593,"c3":3544613368,"c4":"ByZRmalvDsZD","c5":"XgulsUB2lwxhZaMz0POOTcxIAt7mPaBialPFYzth6Aa54OPBoTY9nlJpXWf89oIczO0QVg5lbkmQQUlljIMEe2Mh8rjtDoYox2uDykAXeH0BXGVtOZ12EIwe","c6":1456292105546,"c7":-549158400000,"c8":true,"c9":0.189172246469} +{"c1":56,"c2":420230528,"c3":7209802880,"c4":"ZwiFgCchxUqJTEShealG","c5":"K7RQD052LFrkPBfuL9hIihch","c6":1456292105546,"c7":1149811200000,"c8":true,"c9":0.462810537514} +{"c1":57,"c2":-1894216997,"c3":5384805744,"c4":"eBKZaQYxrdPMRRwgeqPtJjxjqOBqUH","c5":"z7cJERSftFvbtxxgVbdF31jZCHmkHBmazWaHAa3eKAp1PPEObxkcuhjA1vf6cSlGe4TT8Yektdwqcr1Jd","c6":1456292105546,"c7":-1227484800000,"c8":true,"c9":0.604145500315} +{"c1":58,"c2":-263032861,"c3":56565941904,"c4":"dqXVKoNjMFKcIszKFRXCotmbejjxGU","c5":"3QDSogOdQYe6NfDPkEPIHhtECUKeT3eXsRdbwCj3omYIHz3HX","c6":1456292105547,"c7":-1192147200000,"c8":false,"c9":0.472430725946} +{"c1":59,"c2":-1661674051,"c3":9595495792,"c4":"atHZ","c5":"7SWlwF1qaCNs3Jea1NollJfLsgTK95yuo","c6":1456292105547,"c7":138240000000,"c8":true,"c9":0.0923867105652} +{"c1":60,"c2":-1129907976,"c3":6907332688,"c4":"Mc","c5":"4FN","c6":1456292105547,"c7":-192585600000,"c8":false,"c9":0.701638057307} +{"c1":61,"c2":-514254242,"c3":195633336,"c4":"kenCPo","c5":"eqVUbd9TzGrQS2W1HuV1apKCt1rkYLRTHgED2RIk1jhowGYzlRa2DVMpzTVc8RVO44rm9stOD9uvJPLNE48jVMQfhLwtNDlKXUchY8N0FOEsMW2G37X7dYnzod","c6":1456292105547,"c7":698716800000,"c8":true,"c9":0.511369563541} {"c1":62,"c2":null,"c3":null,"c4":null,"c5":null,"c6":null,"c7":null,"c8":null,"c9":null} -{"c1":63,"c2":-535361692,"c3":19818839912000624,"c4":"aAbGSv","c5":"yNVubV8dlc8KC8aijpwxEFyYSqtI8iiAQeZJAQMKUqTKMD5EqFSG4z3AraZ9dX","c6":1456292105548,"c7":422959968000000,"c8":true,"c9":0.587212121798} -{"c1":64,"c2":-960817944,"c3":295389276379024,"c4":"ToiqQVHAUTzNXZxhxT","c5":"pf7gyDzSkNhMD1Cio9GnoDpe7tFF6lPUfskrdSBrdmkPrwPN5YrF8cYArRXsqeiWiUwhbQ","c6":1456292105548,"c7":422200339200000,"c8":false,"c9":0.425544340933} -{"c1":65,"c2":2079413364,"c3":875274205296,"c4":"QiyvPpHnbIkkLsJeF","c5":"cuX2mutS6Dt23LpVfQ4RhuZ2upAZZxSDl3NQjHYecHX6UpRud5J5GUO2yKikEShe1AeqzPbW2W3GVn3pwpt5k9SFo5BbAQSgBpXWe7OUYuLB6XE0VPbSk6RK","c6":1456292105549,"c7":420572131200000,"c8":false,"c9":0.201374181269} -{"c1":66,"c2":-786819448,"c3":64868238648,"c4":"h","c5":"ScG74UmrI77csOCHrVQQMGnAtWw0IOpLtDTp6xF8eH2MxrvjwW5BhxTKxSelvvDK4LJlO0YhFW6CoZM62aUEkOmtXnTBp7KH4lAiCPW2lnC5DEHn","c6":1456292105549,"c7":420490656000000,"c8":true,"c9":0.795415967241} -{"c1":67,"c2":1250142164,"c3":6916992,"c4":"vpZxaEsZkTwnMmmWa","c5":"dQxtZCEAviuZYSdc4sgLjb","c6":1456292105549,"c7":422506281600000,"c8":false,"c9":0.556953353203} -{"c1":68,"c2":-38579995,"c3":233512622900,"c4":"JgwMVLyvDb","c5":"fwuYfa4PJlIQoxfetQdAaVY4VvbbX0N9E641","c6":1456292105550,"c7":423074275200000,"c8":false,"c9":0.453069633364} -{"c1":69,"c2":-1027828328,"c3":40672068432544096,"c4":"TZGIkEEfJb","c5":"RftjjeGNAAGDTikjbjJ6U4GxR8ZRGclneqPib9RyyXI37Rr8E3DstJ55jBpN4KY1j2Dp44l6NkRvDHQO","c6":1456292105550,"c7":422924976000000,"c8":false,"c9":0.998189105778} -{"c1":70,"c2":-569066973,"c3":8907959568528,"c4":"eah","c5":"2svWW0","c6":1456292148510,"c7":423076867200000,"c8":false,"c9":0.264360661376} -{"c1":71,"c2":815270244,"c3":15558122158,"c4":"l","c5":"5rZomJ4gkusDD","c6":1456292148510,"c7":420599520000000,"c8":true,"c9":0.85139020182} -{"c1":72,"c2":-2085514660,"c3":1345100636752,"c4":"Gb","c5":"VecY","c6":1456292148511,"c7":421304976000000,"c8":false,"c9":0.159955514577} -{"c1":73,"c2":-2086451715,"c3":778834561344,"c4":"LGGRD","c5":"Dv1","c6":1456292148511,"c7":422926185600000,"c8":false,"c9":0.892355433411} -{"c1":74,"c2":1317696149,"c3":13607285678,"c4":"Zf","c5":"h6rrw","c6":1456292148512,"c7":420486422400000,"c8":true,"c9":0.226484732} -{"c1":75,"c2":-475078365,"c3":591562831712,"c4":"Gas","c5":"Az81IOQGM1","c6":1456292148512,"c7":420418166400000,"c8":true,"c9":0.519357968945} -{"c1":76,"c2":-1610419012,"c3":987957004000,"c4":"u","c5":"RUUA2mTktOtMRpv","c6":1456292148512,"c7":421413148800000,"c8":true,"c9":0.283395173271} -{"c1":77,"c2":1,"c3":1,"c4":"gn","c5":"iyZ","c6":1456295748512,"c7":420525820800000,"c8":false,"c9":0.714975974474} -{"c1":78,"c2":841562550,"c3":5229353983640,"c4":"kfJpOk","c5":"BP4z","c6":1456292148512,"c7":420504998400000,"c8":true,"c9":0.773946465042} -{"c1":79,"c2":2049749227,"c3":7886072056192,"c4":"lJroaw","c5":"CS","c6":1456292148512,"c7":420713740800000,"c8":true,"c9":0.866341378246} -{"c1":80,"c2":-514969856,"c3":15673249928080,"c4":"BNnLZdPW","c5":"l5a0ZpK","c6":1456292148513,"c7":421434748800000,"c8":false,"c9":0.500069917482} -{"c1":81,"c2":-889738985,"c3":32298052657224,"c4":"xFDhe","c5":"yqzQp2PYA2w0r","c6":1456292148513,"c7":421726176000000,"c8":true,"c9":0.534138245169} -{"c1":82,"c2":743747735,"c3":42939563416968,"c4":"kcRx","c5":"ALGpDH9qQ","c6":1456292148513,"c7":420958080000000,"c8":true,"c9":0.0339015122383} -{"c1":83,"c2":-991380116,"c3":7521740669440,"c4":"I","c5":"VnrhwXNs5I7A","c6":1456292148513,"c7":420505689600000,"c8":false,"c9":0.729684203442} -{"c1":84,"c2":-605301438,"c3":6462434675584,"c4":"OXqgNP","c5":"XvH","c6":1456292148513,"c7":422746128000000,"c8":false,"c9":0.741169865031} -{"c1":85,"c2":-9600122,"c3":1150794800496,"c4":"XC","c5":"EZ5w","c6":1456292148545,"c7":421704144000000,"c8":false,"c9":0.00508533320078} -{"c1":86,"c2":-1304870155,"c3":7762439826812,"c4":"fOJJ","c5":"3kmqcAhqRv1JKH","c6":1456292148546,"c7":421705440000000,"c8":false,"c9":0.857763501506} -{"c1":87,"c2":-1151111063,"c3":3026615127030,"c4":"B","c5":"UsA2ijb5Q472","c6":1456292148546,"c7":422321299200000,"c8":true,"c9":0.852362483167} -{"c1":88,"c2":726080083,"c3":352777524,"c4":"pvjG","c5":"6DTPUc0kPXNDI","c6":1456292148546,"c7":422189884800000,"c8":true,"c9":0.499536294849} -{"c1":89,"c2":1269715757,"c3":298591968,"c4":"wUJle","c5":"P9hocYIzILG4d","c6":1456292148547,"c7":421664400000000,"c8":false,"c9":0.731465721495} -{"c1":90,"c2":469877353,"c3":5604128205368,"c4":"cgBBlTOr","c5":"0S09","c6":1456292148547,"c7":420358291200000,"c8":false,"c9":0.930318601251} -{"c1":91,"c2":1439825179,"c3":8448247315360,"c4":"GhHH","c5":"xU","c6":1456292148547,"c7":422529177600000,"c8":true,"c9":0.365706281943} -{"c1":92,"c2":-965564746,"c3":6461158733064,"c4":"ApKK","c5":"2MT6vQK7ILiMym","c6":1456292148548,"c7":422356982400000,"c8":true,"c9":0.160958193689} -{"c1":93,"c2":1396621364,"c3":5965143616184,"c4":"zavaYRc","c5":"m9Y","c6":1456292148548,"c7":421204924800000,"c8":true,"c9":0.123678441424} -{"c1":94,"c2":811012151,"c3":1275108040346,"c4":"yQmKx","c5":"ZHAGJpfsrd","c6":1456292148548,"c7":421316812800000,"c8":false,"c9":0.784623709976} -{"c1":95,"c2":1117245576,"c3":7044500976200,"c4":"fePHGR","c5":"pjLll4ngRT36ygIV","c6":1456292148548,"c7":421281993600000,"c8":false,"c9":0.187592141607} -{"c1":96,"c2":-450291430,"c3":7186198454512,"c4":"KhTYcGy","c5":"Y1l","c6":1456292148548,"c7":420275001600000,"c8":true,"c9":0.348668175704} -{"c1":97,"c2":0,"c3":2668027779964,"c4":"kdhbgC","c5":"ejaUw7d3DEIC7axy","c6":1456292148549,"c7":420673132800000,"c8":false,"c9":0.690933751147} -{"c1":98,"c2":-1807107154,"c3":667184558696,"c4":"C","c5":"M4GVt6c4s","c6":1456292148549,"c7":420653260800000,"c8":false,"c9":0.868842793576} -{"c1":99,"c2":-1610548040,"c3":269450722304,"c4":"UECld","c5":"DzTXQDe","c6":1456292148549,"c7":421828905600000,"c8":false,"c9":0.866249476344} -{"c1":100,"c2":1968262238,"c3":1853568964,"c4":"XY","c5":"O61h","c6":1456292148549,"c7":421527456000000,"c8":false,"c9":0.771913788526} -{"c1":101,"c2":547475512,"c3":5424751352,"c4":"Myf","c5":"u91","c6":1456292148549,"c7":421646774400000,"c8":false,"c9":0.020687569041} -{"c1":102,"c2":1347375329,"c3":7601674368,"c4":"PubTJ","c5":"B72D746BYbWS","c6":1456292148550,"c7":420430867200000,"c8":true,"c9":0.669628865357} -{"c1":103,"c2":-730200083,"c3":3734160392,"c4":"R","c5":"JF5b","c6":1456292148550,"c7":421339708800000,"c8":false,"c9":0.754789130107} -{"c1":104,"c2":-2145249349,"c3":8153964480,"c4":"BAGi","c5":"o69DyecndyD47","c6":1456292148550,"c7":421667078400000,"c8":true,"c9":0.669354481782} -{"c1":105,"c2":1922528937,"c3":36022570792,"c4":"lJy","c5":"8iirhhzlmMQeL","c6":1456292148550,"c7":422019849600000,"c8":false,"c9":0.53821290676} -{"c1":106,"c2":353908213,"c3":108831386830,"c4":"Z","c5":"vb32KaNk","c6":1456292148550,"c7":422921692800000,"c8":true,"c9":0.772742852356} -{"c1":107,"c2":-1,"c3":null,"c4":"kYEB","c5":"UY1wFk3nA6uIg","c6":1456292148550,"c7":423125078400000,"c8":false,"c9":0.00471118728446} -{"c1":108,"c2":465659139,"c3":196283469856,"c4":"RVVu","c5":"crs1mym","c6":1456292148550,"c7":422165001600000,"c8":true,"c9":0.510821715226} -{"c1":109,"c2":-1330396672,"c3":584831936,"c4":null,"c5":"XxXpO","c6":1456292302869,"c7":420166310400000,"c8":false,"c9":0.0730700338964} -{"c1":110,"c2":-220130197,"c3":17601020,"c4":"hjeac","c5":"n","c6":1456292302869,"c7":422323977600000,"c8":true,"c9":0.96826163121} -{"c1":111,"c2":-42902,"c3":562336,"c4":"f","c5":"LykOIn","c6":1456292302870,"c7":420630624000000,"c8":true,"c9":0.554463637243} -{"c1":112,"c2":765525961,"c3":28891422336265300,"c4":"demso","c5":"LZoFRza","c6":1456292302870,"c7":420894057600000,"c8":true,"c9":0.850373501413} -{"c1":113,"c2":-68207234,"c3":90427790806585824,"c4":"pnmrt","c5":"HFo","c6":null,"c7":420867878400000,"c8":true,"c9":0.71373237138} -{"c1":114,"c2":-1380251111,"c3":63163365737847936,"c4":"krg","c5":"JXzNCV","c6":1456292302871,"c7":421624569600000,"c8":true,"c9":0.445900639514} -{"c1":115,"c2":-1844719171,"c3":60499043191834664,"c4":"tdqfhi","c5":"SzMEux","c6":1456292302871,"c7":422194809600000,"c8":true,"c9":0.520455630248} -{"c1":116,"c2":1790522162,"c3":47372667161181496,"c4":"orbowc","c5":"n","c6":1456292302871,"c7":421364678400000,"c8":true,"c9":0.891198277906} -{"c1":117,"c2":-27319471,"c3":37102817894137256,"c4":"w","c5":"SYvkQ","c6":1456292302871,"c7":420918681600000,"c8":false,"c9":0.0359150523033} -{"c1":118,"c2":1317690445,"c3":61958708627376736,"c4":"aehmn","c5":"Vioq","c6":1456292302871,"c7":420640732800000,"c8":false,"c9":0.943878204382} -{"c1":119,"c2":1965293154,"c3":4216354032582922,"c4":"d","c5":"RSlQvgb","c6":1456292302872,"c7":420903561600000,"c8":true,"c9":0.300925886142} -{"c1":120,"c2":-1998776606,"c3":80843181252171408,"c4":"hntp","c5":"lt","c6":1456292302872,"c7":422202499200000,"c8":true,"c9":0.81436509096} +{"c1":63,"c2":-535361692,"c3":19818839912000624,"c4":"aAbGSv","c5":"yNVubV8dlc8KC8aijpwxEFyYSqtI8iiAQeZJAQMKUqTKMD5EqFSG4z3AraZ9dX","c6":1456292105548,"c7":1226361600000,"c8":true,"c9":0.587212121798} +{"c1":64,"c2":-960817944,"c3":295389276379024,"c4":"ToiqQVHAUTzNXZxhxT","c5":"pf7gyDzSkNhMD1Cio9GnoDpe7tFF6lPUfskrdSBrdmkPrwPN5YrF8cYArRXsqeiWiUwhbQ","c6":1456292105548,"c7":466732800000,"c8":false,"c9":0.425544340933} +{"c1":65,"c2":2079413364,"c3":875274205296,"c4":"QiyvPpHnbIkkLsJeF","c5":"cuX2mutS6Dt23LpVfQ4RhuZ2upAZZxSDl3NQjHYecHX6UpRud5J5GUO2yKikEShe1AeqzPbW2W3GVn3pwpt5k9SFo5BbAQSgBpXWe7OUYuLB6XE0VPbSk6RK","c6":1456292105549,"c7":-1161475200000,"c8":false,"c9":0.201374181269} +{"c1":66,"c2":-786819448,"c3":64868238648,"c4":"h","c5":"ScG74UmrI77csOCHrVQQMGnAtWw0IOpLtDTp6xF8eH2MxrvjwW5BhxTKxSelvvDK4LJlO0YhFW6CoZM62aUEkOmtXnTBp7KH4lAiCPW2lnC5DEHn","c6":1456292105549,"c7":-1242950400000,"c8":true,"c9":0.795415967241} +{"c1":67,"c2":1250142164,"c3":6916992,"c4":"vpZxaEsZkTwnMmmWa","c5":"dQxtZCEAviuZYSdc4sgLjb","c6":1456292105549,"c7":772675200000,"c8":false,"c9":0.556953353203} +{"c1":68,"c2":-38579995,"c3":233512622900,"c4":"JgwMVLyvDb","c5":"fwuYfa4PJlIQoxfetQdAaVY4VvbbX0N9E641","c6":1456292105550,"c7":1340668800000,"c8":false,"c9":0.453069633364} +{"c1":69,"c2":-1027828328,"c3":40672068432544096,"c4":"TZGIkEEfJb","c5":"RftjjeGNAAGDTikjbjJ6U4GxR8ZRGclneqPib9RyyXI37Rr8E3DstJ55jBpN4KY1j2Dp44l6NkRvDHQO","c6":1456292105550,"c7":1191369600000,"c8":false,"c9":0.998189105778} +{"c1":70,"c2":-569066973,"c3":8907959568528,"c4":"eah","c5":"2svWW0","c6":1456292148510,"c7":1343260800000,"c8":false,"c9":0.264360661376} +{"c1":71,"c2":815270244,"c3":15558122158,"c4":"l","c5":"5rZomJ4gkusDD","c6":1456292148510,"c7":-1134086400000,"c8":true,"c9":0.85139020182} +{"c1":72,"c2":-2085514660,"c3":1345100636752,"c4":"Gb","c5":"VecY","c6":1456292148511,"c7":-428630400000,"c8":false,"c9":0.159955514577} +{"c1":73,"c2":-2086451715,"c3":778834561344,"c4":"LGGRD","c5":"Dv1","c6":1456292148511,"c7":1192579200000,"c8":false,"c9":0.892355433411} +{"c1":74,"c2":1317696149,"c3":13607285678,"c4":"Zf","c5":"h6rrw","c6":1456292148512,"c7":-1247184000000,"c8":true,"c9":0.226484732} +{"c1":75,"c2":-475078365,"c3":591562831712,"c4":"Gas","c5":"Az81IOQGM1","c6":1456292148512,"c7":-1315440000000,"c8":true,"c9":0.519357968945} +{"c1":76,"c2":-1610419012,"c3":987957004000,"c4":"u","c5":"RUUA2mTktOtMRpv","c6":1456292148512,"c7":-320457600000,"c8":true,"c9":0.283395173271} +{"c1":77,"c2":1,"c3":1,"c4":"gn","c5":"iyZ","c6":1456295748512,"c7":-1207785600000,"c8":false,"c9":0.714975974474} +{"c1":78,"c2":841562550,"c3":5229353983640,"c4":"kfJpOk","c5":"BP4z","c6":1456292148512,"c7":-1228608000000,"c8":true,"c9":0.773946465042} +{"c1":79,"c2":2049749227,"c3":7886072056192,"c4":"lJroaw","c5":"CS","c6":1456292148512,"c7":-1019865600000,"c8":true,"c9":0.866341378246} +{"c1":80,"c2":-514969856,"c3":15673249928080,"c4":"BNnLZdPW","c5":"l5a0ZpK","c6":1456292148513,"c7":-298857600000,"c8":false,"c9":0.500069917482} +{"c1":81,"c2":-889738985,"c3":32298052657224,"c4":"xFDhe","c5":"yqzQp2PYA2w0r","c6":1456292148513,"c7":-7430400000,"c8":true,"c9":0.534138245169} +{"c1":82,"c2":743747735,"c3":42939563416968,"c4":"kcRx","c5":"ALGpDH9qQ","c6":1456292148513,"c7":-775526400000,"c8":true,"c9":0.0339015122383} +{"c1":83,"c2":-991380116,"c3":7521740669440,"c4":"I","c5":"VnrhwXNs5I7A","c6":1456292148513,"c7":-1227916800000,"c8":false,"c9":0.729684203442} +{"c1":84,"c2":-605301438,"c3":6462434675584,"c4":"OXqgNP","c5":"XvH","c6":1456292148513,"c7":1012521600000,"c8":false,"c9":0.741169865031} +{"c1":85,"c2":-9600122,"c3":1150794800496,"c4":"XC","c5":"EZ5w","c6":1456292148545,"c7":-29462400000,"c8":false,"c9":0.00508533320078} +{"c1":86,"c2":-1304870155,"c3":7762439826812,"c4":"fOJJ","c5":"3kmqcAhqRv1JKH","c6":1456292148546,"c7":-28166400000,"c8":false,"c9":0.857763501506} +{"c1":87,"c2":-1151111063,"c3":3026615127030,"c4":"B","c5":"UsA2ijb5Q472","c6":1456292148546,"c7":587692800000,"c8":true,"c9":0.852362483167} +{"c1":88,"c2":726080083,"c3":352777524,"c4":"pvjG","c5":"6DTPUc0kPXNDI","c6":1456292148546,"c7":456278400000,"c8":true,"c9":0.499536294849} +{"c1":89,"c2":1269715757,"c3":298591968,"c4":"wUJle","c5":"P9hocYIzILG4d","c6":1456292148547,"c7":-69206400000,"c8":false,"c9":0.731465721495} +{"c1":90,"c2":469877353,"c3":5604128205368,"c4":"cgBBlTOr","c5":"0S09","c6":1456292148547,"c7":-1375315200000,"c8":false,"c9":0.930318601251} +{"c1":91,"c2":1439825179,"c3":8448247315360,"c4":"GhHH","c5":"xU","c6":1456292148547,"c7":795571200000,"c8":true,"c9":0.365706281943} +{"c1":92,"c2":-965564746,"c3":6461158733064,"c4":"ApKK","c5":"2MT6vQK7ILiMym","c6":1456292148548,"c7":623376000000,"c8":true,"c9":0.160958193689} +{"c1":93,"c2":1396621364,"c3":5965143616184,"c4":"zavaYRc","c5":"m9Y","c6":1456292148548,"c7":-528681600000,"c8":true,"c9":0.123678441424} +{"c1":94,"c2":811012151,"c3":1275108040346,"c4":"yQmKx","c5":"ZHAGJpfsrd","c6":1456292148548,"c7":-416793600000,"c8":false,"c9":0.784623709976} +{"c1":95,"c2":1117245576,"c3":7044500976200,"c4":"fePHGR","c5":"pjLll4ngRT36ygIV","c6":1456292148548,"c7":-451612800000,"c8":false,"c9":0.187592141607} +{"c1":96,"c2":-450291430,"c3":7186198454512,"c4":"KhTYcGy","c5":"Y1l","c6":1456292148548,"c7":-1458604800000,"c8":true,"c9":0.348668175704} +{"c1":97,"c2":0,"c3":2668027779964,"c4":"kdhbgC","c5":"ejaUw7d3DEIC7axy","c6":1456292148549,"c7":-1060473600000,"c8":false,"c9":0.690933751147} +{"c1":98,"c2":-1807107154,"c3":667184558696,"c4":"C","c5":"M4GVt6c4s","c6":1456292148549,"c7":-1080345600000,"c8":false,"c9":0.868842793576} +{"c1":99,"c2":-1610548040,"c3":269450722304,"c4":"UECld","c5":"DzTXQDe","c6":1456292148549,"c7":95299200000,"c8":false,"c9":0.866249476344} +{"c1":100,"c2":1968262238,"c3":1853568964,"c4":"XY","c5":"O61h","c6":1456292148549,"c7":-206150400000,"c8":false,"c9":0.771913788526} +{"c1":101,"c2":547475512,"c3":5424751352,"c4":"Myf","c5":"u91","c6":1456292148549,"c7":-86832000000,"c8":false,"c9":0.020687569041} +{"c1":102,"c2":1347375329,"c3":7601674368,"c4":"PubTJ","c5":"B72D746BYbWS","c6":1456292148550,"c7":-1302739200000,"c8":true,"c9":0.669628865357} +{"c1":103,"c2":-730200083,"c3":3734160392,"c4":"R","c5":"JF5b","c6":1456292148550,"c7":-393897600000,"c8":false,"c9":0.754789130107} +{"c1":104,"c2":-2145249349,"c3":8153964480,"c4":"BAGi","c5":"o69DyecndyD47","c6":1456292148550,"c7":-66528000000,"c8":true,"c9":0.669354481782} +{"c1":105,"c2":1922528937,"c3":36022570792,"c4":"lJy","c5":"8iirhhzlmMQeL","c6":1456292148550,"c7":286243200000,"c8":false,"c9":0.53821290676} +{"c1":106,"c2":353908213,"c3":108831386830,"c4":"Z","c5":"vb32KaNk","c6":1456292148550,"c7":1188086400000,"c8":true,"c9":0.772742852356} +{"c1":107,"c2":-1,"c3":null,"c4":"kYEB","c5":"UY1wFk3nA6uIg","c6":1456292148550,"c7":1391472000000,"c8":false,"c9":0.00471118728446} +{"c1":108,"c2":465659139,"c3":196283469856,"c4":"RVVu","c5":"crs1mym","c6":1456292148550,"c7":431395200000,"c8":true,"c9":0.510821715226} +{"c1":109,"c2":-1330396672,"c3":584831936,"c4":null,"c5":"XxXpO","c6":1456292302869,"c7":-1567296000000,"c8":false,"c9":0.0730700338964} +{"c1":110,"c2":-220130197,"c3":17601020,"c4":"hjeac","c5":"n","c6":1456292302869,"c7":590371200000,"c8":true,"c9":0.96826163121} +{"c1":111,"c2":-42902,"c3":562336,"c4":"f","c5":"LykOIn","c6":1456292302870,"c7":-1102982400000,"c8":true,"c9":0.554463637243} +{"c1":112,"c2":765525961,"c3":28891422336265300,"c4":"demso","c5":"LZoFRza","c6":1456292302870,"c7":-839548800000,"c8":true,"c9":0.850373501413} +{"c1":113,"c2":-68207234,"c3":90427790806585824,"c4":"pnmrt","c5":"HFo","c6":null,"c7":-865728000000,"c8":true,"c9":0.71373237138} +{"c1":114,"c2":-1380251111,"c3":63163365737847936,"c4":"krg","c5":"JXzNCV","c6":1456292302871,"c7":-109036800000,"c8":true,"c9":0.445900639514} +{"c1":115,"c2":-1844719171,"c3":60499043191834664,"c4":"tdqfhi","c5":"SzMEux","c6":1456292302871,"c7":461203200000,"c8":true,"c9":0.520455630248} +{"c1":116,"c2":1790522162,"c3":47372667161181496,"c4":"orbowc","c5":"n","c6":1456292302871,"c7":-368928000000,"c8":true,"c9":0.891198277906} +{"c1":117,"c2":-27319471,"c3":37102817894137256,"c4":"w","c5":"SYvkQ","c6":1456292302871,"c7":-814924800000,"c8":false,"c9":0.0359150523033} +{"c1":118,"c2":1317690445,"c3":61958708627376736,"c4":"aehmn","c5":"Vioq","c6":1456292302871,"c7":-1092873600000,"c8":false,"c9":0.943878204382} +{"c1":119,"c2":1965293154,"c3":4216354032582922,"c4":"d","c5":"RSlQvgb","c6":1456292302872,"c7":-830044800000,"c8":true,"c9":0.300925886142} +{"c1":120,"c2":-1998776606,"c3":80843181252171408,"c4":"hntp","c5":"lt","c6":1456292302872,"c7":468892800000,"c8":true,"c9":0.81436509096} {"c1":121,"c2":-759066837,"c3":56811856992247296,"c4":"gbfr","c5":"RXwJTUQL","c6":1456292302872,"c7":null,"c8":true,"c9":0.602622967887} -{"c1":122,"c2":null,"c3":16079752775114834,"c4":"i","c5":"wOKgo","c6":1456292302872,"c7":421707340800000,"c8":false,"c9":0.314540418049} -{"c1":123,"c2":-1393583038,"c3":21198598572581100,"c4":"cg","c5":"jJo","c6":1456292302872,"c7":421884806400000,"c8":true,"c9":0.863833034127} -{"c1":124,"c2":-359360839,"c3":10812882337641318,"c4":"yn","c5":"pp","c6":1456292302872,"c7":422279222400000,"c8":true,"c9":0.984219215225} -{"c1":125,"c2":-135559442,"c3":26079995480527800,"c4":"u","c5":"PpZ","c6":1456292302872,"c7":421338672000000,"c8":true,"c9":0.112619968355} -{"c1":126,"c2":-963314220,"c3":39899141337524696,"c4":"r","c5":"Kv","c6":1456292302872,"c7":421093209600000,"c8":false,"c9":0.928182322637} -{"c1":127,"c2":-603033839,"c3":91979853046157776,"c4":"iknfwt","c5":"UxrG","c6":1456292302873,"c7":422923593600000,"c8":true,"c9":0.436065582677} -{"c1":128,"c2":1730933740,"c3":52988296769210224,"c4":null,"c5":"SQreQ","c6":1456292338864,"c7":420160435200000,"c8":null,"c9":0.709400581459} -{"c1":129,"c2":-1190592568,"c3":29537626363643852,"c4":null,"c5":"JNq","c6":1456292338864,"c7":421020892800000,"c8":false,"c9":0.426506472311} -{"c1":130,"c2":-1311152842,"c3":24893851443643300,"c4":null,"c5":"VZmvS","c6":1456292338865,"c7":421283203200000,"c8":true,"c9":0.891508292788} -{"c1":131,"c2":-612732312,"c3":14972217694439486,"c4":null,"c5":"yZYlsOH","c6":1456292338865,"c7":420777676800000,"c8":true,"c9":0.682577092573} -{"c1":132,"c2":-1948510563,"c3":2686369516214692,"c4":null,"c5":"HcwNvv","c6":1456292338865,"c7":420190416000000,"c8":true,"c9":0.496484780895} -{"c1":133,"c2":1726823975,"c3":64270113732818952,"c4":null,"c5":"UPYQXFGA","c6":1456292338865,"c7":421703625600000,"c8":true,"c9":0.858728636519} -{"c1":134,"c2":220535371,"c3":15937600070688400,"c4":null,"c5":"tTsISx","c6":1456292338866,"c7":421422134400000,"c8":null,"c9":0.0978174122941} -{"c1":135,"c2":-278647794,"c3":52598911986023288,"c4":null,"c5":"JzQY","c6":1456292338866,"c7":421918416000000,"c8":false,"c9":0.0918752610245} -{"c1":136,"c2":288414270,"c3":8695637273720187,"c4":null,"c5":"OpFxaJUK","c6":1456292338866,"c7":420629328000000,"c8":true,"c9":0.0241314164497} -{"c1":137,"c2":-1033256154,"c3":2397384354833930,"c4":null,"c5":null,"c6":1456292369484,"c7":421578086400000,"c8":true,"c9":0.817785665204} -{"c1":138,"c2":-461761396,"c3":78042258136748336,"c4":null,"c5":null,"c6":1456292369484,"c7":421621632000000,"c8":true,"c9":0.514550507111} -{"c1":139,"c2":-1863053037,"c3":35173898947893320,"c4":null,"c5":null,"c6":1456292369485,"c7":420712531200000,"c8":true,"c9":0.641830766022} -{"c1":140,"c2":-775202711,"c3":21011901540311080,"c4":null,"c5":null,"c6":1456292369485,"c7":420698448000000,"c8":false,"c9":0.563296650508} -{"c1":141,"c2":-109815917,"c3":46096729108954488,"c4":null,"c5":null,"c6":1456292369485,"c7":421707772800000,"c8":null,"c9":0.82789510574} -{"c1":142,"c2":1733733471,"c3":84276421148786976,"c4":null,"c5":null,"c6":1456292369485,"c7":420706828800000,"c8":true,"c9":0.737905542986} -{"c1":143,"c2":1559768019,"c3":91381489058732016,"c4":null,"c5":null,"c6":1456292369486,"c7":422574019200000,"c8":true,"c9":null} -{"c1":144,"c2":-1054306811,"c3":null,"c4":null,"c5":null,"c6":1456292369486,"c7":421684876800000,"c8":false,"c9":0.273335960277} -{"c1":145,"c2":1580853588,"c3":17990322900862228,"c4":null,"c5":null,"c6":1456292369486,"c7":422222025600000,"c8":false,"c9":0.668501595682} \ No newline at end of file +{"c1":122,"c2":null,"c3":16079752775114834,"c4":"i","c5":"wOKgo","c6":1456292302872,"c7":-26265600000,"c8":false,"c9":0.314540418049} +{"c1":123,"c2":-1393583038,"c3":21198598572581100,"c4":"cg","c5":"jJo","c6":1456292302872,"c7":151200000000,"c8":true,"c9":0.863833034127} +{"c1":124,"c2":-359360839,"c3":10812882337641318,"c4":"yn","c5":"pp","c6":1456292302872,"c7":545616000000,"c8":true,"c9":0.984219215225} +{"c1":125,"c2":-135559442,"c3":26079995480527800,"c4":"u","c5":"PpZ","c6":1456292302872,"c7":-394934400000,"c8":true,"c9":0.112619968355} +{"c1":126,"c2":-963314220,"c3":39899141337524696,"c4":"r","c5":"Kv","c6":1456292302872,"c7":-640396800000,"c8":false,"c9":0.928182322637} +{"c1":127,"c2":-603033839,"c3":91979853046157776,"c4":"iknfwt","c5":"UxrG","c6":1456292302873,"c7":1189987200000,"c8":true,"c9":0.436065582677} +{"c1":128,"c2":1730933740,"c3":52988296769210224,"c4":null,"c5":"SQreQ","c6":1456292338864,"c7":-1573171200000,"c8":null,"c9":0.709400581459} +{"c1":129,"c2":-1190592568,"c3":29537626363643852,"c4":null,"c5":"JNq","c6":1456292338864,"c7":-712713600000,"c8":false,"c9":0.426506472311} +{"c1":130,"c2":-1311152842,"c3":24893851443643300,"c4":null,"c5":"VZmvS","c6":1456292338865,"c7":-450403200000,"c8":true,"c9":0.891508292788} +{"c1":131,"c2":-612732312,"c3":14972217694439486,"c4":null,"c5":"yZYlsOH","c6":1456292338865,"c7":-955929600000,"c8":true,"c9":0.682577092573} +{"c1":132,"c2":-1948510563,"c3":2686369516214692,"c4":null,"c5":"HcwNvv","c6":1456292338865,"c7":-1543190400000,"c8":true,"c9":0.496484780895} +{"c1":133,"c2":1726823975,"c3":64270113732818952,"c4":null,"c5":"UPYQXFGA","c6":1456292338865,"c7":-29980800000,"c8":true,"c9":0.858728636519} +{"c1":134,"c2":220535371,"c3":15937600070688400,"c4":null,"c5":"tTsISx","c6":1456292338866,"c7":-311472000000,"c8":null,"c9":0.0978174122941} +{"c1":135,"c2":-278647794,"c3":52598911986023288,"c4":null,"c5":"JzQY","c6":1456292338866,"c7":184809600000,"c8":false,"c9":0.0918752610245} +{"c1":136,"c2":288414270,"c3":8695637273720187,"c4":null,"c5":"OpFxaJUK","c6":1456292338866,"c7":-1104278400000,"c8":true,"c9":0.0241314164497} +{"c1":137,"c2":-1033256154,"c3":2397384354833930,"c4":null,"c5":null,"c6":1456292369484,"c7":-155520000000,"c8":true,"c9":0.817785665204} +{"c1":138,"c2":-461761396,"c3":78042258136748336,"c4":null,"c5":null,"c6":1456292369484,"c7":-111974400000,"c8":true,"c9":0.514550507111} +{"c1":139,"c2":-1863053037,"c3":35173898947893320,"c4":null,"c5":null,"c6":1456292369485,"c7":-1021075200000,"c8":true,"c9":0.641830766022} +{"c1":140,"c2":-775202711,"c3":21011901540311080,"c4":null,"c5":null,"c6":1456292369485,"c7":-1035158400000,"c8":false,"c9":0.563296650508} +{"c1":141,"c2":-109815917,"c3":46096729108954488,"c4":null,"c5":null,"c6":1456292369485,"c7":-25833600000,"c8":null,"c9":0.82789510574} +{"c1":142,"c2":1733733471,"c3":84276421148786976,"c4":null,"c5":null,"c6":1456292369485,"c7":-1026777600000,"c8":true,"c9":0.737905542986} +{"c1":143,"c2":1559768019,"c3":91381489058732016,"c4":null,"c5":null,"c6":1456292369486,"c7":840412800000,"c8":true,"c9":null} +{"c1":144,"c2":-1054306811,"c3":null,"c4":null,"c5":null,"c6":1456292369486,"c7":-48729600000,"c8":false,"c9":0.273335960277} +{"c1":145,"c2":1580853588,"c3":17990322900862228,"c4":null,"c5":null,"c6":1456292369486,"c7":488419200000,"c8":false,"c9":0.668501595682} \ No newline at end of file diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx index 447a6379487b..af39c2d0385b 100644 --- a/web-console/src/views/load-data-view/load-data-view.tsx +++ b/web-console/src/views/load-data-view/load-data-view.tsx @@ -3178,8 +3178,24 @@ export class LoadDataView extends React.PureComponent (v ? (appendToExisting ? 'APPEND' : 'REPLACE') : undefined), - adjustValue: v => v === (appendToExisting ? 'APPEND' : 'REPLACE'), + valueAdjustment: v => { + if (!v) return undefined; + + if (isStreamingSpec(spec)) { + return 'APPEND'; + } else { + return appendToExisting ? 'APPEND' : 'REPLACE'; + } + }, + adjustValue: v => { + if (v === undefined) return false; + + if (isStreamingSpec(spec)) { + return v === 'APPEND'; + } + + return v === (appendToExisting ? 'APPEND' : 'REPLACE'); + }, info:

    Allows or forbids concurrent tasks.

    , }, ]}