Search in sources :

Example 11 with Pair

use of io.druid.java.util.common.Pair in project druid by druid-io.

the class ServerManagerTest method testDelete1.

@Test
public void testDelete1() throws Exception {
    final String dataSouce = "test";
    final Interval interval = new Interval("2011-04-01/2011-04-02");
    Future future = assertQueryable(Granularities.DAY, dataSouce, interval, ImmutableList.<Pair<String, Interval>>of(new Pair<String, Interval>("2", interval)));
    waitForTestVerificationAndCleanup(future);
    dropQueryable(dataSouce, "2", interval);
    future = assertQueryable(Granularities.DAY, dataSouce, interval, ImmutableList.<Pair<String, Interval>>of(new Pair<String, Interval>("1", interval)));
    waitForTestVerificationAndCleanup(future);
}
Also used : Future(java.util.concurrent.Future) Interval(org.joda.time.Interval) Pair(io.druid.java.util.common.Pair) Test(org.junit.Test)

Example 12 with Pair

use of io.druid.java.util.common.Pair in project druid by druid-io.

the class ServerManagerTest method testSimpleGet.

@Test
public void testSimpleGet() {
    Future future = assertQueryable(Granularities.DAY, "test", new Interval("P1d/2011-04-01"), ImmutableList.<Pair<String, Interval>>of(new Pair<String, Interval>("1", new Interval("P1d/2011-04-01"))));
    waitForTestVerificationAndCleanup(future);
    future = assertQueryable(Granularities.DAY, "test", new Interval("P2d/2011-04-02"), ImmutableList.<Pair<String, Interval>>of(new Pair<String, Interval>("1", new Interval("P1d/2011-04-01")), new Pair<String, Interval>("2", new Interval("P1d/2011-04-02"))));
    waitForTestVerificationAndCleanup(future);
}
Also used : Future(java.util.concurrent.Future) Interval(org.joda.time.Interval) Pair(io.druid.java.util.common.Pair) Test(org.junit.Test)

Example 13 with Pair

use of io.druid.java.util.common.Pair in project druid by druid-io.

the class ServerManagerTest method testDelete2.

@Test
public void testDelete2() throws Exception {
    loadQueryable("test", "3", new Interval("2011-04-04/2011-04-05"));
    Future future = assertQueryable(Granularities.DAY, "test", new Interval("2011-04-04/2011-04-06"), ImmutableList.<Pair<String, Interval>>of(new Pair<String, Interval>("3", new Interval("2011-04-04/2011-04-05"))));
    waitForTestVerificationAndCleanup(future);
    dropQueryable("test", "3", new Interval("2011-04-04/2011-04-05"));
    dropQueryable("test", "1", new Interval("2011-04-04/2011-04-05"));
    future = assertQueryable(Granularities.HOUR, "test", new Interval("2011-04-04/2011-04-04T06"), ImmutableList.<Pair<String, Interval>>of(new Pair<String, Interval>("2", new Interval("2011-04-04T00/2011-04-04T01")), new Pair<String, Interval>("2", new Interval("2011-04-04T01/2011-04-04T02")), new Pair<String, Interval>("2", new Interval("2011-04-04T02/2011-04-04T03")), new Pair<String, Interval>("2", new Interval("2011-04-04T04/2011-04-04T05")), new Pair<String, Interval>("2", new Interval("2011-04-04T05/2011-04-04T06"))));
    waitForTestVerificationAndCleanup(future);
    future = assertQueryable(Granularities.HOUR, "test", new Interval("2011-04-04/2011-04-04T03"), ImmutableList.<Pair<String, Interval>>of(new Pair<String, Interval>("2", new Interval("2011-04-04T00/2011-04-04T01")), new Pair<String, Interval>("2", new Interval("2011-04-04T01/2011-04-04T02")), new Pair<String, Interval>("2", new Interval("2011-04-04T02/2011-04-04T03"))));
    waitForTestVerificationAndCleanup(future);
    future = assertQueryable(Granularities.HOUR, "test", new Interval("2011-04-04T04/2011-04-04T06"), ImmutableList.<Pair<String, Interval>>of(new Pair<String, Interval>("2", new Interval("2011-04-04T04/2011-04-04T05")), new Pair<String, Interval>("2", new Interval("2011-04-04T05/2011-04-04T06"))));
    waitForTestVerificationAndCleanup(future);
}
Also used : Future(java.util.concurrent.Future) Interval(org.joda.time.Interval) Pair(io.druid.java.util.common.Pair) Test(org.junit.Test)

Example 14 with Pair

use of io.druid.java.util.common.Pair in project druid by druid-io.

the class QuotableWhiteSpaceSplitter method restore.

@Override
public List<Pair<Task, ListenableFuture<TaskStatus>>> restore() {
    final File restoreFile = getRestoreFile();
    final TaskRestoreInfo taskRestoreInfo;
    if (restoreFile.exists()) {
        try {
            taskRestoreInfo = jsonMapper.readValue(restoreFile, TaskRestoreInfo.class);
        } catch (Exception e) {
            log.error(e, "Failed to read restorable tasks from file[%s]. Skipping restore.", restoreFile);
            return ImmutableList.of();
        }
    } else {
        return ImmutableList.of();
    }
    final List<Pair<Task, ListenableFuture<TaskStatus>>> retVal = Lists.newArrayList();
    for (final String taskId : taskRestoreInfo.getRunningTasks()) {
        try {
            final File taskFile = new File(taskConfig.getTaskDir(taskId), "task.json");
            final Task task = jsonMapper.readValue(taskFile, Task.class);
            if (!task.getId().equals(taskId)) {
                throw new ISE("WTF?! Task[%s] restore file had wrong id[%s].", taskId, task.getId());
            }
            if (taskConfig.isRestoreTasksOnRestart() && task.canRestore()) {
                log.info("Restoring task[%s].", task.getId());
                retVal.add(Pair.of(task, run(task)));
            }
        } catch (Exception e) {
            log.warn(e, "Failed to restore task[%s]. Trying to restore other tasks.", taskId);
        }
    }
    log.info("Restored %,d tasks.", retVal.size());
    return retVal;
}
Also used : Task(io.druid.indexing.common.task.Task) ISE(io.druid.java.util.common.ISE) TaskStatus(io.druid.indexing.common.TaskStatus) File(java.io.File) IOException(java.io.IOException) Pair(io.druid.java.util.common.Pair)

Example 15 with Pair

use of io.druid.java.util.common.Pair in project druid by druid-io.

the class RealtimeIndexTaskTest method testRestore.

@Test(timeout = 60_000L)
public void testRestore() throws Exception {
    final File directory = tempFolder.newFolder();
    final RealtimeIndexTask task1 = makeRealtimeTask(null);
    final DataSegment publishedSegment;
    // First run:
    {
        final TestIndexerMetadataStorageCoordinator mdc = new TestIndexerMetadataStorageCoordinator();
        final TaskToolbox taskToolbox = makeToolbox(task1, mdc, directory);
        final ListenableFuture<TaskStatus> statusFuture = runTask(task1, taskToolbox);
        // Wait for firehose to show up, it starts off null.
        while (task1.getFirehose() == null) {
            Thread.sleep(50);
        }
        final TestFirehose firehose = (TestFirehose) task1.getFirehose();
        firehose.addRows(ImmutableList.<InputRow>of(new MapBasedInputRow(now, ImmutableList.of("dim1"), ImmutableMap.<String, Object>of("dim1", "foo"))));
        // Trigger graceful shutdown.
        task1.stopGracefully();
        // Wait for the task to finish. The status doesn't really matter, but we'll check it anyway.
        final TaskStatus taskStatus = statusFuture.get();
        Assert.assertEquals(TaskStatus.Status.SUCCESS, taskStatus.getStatusCode());
        // Nothing should be published.
        Assert.assertEquals(Sets.newHashSet(), mdc.getPublished());
    }
    // Second run:
    {
        final TestIndexerMetadataStorageCoordinator mdc = new TestIndexerMetadataStorageCoordinator();
        final RealtimeIndexTask task2 = makeRealtimeTask(task1.getId());
        final TaskToolbox taskToolbox = makeToolbox(task2, mdc, directory);
        final ListenableFuture<TaskStatus> statusFuture = runTask(task2, taskToolbox);
        // Wait for firehose to show up, it starts off null.
        while (task2.getFirehose() == null) {
            Thread.sleep(50);
        }
        // Do a query, at this point the previous data should be loaded.
        Assert.assertEquals(1, sumMetric(task2, "rows"));
        final TestFirehose firehose = (TestFirehose) task2.getFirehose();
        firehose.addRows(ImmutableList.<InputRow>of(new MapBasedInputRow(now, ImmutableList.of("dim2"), ImmutableMap.<String, Object>of("dim2", "bar"))));
        // Stop the firehose, this will drain out existing events.
        firehose.close();
        // Wait for publish.
        while (mdc.getPublished().isEmpty()) {
            Thread.sleep(50);
        }
        publishedSegment = Iterables.getOnlyElement(mdc.getPublished());
        // Do a query.
        Assert.assertEquals(2, sumMetric(task2, "rows"));
        // Simulate handoff.
        for (Map.Entry<SegmentDescriptor, Pair<Executor, Runnable>> entry : handOffCallbacks.entrySet()) {
            final Pair<Executor, Runnable> executorRunnablePair = entry.getValue();
            Assert.assertEquals(new SegmentDescriptor(publishedSegment.getInterval(), publishedSegment.getVersion(), publishedSegment.getShardSpec().getPartitionNum()), entry.getKey());
            executorRunnablePair.lhs.execute(executorRunnablePair.rhs);
        }
        handOffCallbacks.clear();
        // Wait for the task to finish.
        final TaskStatus taskStatus = statusFuture.get();
        Assert.assertEquals(TaskStatus.Status.SUCCESS, taskStatus.getStatusCode());
    }
}
Also used : TaskStatus(io.druid.indexing.common.TaskStatus) DataSegment(io.druid.timeline.DataSegment) TaskToolbox(io.druid.indexing.common.TaskToolbox) Executor(java.util.concurrent.Executor) TestIndexerMetadataStorageCoordinator(io.druid.indexing.test.TestIndexerMetadataStorageCoordinator) SegmentDescriptor(io.druid.query.SegmentDescriptor) MapBasedInputRow(io.druid.data.input.MapBasedInputRow) InputRow(io.druid.data.input.InputRow) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) MapBasedInputRow(io.druid.data.input.MapBasedInputRow) File(java.io.File) Pair(io.druid.java.util.common.Pair) Test(org.junit.Test)

Aggregations

Pair (io.druid.java.util.common.Pair)62 Test (org.junit.Test)26 Interval (org.joda.time.Interval)15 DataSegment (io.druid.timeline.DataSegment)11 Map (java.util.Map)11 ByteBuffer (java.nio.ByteBuffer)10 HashMap (java.util.HashMap)9 SerializablePair (io.druid.collections.SerializablePair)8 SegmentDescriptor (io.druid.query.SegmentDescriptor)8 List (java.util.List)8 ImmutableMap (com.google.common.collect.ImmutableMap)7 Executor (java.util.concurrent.Executor)7 DateTime (org.joda.time.DateTime)7 Function (com.google.common.base.Function)6 ListenableFuture (com.google.common.util.concurrent.ListenableFuture)6 MapBasedInputRow (io.druid.data.input.MapBasedInputRow)6 TaskStatus (io.druid.indexing.common.TaskStatus)6 ISE (io.druid.java.util.common.ISE)6 Access (io.druid.server.security.Access)6 Action (io.druid.server.security.Action)6