Examples with TaskToolbox org.apache.druid.indexing.common.TaskToolbox used on opensource projects

Example 31 with TaskToolbox

use of org.apache.druid.indexing.common.TaskToolbox in project druid by druid-io.

the class IndexTaskTest method testWaitForSegmentAvailabilityInvalidWaitTimeout.

@Test
public void testWaitForSegmentAvailabilityInvalidWaitTimeout() throws IOException {
    final File tmpDir = temporaryFolder.newFolder();
    TaskToolbox mockToolbox = EasyMock.createMock(TaskToolbox.class);
    List<DataSegment> segmentsToWaitFor = new ArrayList<>();
    segmentsToWaitFor.add(EasyMock.createMock(DataSegment.class));
    IndexTask indexTask = new IndexTask(null, null, createDefaultIngestionSpec(jsonMapper, tmpDir, new UniformGranularitySpec(Granularities.HOUR, Granularities.MINUTE, null), null, createTuningConfigWithMaxRowsPerSegment(2, true), false, false), null);
    EasyMock.replay(mockToolbox);
    Assert.assertFalse(indexTask.waitForSegmentAvailability(mockToolbox, segmentsToWaitFor, -1));
    EasyMock.verify(mockToolbox);
}

Example 32 with TaskToolbox

use of org.apache.druid.indexing.common.TaskToolbox in project druid by druid-io.

the class IndexTaskTest method testWaitForSegmentAvailabilityNoSegments.

@Test
public void testWaitForSegmentAvailabilityNoSegments() throws IOException {
    final File tmpDir = temporaryFolder.newFolder();
    TaskToolbox mockToolbox = EasyMock.createMock(TaskToolbox.class);
    List<DataSegment> segmentsToWaitFor = new ArrayList<>();
    IndexTask indexTask = new IndexTask(null, null, createDefaultIngestionSpec(jsonMapper, tmpDir, new UniformGranularitySpec(Granularities.HOUR, Granularities.MINUTE, null), null, createTuningConfigWithMaxRowsPerSegment(2, true), false, false), null);
    EasyMock.replay(mockToolbox);
    Assert.assertTrue(indexTask.waitForSegmentAvailability(mockToolbox, segmentsToWaitFor, 1000));
    EasyMock.verify(mockToolbox);
}

Example 33 with TaskToolbox

use of org.apache.druid.indexing.common.TaskToolbox in project druid by druid-io.

the class IndexTaskTest method testWaitForSegmentAvailabilityEmitsExpectedMetric.

@Test
public void testWaitForSegmentAvailabilityEmitsExpectedMetric() throws IOException, InterruptedException {
    final File tmpDir = temporaryFolder.newFolder();
    LatchableServiceEmitter latchEmitter = new LatchableServiceEmitter();
    latchEmitter.latch = new CountDownLatch(1);
    TaskToolbox mockToolbox = EasyMock.createMock(TaskToolbox.class);
    DataSegment mockDataSegment1 = EasyMock.createMock(DataSegment.class);
    DataSegment mockDataSegment2 = EasyMock.createMock(DataSegment.class);
    List<DataSegment> segmentsToWaitFor = new ArrayList<>();
    segmentsToWaitFor.add(mockDataSegment1);
    segmentsToWaitFor.add(mockDataSegment2);
    IndexTask indexTask = new IndexTask(null, null, createDefaultIngestionSpec(jsonMapper, tmpDir, new UniformGranularitySpec(Granularities.HOUR, Granularities.MINUTE, null), null, createTuningConfigWithMaxRowsPerSegment(2, true), false, false), null);
    EasyMock.expect(mockDataSegment1.getInterval()).andReturn(Intervals.of("1970-01-01/1971-01-01")).once();
    EasyMock.expect(mockDataSegment1.getVersion()).andReturn("dummyString").once();
    EasyMock.expect(mockDataSegment1.getShardSpec()).andReturn(EasyMock.createMock(ShardSpec.class)).once();
    EasyMock.expect(mockDataSegment1.getId()).andReturn(SegmentId.dummy("MockDataSource")).once();
    EasyMock.expect(mockDataSegment2.getInterval()).andReturn(Intervals.of("1971-01-01/1972-01-01")).once();
    EasyMock.expect(mockDataSegment2.getVersion()).andReturn("dummyString").once();
    EasyMock.expect(mockDataSegment2.getShardSpec()).andReturn(EasyMock.createMock(ShardSpec.class)).once();
    EasyMock.expect(mockDataSegment2.getId()).andReturn(SegmentId.dummy("MockDataSource")).once();
    EasyMock.expect(mockToolbox.getSegmentHandoffNotifierFactory()).andReturn(new NoopSegmentHandoffNotifierFactory()).once();
    EasyMock.expect(mockToolbox.getEmitter()).andReturn(latchEmitter).anyTimes();
    EasyMock.expect(mockDataSegment1.getDataSource()).andReturn("MockDataSource").once();
    EasyMock.replay(mockToolbox);
    EasyMock.replay(mockDataSegment1, mockDataSegment2);
    Assert.assertTrue(indexTask.waitForSegmentAvailability(mockToolbox, segmentsToWaitFor, 30000));
    latchEmitter.latch.await(300000, TimeUnit.MILLISECONDS);
    EasyMock.verify(mockToolbox);
    EasyMock.verify(mockDataSegment1, mockDataSegment2);
}

Example 34 with TaskToolbox

use of org.apache.druid.indexing.common.TaskToolbox in project druid by druid-io.

the class HadoopTaskTest method testBuildClassLoader.

@Test
public void testBuildClassLoader() throws Exception {
    final HadoopTask task = new HadoopTask("taskId", "dataSource", ImmutableList.of(), ImmutableMap.of()) {

        @Override
        public String getType() {
            return null;
        }

        @Override
        public boolean isReady(TaskActionClient taskActionClient) {
            return false;
        }

        @Override
        public void stopGracefully(TaskConfig taskConfig) {
        }

        @Override
        public boolean requireLockExistingSegments() {
            return true;
        }

        @Override
        public List<DataSegment> findSegmentsToLock(TaskActionClient taskActionClient, List<Interval> intervals) {
            return Collections.emptyList();
        }

        @Override
        public boolean isPerfectRollup() {
            return true;
        }

        @Nullable
        @Override
        public Granularity getSegmentGranularity() {
            return null;
        }

        @Override
        public TaskStatus runTask(TaskToolbox toolbox) {
            return null;
        }
    };
    final TaskToolbox toolbox = EasyMock.createStrictMock(TaskToolbox.class);
    EasyMock.expect(toolbox.getConfig()).andReturn(new TaskConfig(temporaryFolder.newFolder().toString(), null, null, null, ImmutableList.of("something:hadoop:1"), false, null, null, null, false, false, TaskConfig.BATCH_PROCESSING_MODE_DEFAULT.name())).once();
    EasyMock.replay(toolbox);
    final ClassLoader classLoader = task.buildClassLoader(toolbox);
    assertClassLoaderIsSingular(classLoader);
    final Class<?> hadoopClazz = Class.forName("org.apache.hadoop.fs.FSDataInputStream", false, classLoader);
    assertClassLoaderIsSingular(hadoopClazz.getClassLoader());
    final Class<?> druidHadoopConfigClazz = Class.forName("org.apache.druid.indexer.HadoopDruidIndexerConfig", false, classLoader);
    assertClassLoaderIsSingular(druidHadoopConfigClazz.getClassLoader());
}

Example 35 with TaskToolbox

use of org.apache.druid.indexing.common.TaskToolbox in project druid by druid-io.

the class RealtimeIndexTaskTest method testBasics.

@Test(timeout = 60_000L)
public void testBasics() throws Exception {
    final TestIndexerMetadataStorageCoordinator mdc = new TestIndexerMetadataStorageCoordinator();
    final RealtimeIndexTask task = makeRealtimeTask(null);
    final TaskToolbox taskToolbox = makeToolbox(task, mdc, tempFolder.newFolder());
    final ListenableFuture<TaskStatus> statusFuture = runTask(task, taskToolbox);
    final DataSegment publishedSegment;
    // Wait for firehose to show up, it starts off null.
    while (task.getFirehose() == null) {
        Thread.sleep(50);
    }
    final TestFirehose firehose = (TestFirehose) task.getFirehose();
    firehose.addRows(ImmutableList.of(ImmutableMap.of("t", now.getMillis(), "dim1", "foo", "met1", "1"), ImmutableMap.of("t", now.minus(new Period("P1D")).getMillis(), "dim1", "foo", "met1", 2.0), ImmutableMap.of("t", now.getMillis(), "dim2", "bar", "met1", 2.0)));
    // Stop the firehose, this will drain out existing events.
    firehose.close();
    // Wait for publish.
    while (mdc.getPublished().isEmpty()) {
        Thread.sleep(50);
    }
    publishedSegment = Iterables.getOnlyElement(mdc.getPublished());
    // Check metrics.
    Assert.assertEquals(2, task.getMetrics().processed());
    Assert.assertEquals(1, task.getMetrics().thrownAway());
    Assert.assertEquals(0, task.getMetrics().unparseable());
    // Do some queries.
    Assert.assertEquals(2, sumMetric(task, null, "rows").longValue());
    Assert.assertEquals(3, sumMetric(task, null, "met1").longValue());
    // Simulate handoff.
    for (Map.Entry<SegmentDescriptor, Pair<Executor, Runnable>> entry : handOffCallbacks.entrySet()) {
        final Pair<Executor, Runnable> executorRunnablePair = entry.getValue();
        Assert.assertEquals(new SegmentDescriptor(publishedSegment.getInterval(), publishedSegment.getVersion(), publishedSegment.getShardSpec().getPartitionNum()), entry.getKey());
        executorRunnablePair.lhs.execute(executorRunnablePair.rhs);
    }
    handOffCallbacks.clear();
    // Wait for the task to finish.
    final TaskStatus taskStatus = statusFuture.get();
    Assert.assertEquals(TaskState.SUCCESS, taskStatus.getStatusCode());
}