Examples with SegmentDescriptor io.druid.query.SegmentDescriptor used on opensource projects

Example 31 with SegmentDescriptor

use of io.druid.query.SegmentDescriptor in project druid by druid-io.

the class KafkaIndexTaskTest method testRunWithPauseAndResume.

@Test(timeout = 60_000L)
public void testRunWithPauseAndResume() throws Exception {
    final KafkaIndexTask task = createTask(null, new KafkaIOConfig("sequence0", new KafkaPartitions("topic0", ImmutableMap.of(0, 2L)), new KafkaPartitions("topic0", ImmutableMap.of(0, 5L)), kafkaServer.consumerProperties(), true, false, null), null, null);
    final ListenableFuture<TaskStatus> future = runTask(task);
    // Insert some data, but not enough for the task to finish
    try (final KafkaProducer<byte[], byte[]> kafkaProducer = kafkaServer.newProducer()) {
        for (ProducerRecord<byte[], byte[]> record : Iterables.limit(RECORDS, 4)) {
            kafkaProducer.send(record).get();
        }
    }
    while (countEvents(task) != 2) {
        Thread.sleep(25);
    }
    Assert.assertEquals(2, countEvents(task));
    Assert.assertEquals(KafkaIndexTask.Status.READING, task.getStatus());
    Map<Integer, Long> currentOffsets = objectMapper.readValue(task.pause(0).getEntity().toString(), new TypeReference<Map<Integer, Long>>() {
    });
    Assert.assertEquals(KafkaIndexTask.Status.PAUSED, task.getStatus());
    // Insert remaining data
    try (final KafkaProducer<byte[], byte[]> kafkaProducer = kafkaServer.newProducer()) {
        for (ProducerRecord<byte[], byte[]> record : Iterables.skip(RECORDS, 4)) {
            kafkaProducer.send(record).get();
        }
    }
    try {
        future.get(10, TimeUnit.SECONDS);
        Assert.fail("Task completed when it should have been paused");
    } catch (TimeoutException e) {
    // carry on..
    }
    Assert.assertEquals(currentOffsets, task.getCurrentOffsets());
    task.resume();
    Assert.assertEquals(TaskStatus.Status.SUCCESS, future.get().getStatusCode());
    Assert.assertEquals(task.getEndOffsets(), task.getCurrentOffsets());
    // Check metrics
    Assert.assertEquals(3, task.getFireDepartmentMetrics().processed());
    Assert.assertEquals(0, task.getFireDepartmentMetrics().unparseable());
    Assert.assertEquals(0, task.getFireDepartmentMetrics().thrownAway());
    // Check published metadata
    SegmentDescriptor desc1 = SD(task, "2010/P1D", 0);
    SegmentDescriptor desc2 = SD(task, "2011/P1D", 0);
    Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors());
    Assert.assertEquals(new KafkaDataSourceMetadata(new KafkaPartitions("topic0", ImmutableMap.of(0, 5L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()));
    // Check segments in deep storage
    Assert.assertEquals(ImmutableList.of("c"), readSegmentDim1(desc1));
    Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentDim1(desc2));
}

Example 32 with SegmentDescriptor

use of io.druid.query.SegmentDescriptor in project druid by druid-io.

the class KafkaIndexTaskTest method testRunAfterDataInserted.

@Test(timeout = 60_000L)
public void testRunAfterDataInserted() throws Exception {
    // Insert data
    try (final KafkaProducer<byte[], byte[]> kafkaProducer = kafkaServer.newProducer()) {
        for (ProducerRecord<byte[], byte[]> record : RECORDS) {
            kafkaProducer.send(record).get();
        }
    }
    final KafkaIndexTask task = createTask(null, new KafkaIOConfig("sequence0", new KafkaPartitions("topic0", ImmutableMap.of(0, 2L)), new KafkaPartitions("topic0", ImmutableMap.of(0, 5L)), kafkaServer.consumerProperties(), true, false, null), null, null);
    final ListenableFuture<TaskStatus> future = runTask(task);
    // Wait for task to exit
    Assert.assertEquals(TaskStatus.Status.SUCCESS, future.get().getStatusCode());
    // Check metrics
    Assert.assertEquals(3, task.getFireDepartmentMetrics().processed());
    Assert.assertEquals(0, task.getFireDepartmentMetrics().unparseable());
    Assert.assertEquals(0, task.getFireDepartmentMetrics().thrownAway());
    // Check published metadata
    SegmentDescriptor desc1 = SD(task, "2010/P1D", 0);
    SegmentDescriptor desc2 = SD(task, "2011/P1D", 0);
    Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors());
    Assert.assertEquals(new KafkaDataSourceMetadata(new KafkaPartitions("topic0", ImmutableMap.of(0, 5L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()));
    // Check segments in deep storage
    Assert.assertEquals(ImmutableList.of("c"), readSegmentDim1(desc1));
    Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentDim1(desc2));
}

Example 33 with SegmentDescriptor

use of io.druid.query.SegmentDescriptor in project druid by druid-io.

the class KafkaIndexTaskTest method testRunOneTaskTwoPartitions.

@Test(timeout = 60_000L)
public void testRunOneTaskTwoPartitions() throws Exception {
    final KafkaIndexTask task = createTask(null, new KafkaIOConfig("sequence0", new KafkaPartitions("topic0", ImmutableMap.of(0, 2L, 1, 0L)), new KafkaPartitions("topic0", ImmutableMap.of(0, 5L, 1, 2L)), kafkaServer.consumerProperties(), true, false, null), null, null);
    final ListenableFuture<TaskStatus> future = runTask(task);
    // Insert data
    try (final KafkaProducer<byte[], byte[]> kafkaProducer = kafkaServer.newProducer()) {
        for (ProducerRecord<byte[], byte[]> record : RECORDS) {
            kafkaProducer.send(record).get();
        }
    }
    // Wait for tasks to exit
    Assert.assertEquals(TaskStatus.Status.SUCCESS, future.get().getStatusCode());
    // Check metrics
    Assert.assertEquals(5, task.getFireDepartmentMetrics().processed());
    Assert.assertEquals(0, task.getFireDepartmentMetrics().unparseable());
    Assert.assertEquals(0, task.getFireDepartmentMetrics().thrownAway());
    // Check published segments & metadata
    SegmentDescriptor desc1 = SD(task, "2010/P1D", 0);
    SegmentDescriptor desc2 = SD(task, "2011/P1D", 0);
    SegmentDescriptor desc3 = SD(task, "2011/P1D", 1);
    SegmentDescriptor desc4 = SD(task, "2012/P1D", 0);
    Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3, desc4), publishedDescriptors());
    Assert.assertEquals(new KafkaDataSourceMetadata(new KafkaPartitions("topic0", ImmutableMap.of(0, 5L, 1, 2L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()));
    // Check segments in deep storage
    Assert.assertEquals(ImmutableList.of("c"), readSegmentDim1(desc1));
    Assert.assertEquals(ImmutableList.of("g"), readSegmentDim1(desc4));
    // Check desc2/desc3 without strong ordering because two partitions are interleaved nondeterministically
    Assert.assertEquals(ImmutableSet.of(ImmutableList.of("d", "e"), ImmutableList.of("h")), ImmutableSet.of(readSegmentDim1(desc2), readSegmentDim1(desc3)));
}

Example 34 with SegmentDescriptor

use of io.druid.query.SegmentDescriptor in project druid by druid-io.

the class KafkaIndexTaskTest method testRestore.

@Test(timeout = 60_000L)
public void testRestore() throws Exception {
    final KafkaIndexTask task1 = createTask(null, new KafkaIOConfig("sequence0", new KafkaPartitions("topic0", ImmutableMap.of(0, 2L)), new KafkaPartitions("topic0", ImmutableMap.of(0, 5L)), kafkaServer.consumerProperties(), true, false, null), null, null);
    final ListenableFuture<TaskStatus> future1 = runTask(task1);
    // Insert some data, but not enough for the task to finish
    try (final KafkaProducer<byte[], byte[]> kafkaProducer = kafkaServer.newProducer()) {
        for (ProducerRecord<byte[], byte[]> record : Iterables.limit(RECORDS, 4)) {
            kafkaProducer.send(record).get();
        }
    }
    while (countEvents(task1) != 2) {
        Thread.sleep(25);
    }
    Assert.assertEquals(2, countEvents(task1));
    // Stop without publishing segment
    task1.stopGracefully();
    Assert.assertEquals(TaskStatus.Status.SUCCESS, future1.get().getStatusCode());
    // Start a new task
    final KafkaIndexTask task2 = createTask(task1.getId(), new KafkaIOConfig("sequence0", new KafkaPartitions("topic0", ImmutableMap.of(0, 2L)), new KafkaPartitions("topic0", ImmutableMap.of(0, 5L)), kafkaServer.consumerProperties(), true, false, null), null, null);
    final ListenableFuture<TaskStatus> future2 = runTask(task2);
    // Insert remaining data
    try (final KafkaProducer<byte[], byte[]> kafkaProducer = kafkaServer.newProducer()) {
        for (ProducerRecord<byte[], byte[]> record : Iterables.skip(RECORDS, 4)) {
            kafkaProducer.send(record).get();
        }
    }
    // Wait for task to exit
    Assert.assertEquals(TaskStatus.Status.SUCCESS, future2.get().getStatusCode());
    // Check metrics
    Assert.assertEquals(2, task1.getFireDepartmentMetrics().processed());
    Assert.assertEquals(0, task1.getFireDepartmentMetrics().unparseable());
    Assert.assertEquals(0, task1.getFireDepartmentMetrics().thrownAway());
    Assert.assertEquals(1, task2.getFireDepartmentMetrics().processed());
    Assert.assertEquals(0, task2.getFireDepartmentMetrics().unparseable());
    Assert.assertEquals(0, task2.getFireDepartmentMetrics().thrownAway());
    // Check published segments & metadata
    SegmentDescriptor desc1 = SD(task1, "2010/P1D", 0);
    SegmentDescriptor desc2 = SD(task1, "2011/P1D", 0);
    Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors());
    Assert.assertEquals(new KafkaDataSourceMetadata(new KafkaPartitions("topic0", ImmutableMap.of(0, 5L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()));
    // Check segments in deep storage
    Assert.assertEquals(ImmutableList.of("c"), readSegmentDim1(desc1));
    Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentDim1(desc2));
}

Example 35 with SegmentDescriptor

use of io.druid.query.SegmentDescriptor in project druid by druid-io.

the class KafkaIndexTaskTest method testHandoffConditionTimeoutWhenHandoffDoesNotOccur.

@Test(timeout = 60_000L)
public void testHandoffConditionTimeoutWhenHandoffDoesNotOccur() throws Exception {
    doHandoff = false;
    handoffConditionTimeout = 100;
    // Insert data
    try (final KafkaProducer<byte[], byte[]> kafkaProducer = kafkaServer.newProducer()) {
        for (ProducerRecord<byte[], byte[]> record : RECORDS) {
            kafkaProducer.send(record).get();
        }
    }
    final KafkaIndexTask task = createTask(null, new KafkaIOConfig("sequence0", new KafkaPartitions("topic0", ImmutableMap.of(0, 2L)), new KafkaPartitions("topic0", ImmutableMap.of(0, 5L)), kafkaServer.consumerProperties(), true, false, null), null, null);
    final ListenableFuture<TaskStatus> future = runTask(task);
    // Wait for task to exit
    Assert.assertEquals(TaskStatus.Status.FAILED, future.get().getStatusCode());
    // Check metrics
    Assert.assertEquals(3, task.getFireDepartmentMetrics().processed());
    Assert.assertEquals(0, task.getFireDepartmentMetrics().unparseable());
    Assert.assertEquals(0, task.getFireDepartmentMetrics().thrownAway());
    // Check published metadata
    SegmentDescriptor desc1 = SD(task, "2010/P1D", 0);
    SegmentDescriptor desc2 = SD(task, "2011/P1D", 0);
    Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors());
    Assert.assertEquals(new KafkaDataSourceMetadata(new KafkaPartitions("topic0", ImmutableMap.of(0, 5L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()));
    // Check segments in deep storage
    Assert.assertEquals(ImmutableList.of("c"), readSegmentDim1(desc1));
    Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentDim1(desc2));
}