Examples with NumberedShardSpec org.apache.druid.timeline.partition.NumberedShardSpec used on opensource projects

Example 56 with NumberedShardSpec

use of org.apache.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class StreamAppenderatorDriverTest method testPublishPerRow.

@Test
public void testPublishPerRow() throws IOException, InterruptedException, TimeoutException, ExecutionException {
    final TestCommitterSupplier<Integer> committerSupplier = new TestCommitterSupplier<>();
    Assert.assertNull(driver.startJob(null));
    // Add the first row and publish immediately
    {
        committerSupplier.setMetadata(1);
        Assert.assertTrue(driver.add(ROWS.get(0), "dummy", committerSupplier, false, true).isOk());
        final SegmentsAndCommitMetadata segmentsAndCommitMetadata = driver.publishAndRegisterHandoff(makeOkPublisher(), committerSupplier.get(), ImmutableList.of("dummy")).get(PUBLISH_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
        Assert.assertEquals(ImmutableSet.of(new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0))), asIdentifiers(segmentsAndCommitMetadata.getSegments()));
        Assert.assertEquals(1, segmentsAndCommitMetadata.getCommitMetadata());
    }
    // Add the second and third rows and publish immediately
    for (int i = 1; i < ROWS.size(); i++) {
        committerSupplier.setMetadata(i + 1);
        Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk());
        final SegmentsAndCommitMetadata segmentsAndCommitMetadata = driver.publishAndRegisterHandoff(makeOkPublisher(), committerSupplier.get(), ImmutableList.of("dummy")).get(PUBLISH_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
        Assert.assertEquals(ImmutableSet.of(// different partitionNum
        new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(i - 1, 0))), asIdentifiers(segmentsAndCommitMetadata.getSegments()));
        Assert.assertEquals(i + 1, segmentsAndCommitMetadata.getCommitMetadata());
    }
    driver.persist(committerSupplier.get());
    // There is no remaining rows in the driver, and thus the result must be empty
    final SegmentsAndCommitMetadata segmentsAndCommitMetadata = driver.publishAndRegisterHandoff(makeOkPublisher(), committerSupplier.get(), ImmutableList.of("dummy")).get(PUBLISH_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
    Assert.assertEquals(ImmutableSet.of(), asIdentifiers(segmentsAndCommitMetadata.getSegments()));
    Assert.assertEquals(3, segmentsAndCommitMetadata.getCommitMetadata());
}

Example 57 with NumberedShardSpec

use of org.apache.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class StreamAppenderatorDriverTest method testIncrementalHandoff.

@Test
public void testIncrementalHandoff() throws Exception {
    final TestCommitterSupplier<Integer> committerSupplier = new TestCommitterSupplier<>();
    Assert.assertNull(driver.startJob(null));
    committerSupplier.setMetadata(1);
    Assert.assertTrue(driver.add(ROWS.get(0), "sequence_0", committerSupplier, false, true).isOk());
    for (int i = 1; i < ROWS.size(); i++) {
        committerSupplier.setMetadata(i + 1);
        Assert.assertTrue(driver.add(ROWS.get(i), "sequence_1", committerSupplier, false, true).isOk());
    }
    final ListenableFuture<SegmentsAndCommitMetadata> futureForSequence0 = driver.publishAndRegisterHandoff(makeOkPublisher(), committerSupplier.get(), ImmutableList.of("sequence_0"));
    final ListenableFuture<SegmentsAndCommitMetadata> futureForSequence1 = driver.publishAndRegisterHandoff(makeOkPublisher(), committerSupplier.get(), ImmutableList.of("sequence_1"));
    final SegmentsAndCommitMetadata handedoffFromSequence0 = futureForSequence0.get(HANDOFF_CONDITION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
    final SegmentsAndCommitMetadata handedoffFromSequence1 = futureForSequence1.get(HANDOFF_CONDITION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
    Assert.assertEquals(ImmutableSet.of(new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0))), asIdentifiers(handedoffFromSequence0.getSegments()));
    Assert.assertEquals(ImmutableSet.of(new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0))), asIdentifiers(handedoffFromSequence1.getSegments()));
    Assert.assertEquals(3, handedoffFromSequence0.getCommitMetadata());
    Assert.assertEquals(3, handedoffFromSequence1.getCommitMetadata());
}

Example 58 with NumberedShardSpec

use of org.apache.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class StreamAppenderatorDriverTest method testSimple.

@Test(timeout = 60_000L)
public void testSimple() throws Exception {
    final TestCommitterSupplier<Integer> committerSupplier = new TestCommitterSupplier<>();
    Assert.assertNull(driver.startJob(null));
    for (int i = 0; i < ROWS.size(); i++) {
        committerSupplier.setMetadata(i + 1);
        Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk());
    }
    final SegmentsAndCommitMetadata published = driver.publish(makeOkPublisher(), committerSupplier.get(), ImmutableList.of("dummy")).get(PUBLISH_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
    while (driver.getSegments().containsKey("dummy")) {
        Thread.sleep(100);
    }
    final SegmentsAndCommitMetadata segmentsAndCommitMetadata = driver.registerHandoff(published).get(HANDOFF_CONDITION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
    Assert.assertEquals(ImmutableSet.of(new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)), new SegmentIdWithShardSpec(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0))), asIdentifiers(segmentsAndCommitMetadata.getSegments()));
    Assert.assertEquals(3, segmentsAndCommitMetadata.getCommitMetadata());
}