Examples with DatasourceIngestionSpec org.apache.druid.indexer.hadoop.DatasourceIngestionSpec used on opensource projects

Example 1 with DatasourceIngestionSpec

use of org.apache.druid.indexer.hadoop.DatasourceIngestionSpec in project druid by druid-io.

the class HadoopIngestionSpecUpdateDatasourcePathSpecSegmentsTest method testUpdateSegmentListThrowsExceptionWithUserSegmentsMismatch.

@Test(expected = IOException.class)
public void testUpdateSegmentListThrowsExceptionWithUserSegmentsMismatch() throws Exception {
    PathSpec pathSpec = new DatasourcePathSpec(null, new DatasourceIngestionSpec(TEST_DATA_SOURCE, TEST_DATA_SOURCE_INTERVAL, null, ImmutableList.of(SEGMENT.withVersion("v2")), null, null, null, false, null), null, false);
    testRunUpdateSegmentListIfDatasourcePathSpecIsUsed(pathSpec, TEST_DATA_SOURCE_INTERVAL);
}

Example 2 with DatasourceIngestionSpec

use of org.apache.druid.indexer.hadoop.DatasourceIngestionSpec in project druid by druid-io.

the class HadoopIngestionSpecUpdateDatasourcePathSpecSegmentsTest method testUpdateSegmentListIfDatasourcePathSpecIsUsedWithJustDatasourcePathSpecAndPartialInterval.

@Test
public void testUpdateSegmentListIfDatasourcePathSpecIsUsedWithJustDatasourcePathSpecAndPartialInterval() throws Exception {
    PathSpec pathSpec = new DatasourcePathSpec(null, new DatasourceIngestionSpec(TEST_DATA_SOURCE, TEST_DATA_SOURCE_INTERVAL_PARTIAL, null, null, null, null, null, false, null), null, false);
    HadoopDruidIndexerConfig config = testRunUpdateSegmentListIfDatasourcePathSpecIsUsed(pathSpec, TEST_DATA_SOURCE_INTERVAL_PARTIAL);
    Assert.assertEquals(ImmutableList.of(new WindowedDataSegment(SEGMENT, TEST_DATA_SOURCE_INTERVAL_PARTIAL)), ((DatasourcePathSpec) config.getPathSpec()).getSegments());
}

Example 3 with DatasourceIngestionSpec

use of org.apache.druid.indexer.hadoop.DatasourceIngestionSpec in project druid by druid-io.

the class HadoopIngestionSpecUpdateDatasourcePathSpecSegmentsTest method testUpdateSegmentListIfDatasourcePathSpecIsUsedWithJustDatasourcePathSpec.

@Test
public void testUpdateSegmentListIfDatasourcePathSpecIsUsedWithJustDatasourcePathSpec() throws Exception {
    PathSpec pathSpec = new DatasourcePathSpec(null, new DatasourceIngestionSpec(TEST_DATA_SOURCE, TEST_DATA_SOURCE_INTERVAL, null, null, null, null, null, false, null), null, false);
    HadoopDruidIndexerConfig config = testRunUpdateSegmentListIfDatasourcePathSpecIsUsed(pathSpec, TEST_DATA_SOURCE_INTERVAL);
    Assert.assertEquals(ImmutableList.of(WindowedDataSegment.of(SEGMENT)), ((DatasourcePathSpec) config.getPathSpec()).getSegments());
}

Example 4 with DatasourceIngestionSpec

use of org.apache.druid.indexer.hadoop.DatasourceIngestionSpec in project druid by druid-io.

the class HadoopIngestionSpecUpdateDatasourcePathSpecSegmentsTest method testUpdateSegmentListIfDatasourcePathSpecWithMatchingUserSegments.

@Test
public void testUpdateSegmentListIfDatasourcePathSpecWithMatchingUserSegments() throws Exception {
    PathSpec pathSpec = new DatasourcePathSpec(null, new DatasourceIngestionSpec(TEST_DATA_SOURCE, TEST_DATA_SOURCE_INTERVAL, null, ImmutableList.of(SEGMENT), null, null, null, false, null), null, false);
    HadoopDruidIndexerConfig config = testRunUpdateSegmentListIfDatasourcePathSpecIsUsed(pathSpec, TEST_DATA_SOURCE_INTERVAL);
    Assert.assertEquals(ImmutableList.of(WindowedDataSegment.of(SEGMENT)), ((DatasourcePathSpec) config.getPathSpec()).getSegments());
}

Example 5 with DatasourceIngestionSpec

use of org.apache.druid.indexer.hadoop.DatasourceIngestionSpec in project druid by druid-io.

the class HadoopIngestionSpecUpdateDatasourcePathSpecSegmentsTest method testUpdateSegmentListIfDatasourcePathSpecIsUsedWithMultiplePathSpec.

@Test
public void testUpdateSegmentListIfDatasourcePathSpecIsUsedWithMultiplePathSpec() throws Exception {
    PathSpec pathSpec = new MultiplePathSpec(ImmutableList.of(new StaticPathSpec("/xyz", null), new DatasourcePathSpec(null, new DatasourceIngestionSpec(TEST_DATA_SOURCE, TEST_DATA_SOURCE_INTERVAL, null, null, null, null, null, false, null), null, false), new DatasourcePathSpec(null, new DatasourceIngestionSpec(TEST_DATA_SOURCE2, TEST_DATA_SOURCE_INTERVAL2, null, null, null, null, null, false, null), null, false)));
    HadoopDruidIndexerConfig config = testRunUpdateSegmentListIfDatasourcePathSpecIsUsed(pathSpec, TEST_DATA_SOURCE_INTERVAL);
    Assert.assertEquals(ImmutableList.of(WindowedDataSegment.of(SEGMENT)), ((DatasourcePathSpec) ((MultiplePathSpec) config.getPathSpec()).getChildren().get(1)).getSegments());
    Assert.assertEquals(ImmutableList.of(new WindowedDataSegment(SEGMENT2, TEST_DATA_SOURCE_INTERVAL2)), ((DatasourcePathSpec) ((MultiplePathSpec) config.getPathSpec()).getChildren().get(2)).getSegments());
}