Examples with SupervisorStateManagerConfig org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig used on opensource projects

Example 11 with SupervisorStateManagerConfig

use of org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig in project druid by druid-io.

the class MaterializedViewSupervisorSpecTest method testMaterializedViewSupervisorSpecCreated.

@Test
public void testMaterializedViewSupervisorSpecCreated() {
    Exception ex = null;
    try {
        MaterializedViewSupervisorSpec spec = new MaterializedViewSupervisorSpec("wikiticker", new DimensionsSpec(Lists.newArrayList(new StringDimensionSchema("isUnpatrolled"), new StringDimensionSchema("metroCode"), new StringDimensionSchema("namespace"), new StringDimensionSchema("page"), new StringDimensionSchema("regionIsoCode"), new StringDimensionSchema("regionName"), new StringDimensionSchema("user"))), new AggregatorFactory[] { new CountAggregatorFactory("count"), new LongSumAggregatorFactory("added", "added") }, HadoopTuningConfig.makeDefaultTuningConfig(), null, null, null, null, null, false, objectMapper, null, null, null, null, null, new MaterializedViewTaskConfig(), EasyMock.createMock(AuthorizerMapper.class), new NoopChatHandlerProvider(), new SupervisorStateManagerConfig());
        Supervisor supervisor = spec.createSupervisor();
        Assert.assertTrue(supervisor instanceof MaterializedViewSupervisor);
        SupervisorTaskAutoScaler autoscaler = spec.createAutoscaler(supervisor);
        Assert.assertNull(autoscaler);
        try {
            supervisor.computeLagStats();
        } catch (Exception e) {
            Assert.assertTrue(e instanceof UnsupportedOperationException);
        }
        try {
            int count = supervisor.getActiveTaskGroupsCount();
        } catch (Exception e) {
            Assert.assertTrue(e instanceof UnsupportedOperationException);
        }
        Callable<Integer> noop = new Callable<Integer>() {

            @Override
            public Integer call() {
                return -1;
            }
        };
    } catch (Exception e) {
        ex = e;
    }
    Assert.assertNull(ex);
}

Example 12 with SupervisorStateManagerConfig

use of org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig in project druid by druid-io.

the class MaterializedViewSupervisorSpecTest method testNullBaseDataSource.

@Test
public void testNullBaseDataSource() {
    expectedException.expect(CoreMatchers.instanceOf(IllegalArgumentException.class));
    expectedException.expectMessage("baseDataSource cannot be null or empty. Please provide a baseDataSource.");
    // noinspection ResultOfObjectAllocationIgnored (this method call will trigger the expected exception)
    new MaterializedViewSupervisorSpec(null, new DimensionsSpec(Lists.newArrayList(new StringDimensionSchema("isUnpatrolled"), new StringDimensionSchema("metroCode"), new StringDimensionSchema("namespace"), new StringDimensionSchema("page"), new StringDimensionSchema("regionIsoCode"), new StringDimensionSchema("regionName"), new StringDimensionSchema("user"))), new AggregatorFactory[] { new CountAggregatorFactory("count"), new LongSumAggregatorFactory("added", "added") }, HadoopTuningConfig.makeDefaultTuningConfig(), null, null, null, null, null, false, objectMapper, null, null, null, null, null, new MaterializedViewTaskConfig(), EasyMock.createMock(AuthorizerMapper.class), new NoopChatHandlerProvider(), new SupervisorStateManagerConfig());
}

Example 13 with SupervisorStateManagerConfig

use of org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig in project druid by druid-io.

the class KinesisSupervisorTest method getTestableSupervisor.

private TestableKinesisSupervisor getTestableSupervisor(int replicas, int taskCount, boolean useEarliestOffset, boolean resetOffsetAutomatically, String duration, Period lateMessageRejectionPeriod, Period earlyMessageRejectionPeriod, boolean suspended) {
    KinesisSupervisorIOConfig kinesisSupervisorIOConfig = new KinesisSupervisorIOConfig(STREAM, INPUT_FORMAT, "awsEndpoint", null, replicas, taskCount, new Period(duration), new Period("P1D"), new Period("PT30S"), useEarliestOffset, new Period("PT30M"), lateMessageRejectionPeriod, earlyMessageRejectionPeriod, null, null, null, null, null, null, false);
    KinesisIndexTaskClientFactory taskClientFactory = new KinesisIndexTaskClientFactory(null, null) {

        @Override
        public KinesisIndexTaskClient build(TaskInfoProvider taskInfoProvider, String dataSource, int numThreads, Duration httpTimeout, long numRetries) {
            Assert.assertEquals(TEST_CHAT_THREADS, numThreads);
            Assert.assertEquals(TEST_HTTP_TIMEOUT.toStandardDuration(), httpTimeout);
            Assert.assertEquals(TEST_CHAT_RETRIES, numRetries);
            return taskClient;
        }
    };
    final KinesisSupervisorTuningConfig tuningConfig = new KinesisSupervisorTuningConfig(null, 1000, null, null, 50000, null, new Period("P1Y"), new File("/test"), null, null, null, false, null, resetOffsetAutomatically, null, null, numThreads, TEST_CHAT_THREADS, TEST_CHAT_RETRIES, TEST_HTTP_TIMEOUT, TEST_SHUTDOWN_TIMEOUT, null, null, null, 5000, null, null, null, null, null, null, null, null, null);
    return new TestableKinesisSupervisor(taskStorage, taskMaster, indexerMetadataStorageCoordinator, taskClientFactory, OBJECT_MAPPER, new KinesisSupervisorSpec(null, dataSchema, tuningConfig, kinesisSupervisorIOConfig, null, suspended, taskStorage, taskMaster, indexerMetadataStorageCoordinator, taskClientFactory, OBJECT_MAPPER, new NoopServiceEmitter(), new DruidMonitorSchedulerConfig(), rowIngestionMetersFactory, null, new SupervisorStateManagerConfig()), rowIngestionMetersFactory);
}

Example 14 with SupervisorStateManagerConfig

use of org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig in project druid by druid-io.

the class KinesisSupervisorTest method testRecordSupplier.

@Test
public void testRecordSupplier() {
    KinesisSupervisorIOConfig kinesisSupervisorIOConfig = new KinesisSupervisorIOConfig(STREAM, INPUT_FORMAT, "awsEndpoint", null, 1, 1, new Period("PT30M"), new Period("P1D"), new Period("PT30S"), false, new Period("PT30M"), null, null, null, 100, 1000, null, null, null, false);
    KinesisIndexTaskClientFactory clientFactory = new KinesisIndexTaskClientFactory(null, OBJECT_MAPPER);
    KinesisSupervisor supervisor = new KinesisSupervisor(taskStorage, taskMaster, indexerMetadataStorageCoordinator, clientFactory, OBJECT_MAPPER, new KinesisSupervisorSpec(null, dataSchema, tuningConfig, kinesisSupervisorIOConfig, null, false, taskStorage, taskMaster, indexerMetadataStorageCoordinator, clientFactory, OBJECT_MAPPER, new NoopServiceEmitter(), new DruidMonitorSchedulerConfig(), rowIngestionMetersFactory, null, new SupervisorStateManagerConfig()), rowIngestionMetersFactory, null);
    KinesisRecordSupplier supplier = (KinesisRecordSupplier) supervisor.setupRecordSupplier();
    Assert.assertNotNull(supplier);
    Assert.assertEquals(0, supplier.bufferSize());
    Assert.assertEquals(Collections.emptySet(), supplier.getAssignment());
    // background fetch should not be enabled for supervisor supplier
    supplier.start();
    Assert.assertFalse(supplier.isBackgroundFetchRunning());
}

Example 15 with SupervisorStateManagerConfig

use of org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig in project druid by druid-io.

the class KinesisSupervisorTest method setupTest.

@Before
public void setupTest() {
    taskStorage = createMock(TaskStorage.class);
    taskMaster = createMock(TaskMaster.class);
    taskRunner = createMock(TaskRunner.class);
    indexerMetadataStorageCoordinator = createMock(IndexerMetadataStorageCoordinator.class);
    taskClient = createMock(KinesisIndexTaskClient.class);
    taskQueue = createMock(TaskQueue.class);
    supervisorRecordSupplier = createMock(KinesisRecordSupplier.class);
    tuningConfig = new KinesisSupervisorTuningConfig(null, 1000, null, null, 50000, null, new Period("P1Y"), new File("/test"), null, null, null, false, null, null, null, null, numThreads, TEST_CHAT_THREADS, TEST_CHAT_RETRIES, TEST_HTTP_TIMEOUT, TEST_SHUTDOWN_TIMEOUT, null, null, null, 5000, null, null, null, null, null, null, null, null, null);
    rowIngestionMetersFactory = new TestUtils().getRowIngestionMetersFactory();
    serviceEmitter = new ExceptionCapturingServiceEmitter();
    EmittingLogger.registerEmitter(serviceEmitter);
    supervisorConfig = new SupervisorStateManagerConfig();
}