Search in sources :

Example 6 with SupervisorStateManagerConfig

use of org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig in project druid by druid-io.

the class KafkaSupervisorTest method getTestableSupervisor.

private TestableKafkaSupervisor getTestableSupervisor(int replicas, int taskCount, boolean useEarliestOffset, boolean resetOffsetAutomatically, String duration, Period lateMessageRejectionPeriod, Period earlyMessageRejectionPeriod, boolean suspended, String kafkaHost) {
    final Map<String, Object> consumerProperties = KafkaConsumerConfigs.getConsumerProperties();
    consumerProperties.put("myCustomKey", "myCustomValue");
    consumerProperties.put("bootstrap.servers", kafkaHost);
    KafkaSupervisorIOConfig kafkaSupervisorIOConfig = new KafkaSupervisorIOConfig(topic, INPUT_FORMAT, replicas, taskCount, new Period(duration), consumerProperties, null, KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, new Period("P1D"), new Period("PT30S"), useEarliestOffset, new Period("PT30M"), lateMessageRejectionPeriod, earlyMessageRejectionPeriod, null);
    KafkaIndexTaskClientFactory taskClientFactory = new KafkaIndexTaskClientFactory(null, null) {

        @Override
        public KafkaIndexTaskClient build(TaskInfoProvider taskInfoProvider, String dataSource, int numThreads, Duration httpTimeout, long numRetries) {
            Assert.assertEquals(TEST_CHAT_THREADS, numThreads);
            Assert.assertEquals(TEST_HTTP_TIMEOUT.toStandardDuration(), httpTimeout);
            Assert.assertEquals(TEST_CHAT_RETRIES, numRetries);
            return taskClient;
        }
    };
    final KafkaSupervisorTuningConfig tuningConfig = new KafkaSupervisorTuningConfig(null, 1000, null, null, 50000, null, new Period("P1Y"), new File("/test"), null, null, null, false, null, resetOffsetAutomatically, null, numThreads, TEST_CHAT_THREADS, TEST_CHAT_RETRIES, TEST_HTTP_TIMEOUT, TEST_SHUTDOWN_TIMEOUT, null, null, null, null, 10);
    return new TestableKafkaSupervisor(taskStorage, taskMaster, indexerMetadataStorageCoordinator, taskClientFactory, OBJECT_MAPPER, new KafkaSupervisorSpec(null, dataSchema, tuningConfig, kafkaSupervisorIOConfig, null, suspended, taskStorage, taskMaster, indexerMetadataStorageCoordinator, taskClientFactory, OBJECT_MAPPER, new NoopServiceEmitter(), new DruidMonitorSchedulerConfig(), rowIngestionMetersFactory, new SupervisorStateManagerConfig()), rowIngestionMetersFactory);
}
Also used : Period(org.joda.time.Period) Duration(org.joda.time.Duration) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) DruidMonitorSchedulerConfig(org.apache.druid.server.metrics.DruidMonitorSchedulerConfig) SupervisorStateManagerConfig(org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig) KafkaIndexTaskClientFactory(org.apache.druid.indexing.kafka.KafkaIndexTaskClientFactory) TaskInfoProvider(org.apache.druid.indexing.common.TaskInfoProvider) File(java.io.File)

Example 7 with SupervisorStateManagerConfig

use of org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig in project druid by druid-io.

the class KafkaSupervisorTest method setupTest.

@Before
public void setupTest() {
    taskStorage = createMock(TaskStorage.class);
    taskMaster = createMock(TaskMaster.class);
    taskRunner = createMock(TaskRunner.class);
    indexerMetadataStorageCoordinator = createMock(IndexerMetadataStorageCoordinator.class);
    taskClient = createMock(KafkaIndexTaskClient.class);
    taskQueue = createMock(TaskQueue.class);
    topic = getTopic();
    rowIngestionMetersFactory = new TestUtils().getRowIngestionMetersFactory();
    serviceEmitter = new ExceptionCapturingServiceEmitter();
    EmittingLogger.registerEmitter(serviceEmitter);
    supervisorConfig = new SupervisorStateManagerConfig();
    ingestionSchema = EasyMock.createMock(KafkaSupervisorIngestionSpec.class);
}
Also used : IndexerMetadataStorageCoordinator(org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator) TestUtils(org.apache.druid.indexing.common.TestUtils) TaskStorage(org.apache.druid.indexing.overlord.TaskStorage) SupervisorStateManagerConfig(org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig) KafkaIndexTaskClient(org.apache.druid.indexing.kafka.KafkaIndexTaskClient) TaskQueue(org.apache.druid.indexing.overlord.TaskQueue) TaskMaster(org.apache.druid.indexing.overlord.TaskMaster) ExceptionCapturingServiceEmitter(org.apache.druid.server.metrics.ExceptionCapturingServiceEmitter) TaskRunner(org.apache.druid.indexing.overlord.TaskRunner) Before(org.junit.Before)

Example 8 with SupervisorStateManagerConfig

use of org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig in project druid by druid-io.

the class MaterializedViewSupervisorTest method setUp.

@Before
public void setUp() {
    TestDerbyConnector derbyConnector = derbyConnectorRule.getConnector();
    derbyConnector.createDataSourceTable();
    derbyConnector.createSegmentTable();
    taskStorage = EasyMock.createMock(TaskStorage.class);
    taskMaster = EasyMock.createMock(TaskMaster.class);
    indexerMetadataStorageCoordinator = new IndexerSQLMetadataStorageCoordinator(objectMapper, derbyConnectorRule.metadataTablesConfigSupplier().get(), derbyConnector);
    metadataSupervisorManager = EasyMock.createMock(MetadataSupervisorManager.class);
    sqlSegmentsMetadataManager = EasyMock.createMock(SqlSegmentsMetadataManager.class);
    taskQueue = EasyMock.createMock(TaskQueue.class);
    taskQueue.start();
    objectMapper.registerSubtypes(new NamedType(HashBasedNumberedShardSpec.class, "hashed"));
    spec = new MaterializedViewSupervisorSpec("base", new DimensionsSpec(Collections.singletonList(new StringDimensionSchema("dim"))), new AggregatorFactory[] { new LongSumAggregatorFactory("m1", "m1") }, HadoopTuningConfig.makeDefaultTuningConfig(), null, null, null, null, null, false, objectMapper, taskMaster, taskStorage, metadataSupervisorManager, sqlSegmentsMetadataManager, indexerMetadataStorageCoordinator, new MaterializedViewTaskConfig(), EasyMock.createMock(AuthorizerMapper.class), EasyMock.createMock(ChatHandlerProvider.class), new SupervisorStateManagerConfig());
    derivativeDatasourceName = spec.getDataSourceName();
    supervisor = (MaterializedViewSupervisor) spec.createSupervisor();
}
Also used : IndexerSQLMetadataStorageCoordinator(org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator) HashBasedNumberedShardSpec(org.apache.druid.timeline.partition.HashBasedNumberedShardSpec) NamedType(com.fasterxml.jackson.databind.jsontype.NamedType) LongSumAggregatorFactory(org.apache.druid.query.aggregation.LongSumAggregatorFactory) TestDerbyConnector(org.apache.druid.metadata.TestDerbyConnector) LongSumAggregatorFactory(org.apache.druid.query.aggregation.LongSumAggregatorFactory) AggregatorFactory(org.apache.druid.query.aggregation.AggregatorFactory) StringDimensionSchema(org.apache.druid.data.input.impl.StringDimensionSchema) TaskStorage(org.apache.druid.indexing.overlord.TaskStorage) SupervisorStateManagerConfig(org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig) TaskQueue(org.apache.druid.indexing.overlord.TaskQueue) DimensionsSpec(org.apache.druid.data.input.impl.DimensionsSpec) MetadataSupervisorManager(org.apache.druid.metadata.MetadataSupervisorManager) TaskMaster(org.apache.druid.indexing.overlord.TaskMaster) SqlSegmentsMetadataManager(org.apache.druid.metadata.SqlSegmentsMetadataManager) Before(org.junit.Before)

Example 9 with SupervisorStateManagerConfig

use of org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig in project druid by druid-io.

the class MaterializedViewSupervisorTest method testSuspendedDoesntRun.

@Test
public void testSuspendedDoesntRun() {
    MaterializedViewSupervisorSpec suspended = new MaterializedViewSupervisorSpec("base", new DimensionsSpec(Collections.singletonList(new StringDimensionSchema("dim"))), new AggregatorFactory[] { new LongSumAggregatorFactory("m1", "m1") }, HadoopTuningConfig.makeDefaultTuningConfig(), null, null, null, null, null, true, objectMapper, taskMaster, taskStorage, metadataSupervisorManager, sqlSegmentsMetadataManager, indexerMetadataStorageCoordinator, new MaterializedViewTaskConfig(), EasyMock.createMock(AuthorizerMapper.class), EasyMock.createMock(ChatHandlerProvider.class), new SupervisorStateManagerConfig());
    MaterializedViewSupervisor supervisor = (MaterializedViewSupervisor) suspended.createSupervisor();
    // mock IndexerSQLMetadataStorageCoordinator to ensure that retrieveDataSourceMetadata is not called
    // which will be true if truly suspended, since this is the first operation of the 'run' method otherwise
    IndexerSQLMetadataStorageCoordinator mock = EasyMock.createMock(IndexerSQLMetadataStorageCoordinator.class);
    EasyMock.expect(mock.retrieveDataSourceMetadata(suspended.getDataSourceName())).andAnswer(() -> {
        Assert.fail();
        return null;
    }).anyTimes();
    EasyMock.replay(mock);
    supervisor.run();
}
Also used : IndexerSQLMetadataStorageCoordinator(org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator) SupervisorStateManagerConfig(org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig) LongSumAggregatorFactory(org.apache.druid.query.aggregation.LongSumAggregatorFactory) DimensionsSpec(org.apache.druid.data.input.impl.DimensionsSpec) AuthorizerMapper(org.apache.druid.server.security.AuthorizerMapper) ChatHandlerProvider(org.apache.druid.segment.realtime.firehose.ChatHandlerProvider) StringDimensionSchema(org.apache.druid.data.input.impl.StringDimensionSchema) Test(org.junit.Test)

Example 10 with SupervisorStateManagerConfig

use of org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig in project druid by druid-io.

the class MaterializedViewSupervisorSpecTest method setup.

@Before
public void setup() {
    objectMapper.registerSubtypes(new NamedType(MaterializedViewSupervisorSpec.class, "derivativeDataSource"));
    objectMapper.setInjectableValues(new InjectableValues.Std().addValue(TaskMaster.class, null).addValue(TaskStorage.class, null).addValue(ExprMacroTable.class.getName(), LookupEnabledTestExprMacroTable.INSTANCE).addValue(ObjectMapper.class, objectMapper).addValue(MetadataSupervisorManager.class, null).addValue(SqlSegmentsMetadataManager.class, null).addValue(IndexerMetadataStorageCoordinator.class, null).addValue(MaterializedViewTaskConfig.class, new MaterializedViewTaskConfig()).addValue(AuthorizerMapper.class, EasyMock.createMock(AuthorizerMapper.class)).addValue(ChatHandlerProvider.class, new NoopChatHandlerProvider()).addValue(SupervisorStateManagerConfig.class, new SupervisorStateManagerConfig()));
}
Also used : IndexerMetadataStorageCoordinator(org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator) NamedType(com.fasterxml.jackson.databind.jsontype.NamedType) NoopChatHandlerProvider(org.apache.druid.segment.realtime.firehose.NoopChatHandlerProvider) SupervisorStateManagerConfig(org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig) AuthorizerMapper(org.apache.druid.server.security.AuthorizerMapper) MetadataSupervisorManager(org.apache.druid.metadata.MetadataSupervisorManager) TaskMaster(org.apache.druid.indexing.overlord.TaskMaster) Before(org.junit.Before)

Aggregations

SupervisorStateManagerConfig (org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig)16 Before (org.junit.Before)7 Test (org.junit.Test)7 DimensionsSpec (org.apache.druid.data.input.impl.DimensionsSpec)6 StringDimensionSchema (org.apache.druid.data.input.impl.StringDimensionSchema)6 TaskMaster (org.apache.druid.indexing.overlord.TaskMaster)6 LongSumAggregatorFactory (org.apache.druid.query.aggregation.LongSumAggregatorFactory)6 AuthorizerMapper (org.apache.druid.server.security.AuthorizerMapper)6 IndexerMetadataStorageCoordinator (org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator)5 TaskStorage (org.apache.druid.indexing.overlord.TaskStorage)5 NoopChatHandlerProvider (org.apache.druid.segment.realtime.firehose.NoopChatHandlerProvider)5 DruidMonitorSchedulerConfig (org.apache.druid.server.metrics.DruidMonitorSchedulerConfig)5 Period (org.joda.time.Period)5 File (java.io.File)4 TaskQueue (org.apache.druid.indexing.overlord.TaskQueue)4 CountAggregatorFactory (org.apache.druid.query.aggregation.CountAggregatorFactory)4 NoopServiceEmitter (org.apache.druid.server.metrics.NoopServiceEmitter)4 TaskInfoProvider (org.apache.druid.indexing.common.TaskInfoProvider)3 TestUtils (org.apache.druid.indexing.common.TestUtils)3 TaskRunner (org.apache.druid.indexing.overlord.TaskRunner)3