Examples with DruidMonitorSchedulerConfig org.apache.druid.server.metrics.DruidMonitorSchedulerConfig used on opensource projects

Example 6 with DruidMonitorSchedulerConfig

use of org.apache.druid.server.metrics.DruidMonitorSchedulerConfig in project druid by druid-io.

the class KinesisSupervisorTest method getTestableSupervisorCustomIsTaskCurrent.

/**
 * Use when you want to mock the return value of SeekableStreamSupervisor#isTaskCurrent()
 */
private TestableKinesisSupervisor getTestableSupervisorCustomIsTaskCurrent(int replicas, int taskCount, boolean useEarliestOffset, String duration, Period lateMessageRejectionPeriod, Period earlyMessageRejectionPeriod, boolean suspended, Integer recordsPerFetch, Integer fetchDelayMillis, boolean isTaskCurrentReturn) {
    KinesisSupervisorIOConfig kinesisSupervisorIOConfig = new KinesisSupervisorIOConfig(STREAM, INPUT_FORMAT, "awsEndpoint", null, replicas, taskCount, new Period(duration), new Period("P1D"), new Period("PT30S"), useEarliestOffset, new Period("PT30M"), lateMessageRejectionPeriod, earlyMessageRejectionPeriod, null, recordsPerFetch, fetchDelayMillis, null, null, null, false);
    KinesisIndexTaskClientFactory taskClientFactory = new KinesisIndexTaskClientFactory(null, null) {

        @Override
        public KinesisIndexTaskClient build(TaskInfoProvider taskInfoProvider, String dataSource, int numThreads, Duration httpTimeout, long numRetries) {
            Assert.assertEquals(TEST_CHAT_THREADS, numThreads);
            Assert.assertEquals(TEST_HTTP_TIMEOUT.toStandardDuration(), httpTimeout);
            Assert.assertEquals(TEST_CHAT_RETRIES, numRetries);
            return taskClient;
        }
    };
    return new TestableKinesisSupervisorWithCustomIsTaskCurrent(taskStorage, taskMaster, indexerMetadataStorageCoordinator, taskClientFactory, OBJECT_MAPPER, new KinesisSupervisorSpec(null, dataSchema, tuningConfig, kinesisSupervisorIOConfig, null, suspended, taskStorage, taskMaster, indexerMetadataStorageCoordinator, taskClientFactory, OBJECT_MAPPER, new NoopServiceEmitter(), new DruidMonitorSchedulerConfig(), rowIngestionMetersFactory, null, supervisorConfig), rowIngestionMetersFactory, isTaskCurrentReturn);
}

Example 7 with DruidMonitorSchedulerConfig

use of org.apache.druid.server.metrics.DruidMonitorSchedulerConfig in project druid by apache.

the class KafkaSupervisorTest method getSupervisor.

/**
 * Use when you don't want generateSequenceNumber overridden
 */
private KafkaSupervisor getSupervisor(int replicas, int taskCount, boolean useEarliestOffset, String duration, Period lateMessageRejectionPeriod, Period earlyMessageRejectionPeriod, boolean suspended, String kafkaHost, DataSchema dataSchema, KafkaSupervisorTuningConfig tuningConfig) {
    Map<String, Object> consumerProperties = new HashMap<>();
    consumerProperties.put("myCustomKey", "myCustomValue");
    consumerProperties.put("bootstrap.servers", kafkaHost);
    consumerProperties.put("isolation.level", "read_committed");
    KafkaSupervisorIOConfig kafkaSupervisorIOConfig = new KafkaSupervisorIOConfig(topic, INPUT_FORMAT, replicas, taskCount, new Period(duration), consumerProperties, null, KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, new Period("P1D"), new Period("PT30S"), useEarliestOffset, new Period("PT30M"), lateMessageRejectionPeriod, earlyMessageRejectionPeriod, null);
    KafkaIndexTaskClientFactory taskClientFactory = new KafkaIndexTaskClientFactory(null, null) {

        @Override
        public KafkaIndexTaskClient build(TaskInfoProvider taskInfoProvider, String dataSource, int numThreads, Duration httpTimeout, long numRetries) {
            Assert.assertEquals(TEST_CHAT_THREADS, numThreads);
            Assert.assertEquals(TEST_HTTP_TIMEOUT.toStandardDuration(), httpTimeout);
            Assert.assertEquals(TEST_CHAT_RETRIES, numRetries);
            return taskClient;
        }
    };
    return new KafkaSupervisor(taskStorage, taskMaster, indexerMetadataStorageCoordinator, taskClientFactory, OBJECT_MAPPER, new KafkaSupervisorSpec(null, dataSchema, tuningConfig, kafkaSupervisorIOConfig, null, suspended, taskStorage, taskMaster, indexerMetadataStorageCoordinator, taskClientFactory, OBJECT_MAPPER, new NoopServiceEmitter(), new DruidMonitorSchedulerConfig(), rowIngestionMetersFactory, supervisorConfig), rowIngestionMetersFactory);
}

Example 8 with DruidMonitorSchedulerConfig

use of org.apache.druid.server.metrics.DruidMonitorSchedulerConfig in project druid by apache.

the class KafkaSupervisorTest method getTestableSupervisorCustomIsTaskCurrent.

/**
 * Use when you want to mock the return value of SeekableStreamSupervisor#isTaskCurrent()
 */
private TestableKafkaSupervisor getTestableSupervisorCustomIsTaskCurrent(int replicas, int taskCount, boolean useEarliestOffset, String duration, Period lateMessageRejectionPeriod, Period earlyMessageRejectionPeriod, boolean suspended, boolean isTaskCurrentReturn) {
    Map<String, Object> consumerProperties = new HashMap<>();
    consumerProperties.put("myCustomKey", "myCustomValue");
    consumerProperties.put("bootstrap.servers", kafkaHost);
    consumerProperties.put("isolation.level", "read_committed");
    KafkaSupervisorIOConfig kafkaSupervisorIOConfig = new KafkaSupervisorIOConfig(topic, INPUT_FORMAT, replicas, taskCount, new Period(duration), consumerProperties, null, KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, new Period("P1D"), new Period("PT30S"), useEarliestOffset, new Period("PT30M"), lateMessageRejectionPeriod, earlyMessageRejectionPeriod, null);
    KafkaIndexTaskClientFactory taskClientFactory = new KafkaIndexTaskClientFactory(null, null) {

        @Override
        public KafkaIndexTaskClient build(TaskInfoProvider taskInfoProvider, String dataSource, int numThreads, Duration httpTimeout, long numRetries) {
            Assert.assertEquals(TEST_CHAT_THREADS, numThreads);
            Assert.assertEquals(TEST_HTTP_TIMEOUT.toStandardDuration(), httpTimeout);
            Assert.assertEquals(TEST_CHAT_RETRIES, numRetries);
            return taskClient;
        }
    };
    final KafkaSupervisorTuningConfig tuningConfig = new KafkaSupervisorTuningConfig(null, 1000, null, null, 50000, null, new Period("P1Y"), new File("/test"), null, null, null, false, null, false, null, numThreads, TEST_CHAT_THREADS, TEST_CHAT_RETRIES, TEST_HTTP_TIMEOUT, TEST_SHUTDOWN_TIMEOUT, null, null, null, null, null);
    return new TestableKafkaSupervisorWithCustomIsTaskCurrent(taskStorage, taskMaster, indexerMetadataStorageCoordinator, taskClientFactory, OBJECT_MAPPER, new KafkaSupervisorSpec(null, dataSchema, tuningConfig, kafkaSupervisorIOConfig, null, suspended, taskStorage, taskMaster, indexerMetadataStorageCoordinator, taskClientFactory, OBJECT_MAPPER, new NoopServiceEmitter(), new DruidMonitorSchedulerConfig(), rowIngestionMetersFactory, supervisorConfig), rowIngestionMetersFactory, isTaskCurrentReturn);
}

Example 9 with DruidMonitorSchedulerConfig

use of org.apache.druid.server.metrics.DruidMonitorSchedulerConfig in project druid by apache.

the class KinesisSupervisorTest method getTestableSupervisor.

private TestableKinesisSupervisor getTestableSupervisor(int replicas, int taskCount, boolean useEarliestOffset, String duration, Period lateMessageRejectionPeriod, Period earlyMessageRejectionPeriod, boolean suspended, Integer recordsPerFetch, Integer fetchDelayMillis, AutoScalerConfig autoScalerConfig) {
    KinesisSupervisorIOConfig kinesisSupervisorIOConfig = new KinesisSupervisorIOConfig(STREAM, INPUT_FORMAT, "awsEndpoint", null, replicas, taskCount, new Period(duration), new Period("P1D"), new Period("PT30S"), useEarliestOffset, new Period("PT30M"), lateMessageRejectionPeriod, earlyMessageRejectionPeriod, null, recordsPerFetch, fetchDelayMillis, null, null, autoScalerConfig, false);
    KinesisIndexTaskClientFactory taskClientFactory = new KinesisIndexTaskClientFactory(null, null) {

        @Override
        public KinesisIndexTaskClient build(TaskInfoProvider taskInfoProvider, String dataSource, int numThreads, Duration httpTimeout, long numRetries) {
            Assert.assertEquals(TEST_CHAT_THREADS, numThreads);
            Assert.assertEquals(TEST_HTTP_TIMEOUT.toStandardDuration(), httpTimeout);
            Assert.assertEquals(TEST_CHAT_RETRIES, numRetries);
            return taskClient;
        }
    };
    return new TestableKinesisSupervisor(taskStorage, taskMaster, indexerMetadataStorageCoordinator, taskClientFactory, OBJECT_MAPPER, new KinesisSupervisorSpec(null, dataSchema, tuningConfig, kinesisSupervisorIOConfig, null, suspended, taskStorage, taskMaster, indexerMetadataStorageCoordinator, taskClientFactory, OBJECT_MAPPER, new NoopServiceEmitter(), new DruidMonitorSchedulerConfig(), rowIngestionMetersFactory, null, supervisorConfig), rowIngestionMetersFactory);
}

Example 10 with DruidMonitorSchedulerConfig

use of org.apache.druid.server.metrics.DruidMonitorSchedulerConfig in project druid by apache.

the class KinesisSupervisorTest method getTestableSupervisorCustomIsTaskCurrent.

/**
 * Use when you want to mock the return value of SeekableStreamSupervisor#isTaskCurrent()
 */
private TestableKinesisSupervisor getTestableSupervisorCustomIsTaskCurrent(int replicas, int taskCount, boolean useEarliestOffset, String duration, Period lateMessageRejectionPeriod, Period earlyMessageRejectionPeriod, boolean suspended, Integer recordsPerFetch, Integer fetchDelayMillis, boolean isTaskCurrentReturn) {
    KinesisSupervisorIOConfig kinesisSupervisorIOConfig = new KinesisSupervisorIOConfig(STREAM, INPUT_FORMAT, "awsEndpoint", null, replicas, taskCount, new Period(duration), new Period("P1D"), new Period("PT30S"), useEarliestOffset, new Period("PT30M"), lateMessageRejectionPeriod, earlyMessageRejectionPeriod, null, recordsPerFetch, fetchDelayMillis, null, null, null, false);
    KinesisIndexTaskClientFactory taskClientFactory = new KinesisIndexTaskClientFactory(null, null) {

        @Override
        public KinesisIndexTaskClient build(TaskInfoProvider taskInfoProvider, String dataSource, int numThreads, Duration httpTimeout, long numRetries) {
            Assert.assertEquals(TEST_CHAT_THREADS, numThreads);
            Assert.assertEquals(TEST_HTTP_TIMEOUT.toStandardDuration(), httpTimeout);
            Assert.assertEquals(TEST_CHAT_RETRIES, numRetries);
            return taskClient;
        }
    };
    return new TestableKinesisSupervisorWithCustomIsTaskCurrent(taskStorage, taskMaster, indexerMetadataStorageCoordinator, taskClientFactory, OBJECT_MAPPER, new KinesisSupervisorSpec(null, dataSchema, tuningConfig, kinesisSupervisorIOConfig, null, suspended, taskStorage, taskMaster, indexerMetadataStorageCoordinator, taskClientFactory, OBJECT_MAPPER, new NoopServiceEmitter(), new DruidMonitorSchedulerConfig(), rowIngestionMetersFactory, null, supervisorConfig), rowIngestionMetersFactory, isTaskCurrentReturn);
}