Examples with TaskMaster io.druid.indexing.overlord.TaskMaster used on opensource projects

Example 1 with TaskMaster

use of io.druid.indexing.overlord.TaskMaster in project druid by druid-io.

the class OverlordTest method setUp.

@Before
public void setUp() throws Exception {
    req = EasyMock.createStrictMock(HttpServletRequest.class);
    supervisorManager = EasyMock.createMock(SupervisorManager.class);
    taskLockbox = EasyMock.createStrictMock(TaskLockbox.class);
    taskLockbox.syncFromStorage();
    EasyMock.expectLastCall().atLeastOnce();
    taskLockbox.add(EasyMock.<Task>anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    taskLockbox.remove(EasyMock.<Task>anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    // for second Noop Task directly added to deep storage.
    taskLockbox.add(EasyMock.<Task>anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    taskLockbox.remove(EasyMock.<Task>anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    taskActionClientFactory = EasyMock.createStrictMock(TaskActionClientFactory.class);
    EasyMock.expect(taskActionClientFactory.create(EasyMock.<Task>anyObject())).andReturn(null).anyTimes();
    EasyMock.replay(taskLockbox, taskActionClientFactory);
    taskStorage = new HeapMemoryTaskStorage(new TaskStorageConfig(null));
    runTaskCountDownLatches = new CountDownLatch[2];
    runTaskCountDownLatches[0] = new CountDownLatch(1);
    runTaskCountDownLatches[1] = new CountDownLatch(1);
    taskCompletionCountDownLatches = new CountDownLatch[2];
    taskCompletionCountDownLatches[0] = new CountDownLatch(1);
    taskCompletionCountDownLatches[1] = new CountDownLatch(1);
    announcementLatch = new CountDownLatch(1);
    IndexerZkConfig indexerZkConfig = new IndexerZkConfig(new ZkPathsConfig(), null, null, null, null, null);
    setupServerAndCurator();
    curator.start();
    curator.blockUntilConnected();
    curator.create().creatingParentsIfNeeded().forPath(indexerZkConfig.getLeaderLatchPath());
    druidNode = new DruidNode("hey", "what", 1234);
    ServiceEmitter serviceEmitter = new NoopServiceEmitter();
    taskMaster = new TaskMaster(new TaskQueueConfig(null, new Period(1), null, new Period(10)), taskLockbox, taskStorage, taskActionClientFactory, druidNode, indexerZkConfig, new TaskRunnerFactory<MockTaskRunner>() {

        @Override
        public MockTaskRunner build() {
            return new MockTaskRunner(runTaskCountDownLatches, taskCompletionCountDownLatches);
        }
    }, curator, new NoopServiceAnnouncer() {

        @Override
        public void announce(DruidNode node) {
            announcementLatch.countDown();
        }
    }, new CoordinatorOverlordServiceConfig(null, null), serviceEmitter, supervisorManager, EasyMock.createNiceMock(OverlordHelperManager.class));
    EmittingLogger.registerEmitter(serviceEmitter);
}

Example 2 with TaskMaster

use of io.druid.indexing.overlord.TaskMaster in project druid by druid-io.

the class KafkaSupervisorTest method setUp.

@Before
public void setUp() throws Exception {
    taskStorage = createMock(TaskStorage.class);
    taskMaster = createMock(TaskMaster.class);
    taskRunner = createMock(TaskRunner.class);
    indexerMetadataStorageCoordinator = createMock(IndexerMetadataStorageCoordinator.class);
    taskClient = createMock(KafkaIndexTaskClient.class);
    taskQueue = createMock(TaskQueue.class);
    zkServer = new TestingCluster(1);
    zkServer.start();
    kafkaServer = new TestBroker(zkServer.getConnectString(), tempFolder.newFolder(), 1, ImmutableMap.of("num.partitions", String.valueOf(NUM_PARTITIONS)));
    kafkaServer.start();
    kafkaHost = String.format("localhost:%d", kafkaServer.getPort());
    dataSchema = getDataSchema(DATASOURCE);
    tuningConfig = new KafkaSupervisorTuningConfig(1000, 50000, new Period("P1Y"), new File("/test"), null, null, true, false, null, null, numThreads, TEST_CHAT_THREADS, TEST_CHAT_RETRIES, TEST_HTTP_TIMEOUT, TEST_SHUTDOWN_TIMEOUT);
}

Example 3 with TaskMaster

use of io.druid.indexing.overlord.TaskMaster in project druid by druid-io.

the class OverlordResourceTest method setUp.

@Before
public void setUp() throws Exception {
    taskRunner = EasyMock.createMock(TaskRunner.class);
    taskMaster = EasyMock.createStrictMock(TaskMaster.class);
    tsqa = EasyMock.createStrictMock(TaskStorageQueryAdapter.class);
    req = EasyMock.createStrictMock(HttpServletRequest.class);
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(taskRunner)).anyTimes();
    overlordResource = new OverlordResource(taskMaster, tsqa, null, null, null, new AuthConfig(true));
    EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTH_TOKEN)).andReturn(new AuthorizationInfo() {

        @Override
        public Access isAuthorized(Resource resource, Action action) {
            if (resource.getName().equals("allow")) {
                return new Access(true);
            } else {
                return new Access(false);
            }
        }
    });
}