Search in sources :

Example 6 with TaskMaster

use of org.apache.druid.indexing.overlord.TaskMaster in project druid by druid-io.

the class SeekableStreamSupervisorStateTest method setupTest.

@Before
public void setupTest() {
    taskStorage = createMock(TaskStorage.class);
    taskMaster = createMock(TaskMaster.class);
    taskRunner = createMock(TaskRunner.class);
    taskQueue = createMock(TaskQueue.class);
    indexerMetadataStorageCoordinator = createMock(IndexerMetadataStorageCoordinator.class);
    taskClientFactory = createMock(SeekableStreamIndexTaskClientFactory.class);
    spec = createMock(SeekableStreamSupervisorSpec.class);
    indexTaskClient = createMock(SeekableStreamIndexTaskClient.class);
    recordSupplier = (RecordSupplier<String, String, ByteEntity>) createMock(RecordSupplier.class);
    rowIngestionMetersFactory = new TestUtils().getRowIngestionMetersFactory();
    supervisorConfig = new SupervisorStateManagerConfig();
    emitter = new TestEmitter();
    EasyMock.expect(spec.getSupervisorStateManagerConfig()).andReturn(supervisorConfig).anyTimes();
    EasyMock.expect(spec.getDataSchema()).andReturn(getDataSchema()).anyTimes();
    EasyMock.expect(spec.getIoConfig()).andReturn(getIOConfig()).anyTimes();
    EasyMock.expect(spec.getTuningConfig()).andReturn(getTuningConfig()).anyTimes();
    EasyMock.expect(spec.getEmitter()).andReturn(emitter).anyTimes();
    EasyMock.expect(taskClientFactory.build(EasyMock.anyObject(), EasyMock.anyString(), EasyMock.anyInt(), EasyMock.anyObject(), EasyMock.anyLong())).andReturn(indexTaskClient).anyTimes();
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(taskRunner)).anyTimes();
    EasyMock.expect(taskMaster.getTaskQueue()).andReturn(Optional.of(taskQueue)).anyTimes();
    taskRunner.registerListener(EasyMock.anyObject(TaskRunnerListener.class), EasyMock.anyObject(Executor.class));
    EasyMock.expect(indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(null).anyTimes();
    EasyMock.expect(recordSupplier.getAssignment()).andReturn(ImmutableSet.of(SHARD0_PARTITION)).anyTimes();
    EasyMock.expect(recordSupplier.getLatestSequenceNumber(EasyMock.anyObject())).andReturn("10").anyTimes();
}
Also used : SeekableStreamIndexTaskClient(org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskClient) IndexerMetadataStorageCoordinator(org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator) TaskRunnerListener(org.apache.druid.indexing.overlord.TaskRunnerListener) ByteEntity(org.apache.druid.data.input.impl.ByteEntity) SeekableStreamIndexTaskClientFactory(org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskClientFactory) TaskRunner(org.apache.druid.indexing.overlord.TaskRunner) SeekableStreamIndexTaskRunner(org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner) TestUtils(org.apache.druid.indexing.common.TestUtils) Executor(java.util.concurrent.Executor) TaskStorage(org.apache.druid.indexing.overlord.TaskStorage) SupervisorStateManagerConfig(org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig) TaskQueue(org.apache.druid.indexing.overlord.TaskQueue) TaskMaster(org.apache.druid.indexing.overlord.TaskMaster) Before(org.junit.Before)

Example 7 with TaskMaster

use of org.apache.druid.indexing.overlord.TaskMaster in project druid by druid-io.

the class OverlordResourceTest method setUp.

@Before
public void setUp() {
    taskRunner = EasyMock.createMock(TaskRunner.class);
    configManager = EasyMock.createMock(JacksonConfigManager.class);
    provisioningStrategy = EasyMock.createMock(ProvisioningStrategy.class);
    taskMaster = EasyMock.createStrictMock(TaskMaster.class);
    taskStorageQueryAdapter = EasyMock.createStrictMock(TaskStorageQueryAdapter.class);
    indexerMetadataStorageAdapter = EasyMock.createStrictMock(IndexerMetadataStorageAdapter.class);
    req = EasyMock.createStrictMock(HttpServletRequest.class);
    workerTaskRunnerQueryAdapter = EasyMock.createStrictMock(WorkerTaskRunnerQueryAdapter.class);
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(taskRunner)).anyTimes();
    AuthorizerMapper authMapper = new AuthorizerMapper(null) {

        @Override
        public Authorizer getAuthorizer(String name) {
            return new Authorizer() {

                @Override
                public Access authorize(AuthenticationResult authenticationResult, Resource resource, Action action) {
                    final String username = authenticationResult.getIdentity();
                    switch(resource.getName()) {
                        case "allow":
                            return new Access(true);
                        case Datasources.WIKIPEDIA:
                            // Only "Wiki Reader" can read "wikipedia"
                            return new Access(action == Action.READ && Users.WIKI_READER.equals(username));
                        case Datasources.BUZZFEED:
                            // Only "Buzz Reader" can read "buzzfeed"
                            return new Access(action == Action.READ && Users.BUZZ_READER.equals(username));
                        default:
                            return new Access(false);
                    }
                }
            };
        }
    };
    overlordResource = new OverlordResource(taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, null, configManager, null, authMapper, workerTaskRunnerQueryAdapter, provisioningStrategy);
}
Also used : IndexerMetadataStorageAdapter(org.apache.druid.indexing.overlord.IndexerMetadataStorageAdapter) Action(org.apache.druid.server.security.Action) JacksonConfigManager(org.apache.druid.common.config.JacksonConfigManager) Resource(org.apache.druid.server.security.Resource) Access(org.apache.druid.server.security.Access) TaskStorageQueryAdapter(org.apache.druid.indexing.overlord.TaskStorageQueryAdapter) TaskRunner(org.apache.druid.indexing.overlord.TaskRunner) WorkerTaskRunner(org.apache.druid.indexing.overlord.WorkerTaskRunner) ProvisioningStrategy(org.apache.druid.indexing.overlord.autoscaling.ProvisioningStrategy) AuthenticationResult(org.apache.druid.server.security.AuthenticationResult) HttpServletRequest(javax.servlet.http.HttpServletRequest) Authorizer(org.apache.druid.server.security.Authorizer) AuthorizerMapper(org.apache.druid.server.security.AuthorizerMapper) TaskMaster(org.apache.druid.indexing.overlord.TaskMaster) WorkerTaskRunnerQueryAdapter(org.apache.druid.indexing.overlord.WorkerTaskRunnerQueryAdapter) Before(org.junit.Before)

Aggregations

TaskMaster (org.apache.druid.indexing.overlord.TaskMaster)7 Before (org.junit.Before)7 TaskStorage (org.apache.druid.indexing.overlord.TaskStorage)5 SupervisorStateManagerConfig (org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig)5 IndexerMetadataStorageCoordinator (org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator)4 TaskQueue (org.apache.druid.indexing.overlord.TaskQueue)4 TaskRunner (org.apache.druid.indexing.overlord.TaskRunner)4 TestUtils (org.apache.druid.indexing.common.TestUtils)3 HttpServletRequest (javax.servlet.http.HttpServletRequest)2 SeekableStreamIndexTaskRunner (org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner)2 AuthenticationResult (org.apache.druid.server.security.AuthenticationResult)2 Period (org.joda.time.Period)2 NamedType (com.fasterxml.jackson.databind.jsontype.NamedType)1 File (java.io.File)1 CountDownLatch (java.util.concurrent.CountDownLatch)1 Executor (java.util.concurrent.Executor)1 JacksonConfigManager (org.apache.druid.common.config.JacksonConfigManager)1 NoopServiceAnnouncer (org.apache.druid.curator.discovery.NoopServiceAnnouncer)1 ByteEntity (org.apache.druid.data.input.impl.ByteEntity)1 DimensionsSpec (org.apache.druid.data.input.impl.DimensionsSpec)1