Examples with ProvisioningStrategy org.apache.druid.indexing.overlord.autoscaling.ProvisioningStrategy used on opensource projects

Example 1 with ProvisioningStrategy

use of org.apache.druid.indexing.overlord.autoscaling.ProvisioningStrategy in project druid by druid-io.

the class HttpRemoteTaskRunnerTest method testFreshStartAndStopNoProvisioner.

/*
  Simulates startup of Overlord with no provisoner. Overlord is then stopped and is expected to close down certain
  things.
   */
@Test(timeout = 60_000L)
public void testFreshStartAndStopNoProvisioner() {
    TestDruidNodeDiscovery druidNodeDiscovery = new TestDruidNodeDiscovery();
    ProvisioningStrategy provisioningStrategy = EasyMock.createMock(ProvisioningStrategy.class);
    DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class);
    EasyMock.expect(druidNodeDiscoveryProvider.getForService(WorkerNodeService.DISCOVERY_SERVICE_KEY)).andReturn(druidNodeDiscovery).times(2);
    EasyMock.expect(provisioningStrategy.makeProvisioningService(isA(HttpRemoteTaskRunner.class))).andReturn(null);
    EasyMock.expectLastCall();
    EasyMock.replay(druidNodeDiscoveryProvider, provisioningStrategy);
    HttpRemoteTaskRunner taskRunner = new HttpRemoteTaskRunner(TestHelper.makeJsonMapper(), new HttpRemoteTaskRunnerConfig() {

        @Override
        public int getPendingTasksRunnerNumThreads() {
            return 3;
        }
    }, EasyMock.createNiceMock(HttpClient.class), DSuppliers.of(new AtomicReference<>(DefaultWorkerBehaviorConfig.defaultConfig())), provisioningStrategy, druidNodeDiscoveryProvider, EasyMock.createNiceMock(TaskStorage.class), EasyMock.createNiceMock(CuratorFramework.class), new IndexerZkConfig(new ZkPathsConfig(), null, null, null, null)) {

        @Override
        protected WorkerHolder createWorkerHolder(ObjectMapper smileMapper, HttpClient httpClient, HttpRemoteTaskRunnerConfig config, ScheduledExecutorService workersSyncExec, WorkerHolder.Listener listener, Worker worker, List<TaskAnnouncement> knownAnnouncements) {
            return HttpRemoteTaskRunnerTest.createWorkerHolder(smileMapper, httpClient, config, workersSyncExec, listener, worker, ImmutableList.of(), ImmutableList.of(), ImmutableMap.of(), new AtomicInteger(), ImmutableSet.of());
        }
    };
    taskRunner.start();
    taskRunner.stop();
    EasyMock.verify(druidNodeDiscoveryProvider, provisioningStrategy);
}

Example 2 with ProvisioningStrategy

use of org.apache.druid.indexing.overlord.autoscaling.ProvisioningStrategy in project druid by druid-io.

the class HttpRemoteTaskRunnerTest method testFreshStartAndStop.

/*
  Simulates startup of Overlord. Overlord is then stopped and is expected to close down certain things.
   */
@Test(timeout = 60_000L)
public void testFreshStartAndStop() {
    TestDruidNodeDiscovery druidNodeDiscovery = new TestDruidNodeDiscovery();
    DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class);
    EasyMock.expect(druidNodeDiscoveryProvider.getForService(WorkerNodeService.DISCOVERY_SERVICE_KEY)).andReturn(druidNodeDiscovery).times(2);
    ProvisioningStrategy provisioningStrategy = EasyMock.createMock(ProvisioningStrategy.class);
    ProvisioningService provisioningService = EasyMock.createNiceMock(ProvisioningService.class);
    EasyMock.expect(provisioningStrategy.makeProvisioningService(isA(HttpRemoteTaskRunner.class))).andReturn(provisioningService);
    provisioningService.close();
    EasyMock.expectLastCall();
    EasyMock.replay(druidNodeDiscoveryProvider, provisioningStrategy, provisioningService);
    DiscoveryDruidNode druidNode1 = new DiscoveryDruidNode(new DruidNode("service", "host1", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_CATEGORY)));
    DiscoveryDruidNode druidNode2 = new DiscoveryDruidNode(new DruidNode("service", "host2", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 2, "0", WorkerConfig.DEFAULT_CATEGORY)));
    HttpRemoteTaskRunner taskRunner = newHttpTaskRunnerInstance(druidNodeDiscoveryProvider, provisioningStrategy);
    taskRunner.start();
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode1, druidNode2));
    ConcurrentMap<String, WorkerHolder> workers = taskRunner.getWorkersForTestingReadOnly();
    Assert.assertEquals(2, workers.size());
    Assert.assertTrue(workers.values().stream().noneMatch(w -> w.getUnderlyingSyncer().isExecutorShutdown()));
    workers.values().iterator().next().stop();
    taskRunner.stop();
    Assert.assertTrue(druidNodeDiscovery.getListeners().isEmpty());
    Assert.assertEquals(2, workers.size());
    Assert.assertTrue(workers.values().stream().allMatch(w -> w.getUnderlyingSyncer().isExecutorShutdown()));
    EasyMock.verify(druidNodeDiscoveryProvider, provisioningStrategy, provisioningService);
}

Example 3 with ProvisioningStrategy

use of org.apache.druid.indexing.overlord.autoscaling.ProvisioningStrategy in project druid by druid-io.

the class OverlordResourceTest method setUp.

@Before
public void setUp() {
    taskRunner = EasyMock.createMock(TaskRunner.class);
    configManager = EasyMock.createMock(JacksonConfigManager.class);
    provisioningStrategy = EasyMock.createMock(ProvisioningStrategy.class);
    taskMaster = EasyMock.createStrictMock(TaskMaster.class);
    taskStorageQueryAdapter = EasyMock.createStrictMock(TaskStorageQueryAdapter.class);
    indexerMetadataStorageAdapter = EasyMock.createStrictMock(IndexerMetadataStorageAdapter.class);
    req = EasyMock.createStrictMock(HttpServletRequest.class);
    workerTaskRunnerQueryAdapter = EasyMock.createStrictMock(WorkerTaskRunnerQueryAdapter.class);
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(taskRunner)).anyTimes();
    AuthorizerMapper authMapper = new AuthorizerMapper(null) {

        @Override
        public Authorizer getAuthorizer(String name) {
            return new Authorizer() {

                @Override
                public Access authorize(AuthenticationResult authenticationResult, Resource resource, Action action) {
                    final String username = authenticationResult.getIdentity();
                    switch(resource.getName()) {
                        case "allow":
                            return new Access(true);
                        case Datasources.WIKIPEDIA:
                            // Only "Wiki Reader" can read "wikipedia"
                            return new Access(action == Action.READ && Users.WIKI_READER.equals(username));
                        case Datasources.BUZZFEED:
                            // Only "Buzz Reader" can read "buzzfeed"
                            return new Access(action == Action.READ && Users.BUZZ_READER.equals(username));
                        default:
                            return new Access(false);
                    }
                }
            };
        }
    };
    overlordResource = new OverlordResource(taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, null, configManager, null, authMapper, workerTaskRunnerQueryAdapter, provisioningStrategy);
}