Search in sources :

Example 26 with HttpClient

use of org.apache.druid.java.util.http.client.HttpClient in project druid by druid-io.

the class IndexTaskClientTest method dontRetryIfRetryFalse.

@Test
public void dontRetryIfRetryFalse() {
    final HttpClient httpClient = EasyMock.createMock(HttpClient.class);
    EasyMock.expect(httpClient.go(EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Futures.immediateFuture(Either.error(new StringFullResponseHolder(new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.INTERNAL_SERVER_ERROR), StandardCharsets.UTF_8).addChunk("Error")))).times(1);
    EasyMock.replay(httpClient);
    try (IndexTaskClient indexTaskClient = buildIndexTaskClient(httpClient, id -> TaskLocation.create(id, 8000, -1))) {
        final IOException e = Assert.assertThrows(IOException.class, () -> indexTaskClient.submitRequestWithEmptyContent("taskId", HttpMethod.GET, "test", null, false));
        Assert.assertEquals("Received server error with status [500 Internal Server Error]; first 1KB of body: Error", e.getMessage());
    }
    EasyMock.verify(httpClient);
}
Also used : StringFullResponseHolder(org.apache.druid.java.util.http.client.response.StringFullResponseHolder) DefaultHttpResponse(org.jboss.netty.handler.codec.http.DefaultHttpResponse) HttpClient(org.apache.druid.java.util.http.client.HttpClient) IOException(java.io.IOException) Test(org.junit.Test)

Example 27 with HttpClient

use of org.apache.druid.java.util.http.client.HttpClient in project druid by druid-io.

the class IndexTaskClientTest method retryOnChannelException.

@Test
public void retryOnChannelException() throws IOException {
    final HttpClient httpClient = EasyMock.createNiceMock(HttpClient.class);
    EasyMock.expect(httpClient.go(EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Futures.immediateFailedFuture(new ChannelException("IndexTaskClientTest"))).times(2);
    EasyMock.expect(httpClient.go(EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Futures.immediateFuture(Either.value(new StringFullResponseHolder(new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK), StandardCharsets.UTF_8)))).once();
    EasyMock.replay(httpClient);
    try (IndexTaskClient indexTaskClient = buildIndexTaskClient(httpClient, id -> TaskLocation.create(id, 8000, -1))) {
        final StringFullResponseHolder response = indexTaskClient.submitRequestWithEmptyContent("taskId", HttpMethod.GET, "test", null, true);
        Assert.assertEquals(HttpResponseStatus.OK, response.getStatus());
    }
}
Also used : StringFullResponseHolder(org.apache.druid.java.util.http.client.response.StringFullResponseHolder) DefaultHttpResponse(org.jboss.netty.handler.codec.http.DefaultHttpResponse) HttpClient(org.apache.druid.java.util.http.client.HttpClient) ChannelException(org.jboss.netty.channel.ChannelException) Test(org.junit.Test)

Example 28 with HttpClient

use of org.apache.druid.java.util.http.client.HttpClient in project druid by druid-io.

the class IndexTaskClientTest method dontRetryOnBadRequest.

@Test
public void dontRetryOnBadRequest() {
    final HttpClient httpClient = EasyMock.createMock(HttpClient.class);
    EasyMock.expect(httpClient.go(EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Futures.immediateFuture(Either.error(new StringFullResponseHolder(new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.BAD_REQUEST), StandardCharsets.UTF_8).addChunk("Error")))).times(1);
    EasyMock.replay(httpClient);
    try (IndexTaskClient indexTaskClient = buildIndexTaskClient(httpClient, id -> TaskLocation.create(id, 8000, -1))) {
        final IllegalArgumentException e = Assert.assertThrows(IllegalArgumentException.class, () -> indexTaskClient.submitRequestWithEmptyContent("taskId", HttpMethod.GET, "test", null, true));
        Assert.assertEquals("Received server error with status [400 Bad Request]; first 1KB of body: Error", e.getMessage());
    }
    EasyMock.verify(httpClient);
}
Also used : StringFullResponseHolder(org.apache.druid.java.util.http.client.response.StringFullResponseHolder) DefaultHttpResponse(org.jboss.netty.handler.codec.http.DefaultHttpResponse) HttpClient(org.apache.druid.java.util.http.client.HttpClient) Test(org.junit.Test)

Example 29 with HttpClient

use of org.apache.druid.java.util.http.client.HttpClient in project druid by druid-io.

the class HttpRemoteTaskRunnerTest method testOneStuckTaskAssignmentDoesntBlockOthers.

/*
  Simulates one task not getting acknowledged to be running after assigning it to a worker. But, other tasks are
  successfully assigned to other worker and get completed.
   */
@Test(timeout = 60_000L)
public void testOneStuckTaskAssignmentDoesntBlockOthers() throws Exception {
    TestDruidNodeDiscovery druidNodeDiscovery = new TestDruidNodeDiscovery();
    DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class);
    EasyMock.expect(druidNodeDiscoveryProvider.getForService(WorkerNodeService.DISCOVERY_SERVICE_KEY)).andReturn(druidNodeDiscovery);
    EasyMock.replay(druidNodeDiscoveryProvider);
    Task task1 = NoopTask.create("task-id-1", 0);
    Task task2 = NoopTask.create("task-id-2", 0);
    Task task3 = NoopTask.create("task-id-3", 0);
    HttpRemoteTaskRunner taskRunner = new HttpRemoteTaskRunner(TestHelper.makeJsonMapper(), new HttpRemoteTaskRunnerConfig() {

        @Override
        public int getPendingTasksRunnerNumThreads() {
            return 3;
        }
    }, EasyMock.createNiceMock(HttpClient.class), DSuppliers.of(new AtomicReference<>(DefaultWorkerBehaviorConfig.defaultConfig())), new NoopProvisioningStrategy<>(), druidNodeDiscoveryProvider, EasyMock.createNiceMock(TaskStorage.class), EasyMock.createNiceMock(CuratorFramework.class), new IndexerZkConfig(new ZkPathsConfig(), null, null, null, null)) {

        @Override
        protected WorkerHolder createWorkerHolder(ObjectMapper smileMapper, HttpClient httpClient, HttpRemoteTaskRunnerConfig config, ScheduledExecutorService workersSyncExec, WorkerHolder.Listener listener, Worker worker, List<TaskAnnouncement> knownAnnouncements) {
            return HttpRemoteTaskRunnerTest.createWorkerHolder(smileMapper, httpClient, config, workersSyncExec, listener, worker, ImmutableList.of(), ImmutableList.of(), // no announcements would be received for task1
            ImmutableMap.of(task1, ImmutableList.of()), new AtomicInteger(), ImmutableSet.of());
        }
    };
    taskRunner.start();
    DiscoveryDruidNode druidNode1 = new DiscoveryDruidNode(new DruidNode("service", "host1", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_CATEGORY)));
    DiscoveryDruidNode druidNode2 = new DiscoveryDruidNode(new DruidNode("service", "host2", false, 8080, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 2, "0", WorkerConfig.DEFAULT_CATEGORY)));
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode1, druidNode2));
    taskRunner.run(task1);
    Future<TaskStatus> future2 = taskRunner.run(task2);
    Future<TaskStatus> future3 = taskRunner.run(task3);
    Assert.assertTrue(future2.get().isSuccess());
    Assert.assertTrue(future3.get().isSuccess());
    Assert.assertEquals(task1.getId(), Iterables.getOnlyElement(taskRunner.getPendingTasks()).getTaskId());
}
Also used : IndexerZkConfig(org.apache.druid.server.initialization.IndexerZkConfig) Task(org.apache.druid.indexing.common.task.Task) NoopTask(org.apache.druid.indexing.common.task.NoopTask) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) TaskRunnerListener(org.apache.druid.indexing.overlord.TaskRunnerListener) AtomicReference(java.util.concurrent.atomic.AtomicReference) TaskStatus(org.apache.druid.indexer.TaskStatus) HttpRemoteTaskRunnerConfig(org.apache.druid.indexing.overlord.config.HttpRemoteTaskRunnerConfig) WorkerNodeService(org.apache.druid.discovery.WorkerNodeService) CuratorFramework(org.apache.curator.framework.CuratorFramework) TaskStorage(org.apache.druid.indexing.overlord.TaskStorage) DiscoveryDruidNode(org.apache.druid.discovery.DiscoveryDruidNode) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DruidNodeDiscoveryProvider(org.apache.druid.discovery.DruidNodeDiscoveryProvider) ZkPathsConfig(org.apache.druid.server.initialization.ZkPathsConfig) HttpClient(org.apache.druid.java.util.http.client.HttpClient) Worker(org.apache.druid.indexing.worker.Worker) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) DiscoveryDruidNode(org.apache.druid.discovery.DiscoveryDruidNode) DruidNode(org.apache.druid.server.DruidNode) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Test(org.junit.Test)

Example 30 with HttpClient

use of org.apache.druid.java.util.http.client.HttpClient in project druid by druid-io.

the class HttpRemoteTaskRunnerTest method testWorkerDisapperAndReappearBeforeItsCleanup.

@Test(timeout = 60_000L)
public void testWorkerDisapperAndReappearBeforeItsCleanup() throws Exception {
    TestDruidNodeDiscovery druidNodeDiscovery = new TestDruidNodeDiscovery();
    DruidNodeDiscoveryProvider druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class);
    EasyMock.expect(druidNodeDiscoveryProvider.getForService(WorkerNodeService.DISCOVERY_SERVICE_KEY)).andReturn(druidNodeDiscovery);
    EasyMock.replay(druidNodeDiscoveryProvider);
    ConcurrentMap<String, CustomFunction> workerHolders = new ConcurrentHashMap<>();
    HttpRemoteTaskRunner taskRunner = new HttpRemoteTaskRunner(TestHelper.makeJsonMapper(), new HttpRemoteTaskRunnerConfig() {

        @Override
        public int getPendingTasksRunnerNumThreads() {
            return 3;
        }
    }, EasyMock.createNiceMock(HttpClient.class), DSuppliers.of(new AtomicReference<>(DefaultWorkerBehaviorConfig.defaultConfig())), new NoopProvisioningStrategy<>(), druidNodeDiscoveryProvider, EasyMock.createNiceMock(TaskStorage.class), EasyMock.createNiceMock(CuratorFramework.class), new IndexerZkConfig(new ZkPathsConfig(), null, null, null, null)) {

        @Override
        protected WorkerHolder createWorkerHolder(ObjectMapper smileMapper, HttpClient httpClient, HttpRemoteTaskRunnerConfig config, ScheduledExecutorService workersSyncExec, WorkerHolder.Listener listener, Worker worker, List<TaskAnnouncement> knownAnnouncements) {
            if (workerHolders.containsKey(worker.getHost())) {
                return workerHolders.get(worker.getHost()).apply(smileMapper, httpClient, config, workersSyncExec, listener, worker, knownAnnouncements);
            } else {
                throw new ISE("No WorkerHolder for [%s].", worker.getHost());
            }
        }
    };
    taskRunner.start();
    Task task1 = NoopTask.create("task-id-1", 0);
    Task task2 = NoopTask.create("task-id-2", 0);
    DiscoveryDruidNode druidNode = new DiscoveryDruidNode(new DruidNode("service", "host", false, 1234, null, true, false), NodeRole.MIDDLE_MANAGER, ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_CATEGORY)));
    workerHolders.put("host:1234", (mapper, httpClient, config, exec, listener, worker, knownAnnouncements) -> createWorkerHolder(mapper, httpClient, config, exec, listener, worker, knownAnnouncements, ImmutableList.of(), ImmutableMap.of(task1, ImmutableList.of(TaskAnnouncement.create(task1, TaskStatus.running(task1.getId()), TaskLocation.unknown()), TaskAnnouncement.create(task1, TaskStatus.running(task1.getId()), TaskLocation.create("host", 1234, 1235)), TaskAnnouncement.create(task1, TaskStatus.success(task1.getId()), TaskLocation.create("host", 1234, 1235))), task2, ImmutableList.of(TaskAnnouncement.create(task2, TaskStatus.running(task2.getId()), TaskLocation.unknown()), TaskAnnouncement.create(task2, TaskStatus.running(task2.getId()), TaskLocation.create("host", 1234, 1235)))), new AtomicInteger(), ImmutableSet.of()));
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode));
    Future<TaskStatus> future1 = taskRunner.run(task1);
    Future<TaskStatus> future2 = taskRunner.run(task2);
    while (taskRunner.getPendingTasks().size() > 0) {
        Thread.sleep(100);
    }
    druidNodeDiscovery.getListeners().get(0).nodesRemoved(ImmutableList.of(druidNode));
    workerHolders.put("host:1234", (mapper, httpClient, config, exec, listener, worker, knownAnnouncements) -> createWorkerHolder(mapper, httpClient, config, exec, listener, worker, knownAnnouncements, ImmutableList.of(TaskAnnouncement.create(task2, TaskStatus.running(task2.getId()), TaskLocation.create("host", 1234, 1235)), TaskAnnouncement.create(task2, TaskStatus.success(task2.getId()), TaskLocation.create("host", 1234, 1235))), ImmutableMap.of(), new AtomicInteger(), ImmutableSet.of()));
    druidNodeDiscovery.getListeners().get(0).nodesAdded(ImmutableList.of(druidNode));
    Assert.assertTrue(future1.get().isSuccess());
    Assert.assertTrue(future2.get().isSuccess());
}
Also used : IndexerZkConfig(org.apache.druid.server.initialization.IndexerZkConfig) Task(org.apache.druid.indexing.common.task.Task) NoopTask(org.apache.druid.indexing.common.task.NoopTask) TaskRunnerListener(org.apache.druid.indexing.overlord.TaskRunnerListener) WorkerNodeService(org.apache.druid.discovery.WorkerNodeService) CuratorFramework(org.apache.curator.framework.CuratorFramework) ZkPathsConfig(org.apache.druid.server.initialization.ZkPathsConfig) Worker(org.apache.druid.indexing.worker.Worker) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) ISE(org.apache.druid.java.util.common.ISE) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) AtomicReference(java.util.concurrent.atomic.AtomicReference) TaskStatus(org.apache.druid.indexer.TaskStatus) HttpRemoteTaskRunnerConfig(org.apache.druid.indexing.overlord.config.HttpRemoteTaskRunnerConfig) TaskStorage(org.apache.druid.indexing.overlord.TaskStorage) DiscoveryDruidNode(org.apache.druid.discovery.DiscoveryDruidNode) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DruidNodeDiscoveryProvider(org.apache.druid.discovery.DruidNodeDiscoveryProvider) HttpClient(org.apache.druid.java.util.http.client.HttpClient) DiscoveryDruidNode(org.apache.druid.discovery.DiscoveryDruidNode) DruidNode(org.apache.druid.server.DruidNode) Test(org.junit.Test)

Aggregations

HttpClient (org.apache.druid.java.util.http.client.HttpClient)36 Test (org.junit.Test)16 CredentialedHttpClient (org.apache.druid.java.util.http.client.CredentialedHttpClient)15 ArrayList (java.util.ArrayList)10 Test (org.testng.annotations.Test)10 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)9 List (java.util.List)9 CuratorFramework (org.apache.curator.framework.CuratorFramework)9 ZkPathsConfig (org.apache.druid.server.initialization.ZkPathsConfig)9 ImmutableList (com.google.common.collect.ImmutableList)8 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)8 AtomicReference (java.util.concurrent.atomic.AtomicReference)8 DruidNodeDiscoveryProvider (org.apache.druid.discovery.DruidNodeDiscoveryProvider)8 TaskRunnerListener (org.apache.druid.indexing.overlord.TaskRunnerListener)8 TaskStorage (org.apache.druid.indexing.overlord.TaskStorage)8 HttpRemoteTaskRunnerConfig (org.apache.druid.indexing.overlord.config.HttpRemoteTaskRunnerConfig)8 Worker (org.apache.druid.indexing.worker.Worker)8 IndexerZkConfig (org.apache.druid.server.initialization.IndexerZkConfig)8 DiscoveryDruidNode (org.apache.druid.discovery.DiscoveryDruidNode)7 WorkerNodeService (org.apache.druid.discovery.WorkerNodeService)7