Examples with InputStreamResponseHandler org.apache.druid.java.util.http.client.response.InputStreamResponseHandler used on opensource projects

Example 1 with InputStreamResponseHandler

use of org.apache.druid.java.util.http.client.response.InputStreamResponseHandler in project druid by druid-io.

the class JettyTest method testNumConnectionsMetricHttp.

@Test
public void testNumConnectionsMetricHttp() throws Exception {
    String text = "hello";
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    try (GZIPOutputStream gzipOutputStream = new GZIPOutputStream(out)) {
        gzipOutputStream.write(text.getBytes(Charset.defaultCharset()));
    }
    Request request = new Request(HttpMethod.GET, new URL("http://localhost:" + port + "/latched/hello"));
    request.setHeader("Content-Encoding", "gzip");
    request.setContent(MediaType.TEXT_PLAIN, out.toByteArray());
    JettyServerModule jsm = injector.getInstance(JettyServerModule.class);
    latchedRequestState.reset();
    waitForJettyServerModuleActiveConnectionsZero(jsm);
    Assert.assertEquals(0, jsm.getActiveConnections());
    ListenableFuture<InputStream> go = client.go(request, new InputStreamResponseHandler());
    latchedRequestState.clientWaitForServerToStartRequest();
    Assert.assertEquals(1, jsm.getActiveConnections());
    latchedRequestState.clientReadyToFinishRequest();
    go.get();
    waitForJettyServerModuleActiveConnectionsZero(jsm);
    Assert.assertEquals(0, jsm.getActiveConnections());
}

Example 2 with InputStreamResponseHandler

use of org.apache.druid.java.util.http.client.response.InputStreamResponseHandler in project druid by druid-io.

the class HttpRemoteTaskRunner method streamTaskReports.

@Override
public Optional<ByteSource> streamTaskReports(String taskId) {
    // Read on tasks is safe
    @SuppressWarnings("GuardedBy") HttpRemoteTaskRunnerWorkItem taskRunnerWorkItem = tasks.get(taskId);
    Worker worker = null;
    if (taskRunnerWorkItem != null && taskRunnerWorkItem.getState() != HttpRemoteTaskRunnerWorkItem.State.COMPLETE) {
        worker = taskRunnerWorkItem.getWorker();
    }
    if (worker == null || !workers.containsKey(worker.getHost())) {
        // Worker is not running this task, it might be available in deep storage
        return Optional.absent();
    } else {
        // Worker is still running this task
        TaskLocation taskLocation = taskRunnerWorkItem.getLocation();
        final URL url = TaskRunnerUtils.makeTaskLocationURL(taskLocation, "/druid/worker/v1/chat/%s/liveReports", taskId);
        return Optional.of(new ByteSource() {

            @Override
            public InputStream openStream() throws IOException {
                try {
                    return httpClient.go(new Request(HttpMethod.GET, url), new InputStreamResponseHandler()).get();
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                } catch (ExecutionException e) {
                    // Unwrap if possible
                    Throwables.propagateIfPossible(e.getCause(), IOException.class);
                    throw new RuntimeException(e);
                }
            }
        });
    }
}

Example 3 with InputStreamResponseHandler

use of org.apache.druid.java.util.http.client.response.InputStreamResponseHandler in project druid by druid-io.

the class HttpRemoteTaskRunner method streamTaskLog.

@Override
public Optional<ByteSource> streamTaskLog(String taskId, long offset) {
    // Read on tasks is safe
    @SuppressWarnings("GuardedBy") HttpRemoteTaskRunnerWorkItem taskRunnerWorkItem = tasks.get(taskId);
    Worker worker = null;
    if (taskRunnerWorkItem != null && taskRunnerWorkItem.getState() != HttpRemoteTaskRunnerWorkItem.State.COMPLETE) {
        worker = taskRunnerWorkItem.getWorker();
    }
    if (worker == null || !workers.containsKey(worker.getHost())) {
        // Worker is not running this task, it might be available in deep storage
        return Optional.absent();
    } else {
        // Worker is still running this task
        final URL url = TaskRunnerUtils.makeWorkerURL(worker, "/druid/worker/v1/task/%s/log?offset=%s", taskId, Long.toString(offset));
        return Optional.of(new ByteSource() {

            @Override
            public InputStream openStream() throws IOException {
                try {
                    return httpClient.go(new Request(HttpMethod.GET, url), new InputStreamResponseHandler()).get();
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                } catch (ExecutionException e) {
                    // Unwrap if possible
                    Throwables.propagateIfPossible(e.getCause(), IOException.class);
                    throw new RuntimeException(e);
                }
            }
        });
    }
}

Example 4 with InputStreamResponseHandler

use of org.apache.druid.java.util.http.client.response.InputStreamResponseHandler in project druid by druid-io.

the class HttpShuffleClient method fetchSegmentFile.

@Override
public File fetchSegmentFile(File partitionDir, String supervisorTaskId, GenericPartitionLocation location) throws IOException {
    // Create a local buffer since this class is not thread-safe.
    // Note that this method can be called by different threads at the same time with ThreadingTaskRunner.
    final byte[] buffer = new byte[BUFFER_SIZE];
    final File zippedFile = new File(partitionDir, StringUtils.format("temp_%s", location.getSubTaskId()));
    final URI uri = location.toIntermediaryDataServerURI(supervisorTaskId);
    FileUtils.copyLarge(uri, u -> {
        try {
            return httpClient.go(new Request(HttpMethod.GET, u.toURL()), new InputStreamResponseHandler()).get();
        } catch (InterruptedException | ExecutionException e) {
            throw new IOException(e);
        }
    }, zippedFile, buffer, t -> t instanceof IOException, NUM_FETCH_RETRIES, StringUtils.format("Failed to fetch file[%s]", uri));
    final File unzippedDir = new File(partitionDir, StringUtils.format("unzipped_%s", location.getSubTaskId()));
    try {
        FileUtils.mkdirp(unzippedDir);
        CompressionUtils.unzip(zippedFile, unzippedDir);
    } finally {
        if (!zippedFile.delete()) {
            LOG.warn("Failed to delete temp file[%s]", zippedFile);
        }
    }
    return unzippedDir;
}

Example 5 with InputStreamResponseHandler

use of org.apache.druid.java.util.http.client.response.InputStreamResponseHandler in project druid by druid-io.

the class RemoteTaskRunner method streamTaskReports.

@Override
public Optional<ByteSource> streamTaskReports(final String taskId) {
    final ZkWorker zkWorker = findWorkerRunningTask(taskId);
    if (zkWorker == null) {
        // Worker is not running this task, it might be available in deep storage
        return Optional.absent();
    } else {
        TaskLocation taskLocation = runningTasks.get(taskId).getLocation();
        final URL url = TaskRunnerUtils.makeTaskLocationURL(taskLocation, "/druid/worker/v1/chat/%s/liveReports", taskId);
        return Optional.of(new ByteSource() {

            @Override
            public InputStream openStream() throws IOException {
                try {
                    return httpClient.go(new Request(HttpMethod.GET, url), new InputStreamResponseHandler()).get();
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                } catch (ExecutionException e) {
                    // Unwrap if possible
                    Throwables.propagateIfPossible(e.getCause(), IOException.class);
                    throw new RuntimeException(e);
                }
            }
        });
    }
}