Examples with HttpContentConsumer io.cdap.common.http.HttpContentConsumer used on opensource projects

Example 1 with HttpContentConsumer

use of io.cdap.common.http.HttpContentConsumer in project cdap by caskdata.

the class RemoteLogsFetcher method execute.

private void execute(String path, File file) throws IOException, UnauthorizedException {
    try (FileChannel channel = new FileOutputStream(file, false).getChannel()) {
        URL url = remoteClient.resolve(path);
        HttpRequest request = HttpRequest.get(url).withContentConsumer(new HttpContentConsumer() {

            @Override
            public boolean onReceived(ByteBuffer buffer) {
                try {
                    channel.write(buffer);
                } catch (IOException e) {
                    LOG.error("Failed write to file {}", file);
                    return false;
                }
                return true;
            }

            @Override
            public void onFinished() {
                Closeables.closeQuietly(channel);
            }
        }).build();
        remoteClient.executeStreamingRequest(request);
    }
}

Example 2 with HttpContentConsumer

use of io.cdap.common.http.HttpContentConsumer in project cdap by caskdata.

the class FileFetcherHttpHandlerInternalTest method download.

private HttpResponse download(File src, Location dst) throws IOException {
    // Make a request to download the source file.
    URL url = new URL(String.format("%s/v3Internal/location/%s", baseURL, src.toURI().getPath()));
    OutputStream outputStream = dst.getOutputStream();
    HttpRequest request = HttpRequest.builder(HttpMethod.GET, url).withContentConsumer(new HttpContentConsumer() {

        @Override
        public boolean onReceived(ByteBuffer chunk) {
            try {
                byte[] bytes = new byte[chunk.remaining()];
                chunk.get(bytes, 0, bytes.length);
                outputStream.write(bytes);
            } catch (IOException e) {
                LOG.error("Failed to write to {}", dst.toURI());
                return false;
            }
            return true;
        }

        @Override
        public void onFinished() {
            try {
                outputStream.close();
            } catch (Exception e) {
                LOG.error("Failed to close {}", dst.toURI());
            }
        }
    }).build();
    HttpResponse httpResponse = HttpRequests.execute(request, new DefaultHttpRequestConfig(false));
    httpResponse.consumeContent();
    return httpResponse;
}

Example 3 with HttpContentConsumer

use of io.cdap.common.http.HttpContentConsumer in project cdap by caskdata.

the class HubPackage method installPlugin.

/**
 * Downloads the plugin from the given URL and installs it in the artifact repository
 *
 * @param url URL that points to the plugin directory on the hub
 * @param artifactRepository {@link ArtifactRepository} in which the plugin will be installed
 * @param tmpDir temporary directory where plugin jar is downloaded from the hub
 */
public void installPlugin(URL url, ArtifactRepository artifactRepository, File tmpDir) throws Exception {
    // Deserialize spec.json
    URL specURL = new URL(url.getProtocol(), url.getHost(), url.getPort(), url.getPath() + "/packages/" + name + "/" + version + "/spec.json");
    Spec spec = GSON.fromJson(HttpClients.doGetAsString(specURL), Spec.class);
    for (Spec.Action action : spec.getActions()) {
        // See https://cdap.atlassian.net/wiki/spaces/DOCS/pages/554401840/Hub+API?src=search#one_step_deploy_plugin
        if (!action.getType().equals("one_step_deploy_plugin")) {
            continue;
        }
        String configFilename = action.getConfigFilename();
        if (configFilename == null) {
            LOG.warn("Ignoring plugin {} due to missing config", name);
            continue;
        }
        URL configURL = new URL(url.getProtocol(), url.getHost(), url.getPort(), url.getPath() + Joiner.on("/").join(Arrays.asList("/packages", name, version, configFilename)));
        // Download plugin json from hub
        JsonObject jsonObj = GSON.fromJson(HttpClients.doGetAsString(configURL), JsonObject.class);
        List<String> parents = GSON.fromJson(jsonObj.get("parents"), new TypeToken<List<String>>() {
        }.getType());
        String jarName = action.getJarName();
        if (jarName == null) {
            LOG.warn("Ignoring plugin {} due to missing jar", name);
            continue;
        }
        // Download plugin jar from hub
        File destination = File.createTempFile("artifact-", ".jar", tmpDir);
        FileChannel channel = new FileOutputStream(destination, false).getChannel();
        URL jarURL = new URL(url.getProtocol(), url.getHost(), url.getPort(), url.getPath() + Joiner.on("/").join(Arrays.asList("/packages", name, version, jarName)));
        HttpRequest request = HttpRequest.get(jarURL).withContentConsumer(new HttpContentConsumer() {

            @Override
            public boolean onReceived(ByteBuffer buffer) {
                try {
                    channel.write(buffer);
                } catch (IOException e) {
                    LOG.error("Failed write to file {}", destination);
                    return false;
                }
                return true;
            }

            @Override
            public void onFinished() {
                Closeables.closeQuietly(channel);
            }
        }).build();
        HttpClients.executeStreamingRequest(request);
        Set<ArtifactRange> parentArtifacts = new HashSet<>();
        for (String parent : parents) {
            try {
                // try parsing it as a namespaced range like system:cdap-data-pipeline[6.3 1.1,7.0.0)
                parentArtifacts.add(ArtifactRanges.parseArtifactRange(parent));
            } catch (InvalidArtifactRangeException e) {
                // if this failed, try parsing as a non-namespaced range like cdap-data-pipeline[6.3 1.1,7.0.0)
                parentArtifacts.add(ArtifactRanges.parseArtifactRange(NamespaceId.DEFAULT.getNamespace(), parent));
            }
        }
        // add the artifact to the repo
        io.cdap.cdap.proto.id.ArtifactId artifactId = NamespaceId.DEFAULT.artifact(name, version);
        Id.Artifact artifact = Id.Artifact.fromEntityId(artifactId);
        try {
            artifactRepository.addArtifact(artifact, destination, parentArtifacts, ImmutableSet.of());
        } catch (ArtifactAlreadyExistsException e) {
            LOG.debug("Artifact artifact {}-{} already exists", name, version);
        }
        Map<String, String> properties = GSON.fromJson(jsonObj.get("properties"), new TypeToken<Map<String, String>>() {
        }.getType());
        artifactRepository.writeArtifactProperties(Id.Artifact.fromEntityId(artifactId), properties);
        if (!java.nio.file.Files.deleteIfExists(Paths.get(destination.getPath()))) {
            LOG.warn("Failed to cleanup file {}", destination);
        }
    }
}