Examples with S3Artifact com.hubspot.deploy.S3Artifact used on opensource projects

Example 1 with S3Artifact

use of com.hubspot.deploy.S3Artifact in project Singularity by HubSpot.

the class SingularityExecutorArtifactVerifier method checkSignatures.

public void checkSignatures(List<S3Artifact> s3Artifacts, List<S3ArtifactSignature> s3ArtifactsWithSignatures) {
    Map<String, S3Artifact> artifactsByFilename = s3Artifacts.stream().collect(Collectors.toMap(S3Artifact::getFilename, Function.identity(), (lhsDuplicate, rhsDuplicate) -> {
        log.warn("Duplicate artifact filenames found ({}; {})", lhsDuplicate, rhsDuplicate);
        return lhsDuplicate;
    }));
    Map<String, S3ArtifactSignature> signaturesByFilename = s3ArtifactsWithSignatures.stream().collect(Collectors.toMap(S3ArtifactSignature::getArtifactFilename, Function.identity(), (lhsDuplicate, rhsDuplicate) -> {
        log.warn("Duplicate signature filenames found ({}; {})", lhsDuplicate, rhsDuplicate);
        return lhsDuplicate;
    }));
    SetView<String> signaturesMissingArtifacts = Sets.difference(signaturesByFilename.keySet(), artifactsByFilename.keySet());
    if (signaturesMissingArtifacts.size() > 0) {
        if (executorConfiguration.isFailOnSignatureWithNoMatchingArtifact()) {
            throw new ArtifactVerificationException(String.format("No matching artifact(s) found for signature(s) %s", signaturesMissingArtifacts));
        } else {
            log.warn("No matching artifact(s) found for signature(s) {}", signaturesMissingArtifacts);
        }
    }
    SetView<String> artifactsMissingSignatures = Sets.difference(artifactsByFilename.keySet(), signaturesByFilename.keySet());
    if (artifactsMissingSignatures.size() > 0) {
        if (executorConfiguration.isFailOnArtifactWithNoMatchingSignature()) {
            throw new ArtifactVerificationException(String.format("No signature(s) found for artifact(s) %s", artifactsMissingSignatures));
        } else {
            log.warn("No signature(s) found for artifact(s) {}", artifactsMissingSignatures);
        }
    }
    if (s3ArtifactsWithSignatures.isEmpty()) {
        log.info("No files containing artifact signatures specified, skipping verification.");
        return;
    }
    for (S3ArtifactSignature s3ArtifactSignature : s3ArtifactsWithSignatures) {
        S3Artifact maybeMatchingForSignature = artifactsByFilename.get(s3ArtifactSignature.getArtifactFilename());
        if (maybeMatchingForSignature != null) {
            checkArtifactSignature(maybeMatchingForSignature, s3ArtifactSignature);
        }
    }
}

Example 2 with S3Artifact

use of com.hubspot.deploy.S3Artifact in project Singularity by HubSpot.

the class UnixLocalDownloadServiceFetcher method downloadFiles.

@Override
public void downloadFiles(List<? extends S3Artifact> s3Artifacts, SingularityExecutorTask task) throws InterruptedException {
    final List<CompletableFutureHolder> futures = Lists.newArrayListWithCapacity(s3Artifacts.size());
    for (S3Artifact s3Artifact : s3Artifacts) {
        String destination = task.getArtifactPath(s3Artifact, task.getTaskDefinition().getTaskDirectoryPath()).toString();
        ArtifactDownloadRequest artifactDownloadRequest = new ArtifactDownloadRequest(destination, s3Artifact, Optional.of(executorConfiguration.getLocalDownloadServiceTimeoutMillis()));
        task.getLog().debug("Requesting {} from {}", artifactDownloadRequest, s3Configuration.getLocalDownloadSocket().get());
        try {
            CompletableFuture<ContentResponse> future = new CompletableFuture<>();
            httpClient.newRequest(localDownloadPath).method(HttpMethod.POST).content(new BytesContentProvider(objectMapper.writeValueAsBytes(artifactDownloadRequest)), "application/json").send(new CompletableFutureResponseListener(future));
            futures.add(new CompletableFutureHolder(future, System.currentTimeMillis(), s3Artifact));
        } catch (JsonProcessingException e) {
            throw new RuntimeException(e);
        }
    }
    for (CompletableFutureHolder future : futures) {
        ContentResponse response = future.getResponse();
        task.getLog().debug("Future for {} got status code {} after {}", future.s3Artifact.getName(), response.getStatus(), JavaUtils.duration(future.start));
        if (response.getStatus() != 200) {
            throw new IllegalStateException("Got status code:" + response.getStatus());
        }
    }
}

Example 3 with S3Artifact

use of com.hubspot.deploy.S3Artifact in project Singularity by HubSpot.

the class HttpLocalDownloadServiceFetcher method downloadFiles.

@Override
public void downloadFiles(List<? extends S3Artifact> s3Artifacts, SingularityExecutorTask task) throws InterruptedException {
    final List<FutureHolder> futures = Lists.newArrayListWithCapacity(s3Artifacts.size());
    for (S3Artifact s3Artifact : s3Artifacts) {
        String destination = task.getArtifactPath(s3Artifact, task.getTaskDefinition().getTaskDirectoryPath()).toString();
        ArtifactDownloadRequest artifactDownloadRequest = new ArtifactDownloadRequest(destination, s3Artifact, Optional.of(executorConfiguration.getLocalDownloadServiceTimeoutMillis()));
        task.getLog().debug("Requesting {} from {}", artifactDownloadRequest, localDownloadUri);
        BoundRequestBuilder postRequestBldr = httpClient.preparePost(localDownloadUri);
        try {
            postRequestBldr.setBody(objectMapper.writeValueAsBytes(artifactDownloadRequest));
        } catch (JsonProcessingException e) {
            throw new RuntimeException(e);
        }
        try {
            ListenableFuture<Response> future = httpClient.executeRequest(postRequestBldr.build());
            futures.add(new FutureHolder(future, System.currentTimeMillis(), s3Artifact));
        } catch (Throwable t) {
            throw new RuntimeException(t);
        }
    }
    for (FutureHolder future : futures) {
        Response response = future.getResponse();
        task.getLog().debug("Future for {} got status code {} after {}", future.s3Artifact.getName(), response.getStatusCode(), JavaUtils.duration(future.start));
        if (response.getStatusCode() != 200) {
            throw new IllegalStateException("Got status code:" + response.getStatusCode());
        }
    }
}

Example 4 with S3Artifact

use of com.hubspot.deploy.S3Artifact in project Singularity by HubSpot.

the class SingularityExecutorTaskProcessBuilder method call.

@Override
public ProcessBuilder call() throws Exception {
    if (task.getTaskInfo().hasContainer() && task.getTaskInfo().getContainer().hasDocker()) {
        executorUtils.sendStatusUpdate(task.getDriver(), task.getTaskInfo().getTaskId(), TaskState.TASK_STARTING, String.format("Pulling image... (executor pid: %s)", executorPid), task.getLog());
        try {
            dockerUtils.pull(task.getTaskInfo().getContainer().getDocker().getImage());
        } catch (DockerException e) {
            throw new ProcessFailedException("Could not pull docker image", e);
        }
    }
    executorUtils.sendStatusUpdate(task.getDriver(), task.getTaskInfo().getTaskId(), TaskState.TASK_STARTING, String.format("Staging files... (executor pid: %s)", executorPid), task.getLog());
    taskArtifactFetcher = Optional.of(artifactFetcher.buildTaskFetcher(task));
    taskArtifactFetcher.get().fetchFiles(executorData.getEmbeddedArtifacts(), executorData.getS3Artifacts(), executorData.getS3ArtifactSignaturesOrEmpty(), executorData.getExternalArtifacts());
    task.getArtifactVerifier().checkSignatures(executorData.getS3Artifacts(), executorData.getS3ArtifactSignaturesOrEmpty());
    List<ArtifactList> artifactLists = new ArrayList<>();
    artifactLists.addAll(checkArtifactsForArtifactLists(executorData.getS3Artifacts()));
    artifactLists.addAll(checkArtifactsForArtifactLists(executorData.getS3ArtifactSignaturesOrEmpty()));
    artifactLists.addAll(checkArtifactsForArtifactLists(executorData.getExternalArtifacts()));
    if (!artifactLists.isEmpty()) {
        List<EmbeddedArtifact> embeddedArtifacts = new ArrayList<>();
        List<S3Artifact> s3Artifacts = new ArrayList<>();
        List<S3ArtifactSignature> s3ArtifactSignatures = new ArrayList<>();
        List<ExternalArtifact> externalArtifacts = new ArrayList<>();
        for (ArtifactList artifactList : artifactLists) {
            embeddedArtifacts.addAll(artifactList.getEmbeddedArtifacts());
            s3Artifacts.addAll(artifactList.getS3Artifacts());
            s3ArtifactSignatures.addAll(artifactList.getS3ArtifactSignatures());
            externalArtifacts.addAll(artifactList.getExternalArtifacts());
        }
        task.getLog().info("Found {} artifact lists with {} embedded, {} s3, {} external, fetching...", artifactLists.size(), embeddedArtifacts.size(), s3Artifacts.size() + s3ArtifactSignatures.size(), externalArtifacts.size());
        taskArtifactFetcher.get().fetchFiles(embeddedArtifacts, s3Artifacts, s3ArtifactSignatures, externalArtifacts);
        task.getArtifactVerifier().checkSignatures(s3Artifacts, s3ArtifactSignatures);
    }
    ProcessBuilder processBuilder = buildProcessBuilder(task.getTaskInfo(), executorData, task.getTaskDefinition().getServiceLogFileName());
    task.getTaskLogManager().setup();
    return processBuilder;
}