Examples with S3UploadMetadata com.hubspot.singularity.runner.base.shared.S3UploadMetadata used on opensource projects

Example 1 with S3UploadMetadata

use of com.hubspot.singularity.runner.base.shared.S3UploadMetadata in project Singularity by HubSpot.

the class SingularityExecutorTaskLogManager method writeS3MetadataFile.

private boolean writeS3MetadataFile(String filenameHint, Path pathToS3Directory, String globForS3Files, Optional<String> s3Bucket, Optional<String> s3KeyPattern, boolean finished, Optional<String> s3StorageClass, Optional<Long> applyS3StorageClassAfterBytes, boolean checkSubdirectories, boolean compressBeforeUpload, boolean checkIfOpen) {
    final String s3UploaderBucket = s3Bucket.orElse(taskDefinition.getExecutorData().getDefaultS3Bucket());
    if (Strings.isNullOrEmpty(s3UploaderBucket)) {
        log.warn("No s3 bucket specified, not writing s3 metadata for file matcher {}", globForS3Files);
        return false;
    }
    S3UploadMetadata s3UploadMetadata = new S3UploadMetadata(pathToS3Directory.toString(), globForS3Files, s3UploaderBucket, getS3KeyPattern(s3KeyPattern.orElse(taskDefinition.getExecutorData().getS3UploaderKeyPattern())), finished, Optional.<String>empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), s3StorageClass, applyS3StorageClassAfterBytes, Optional.of(finished), Optional.of(checkSubdirectories), Optional.of(compressBeforeUpload), Optional.of(checkIfOpen), Optional.empty(), Collections.emptyMap(), Optional.empty(), Optional.empty(), Optional.empty());
    String s3UploadMetadataFileName = String.format("%s-%s%s", taskDefinition.getTaskId(), filenameHint, baseConfiguration.getS3UploaderMetadataSuffix());
    Path s3UploadMetadataPath = Paths.get(baseConfiguration.getS3UploaderMetadataDirectory()).resolve(s3UploadMetadataFileName);
    return jsonObjectFileHelper.writeObject(s3UploadMetadata, s3UploadMetadataPath, log);
}

Example 2 with S3UploadMetadata

use of com.hubspot.singularity.runner.base.shared.S3UploadMetadata in project Singularity by HubSpot.

the class SingularityS3UploaderDriver method checkUploads.

private int checkUploads() {
    if (metadataToUploader.isEmpty() && metadataToImmediateUploader.isEmpty()) {
        return 0;
    }
    int totesUploads = 0;
    // Check results of immediate uploaders
    List<S3UploadMetadata> toRetry = new ArrayList<>();
    List<S3UploadMetadata> toRemove = new ArrayList<>();
    for (Map.Entry<S3UploadMetadata, CompletableFuture<Integer>> entry : immediateUploadersFutures.entrySet()) {
        SingularityUploader uploader = metadataToImmediateUploader.get(entry.getKey());
        if (uploader == null) {
            toRemove.add(entry.getKey());
            continue;
        }
        try {
            int uploadedFiles = entry.getValue().get();
            List<Path> remainingFiles = uploader.filesToUpload(isFinished(uploader));
            if (!remainingFiles.isEmpty() || uploadedFiles == -1) {
                LOG.debug("Immediate uploader had {} remaining files, previously uploaded {}, will retry", remainingFiles.size(), uploadedFiles);
                toRetry.add(entry.getKey());
            } else {
                totesUploads += uploadedFiles;
                toRemove.add(entry.getKey());
            }
        } catch (NoSuchFileException nsfe) {
            LOG.warn("File not found to upload", nsfe);
            toRetry.add(entry.getKey());
        } catch (Throwable t) {
            metrics.error();
            LOG.error("Waiting on future", t);
            exceptionNotifier.notify(String.format("Error waiting on uploader future (%s)", t.getMessage()), t, ImmutableMap.of("metadataPath", uploader.getMetadataPath().toString()));
            toRetry.add(entry.getKey());
        }
    }
    for (S3UploadMetadata uploaderMetadata : toRemove) {
        metrics.getImmediateUploaderCounter().dec();
        SingularityUploader uploader = metadataToImmediateUploader.remove(uploaderMetadata);
        CompletableFuture<Integer> uploaderFuture = immediateUploadersFutures.remove(uploaderMetadata);
        if (uploaderFuture != null) {
            try {
                // All uploaders reaching this point should already be finished, if it isn't done in 30s, it's stuck
                uploaderFuture.get(30, TimeUnit.SECONDS);
            } catch (Throwable t) {
                LOG.error("Exception waiting for immediate uploader to complete for metadata {}", uploaderMetadata, t);
            }
        }
        if (uploader == null) {
            continue;
        }
        expiring.remove(uploader);
        try {
            LOG.info("Deleting finished immediate uploader {}", uploader.getMetadataPath());
            Files.delete(uploader.getMetadataPath());
        } catch (NoSuchFileException nfe) {
            LOG.warn("File {} was already deleted", nfe.getFile());
        } catch (IOException e) {
            LOG.warn("Couldn't delete {}", uploader.getMetadataPath(), e);
            exceptionNotifier.notify("Could not delete metadata file", e, ImmutableMap.of("metadataPath", uploader.getMetadataPath().toString()));
        }
    }
    for (S3UploadMetadata uploaderMetadata : toRetry) {
        SingularityUploader uploader = metadataToImmediateUploader.get(uploaderMetadata);
        if (uploader != null) {
            LOG.debug("Retrying immediate uploader {}", uploaderMetadata);
            performImmediateUpload(uploader);
        } else {
            LOG.debug("Uploader for metadata {} not found to retry upload", uploaderMetadata);
        }
    }
    // Check regular uploaders
    int initialExpectedSize = Math.max(metadataToUploader.size(), 1);
    final Map<SingularityUploader, CompletableFuture<Integer>> futures = Maps.newHashMapWithExpectedSize(initialExpectedSize);
    final Map<SingularityUploader, Boolean> finishing = Maps.newHashMapWithExpectedSize(initialExpectedSize);
    for (final SingularityUploader uploader : metadataToUploader.values()) {
        final boolean isFinished = isFinished(uploader);
        // do this here so we run at least once with isFinished = true
        finishing.put(uploader, isFinished);
        futures.put(uploader, CompletableFuture.supplyAsync(performUploadSupplier(uploader, isFinished, false), executorService));
    }
    LOG.debug("Waiting on {} future(s)", futures.size());
    final long now = System.currentTimeMillis();
    final Set<SingularityUploader> expiredUploaders = Sets.newHashSetWithExpectedSize(initialExpectedSize);
    for (Entry<SingularityUploader, CompletableFuture<Integer>> uploaderToFuture : futures.entrySet()) {
        final SingularityUploader uploader = uploaderToFuture.getKey();
        try {
            LOG.debug("Waiting for future for uploader {}", uploader);
            final int foundFiles = uploaderToFuture.getValue().get(30, TimeUnit.SECONDS);
            final boolean isFinished = finishing.get(uploader);
            if (foundFiles == 0 && shouldExpire(uploader, isFinished)) {
                LOG.info("Expiring {}", uploader);
                expiredUploaders.add(uploader);
            } else {
                LOG.trace("Updating uploader {} last expire time", uploader);
                uploaderLastHadFilesAt.put(uploader, now);
            }
            totesUploads += foundFiles;
        } catch (TimeoutException te) {
            // fuser or another check likely timed out and will retry
            LOG.warn("Timeout exception waiting on future", te);
        } catch (Throwable t) {
            metrics.error();
            LOG.error("Waiting on future", t);
            exceptionNotifier.notify(String.format("Error waiting on uploader future (%s)", t.getMessage()), t, ImmutableMap.of("metadataPath", uploader.getMetadataPath().toString()));
        }
    }
    for (SingularityUploader expiredUploader : expiredUploaders) {
        metrics.getUploaderCounter().dec();
        metadataToUploader.remove(expiredUploader.getUploadMetadata());
        uploaderLastHadFilesAt.remove(expiredUploader);
        expiring.remove(expiredUploader);
        try {
            LOG.info("Deleting expired uploader {}", expiredUploader.getMetadataPath());
            Files.delete(expiredUploader.getMetadataPath());
        } catch (NoSuchFileException nfe) {
            LOG.warn("File {} was already deleted", nfe.getFile());
        } catch (IOException e) {
            LOG.warn("Couldn't delete {}", expiredUploader.getMetadataPath(), e);
            exceptionNotifier.notify("Could not delete metadata file", e, ImmutableMap.of("metadataPath", expiredUploader.getMetadataPath().toString()));
        }
    }
    return totesUploads;
}

Example 3 with S3UploadMetadata

use of com.hubspot.singularity.runner.base.shared.S3UploadMetadata in project Singularity by HubSpot.

the class SingularityS3UploaderDriver method handleNewOrModifiedS3Metadata.

private boolean handleNewOrModifiedS3Metadata(Path filename) throws IOException {
    Optional<S3UploadMetadata> maybeMetadata = readS3UploadMetadata(filename);
    if (!maybeMetadata.isPresent()) {
        return false;
    }
    final S3UploadMetadata metadata = maybeMetadata.get();
    SingularityUploader existingUploader = metadataToUploader.get(metadata);
    SingularityUploader existingImmediateUploader = metadataToImmediateUploader.get(metadata);
    if (metadata.isImmediate()) {
        if (existingUploader != null) {
            LOG.debug("Existing metadata {} from {} changed to be immediate, forcing upload", metadata, filename);
            expiring.remove(existingUploader);
            if (existingImmediateUploader == null) {
                metrics.getUploaderCounter().dec();
                metrics.getImmediateUploaderCounter().inc();
                metadataToImmediateUploader.put(metadata, existingUploader);
                metadataToUploader.remove(existingUploader.getUploadMetadata());
                uploaderLastHadFilesAt.remove(existingUploader);
                performImmediateUpload(existingUploader);
                return true;
            } else {
                performImmediateUpload(existingImmediateUploader);
                return false;
            }
        } else if (existingImmediateUploader != null) {
            LOG.info("Already had an immediate uploader for metadata {}, triggering new upload attempt", metadata);
            performImmediateUpload(existingImmediateUploader);
            return false;
        }
    }
    if (existingUploader != null) {
        if (existingUploader.getUploadMetadata().isFinished() == metadata.isFinished()) {
            LOG.debug("Ignoring metadata {} from {} because there was already one present", metadata, filename);
            return false;
        } else {
            LOG.info("Toggling uploader {} finish state to {}", existingUploader, metadata.isFinished());
            if (metadata.isFinished()) {
                expiring.add(existingUploader);
            } else {
                expiring.remove(existingUploader);
            }
            return true;
        }
    }
    try {
        Optional<BasicAWSCredentials> bucketCreds = Optional.empty();
        if (configuration.getS3BucketCredentials().containsKey(metadata.getS3Bucket())) {
            bucketCreds = Optional.of(configuration.getS3BucketCredentials().get(metadata.getS3Bucket()).toAWSCredentials());
        }
        final BasicAWSCredentials defaultCredentials = new BasicAWSCredentials(configuration.getS3AccessKey().orElse(s3Configuration.getS3AccessKey().get()), configuration.getS3SecretKey().orElse(s3Configuration.getS3SecretKey().get()));
        final SingularityUploader uploader;
        if (metadata.getUploaderType() == SingularityUploaderType.S3) {
            uploader = new SingularityS3Uploader(bucketCreds.orElse(defaultCredentials), s3ClientProvider, metadata, fileSystem, metrics, filename, configuration, hostname, exceptionNotifier, checkFileOpenLock);
        } else {
            uploader = new SingularityGCSUploader(metadata, fileSystem, metrics, filename, configuration, hostname, exceptionNotifier, checkFileOpenLock, jsonObjectFileHelper);
        }
        if (metadata.isFinished()) {
            expiring.add(uploader);
        }
        if (metadata.isImmediate()) {
            LOG.info("Created new immediate uploader {}", uploader);
            metadataToImmediateUploader.put(metadata, uploader);
            metrics.getImmediateUploaderCounter().inc();
            performImmediateUpload(uploader);
            return true;
        } else {
            LOG.info("Created new uploader {}", uploader);
            metrics.getUploaderCounter().inc();
            metadataToUploader.put(metadata, uploader);
            uploaderLastHadFilesAt.put(uploader, System.currentTimeMillis());
            return true;
        }
    } catch (Throwable t) {
        LOG.info("Ignoring metadata {} because uploader couldn't be created", metadata, t);
        return false;
    }
}