Examples with Ds3Client com.spectralogic.ds3client.Ds3Client used on opensource projects

Example 1 with Ds3Client

use of com.spectralogic.ds3client.Ds3Client in project cyberduck by iterate-ch.

the class SpectraObjectListService method list.

@NotNull
protected AttributedList<Path> list(final Path directory, final ListProgressListener listener, final int chunksize) throws BackgroundException {
    try {
        final String prefix = this.createPrefix(directory);
        final AttributedList<Path> objects = new AttributedList<>();
        final Ds3Client client = new SpectraClientBuilder().wrap(session.getClient(), session.getHost());
        final Path bucket = containerService.getContainer(directory);
        long revision = 0L;
        String marker = null;
        String lastKey = null;
        boolean truncated;
        boolean hasDirectoryPlaceholder = containerService.isContainer(directory);
        do {
            final GetBucketResponse response = client.getBucket(new GetBucketRequest(bucket.getName()).withVersions(true).withDelimiter(String.valueOf(Path.DELIMITER)).withMarker(marker).withPrefix(StringUtils.isBlank(prefix) ? StringUtils.EMPTY : prefix).withMaxKeys(chunksize));
            for (final Contents object : response.getListBucketResult().getObjects()) {
                final String key = PathNormalizer.normalize(object.getKey());
                if (String.valueOf(Path.DELIMITER).equals(key)) {
                    log.warn(String.format("Skipping prefix %s", key));
                    continue;
                }
                if (new SimplePathPredicate(new Path(bucket, key, EnumSet.of(Path.Type.directory))).test(directory)) {
                    hasDirectoryPlaceholder = true;
                    // Placeholder object, skip
                    continue;
                }
                objects.add(new Path(directory, PathNormalizer.name(key), object.getKey().endsWith(String.valueOf(Path.DELIMITER)) ? EnumSet.of(Path.Type.directory) : EnumSet.of(Path.Type.file), this.toAttributes(object)));
            }
            for (final Contents object : response.getListBucketResult().getVersionedObjects()) {
                final String key = PathNormalizer.normalize(object.getKey());
                if (String.valueOf(Path.DELIMITER).equals(key)) {
                    log.warn(String.format("Skipping prefix %s", key));
                    continue;
                }
                if (new SimplePathPredicate(new Path(bucket, key, EnumSet.of(Path.Type.directory))).test(directory)) {
                    hasDirectoryPlaceholder = true;
                    // Placeholder object, skip
                    continue;
                }
                if (!StringUtils.equals(lastKey, key)) {
                    // Reset revision for next file
                    revision = 0L;
                }
                final PathAttributes attributes = this.toAttributes(object);
                attributes.setRevision(++revision);
                if (attributes.isDuplicate()) {
                    if (attributes.getRevision() == 1) {
                        // Allow to undelete if latest version is delete marker
                        final Map<String, String> custom = new HashMap<>(attributes.getCustom());
                        custom.put(SpectraVersioningFeature.KEY_REVERTABLE, Boolean.TRUE.toString());
                        attributes.setCustom(custom);
                    }
                }
                objects.add(new Path(directory, PathNormalizer.name(key), object.getKey().endsWith(String.valueOf(Path.DELIMITER)) ? EnumSet.of(Path.Type.directory) : EnumSet.of(Path.Type.file), attributes));
                lastKey = key;
            }
            for (CommonPrefixes common : response.getListBucketResult().getCommonPrefixes()) {
                if (String.valueOf(Path.DELIMITER).equals(common.getPrefix())) {
                    log.warn(String.format("Skipping prefix %s", common.getPrefix()));
                    continue;
                }
                final String key = PathNormalizer.normalize(common.getPrefix());
                if (new Path(bucket, key, EnumSet.of(Path.Type.directory)).equals(directory)) {
                    continue;
                }
                objects.add(new Path(directory, PathNormalizer.name(key), EnumSet.of(Path.Type.directory, Path.Type.placeholder), new PathAttributes()));
            }
            marker = response.getListBucketResult().getNextMarker();
            listener.chunk(directory, objects);
            truncated = response.getListBucketResult().getTruncated();
        } while (truncated);
        if (!hasDirectoryPlaceholder && objects.isEmpty()) {
            throw new NotfoundException(directory.getAbsolute());
        }
        return objects;
    } catch (FailedRequestException e) {
        throw new SpectraExceptionMappingService().map("Listing directory {0} failed", e, directory);
    } catch (IOException e) {
        throw new DefaultIOExceptionMappingService().map("Listing directory {0} failed", e, directory);
    }
}

Example 2 with Ds3Client

use of com.spectralogic.ds3client.Ds3Client in project cyberduck by iterate-ch.

the class SpectraVersioningFeature method revert.

@Override
public void revert(final Path file) throws BackgroundException {
    final Ds3Client client = new SpectraClientBuilder().wrap(session.getClient(), session.getHost());
    final Path container = containerService.getContainer(file);
    try {
        client.undeleteObjectSpectraS3(new UndeleteObjectSpectraS3Request(container.getName(), containerService.getKey(file)));
    } catch (FailedRequestException e) {
        throw new SpectraExceptionMappingService().map("Cannot revert file", e, file);
    } catch (IOException e) {
        throw new DefaultIOExceptionMappingService().map("Cannot revert file", e, file);
    }
}

Example 3 with Ds3Client

use of com.spectralogic.ds3client.Ds3Client in project cyberduck by iterate-ch.

the class SpectraVersioningFeature method setConfiguration.

@Override
public void setConfiguration(final Path file, final PasswordCallback prompt, final VersioningConfiguration configuration) throws BackgroundException {
    final Path container = containerService.getContainer(file);
    try {
        final VersioningConfiguration current = this.getConfiguration(container);
        if (configuration.isEnabled()) {
            if (current.isEnabled()) {
                log.debug(String.format("Versioning already enabled for bucket %s", container));
            } else {
                log.debug(String.format("Enable bucket versioning for %s", container));
                final Ds3Client client = new SpectraClientBuilder().wrap(session.getClient(), session.getHost());
                final GetBucketSpectraS3Response bucket = client.getBucketSpectraS3(new GetBucketSpectraS3Request(container.getName()));
                final UUID id = bucket.getBucketResult().getDataPolicyId();
                client.modifyDataPolicySpectraS3(new ModifyDataPolicySpectraS3Request(id).withVersioning(VersioningLevel.KEEP_MULTIPLE_VERSIONS));
            }
        } else {
            log.warn(String.format("Disable bucket versioning for %s is not supported", container));
        }
        cache.remove(container);
    } catch (FailedRequestException e) {
        throw new SpectraExceptionMappingService().map("Failure to write attributes of {0}", e, container);
    } catch (IOException e) {
        throw new DefaultIOExceptionMappingService().map("Failure to write attributes of {0}", e, container);
    }
}

Example 4 with Ds3Client

use of com.spectralogic.ds3client.Ds3Client in project cyberduck by iterate-ch.

the class SpectraAttributesFinderFeature method details.

protected HeadObjectResponse details(final Path file) throws BackgroundException {
    try {
        final Ds3Client client = new SpectraClientBuilder().wrap(session.getClient(), session.getHost());
        final HeadObjectResponse response = client.headObject(new HeadObjectRequest(containerService.getContainer(file).getName(), containerService.getKey(file)));
        switch(response.getStatus()) {
            case DOESNTEXIST:
                throw new NotfoundException(file.getAbsolute());
        }
        return response;
    } catch (FailedRequestException e) {
        throw new SpectraExceptionMappingService().map("Failure to read attributes of {0}", e, file);
    } catch (IOException e) {
        throw new DefaultIOExceptionMappingService().map("Failure to read attributes of {0}", e, file);
    }
}

Example 5 with Ds3Client

use of com.spectralogic.ds3client.Ds3Client in project cyberduck by iterate-ch.

the class SpectraBulkService method query.

/**
 * Get a list of all job chunks for a given job that are ready for client processing.
 * <p>
 * For PUT jobs, this will allocate a working window of job chunks, if possible, and return the job chunks that the client can upload.
 * Any chunk returned is fully allocated, meaning that you do not have to handle HTTP 307 retries on subsequent PUTs for the chunks.
 * Retries adversely impact BlackPearl gateway performance and require you to provide the object data stream for every PUT retry.
 * <p>
 * For GET jobs, this will respond with which job chunks have been loaded into cache and are ready for download.
 *
 * @param file   File
 * @param status Write job id into status parameters
 * @throws RetriableAccessDeniedException                File is not yet in cache
 * @throws ch.cyberduck.core.exception.RedirectException Should be accessed from different node
 */
public List<TransferStatus> query(final Transfer.Type type, final Path file, final TransferStatus status) throws BackgroundException {
    // This will respond with which job chunks have been loaded into cache and are ready for download.
    try {
        if (!status.getParameters().containsKey(REQUEST_PARAMETER_JOBID_IDENTIFIER)) {
            throw new NotfoundException(String.format("Missing job id parameter in status for %s", file.getName()));
        }
        final String job = status.getParameters().get(REQUEST_PARAMETER_JOBID_IDENTIFIER);
        if (log.isDebugEnabled()) {
            log.debug(String.format("Query status for job %s", job));
        }
        // Fetch current list from server
        final Ds3Client client = new SpectraClientBuilder().wrap(session.getClient(), session.getHost());
        // For GET, the client may need to issue multiple GET requests for a single object if it has
        // been broken up into multiple pieces due to its large size
        // For PUT, This will allocate a working window of job chunks, if possible, and return a list of
        // the job chunks that the client can upload. The client should PUT all of the object parts
        // from the list of job chunks returned and repeat this process until all chunks are transferred
        final GetJobChunksReadyForClientProcessingSpectraS3Response response = client.getJobChunksReadyForClientProcessingSpectraS3(new GetJobChunksReadyForClientProcessingSpectraS3Request(UUID.fromString(job)).withPreferredNumberOfChunks(Integer.MAX_VALUE));
        if (log.isInfoEnabled()) {
            log.info(String.format("Job status %s for job %s", response.getStatus(), job));
        }
        switch(response.getStatus()) {
            case RETRYLATER:
                {
                    final Duration delay = Duration.ofSeconds(response.getRetryAfterSeconds());
                    throw new RetriableAccessDeniedException(String.format("Job %s not yet loaded into cache", job), delay);
                }
        }
        final MasterObjectList master = response.getMasterObjectListResult();
        if (log.isInfoEnabled()) {
            log.info(String.format("Master object list with %d objects for %s", master.getObjects().size(), file));
            log.info(String.format("Master object list status %s for %s", master.getStatus(), file));
        }
        final List<TransferStatus> chunks = query(file, status, job, master);
        if (chunks.isEmpty()) {
            log.info(String.format("Still missing chunks for file %s for job %s", file.getName(), job));
            throw new RetriableAccessDeniedException(String.format("Missing chunks for job %s", job), Duration.ofSeconds(new HostPreferences(session.getHost()).getInteger("spectra.retry.delay")));
        }
        if (log.isInfoEnabled()) {
            log.info(String.format("Server returned %d chunks for %s", chunks.size(), file));
        }
        return chunks;
    } catch (FailedRequestException e) {
        throw new SpectraExceptionMappingService().map(e);
    } catch (IOException e) {
        throw new DefaultIOExceptionMappingService().map(e);
    }
}