Examples with B2UploadPartResponse synapticloop.b2.response.B2UploadPartResponse used on opensource projects

Example 1 with B2UploadPartResponse

use of synapticloop.b2.response.B2UploadPartResponse in project cyberduck by iterate-ch.

the class B2LargeUploadPartService method list.

/**
 * @param fileid File id of unfinished large upload
 * @return List of parts
 */
public List<B2UploadPartResponse> list(final String fileid) throws BackgroundException {
    if (log.isInfoEnabled()) {
        log.info(String.format("List completed parts of file %s", fileid));
    }
    // This operation lists the parts that have been uploaded for a specific multipart upload.
    try {
        // Completed parts
        final List<B2UploadPartResponse> completed = new ArrayList<B2UploadPartResponse>();
        Integer startPartNumber = null;
        do {
            final B2ListPartsResponse response = session.getClient().listParts(fileid, startPartNumber, null);
            completed.addAll(response.getFiles());
            startPartNumber = response.getNextPartNumber();
        } while (startPartNumber != null);
        return completed;
    } catch (B2ApiException e) {
        throw new B2ExceptionMappingService(this.fileid).map(e);
    } catch (IOException e) {
        throw new DefaultIOExceptionMappingService().map(e);
    }
}

Example 2 with B2UploadPartResponse

use of synapticloop.b2.response.B2UploadPartResponse in project cyberduck by iterate-ch.

the class B2LargeUploadService method upload.

@Override
public BaseB2Response upload(final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
    final ThreadPool pool = ThreadPoolFactory.get("largeupload", concurrency);
    try {
        // Get the results of the uploads in the order they were submitted
        // this is important for building the manifest, and is not a problem in terms of performance
        // because we should only continue when all segments have uploaded successfully
        final List<B2UploadPartResponse> completed = new ArrayList<>();
        final Map<String, String> fileinfo = new HashMap<>(status.getMetadata());
        final Checksum checksum = status.getChecksum();
        if (Checksum.NONE != checksum) {
            switch(checksum.algorithm) {
                case sha1:
                    fileinfo.put(X_BZ_INFO_LARGE_FILE_SHA1, status.getChecksum().hash);
                    break;
            }
        }
        if (null != status.getTimestamp()) {
            fileinfo.put(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS, String.valueOf(status.getTimestamp()));
        }
        final String fileId;
        if (status.isAppend()) {
            // Add already completed parts
            final B2LargeUploadPartService partService = new B2LargeUploadPartService(session, fileid);
            final List<B2FileInfoResponse> uploads = partService.find(file);
            if (uploads.isEmpty()) {
                fileId = session.getClient().startLargeFileUpload(fileid.getVersionId(containerService.getContainer(file), new DisabledListProgressListener()), containerService.getKey(file), status.getMime(), fileinfo).getFileId();
            } else {
                fileId = uploads.iterator().next().getFileId();
                completed.addAll(partService.list(fileId));
            }
        } else {
            fileId = session.getClient().startLargeFileUpload(fileid.getVersionId(containerService.getContainer(file), new DisabledListProgressListener()), containerService.getKey(file), status.getMime(), fileinfo).getFileId();
        }
        // Full size of file
        final long size = status.getLength() + status.getOffset();
        // Submit file segments for concurrent upload
        final List<Future<B2UploadPartResponse>> parts = new ArrayList<>();
        long remaining = status.getLength();
        long offset = 0;
        for (int partNumber = 1; remaining > 0; partNumber++) {
            boolean skip = false;
            if (status.isAppend()) {
                if (log.isInfoEnabled()) {
                    log.info(String.format("Determine if part number %d can be skipped", partNumber));
                }
                for (B2UploadPartResponse c : completed) {
                    if (c.getPartNumber().equals(partNumber)) {
                        if (log.isInfoEnabled()) {
                            log.info(String.format("Skip completed part number %d", partNumber));
                        }
                        skip = true;
                        offset += c.getContentLength();
                        break;
                    }
                }
            }
            if (!skip) {
                final long length = Math.min(Math.max((size / B2LargeUploadService.MAXIMUM_UPLOAD_PARTS), partSize), remaining);
                // Submit to queue
                parts.add(this.submit(pool, file, local, throttle, listener, status, fileId, partNumber, offset, length, callback));
                if (log.isDebugEnabled()) {
                    log.debug(String.format("Part %s submitted with size %d and offset %d", partNumber, length, offset));
                }
                remaining -= length;
                offset += length;
            }
        }
        try {
            for (Future<B2UploadPartResponse> f : parts) {
                completed.add(f.get());
            }
        } catch (InterruptedException e) {
            log.error("Part upload failed with interrupt failure");
            status.setCanceled();
            throw new ConnectionCanceledException(e);
        } catch (ExecutionException e) {
            log.warn(String.format("Part upload failed with execution failure %s", e.getMessage()));
            if (e.getCause() instanceof BackgroundException) {
                throw (BackgroundException) e.getCause();
            }
            throw new DefaultExceptionMappingService().map(e.getCause());
        }
        completed.sort(new Comparator<B2UploadPartResponse>() {

            @Override
            public int compare(final B2UploadPartResponse o1, final B2UploadPartResponse o2) {
                return o1.getPartNumber().compareTo(o2.getPartNumber());
            }
        });
        final List<String> checksums = new ArrayList<>();
        for (B2UploadPartResponse part : completed) {
            checksums.add(part.getContentSha1());
        }
        final B2FinishLargeFileResponse response = session.getClient().finishLargeFileUpload(fileId, checksums.toArray(new String[checksums.size()]));
        if (log.isInfoEnabled()) {
            log.info(String.format("Finished large file upload %s with %d parts", file, completed.size()));
        }
        fileid.cache(file, response.getFileId());
        // Mark parent status as complete
        status.withResponse(new B2AttributesFinderFeature(session, fileid).toAttributes(response)).setComplete();
        return response;
    } catch (B2ApiException e) {
        throw new B2ExceptionMappingService(fileid).map("Upload {0} failed", e, file);
    } catch (IOException e) {
        throw new DefaultIOExceptionMappingService().map("Upload {0} failed", e, file);
    } finally {
        pool.shutdown(false);
    }
}

Example 3 with B2UploadPartResponse

use of synapticloop.b2.response.B2UploadPartResponse in project cyberduck by iterate-ch.

the class B2WriteFeature method append.

@Override
public Append append(final Path file, final TransferStatus status) throws BackgroundException {
    final B2LargeUploadPartService partService = new B2LargeUploadPartService(session, fileid);
    final List<B2FileInfoResponse> upload = partService.find(file);
    if (!upload.isEmpty()) {
        Long size = 0L;
        for (B2UploadPartResponse completed : partService.list(upload.iterator().next().getFileId())) {
            size += completed.getContentLength();
        }
        return new Append(true).withStatus(status).withSize(size);
    }
    return new Append(false).withStatus(status);
}

Example 4 with B2UploadPartResponse

use of synapticloop.b2.response.B2UploadPartResponse in project cyberduck by iterate-ch.

the class B2LargeCopyFeature method copy.

@Override
public Path copy(final Path source, final Path target, final TransferStatus status, final ConnectionCallback callback, final StreamListener listener) throws BackgroundException {
    final ThreadPool pool = ThreadPoolFactory.get("largeupload", concurrency);
    try {
        final Map<String, String> fileinfo = new HashMap<>(status.getMetadata());
        if (null != status.getTimestamp()) {
            fileinfo.put(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS, String.valueOf(status.getTimestamp()));
        }
        final Checksum checksum = status.getChecksum();
        if (Checksum.NONE != checksum) {
            switch(checksum.algorithm) {
                case sha1:
                    fileinfo.put(X_BZ_INFO_LARGE_FILE_SHA1, status.getChecksum().hash);
                    break;
            }
        }
        final B2StartLargeFileResponse response = session.getClient().startLargeFileUpload(fileid.getVersionId(containerService.getContainer(target), new DisabledListProgressListener()), containerService.getKey(target), status.getMime(), fileinfo);
        final long size = status.getLength();
        // Submit file segments for concurrent upload
        final List<Future<B2UploadPartResponse>> parts = new ArrayList<Future<B2UploadPartResponse>>();
        long remaining = status.getLength();
        long offset = 0;
        final List<B2UploadPartResponse> completed = new ArrayList<B2UploadPartResponse>();
        for (int partNumber = 1; remaining > 0; partNumber++) {
            final Long length = Math.min(Math.max((size / B2LargeUploadService.MAXIMUM_UPLOAD_PARTS), partSize), remaining);
            // Submit to queue
            parts.add(this.submit(pool, source, response.getFileId(), status, partNumber, offset, length, callback));
            if (log.isDebugEnabled()) {
                log.debug(String.format("Part %s submitted with size %d and offset %d", partNumber, length, offset));
            }
            remaining -= length;
            offset += length;
        }
        try {
            for (Future<B2UploadPartResponse> f : parts) {
                completed.add(f.get());
            }
        } catch (InterruptedException e) {
            log.error("Part upload failed with interrupt failure");
            status.setCanceled();
            throw new ConnectionCanceledException(e);
        } catch (ExecutionException e) {
            log.warn(String.format("Part upload failed with execution failure %s", e.getMessage()));
            if (e.getCause() instanceof BackgroundException) {
                throw (BackgroundException) e.getCause();
            }
            throw new DefaultExceptionMappingService().map(e.getCause());
        }
        completed.sort(new Comparator<B2UploadPartResponse>() {

            @Override
            public int compare(final B2UploadPartResponse o1, final B2UploadPartResponse o2) {
                return o1.getPartNumber().compareTo(o2.getPartNumber());
            }
        });
        final List<String> checksums = new ArrayList<String>();
        for (B2UploadPartResponse part : completed) {
            checksums.add(part.getContentSha1());
        }
        session.getClient().finishLargeFileUpload(response.getFileId(), checksums.toArray(new String[checksums.size()]));
        if (log.isInfoEnabled()) {
            log.info(String.format("Finished large file upload %s with %d parts", target, completed.size()));
        }
        fileid.cache(target, response.getFileId());
        return target.withAttributes(new PathAttributes(source.attributes()).withVersionId(response.getFileId()));
    } catch (B2ApiException e) {
        throw new B2ExceptionMappingService(fileid).map("Cannot copy {0}", e, source);
    } catch (IOException e) {
        throw new DefaultIOExceptionMappingService().map(e);
    } finally {
        pool.shutdown(false);
    }
}

Example 5 with B2UploadPartResponse

use of synapticloop.b2.response.B2UploadPartResponse in project cyberduck by iterate-ch.

the class B2LargeUploadService method submit.

private Future<B2UploadPartResponse> submit(final ThreadPool pool, final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus overall, final String fileId, final int partNumber, final Long offset, final Long length, final ConnectionCallback callback) {
    if (log.isInfoEnabled()) {
        log.info(String.format("Submit part %d of %s to queue with offset %d and length %d", partNumber, file, offset, length));
    }
    final BytecountStreamListener counter = new BytecountStreamListener(listener);
    return pool.execute(new SegmentRetryCallable<>(session.getHost(), new BackgroundExceptionCallable<B2UploadPartResponse>() {

        @Override
        public B2UploadPartResponse call() throws BackgroundException {
            overall.validate();
            final TransferStatus status = new TransferStatus().withLength(length).withOffset(offset);
            final Map<String, String> requestParameters = new HashMap<>();
            requestParameters.put("fileId", fileId);
            status.setParameters(requestParameters);
            status.setHeader(overall.getHeader());
            status.setChecksum(writer.checksum(file, status).compute(local.getInputStream(), status));
            status.setSegment(true);
            status.setPart(partNumber);
            return (B2UploadPartResponse) B2LargeUploadService.super.upload(file, local, throttle, counter, status, overall, status, callback);
        }
    }, overall, counter));
}