Examples with ProgressListener com.qcloud.cos.event.ProgressListener used on opensource projects

Example 1 with ProgressListener

use of com.qcloud.cos.event.ProgressListener in project cos-java-sdk-v5 by tencentyun.

the class TransferManager method uploadFileList.

/**
 * Uploads all specified files to the bucket named, constructing relative keys depending on the
 * commonParentDirectory given.
 * <p>
 * COS will overwrite any existing objects that happen to have the same key, just as when
 * uploading individual files, so use with caution.
 * </p>
 *
 * @param bucketName The name of the bucket to upload objects to.
 * @param virtualDirectoryKeyPrefix The key prefix of the virtual directory to upload to. Use
 *        the null or empty string to upload files to the root of the bucket.
 * @param directory The common parent directory of files to upload. The keys of the files in the
 *        list of files are constructed relative to this directory and the
 *        virtualDirectoryKeyPrefix.
 * @param files A list of files to upload. The keys of the files are calculated relative to the
 *        common parent directory and the virtualDirectoryKeyPrefix.
 * @param metadataProvider A callback of type <code>ObjectMetadataProvider</code> which is used
 *        to provide metadata for each file being uploaded.
 */
public MultipleFileUpload uploadFileList(String bucketName, String virtualDirectoryKeyPrefix, File directory, List<File> files, ObjectMetadataProvider metadataProvider) {
    if (directory == null || !directory.exists() || !directory.isDirectory()) {
        throw new IllegalArgumentException("Must provide a common base directory for uploaded files");
    }
    if (virtualDirectoryKeyPrefix == null || virtualDirectoryKeyPrefix.length() == 0) {
        virtualDirectoryKeyPrefix = "";
    } else if (!virtualDirectoryKeyPrefix.endsWith("/")) {
        virtualDirectoryKeyPrefix = virtualDirectoryKeyPrefix + "/";
    }
    /* This is the hook for adding additional progress listeners */
    ProgressListenerChain additionalListeners = new ProgressListenerChain();
    TransferProgress progress = new TransferProgress();
    /*
         * Bind additional progress listeners to this MultipleFileTransferProgressUpdatingListener
         * to receive ByteTransferred events from each single-file upload implementation.
         */
    ProgressListener listener = new MultipleFileTransferProgressUpdatingListener(progress, additionalListeners);
    List<UploadImpl> uploads = new LinkedList<UploadImpl>();
    MultipleFileUploadImpl multipleFileUpload = new MultipleFileUploadImpl("Uploading etc", progress, additionalListeners, virtualDirectoryKeyPrefix, bucketName, uploads);
    multipleFileUpload.setMonitor(new MultipleFileTransferMonitor(multipleFileUpload, uploads));
    final CountDownLatch latch = new CountDownLatch(1);
    MultipleFileTransferStateChangeListener transferListener = new MultipleFileTransferStateChangeListener(latch, multipleFileUpload);
    if (files == null || files.isEmpty()) {
        multipleFileUpload.setState(TransferState.Completed);
    } else {
        /*
             * If the absolute path for the common/base directory does NOT end in a separator (which
             * is the case for anything but root directories), then we know there's still a
             * separator between the base directory and the rest of the file's path, so we increment
             * the starting position by one.
             */
        int startingPosition = directory.getAbsolutePath().length();
        if (!(directory.getAbsolutePath().endsWith(File.separator)))
            startingPosition++;
        long totalSize = 0;
        for (File f : files) {
            // Check, if file, since only files can be uploaded.
            if (f.isFile()) {
                totalSize += f.length();
                String key = f.getAbsolutePath().substring(startingPosition).replaceAll("\\\\", "/");
                ObjectMetadata metadata = new ObjectMetadata();
                // for each file being uploaded.
                if (metadataProvider != null) {
                    metadataProvider.provideObjectMetadata(f, metadata);
                }
                // All the single-file uploads share the same
                // MultipleFileTransferProgressUpdatingListener and
                // MultipleFileTransferStateChangeListener
                uploads.add((UploadImpl) doUpload(new PutObjectRequest(bucketName, virtualDirectoryKeyPrefix + key, f).withMetadata(metadata).<PutObjectRequest>withGeneralProgressListener(listener), transferListener, null, null));
            }
        }
        progress.setTotalBytesToTransfer(totalSize);
    }
    // Notify all state changes waiting for the uploads to all be queued
    // to wake up and continue
    latch.countDown();
    return multipleFileUpload;
}

Example 2 with ProgressListener

use of com.qcloud.cos.event.ProgressListener in project cos-java-sdk-v5 by tencentyun.

the class DefaultCosHttpClient method createResponse.

private <X extends CosServiceRequest> CosHttpResponse createResponse(HttpRequestBase httpRequestBase, CosHttpRequest<X> request, org.apache.http.HttpResponse apacheHttpResponse) throws IOException {
    ProgressListener progressListener = request.getProgressListener();
    CosHttpResponse httpResponse = new CosHttpResponse(request, httpRequestBase);
    if (apacheHttpResponse.getEntity() != null) {
        InputStream oriIn = apacheHttpResponse.getEntity().getContent();
        InputStream progressIn = null;
        if (oriIn != null) {
            progressIn = ProgressInputStream.inputStreamForResponse(oriIn, progressListener);
            httpResponse.setContent(progressIn);
        }
    }
    httpResponse.setStatusCode(apacheHttpResponse.getStatusLine().getStatusCode());
    httpResponse.setStatusText(apacheHttpResponse.getStatusLine().getReasonPhrase());
    for (Header header : apacheHttpResponse.getAllHeaders()) {
        String value = CodecUtils.convertFromIso88591ToUtf8(header.getValue());
        httpResponse.addHeader(header.getName(), value);
    }
    return httpResponse;
}

Example 3 with ProgressListener

use of com.qcloud.cos.event.ProgressListener in project cos-java-sdk-v5 by tencentyun.

the class DefaultCosHttpClient method exeute.

@Override
public <X, Y extends CosServiceRequest> X exeute(CosHttpRequest<Y> request, HttpResponseHandler<CosServiceResponse<X>> responseHandler) throws CosClientException, CosServiceException {
    HttpResponse httpResponse = null;
    HttpRequestBase httpRequest = null;
    bufferAndResetAbleContent(request);
    // Always mark the input stream before execution.
    ProgressListener progressListener = request.getProgressListener();
    final InputStream originalContent = request.getContent();
    if (originalContent != null) {
        request.setContent(monitorStreamProgress(progressListener, originalContent));
    }
    if (originalContent != null && originalContent.markSupported() && !(originalContent instanceof BufferedInputStream)) {
        final int readLimit = clientConfig.getReadLimit();
        originalContent.mark(readLimit);
    }
    int retryIndex = 0;
    while (true) {
        try {
            checkInterrupted();
            if (originalContent instanceof BufferedInputStream && originalContent.markSupported()) {
                // Mark everytime for BufferedInputStream, since the marker could have been
                // invalidated
                final int readLimit = clientConfig.getReadLimit();
                originalContent.mark(readLimit);
            }
            // 如果是重试的则恢复流
            if (retryIndex != 0 && originalContent != null) {
                originalContent.reset();
            }
            if (retryIndex != 0) {
                long delay = backoffStrategy.computeDelayBeforeNextRetry(retryIndex);
                Thread.sleep(delay);
            }
            HttpContext context = HttpClientContext.create();
            httpRequest = buildHttpRequest(request);
            httpResponse = null;
            httpResponse = executeOneRequest(context, httpRequest);
            checkResponse(request, httpRequest, httpResponse);
            break;
        } catch (CosServiceException cse) {
            if (cse.getStatusCode() >= 500) {
                String errorMsg = String.format("failed to execute http request, due to service exception," + " httpRequest: %s, retryIdx:%d, maxErrorRetry:%d", request.toString(), retryIndex, maxErrorRetry);
                log.error(errorMsg, cse);
            }
            closeHttpResponseStream(httpResponse);
            if (!shouldRetry(request, httpResponse, cse, retryIndex, retryPolicy)) {
                throw cse;
            }
        } catch (CosClientException cce) {
            String errorMsg = String.format("failed to execute http request, due to client exception," + " httpRequest: %s, retryIdx:%d, maxErrorRetry:%d", request.toString(), retryIndex, maxErrorRetry);
            log.info(errorMsg, cce);
            closeHttpResponseStream(httpResponse);
            if (!shouldRetry(request, httpResponse, cce, retryIndex, retryPolicy)) {
                log.error(errorMsg, cce);
                throw cce;
            }
        } catch (Exception exp) {
            String errorMsg = String.format("httpClient execute occur a unknow exception, httpRequest: %s", request.toString());
            closeHttpResponseStream(httpResponse);
            log.error(errorMsg, exp);
            throw new CosClientException(errorMsg, exp);
        } finally {
            ++retryIndex;
        }
    }
    try {
        CosHttpResponse cosHttpResponse = createResponse(httpRequest, request, httpResponse);
        return responseHandler.handle(cosHttpResponse).getResult();
    } catch (Exception e) {
        if (e.getMessage().equals("Premature end of chunk coded message body: closing chunk expected")) {
            throw new ResponseNotCompleteException("response chunk not complete", e);
        }
        String errorMsg = "Unable to execute response handle: " + e.getMessage();
        log.info(errorMsg, e);
        CosClientException cce = new CosClientException(errorMsg, e);
        throw cce;
    } finally {
        if (!responseHandler.needsConnectionLeftOpen()) {
            httpRequest.releaseConnection();
        }
    }
}

Example 4 with ProgressListener

use of com.qcloud.cos.event.ProgressListener in project cos-java-sdk-v5 by tencentyun.

the class TransferManager method downloadDirectory.

/**
 * Downloads all objects in the virtual directory designated by the keyPrefix given to the
 * destination directory given. All virtual subdirectories will be downloaded recursively.
 *
 * @param bucketName The bucket containing the virtual directory
 * @param keyPrefix The key prefix for the virtual directory, or null for the entire bucket. All
 *        subdirectories will be downloaded recursively.
 * @param destinationDirectory The directory to place downloaded files. Subdirectories will be
 *        created as necessary.
 */
public MultipleFileDownload downloadDirectory(String bucketName, String keyPrefix, File destinationDirectory) {
    if (keyPrefix == null)
        keyPrefix = "";
    List<COSObjectSummary> objectSummaries = new LinkedList<COSObjectSummary>();
    Stack<String> commonPrefixes = new Stack<String>();
    commonPrefixes.add(keyPrefix);
    long totalSize = 0;
    // This is a depth-first search.
    do {
        String prefix = commonPrefixes.pop();
        ObjectListing listObjectsResponse = null;
        do {
            if (listObjectsResponse == null) {
                ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withDelimiter(DEFAULT_DELIMITER).withPrefix(prefix);
                listObjectsResponse = cos.listObjects(listObjectsRequest);
            } else {
                listObjectsResponse = cos.listNextBatchOfObjects(listObjectsResponse);
            }
            for (COSObjectSummary s : listObjectsResponse.getObjectSummaries()) {
                // name.
                if (!s.getKey().equals(prefix) && !listObjectsResponse.getCommonPrefixes().contains(s.getKey() + DEFAULT_DELIMITER)) {
                    objectSummaries.add(s);
                    totalSize += s.getSize();
                } else {
                    log.debug("Skipping download for object " + s.getKey() + " since it is also a virtual directory");
                }
            }
            commonPrefixes.addAll(listObjectsResponse.getCommonPrefixes());
        } while (listObjectsResponse.isTruncated());
    } while (!commonPrefixes.isEmpty());
    /* This is the hook for adding additional progress listeners */
    ProgressListenerChain additionalListeners = new ProgressListenerChain();
    TransferProgress transferProgress = new TransferProgress();
    transferProgress.setTotalBytesToTransfer(totalSize);
    /*
         * Bind additional progress listeners to this MultipleFileTransferProgressUpdatingListener
         * to receive ByteTransferred events from each single-file download implementation.
         */
    ProgressListener listener = new MultipleFileTransferProgressUpdatingListener(transferProgress, additionalListeners);
    List<DownloadImpl> downloads = new ArrayList<DownloadImpl>();
    String description = "Downloading from " + bucketName + "/" + keyPrefix;
    final MultipleFileDownloadImpl multipleFileDownload = new MultipleFileDownloadImpl(description, transferProgress, additionalListeners, keyPrefix, bucketName, downloads);
    multipleFileDownload.setMonitor(new MultipleFileTransferMonitor(multipleFileDownload, downloads));
    final CountDownLatch latch = new CountDownLatch(1);
    MultipleFileTransferStateChangeListener transferListener = new MultipleFileTransferStateChangeListener(latch, multipleFileDownload);
    for (COSObjectSummary summary : objectSummaries) {
        // TODO: non-standard delimiters
        File f = new File(destinationDirectory, summary.getKey());
        File parentFile = f.getParentFile();
        if (parentFile == null || !parentFile.exists() && !parentFile.mkdirs()) {
            throw new RuntimeException("Couldn't create parent directories for " + f.getAbsolutePath());
        }
        // All the single-file downloads share the same
        // MultipleFileTransferProgressUpdatingListener and
        // MultipleFileTransferStateChangeListener
        downloads.add((DownloadImpl) doDownload(new GetObjectRequest(summary.getBucketName(), summary.getKey()).<GetObjectRequest>withGeneralProgressListener(listener), f, transferListener, null, false));
    }
    if (downloads.isEmpty()) {
        multipleFileDownload.setState(TransferState.Completed);
        return multipleFileDownload;
    }
    // Notify all state changes waiting for the downloads to all be queued
    // to wake up and continue.
    latch.countDown();
    return multipleFileDownload;
}