use of com.qcloud.cos.event.ProgressListenerChain in project cos-java-sdk-v5 by tencentyun.
the class TransferManager method uploadFileList.
/**
* Uploads all specified files to the bucket named, constructing relative keys depending on the
* commonParentDirectory given.
* <p>
* COS will overwrite any existing objects that happen to have the same key, just as when
* uploading individual files, so use with caution.
* </p>
*
* @param bucketName The name of the bucket to upload objects to.
* @param virtualDirectoryKeyPrefix The key prefix of the virtual directory to upload to. Use
* the null or empty string to upload files to the root of the bucket.
* @param directory The common parent directory of files to upload. The keys of the files in the
* list of files are constructed relative to this directory and the
* virtualDirectoryKeyPrefix.
* @param files A list of files to upload. The keys of the files are calculated relative to the
* common parent directory and the virtualDirectoryKeyPrefix.
* @param metadataProvider A callback of type <code>ObjectMetadataProvider</code> which is used
* to provide metadata for each file being uploaded.
*/
public MultipleFileUpload uploadFileList(String bucketName, String virtualDirectoryKeyPrefix, File directory, List<File> files, ObjectMetadataProvider metadataProvider) {
if (directory == null || !directory.exists() || !directory.isDirectory()) {
throw new IllegalArgumentException("Must provide a common base directory for uploaded files");
}
if (virtualDirectoryKeyPrefix == null || virtualDirectoryKeyPrefix.length() == 0) {
virtualDirectoryKeyPrefix = "";
} else if (!virtualDirectoryKeyPrefix.endsWith("/")) {
virtualDirectoryKeyPrefix = virtualDirectoryKeyPrefix + "/";
}
/* This is the hook for adding additional progress listeners */
ProgressListenerChain additionalListeners = new ProgressListenerChain();
TransferProgress progress = new TransferProgress();
/*
* Bind additional progress listeners to this MultipleFileTransferProgressUpdatingListener
* to receive ByteTransferred events from each single-file upload implementation.
*/
ProgressListener listener = new MultipleFileTransferProgressUpdatingListener(progress, additionalListeners);
List<UploadImpl> uploads = new LinkedList<UploadImpl>();
MultipleFileUploadImpl multipleFileUpload = new MultipleFileUploadImpl("Uploading etc", progress, additionalListeners, virtualDirectoryKeyPrefix, bucketName, uploads);
multipleFileUpload.setMonitor(new MultipleFileTransferMonitor(multipleFileUpload, uploads));
final CountDownLatch latch = new CountDownLatch(1);
MultipleFileTransferStateChangeListener transferListener = new MultipleFileTransferStateChangeListener(latch, multipleFileUpload);
if (files == null || files.isEmpty()) {
multipleFileUpload.setState(TransferState.Completed);
} else {
/*
* If the absolute path for the common/base directory does NOT end in a separator (which
* is the case for anything but root directories), then we know there's still a
* separator between the base directory and the rest of the file's path, so we increment
* the starting position by one.
*/
int startingPosition = directory.getAbsolutePath().length();
if (!(directory.getAbsolutePath().endsWith(File.separator)))
startingPosition++;
long totalSize = 0;
for (File f : files) {
// Check, if file, since only files can be uploaded.
if (f.isFile()) {
totalSize += f.length();
String key = f.getAbsolutePath().substring(startingPosition).replaceAll("\\\\", "/");
ObjectMetadata metadata = new ObjectMetadata();
// for each file being uploaded.
if (metadataProvider != null) {
metadataProvider.provideObjectMetadata(f, metadata);
}
// All the single-file uploads share the same
// MultipleFileTransferProgressUpdatingListener and
// MultipleFileTransferStateChangeListener
uploads.add((UploadImpl) doUpload(new PutObjectRequest(bucketName, virtualDirectoryKeyPrefix + key, f).withMetadata(metadata).<PutObjectRequest>withGeneralProgressListener(listener), transferListener, null, null));
}
}
progress.setTotalBytesToTransfer(totalSize);
}
// Notify all state changes waiting for the uploads to all be queued
// to wake up and continue
latch.countDown();
return multipleFileUpload;
}
use of com.qcloud.cos.event.ProgressListenerChain in project cos-java-sdk-v5 by tencentyun.
the class TransferManager method copy.
/**
* <p>
* Schedules a new transfer to copy data from one object to another . This method is
* non-blocking and returns immediately (i.e. before the copy has finished).
* </p>
* <p>
* Note: You need to use this method if the {@link TransferManager} is created with a regional
* COS client and the source & destination buckets are in different regions.
* </p>
* <p>
* <code>TransferManager</code> doesn't support copying of encrypted objects whose encryption
* materials are stored in an instruction file.
* </p>
* <p>
* Use the returned <code>Copy</code> object to check if the copy is complete.
* </p>
* <p>
* If resources are available, the copy request will begin immediately. Otherwise, the copy is
* scheduled and started as soon as resources become available.
* </p>
*
* <p>
* <b>Note:</b> If the {@link TransferManager} is created with a regional COS client and the
* source & destination buckets are in different regions, use the
* {@link #copy(CopyObjectRequest, COS, TransferStateChangeListener)} method.
* </p>
*
* @param copyObjectRequest The request containing all the parameters for the copy.
* @param srcCOS An COS client constructed for the region in which the source object's bucket is
* located.
* @param stateChangeListener The transfer state change listener to monitor the copy request
* @return A new <code>Copy</code> object to use to check the state of the copy request being
* processed.
* @throws CosClientException If any errors are encountered in the client while making the
* request or handling the response.
* @throws CosServiceException If any errors occurred in Qcloud COS while processing the
* request.
*/
public Copy copy(final CopyObjectRequest copyObjectRequest, final COS srcCOS, final TransferStateChangeListener stateChangeListener) throws CosServiceException, CosClientException {
appendSingleObjectUserAgent(copyObjectRequest);
assertParameterNotNull(copyObjectRequest.getSourceBucketName(), "The source bucket name must be specified when a copy request is initiated.");
assertParameterNotNull(copyObjectRequest.getSourceKey(), "The source object key must be specified when a copy request is initiated.");
assertParameterNotNull(copyObjectRequest.getDestinationBucketName(), "The destination bucket name must be specified when a copy request is initiated.");
assertParameterNotNull(copyObjectRequest.getDestinationKey(), "The destination object key must be specified when a copy request is initiated.");
assertParameterNotNull(srcCOS, "The srcCOS parameter is mandatory");
String description = "Copying object from " + copyObjectRequest.getSourceBucketName() + "/" + copyObjectRequest.getSourceKey() + " to " + copyObjectRequest.getDestinationBucketName() + "/" + copyObjectRequest.getDestinationKey();
GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest(copyObjectRequest.getSourceBucketName(), copyObjectRequest.getSourceKey()).withVersionId(copyObjectRequest.getSourceVersionId());
ObjectMetadata metadata = srcCOS.getObjectMetadata(getObjectMetadataRequest);
TransferProgress transferProgress = new TransferProgress();
transferProgress.setTotalBytesToTransfer(metadata.getContentLength());
ProgressListenerChain listenerChain = new ProgressListenerChain(new TransferProgressUpdatingListener(transferProgress));
CopyImpl copy = new CopyImpl(description, transferProgress, listenerChain, stateChangeListener);
CopyCallable copyCallable = new CopyCallable(this, threadPool, copy, copyObjectRequest, metadata, listenerChain);
CopyMonitor watcher = CopyMonitor.create(this, copy, threadPool, copyCallable, copyObjectRequest, listenerChain);
copy.setMonitor(watcher);
return copy;
}
use of com.qcloud.cos.event.ProgressListenerChain in project cos-java-sdk-v5 by tencentyun.
the class TransferManager method doDownload.
/**
* Same as public interface, but adds a state listener so that callers can be notified of state
* changes to the download.
*
* @see TransferManager#download(GetObjectRequest, File)
*/
private Download doDownload(final GetObjectRequest getObjectRequest, final File file, final TransferStateChangeListener stateListener, final COSProgressListener cosProgressListener, final boolean resumeExistingDownload) {
appendSingleObjectUserAgent(getObjectRequest);
String description = "Downloading from " + getObjectRequest.getBucketName() + "/" + getObjectRequest.getKey();
TransferProgress transferProgress = new TransferProgress();
// COS progress listener to capture the persistable transfer when available
COSProgressListenerChain listenerChain = new COSProgressListenerChain(// The listener for updating transfer progress
new TransferProgressUpdatingListener(transferProgress), getObjectRequest.getGeneralProgressListener(), // Listeners
cosProgressListener);
// included in
// the original
// request
// The listener chain used by the low-level GetObject request.
// This listener chain ignores any COMPLETE event, so that we could
// delay firing the signal until the high-level download fully finishes.
getObjectRequest.setGeneralProgressListener(new ProgressListenerChain(new TransferCompletionFilter(), listenerChain));
long startingByte = 0;
long lastByte;
long[] range = getObjectRequest.getRange();
if (range != null && range.length == 2) {
startingByte = range[0];
lastByte = range[1];
} else {
GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest(getObjectRequest.getBucketName(), getObjectRequest.getKey());
if (getObjectRequest.getSSECustomerKey() != null)
getObjectMetadataRequest.setSSECustomerKey(getObjectRequest.getSSECustomerKey());
if (getObjectRequest.getVersionId() != null)
getObjectMetadataRequest.setVersionId(getObjectRequest.getVersionId());
final ObjectMetadata objectMetadata = cos.getObjectMetadata(getObjectMetadataRequest);
lastByte = objectMetadata.getContentLength() - 1;
}
final long origStartingByte = startingByte;
// We still pass the unfiltered listener chain into DownloadImpl
final DownloadImpl download = new DownloadImpl(description, transferProgress, listenerChain, null, stateListener, getObjectRequest, file);
long totalBytesToDownload = lastByte - startingByte + 1;
transferProgress.setTotalBytesToTransfer(totalBytesToDownload);
long fileLength = -1;
if (resumeExistingDownload) {
if (!FileLocks.lock(file)) {
throw new FileLockException("Fail to lock " + file + " for resume download");
}
try {
if (file.exists()) {
fileLength = file.length();
startingByte = startingByte + fileLength;
getObjectRequest.setRange(startingByte, lastByte);
transferProgress.updateProgress(Math.min(fileLength, totalBytesToDownload));
totalBytesToDownload = lastByte - startingByte + 1;
if (log.isDebugEnabled()) {
log.debug("Resume download: totalBytesToDownload=" + totalBytesToDownload + ", origStartingByte=" + origStartingByte + ", startingByte=" + startingByte + ", lastByte=" + lastByte + ", numberOfBytesRead=" + fileLength + ", file: " + file);
}
}
} finally {
FileLocks.unlock(file);
}
}
if (totalBytesToDownload < 0) {
throw new IllegalArgumentException("Unable to determine the range for download operation.");
}
final CountDownLatch latch = new CountDownLatch(1);
Future<?> future = threadPool.submit(new DownloadCallable(cos, latch, getObjectRequest, resumeExistingDownload, download, file, origStartingByte, fileLength));
download.setMonitor(new DownloadMonitor(download, future));
latch.countDown();
return download;
}
use of com.qcloud.cos.event.ProgressListenerChain in project cos-java-sdk-v5 by tencentyun.
the class TransferManager method downloadDirectory.
/**
* Downloads all objects in the virtual directory designated by the keyPrefix given to the
* destination directory given. All virtual subdirectories will be downloaded recursively.
*
* @param bucketName The bucket containing the virtual directory
* @param keyPrefix The key prefix for the virtual directory, or null for the entire bucket. All
* subdirectories will be downloaded recursively.
* @param destinationDirectory The directory to place downloaded files. Subdirectories will be
* created as necessary.
*/
public MultipleFileDownload downloadDirectory(String bucketName, String keyPrefix, File destinationDirectory) {
if (keyPrefix == null)
keyPrefix = "";
List<COSObjectSummary> objectSummaries = new LinkedList<COSObjectSummary>();
Stack<String> commonPrefixes = new Stack<String>();
commonPrefixes.add(keyPrefix);
long totalSize = 0;
// This is a depth-first search.
do {
String prefix = commonPrefixes.pop();
ObjectListing listObjectsResponse = null;
do {
if (listObjectsResponse == null) {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withDelimiter(DEFAULT_DELIMITER).withPrefix(prefix);
listObjectsResponse = cos.listObjects(listObjectsRequest);
} else {
listObjectsResponse = cos.listNextBatchOfObjects(listObjectsResponse);
}
for (COSObjectSummary s : listObjectsResponse.getObjectSummaries()) {
// name.
if (!s.getKey().equals(prefix) && !listObjectsResponse.getCommonPrefixes().contains(s.getKey() + DEFAULT_DELIMITER)) {
objectSummaries.add(s);
totalSize += s.getSize();
} else {
log.debug("Skipping download for object " + s.getKey() + " since it is also a virtual directory");
}
}
commonPrefixes.addAll(listObjectsResponse.getCommonPrefixes());
} while (listObjectsResponse.isTruncated());
} while (!commonPrefixes.isEmpty());
/* This is the hook for adding additional progress listeners */
ProgressListenerChain additionalListeners = new ProgressListenerChain();
TransferProgress transferProgress = new TransferProgress();
transferProgress.setTotalBytesToTransfer(totalSize);
/*
* Bind additional progress listeners to this MultipleFileTransferProgressUpdatingListener
* to receive ByteTransferred events from each single-file download implementation.
*/
ProgressListener listener = new MultipleFileTransferProgressUpdatingListener(transferProgress, additionalListeners);
List<DownloadImpl> downloads = new ArrayList<DownloadImpl>();
String description = "Downloading from " + bucketName + "/" + keyPrefix;
final MultipleFileDownloadImpl multipleFileDownload = new MultipleFileDownloadImpl(description, transferProgress, additionalListeners, keyPrefix, bucketName, downloads);
multipleFileDownload.setMonitor(new MultipleFileTransferMonitor(multipleFileDownload, downloads));
final CountDownLatch latch = new CountDownLatch(1);
MultipleFileTransferStateChangeListener transferListener = new MultipleFileTransferStateChangeListener(latch, multipleFileDownload);
for (COSObjectSummary summary : objectSummaries) {
// TODO: non-standard delimiters
File f = new File(destinationDirectory, summary.getKey());
File parentFile = f.getParentFile();
if (parentFile == null || !parentFile.exists() && !parentFile.mkdirs()) {
throw new RuntimeException("Couldn't create parent directories for " + f.getAbsolutePath());
}
// All the single-file downloads share the same
// MultipleFileTransferProgressUpdatingListener and
// MultipleFileTransferStateChangeListener
downloads.add((DownloadImpl) doDownload(new GetObjectRequest(summary.getBucketName(), summary.getKey()).<GetObjectRequest>withGeneralProgressListener(listener), f, transferListener, null, false));
}
if (downloads.isEmpty()) {
multipleFileDownload.setState(TransferState.Completed);
return multipleFileDownload;
}
// Notify all state changes waiting for the downloads to all be queued
// to wake up and continue.
latch.countDown();
return multipleFileDownload;
}
use of com.qcloud.cos.event.ProgressListenerChain in project cos-java-sdk-v5 by tencentyun.
the class TransferManager method doResumableDownload.
private Download doResumableDownload(final GetObjectRequest getObjectRequest, final File destFile, final TransferStateChangeListener stateListener, final COSProgressListener cosProgressListener, String resumableTaskFilePath, int multiThreadThreshold, int partSize) {
PersistableResumeDownload downloadRecord = getPersistableResumeRecord(getObjectRequest, destFile, resumableTaskFilePath);
long bytesToDownload = Long.parseLong(downloadRecord.getContentLength());
if (bytesToDownload < multiThreadThreshold) {
downloadRecord.getDumpFile().delete();
return doDownload(getObjectRequest, destFile, stateListener, cosProgressListener, OVERWRITE_MODE);
}
appendSingleObjectUserAgent(getObjectRequest);
String description = "Resumable downloading from " + getObjectRequest.getBucketName() + "/" + getObjectRequest.getKey();
TransferProgress transferProgress = new TransferProgress();
// COS progress listener to capture the persistable transfer when available
COSProgressListenerChain listenerChain = new COSProgressListenerChain(// The listener for updating transfer progress
new TransferProgressUpdatingListener(transferProgress), getObjectRequest.getGeneralProgressListener(), cosProgressListener);
getObjectRequest.setGeneralProgressListener(new ProgressListenerChain(new TransferCompletionFilter(), listenerChain));
RandomAccessFile destRandomAccessFile;
FileChannel destFileChannel;
try {
destRandomAccessFile = new RandomAccessFile(destFile, "rw");
destRandomAccessFile.setLength(bytesToDownload);
destFileChannel = destRandomAccessFile.getChannel();
} catch (Exception e) {
throw new CosClientException("resumable download got exception:" + e.getCause().getMessage() + e.getMessage());
}
transferProgress.setTotalBytesToTransfer(bytesToDownload);
DownloadImpl download = new DownloadImpl(description, transferProgress, listenerChain, null, stateListener, getObjectRequest, destFile);
ResumableDownloadSubmitter submitter = new ResumableDownloadSubmitter(cos, threadPool, getObjectRequest, download, destFile, destRandomAccessFile, destFileChannel, downloadRecord, partSize, multiThreadThreshold, transferProgress, listenerChain);
ResumableDownloadMonitor monitor = ResumableDownloadMonitor.create(listenerChain, submitter, download, threadPool, downloadRecord, destFile, destFileChannel);
download.setMonitor(monitor);
return download;
}
Aggregations