use of com.qcloud.cos.model.ObjectMetadata in project cos-java-sdk-v5 by tencentyun.
the class AbstractCOSClientTest method putObjectFromLocalFileByInputStream.
// 流式上传
protected static void putObjectFromLocalFileByInputStream(File localFile, long uploadSize, String uploadEtag, String key) {
if (!judgeUserInfoValid()) {
return;
}
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setContentLength(uploadSize);
putObjectFromLocalFileByInputStream(localFile, uploadSize, uploadEtag, key, objectMetadata);
}
use of com.qcloud.cos.model.ObjectMetadata in project cos-java-sdk-v5 by tencentyun.
the class AbstractCOSEncryptionClientTest method testTransferManagerUploadDownBigFile.
@Test
public void testTransferManagerUploadDownBigFile() throws IOException, CosServiceException, CosClientException, InterruptedException {
if (!judgeUserInfoValid()) {
return;
}
TransferManager transferManager = new TransferManager(cosclient);
File localFile = buildTestFile(1024 * 1024 * 10L);
File downFile = new File(localFile.getAbsolutePath() + ".down");
String key = "ut/" + localFile.getName();
String destKey = key + ".copy";
try {
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, localFile);
ObjectMetadata objectMetadata = new ObjectMetadata();
putObjectRequest.setMetadata(objectMetadata);
Upload upload = transferManager.upload(putObjectRequest);
UploadResult uploadResult = upload.waitForUploadResult();
assertTrue(uploadResult.getETag().contains("-"));
assertNotNull(uploadResult.getRequestId());
assertNotNull(uploadResult.getDateStr());
GetObjectRequest getObjectRequest = new GetObjectRequest(bucket, key);
Download download = transferManager.download(getObjectRequest, downFile);
download.waitForCompletion();
// check file
assertEquals(Md5Utils.md5Hex(localFile), Md5Utils.md5Hex(downFile));
CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucket, key, bucket, destKey);
Copy copy = transferManager.copy(copyObjectRequest, cosclient, null);
copy.waitForCompletion();
} finally {
// delete file on cos
clearObject(key);
// delete file on cos
clearObject(destKey);
if (localFile.exists()) {
assertTrue(localFile.delete());
}
if (downFile.exists()) {
assertTrue(downFile.delete());
}
}
}
use of com.qcloud.cos.model.ObjectMetadata in project cos-java-sdk-v5 by tencentyun.
the class RangeDownloadCallable method call.
public DownloadPart call() throws Exception {
COSObject object = cos.getObject(request);
InputStream input = object.getObjectContent();
ObjectMetadata meta = object.getObjectMetadata();
long[] range = request.getRange();
long start = range[0];
long end = range[1];
long position = start;
ByteBuffer tmpBuf = ByteBuffer.allocateDirect(1024 * 1024);
byte[] buffer = new byte[1024 * 10];
int bytesRead;
CRC64 crc64 = new CRC64();
while ((bytesRead = input.read(buffer)) > -1) {
start += bytesRead;
crc64.update(buffer, bytesRead);
if (tmpBuf.remaining() < bytesRead) {
tmpBuf.flip();
position += destFileChannel.write(tmpBuf, position);
tmpBuf.clear();
}
tmpBuf.put(buffer, 0, bytesRead);
}
tmpBuf.flip();
destFileChannel.write(tmpBuf, position);
if (start != end + 1) {
destFileChannel.close();
destFile.delete();
String msg = String.format("get object want %d bytes, but got %d bytes, reqeust_id: %s", end + 1, start, meta.getRequestId());
throw new CosClientException(msg);
}
String block = String.format("%d-%d", range[0], range[1]);
downloadRecord.putDownloadedBlocks(block);
downloadRecord.dump();
return new DownloadPart(range[0], range[1], crc64.getValue());
}
use of com.qcloud.cos.model.ObjectMetadata in project cos-java-sdk-v5 by tencentyun.
the class ServiceUtils method downloadToFile.
/**
* Same as {@link #downloadObjectToFile(COSObject, File, boolean, boolean)}
* but has an additional expected file length parameter for integrity
* checking purposes.
*
* @param expectedFileLength
* applicable only when appendData is true; the expected length
* of the file to append to.
*/
public static void downloadToFile(COSObject cosObject, final File dstfile, boolean performIntegrityCheck, final boolean appendData, final long expectedFileLength) {
// attempt to create the parent if it doesn't exist
File parentDirectory = dstfile.getParentFile();
if (parentDirectory != null && !parentDirectory.exists()) {
if (!(parentDirectory.mkdirs())) {
throw new CosClientException("Unable to create directory in the path" + parentDirectory.getAbsolutePath());
}
}
if (!FileLocks.lock(dstfile)) {
throw new FileLockException("Fail to lock " + dstfile + " for appendData=" + appendData);
}
OutputStream outputStream = null;
try {
final long actualLen = dstfile.length();
if (appendData && actualLen != expectedFileLength) {
// Fail fast to prevent data corruption
throw new IllegalStateException("Expected file length to append is " + expectedFileLength + " but actual length is " + actualLen + " for file " + dstfile);
}
outputStream = new BufferedOutputStream(new FileOutputStream(dstfile, appendData));
byte[] buffer = new byte[1024 * 10];
int bytesRead;
while ((bytesRead = cosObject.getObjectContent().read(buffer)) > -1) {
outputStream.write(buffer, 0, bytesRead);
}
} catch (IOException e) {
cosObject.getObjectContent().abort();
throw new CosClientException("Unable to store object contents to disk: " + e.getMessage(), e);
} finally {
IOUtils.closeQuietly(outputStream, log);
FileLocks.unlock(dstfile);
IOUtils.closeQuietly(cosObject.getObjectContent(), log);
}
if (performIntegrityCheck) {
byte[] clientSideHash = null;
byte[] serverSideHash = null;
try {
final ObjectMetadata metadata = cosObject.getObjectMetadata();
if (!skipMd5CheckStrategy.skipClientSideValidationPerGetResponse(metadata)) {
clientSideHash = Md5Utils.computeMD5Hash(dstfile);
serverSideHash = BinaryUtils.fromHex(metadata.getETag());
}
} catch (Exception e) {
log.warn("Unable to calculate MD5 hash to validate download: " + e.getMessage(), e);
}
if (clientSideHash != null && serverSideHash != null && !Arrays.equals(clientSideHash, serverSideHash)) {
throw new CosClientException("Unable to verify integrity of data download. " + "Client calculated content hash didn't match hash calculated by Qcloud COS. " + "The data stored in '" + dstfile.getAbsolutePath() + "' may be corrupt.");
}
}
}
use of com.qcloud.cos.model.ObjectMetadata in project cos-java-sdk-v5 by tencentyun.
the class TransferManagerDemo method copyFileSetMetadata.
public static void copyFileSetMetadata() {
// 1 初始化用户身份信息(secretId, secretKey)
COSCredentials cred = new BasicCOSCredentials("AKIDXXXXXXXX", "1A2Z3YYYYYYYYYY");
// 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
ClientConfig clientConfig = new ClientConfig(new Region("ap-guangzhou"));
// 3 生成cos客户端
COSClient cosclient = new COSClient(cred, clientConfig);
ClientConfig srcClientConfig = new ClientConfig(new Region("ap-shanghai"));
COSClient srcCosclient = new COSClient(cred, srcClientConfig);
ExecutorService threadPool = Executors.newFixedThreadPool(2);
// 传入一个threadpool, 若不传入线程池, 默认TransferManager中会生成一个单线程的线程池。
TransferManager transferManager = new TransferManager(cosclient, threadPool);
TransferManagerConfiguration transferManagerConfiguration = new TransferManagerConfiguration();
transferManagerConfiguration.setMultipartCopyThreshold(5 * 1024 * 1024);
transferManager.setConfiguration(transferManagerConfiguration);
// 要拷贝的bucket region, 支持跨园区拷贝
Region srcBucketRegion = new Region("ap-shanghai");
// 源bucket, bucket名需包含appid
String srcBucketName = "mysrcbucket-123456789";
// 要拷贝的源文件
String srcKey = "aaa/bbb.txt";
// 目的bucket, bucket名需包含appid
String destBucketName = "mydestbucekt-123456789";
// 要拷贝的目的文件
String destKey = "bbb/ccc.txt";
ObjectMetadata objectMetadata = new ObjectMetadata();
Map<String, String> userMeta = new HashMap<String, String>();
userMeta.put("usermeta", "hello-mult-copy");
objectMetadata.setUserMetadata(userMeta);
CopyObjectRequest copyObjectRequest = new CopyObjectRequest(srcBucketRegion, srcBucketName, srcKey, destBucketName, destKey);
System.out.println(copyObjectRequest.getDestinationBucketName());
copyObjectRequest.setNewObjectMetadata(objectMetadata);
try {
Copy copy = transferManager.copy(copyObjectRequest, srcCosclient, null);
// 返回一个异步结果copy, 可同步的调用waitForCopyResult等待copy结束, 成功返回CopyResult, 失败抛出异常.
// showTransferProgress(copy);
CopyResult copyResult = copy.waitForCopyResult();
System.out.println(copyResult.getCrc64Ecma());
} catch (CosServiceException e) {
e.printStackTrace();
} catch (CosClientException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
transferManager.shutdownNow();
cosclient.shutdown();
}
Aggregations