use of com.qcloud.cos.internal.LengthCheckInputStream in project cos-java-sdk-v5 by tencentyun.
the class COSClient method getObject.
@Override
public COSObject getObject(GetObjectRequest getObjectRequest) throws CosClientException, CosServiceException {
rejectNull(getObjectRequest, "The GetObjectRequest parameter must be specified when requesting an object");
rejectNull(getObjectRequest.getBucketName(), "The bucket name parameter must be specified when requesting an object");
rejectNull(getObjectRequest.getKey(), "The key parameter must be specified when requesting an object");
rejectNull(clientConfig.getRegion(), "region is null, region in clientConfig must be specified when requesting an object");
CosHttpRequest<GetObjectRequest> request = createRequest(getObjectRequest.getBucketName(), getObjectRequest.getKey(), getObjectRequest, HttpMethodName.GET);
addParameterIfNotNull(request, "versionId", getObjectRequest.getVersionId());
// Range
long[] range = getObjectRequest.getRange();
if (range != null) {
request.addHeader(Headers.RANGE, "bytes=" + Long.toString(range[0]) + "-" + Long.toString(range[1]));
}
addResponseHeaderParameters(request, getObjectRequest.getResponseHeaders());
addDateHeader(request, Headers.GET_OBJECT_IF_MODIFIED_SINCE, getObjectRequest.getModifiedSinceConstraint());
addDateHeader(request, Headers.GET_OBJECT_IF_UNMODIFIED_SINCE, getObjectRequest.getUnmodifiedSinceConstraint());
addStringListHeader(request, Headers.GET_OBJECT_IF_MATCH, getObjectRequest.getMatchingETagConstraints());
addStringListHeader(request, Headers.GET_OBJECT_IF_NONE_MATCH, getObjectRequest.getNonmatchingETagConstraints());
// Populate the SSE-C parameters to the request header
populateSSE_C(request, getObjectRequest.getSSECustomerKey());
// Populate the traffic limit parameter to the request header
populateTrafficLimit(request, getObjectRequest.getTrafficLimit());
try {
COSObject cosObject = invoke(request, new COSObjectResponseHandler());
cosObject.setBucketName(getObjectRequest.getBucketName());
cosObject.setKey(getObjectRequest.getKey());
InputStream is = cosObject.getObjectContent();
HttpRequestBase httpRequest = cosObject.getObjectContent().getHttpRequest();
is = new ServiceClientHolderInputStream(is, this);
// bytes and complains if what we received doesn't match the Etag.
if (!skipMd5CheckStrategy.skipClientSideValidation(getObjectRequest, cosObject.getObjectMetadata())) {
try {
byte[] serverSideHash = BinaryUtils.fromHex(cosObject.getObjectMetadata().getETag());
// No content length check is performed when the
// MD5 check is enabled, since a correct MD5 check would
// imply a correct content length.
MessageDigest digest = MessageDigest.getInstance("MD5");
is = new DigestValidationInputStream(is, digest, serverSideHash);
} catch (NoSuchAlgorithmException e) {
log.warn("No MD5 digest algorithm available. Unable to calculate " + "checksum and verify data integrity.", e);
} catch (DecoderException e) {
log.warn("BinaryUtils.fromHex error. Unable to calculate " + "checksum and verify data integrity. etag:" + cosObject.getObjectMetadata().getETag(), e);
}
} else {
// Ensures the data received from COS has the same length as the
// expected content-length
is = new LengthCheckInputStream(is, // expected length
cosObject.getObjectMetadata().getContentLength(), // bytes received from cos are all included even if
INCLUDE_SKIPPED_BYTES);
// skipped
}
cosObject.setObjectContent(new COSObjectInputStream(is, httpRequest));
return cosObject;
} catch (CosServiceException cse) {
/*
* If the request failed because one of the specified constraints was not met (ex:
* matching ETag, modified since date, etc.), then return null, so that users don't have
* to wrap their code in try/catch blocks and check for this status code if they want to
* use constraints.
*/
if (cse.getStatusCode() == 412 || cse.getStatusCode() == 304) {
return null;
}
throw cse;
}
}
use of com.qcloud.cos.internal.LengthCheckInputStream in project cos-java-sdk-v5 by tencentyun.
the class COSCryptoModuleBase method newCOSCipherLiteInputStream.
private CipherLiteInputStream newCOSCipherLiteInputStream(AbstractPutObjectRequest req, ContentCryptoMaterial cekMaterial, long plaintextLength) {
final File fileOrig = req.getFile();
final InputStream isOrig = req.getInputStream();
InputStream isCurr = null;
try {
if (fileOrig == null) {
// When input is a FileInputStream, this wrapping enables
// unlimited mark-and-reset
isCurr = isOrig == null ? null : ReleasableInputStream.wrap(isOrig);
} else {
isCurr = new ResettableInputStream(fileOrig);
}
if (plaintextLength > -1) {
// COS allows a single PUT to be no more than 5GB, which
// therefore won't exceed the maximum length that can be
// encrypted either using any cipher such as CBC or GCM.
// This ensures the plain-text read from the underlying data
// stream has the same length as the expected total.
isCurr = new LengthCheckInputStream(isCurr, plaintextLength, EXCLUDE_SKIPPED_BYTES);
}
final CipherLite cipherLite = cekMaterial.getCipherLite();
if (cipherLite.markSupported()) {
return new CipherLiteInputStream(isCurr, cipherLite, DEFAULT_BUFFER_SIZE);
} else {
return new RenewableCipherLiteInputStream(isCurr, cipherLite, DEFAULT_BUFFER_SIZE);
}
} catch (Exception e) {
cleanupDataSource(req, fileOrig, isOrig, isCurr, log);
throw new CosClientException("Unable to create cipher input stream", e);
}
}
use of com.qcloud.cos.internal.LengthCheckInputStream in project cos-java-sdk-v5 by tencentyun.
the class COSClient method uploadObjectInternal.
protected <UploadObjectRequest extends PutObjectRequest> ObjectMetadata uploadObjectInternal(UploadMode uploadMode, UploadObjectRequest uploadObjectRequest) throws CosClientException, CosServiceException {
rejectNull(uploadObjectRequest, "The PutObjectRequest parameter must be specified when uploading an object");
rejectNull(clientConfig.getRegion(), "region is null, region in clientConfig must be specified when uploading an object");
final File file = uploadObjectRequest.getFile();
final InputStream isOrig = uploadObjectRequest.getInputStream();
final String bucketName = uploadObjectRequest.getBucketName();
final String key = uploadObjectRequest.getKey();
ObjectMetadata metadata = uploadObjectRequest.getMetadata();
InputStream input = isOrig;
if (metadata == null)
metadata = new ObjectMetadata();
rejectNull(bucketName, "The bucket name parameter must be specified when uploading an object");
rejectNull(key, "The key parameter must be specified when uploading an object");
// information from it to auto-configure a few options
if (file == null) {
// unlimited mark-and-reset
if (input != null)
input = ReleasableInputStream.wrap(input);
} else {
// Always set the content length, even if it's already set
metadata.setContentLength(file.length());
final long maxAllowdSingleFileSize = 5 * 1024L * 1024L * 1024L;
if (file.length() > maxAllowdSingleFileSize) {
throw new CosClientException("max size 5GB is allowed by putObject Method, your filesize is " + file.length() + ", please use transferManager to upload big file!");
}
final boolean calculateMD5 = metadata.getContentMD5() == null;
if (calculateMD5 && !skipMd5CheckStrategy.skipServerSideValidation(uploadObjectRequest)) {
try {
String contentMd5_b64 = Md5Utils.md5AsBase64(file);
metadata.setContentMD5(contentMd5_b64);
} catch (Exception e) {
throw new CosClientException("Unable to calculate MD5 hash: " + e.getMessage(), e);
}
}
input = ResettableInputStream.newResettableInputStream(file, "Unable to find file to upload");
}
final ObjectMetadata returnedMetadata;
MD5DigestCalculatingInputStream md5DigestStream = null;
try {
CosHttpRequest<UploadObjectRequest> request = null;
if (uploadMode.equals(UploadMode.PUT_OBJECT)) {
request = createRequest(bucketName, key, uploadObjectRequest, HttpMethodName.PUT);
} else if (uploadMode.equals(UploadMode.APPEND_OBJECT)) {
request = createRequest(bucketName, key, uploadObjectRequest, HttpMethodName.POST);
AppendObjectRequest appendObjectRequest = (AppendObjectRequest) uploadObjectRequest;
String positionStr = String.valueOf(appendObjectRequest.getPosition());
request.addParameter("append", null);
request.addParameter("position", positionStr);
}
if (uploadObjectRequest.getAccessControlList() != null) {
addAclHeaders(request, uploadObjectRequest.getAccessControlList());
} else if (uploadObjectRequest.getCannedAcl() != null) {
request.addHeader(Headers.COS_CANNED_ACL, uploadObjectRequest.getCannedAcl().toString());
}
if (uploadObjectRequest.getStorageClass() != null) {
request.addHeader(Headers.STORAGE_CLASS, uploadObjectRequest.getStorageClass());
}
if (uploadObjectRequest.getRedirectLocation() != null) {
request.addHeader(Headers.REDIRECT_LOCATION, uploadObjectRequest.getRedirectLocation());
if (input == null) {
input = new ByteArrayInputStream(new byte[0]);
}
}
// Populate the SSE-C parameters to the request header
populateSSE_C(request, uploadObjectRequest.getSSECustomerKey());
// Populate the SSE KMS parameters to the request header
populateSSE_KMS(request, uploadObjectRequest.getSSECOSKeyManagementParams());
// Populate the traffic limit parameter to the request header
populateTrafficLimit(request, uploadObjectRequest.getTrafficLimit());
// Use internal interface to differentiate 0 from unset.
final Long contentLength = (Long) metadata.getRawMetadataValue(Headers.CONTENT_LENGTH);
if (contentLength == null) {
/*
* There's nothing we can do except for let the HTTP client buffer the input stream
* contents if the caller doesn't tell us how much data to expect in a stream since
* we have to explicitly tell how much we're sending before we start sending any of
* it.
*/
log.warn("No content length specified for stream data. " + "Stream contents will be buffered in memory and could result in " + "out of memory errors.");
} else {
final long expectedLength = contentLength.longValue();
final long maxAllowdSingleFileSize = 5 * 1024L * 1024L * 1024L;
if (expectedLength > maxAllowdSingleFileSize) {
throw new CosClientException("max size 5GB is allowed by putObject Method, your filesize is " + expectedLength + ", please use transferManager to upload big file!");
}
if (expectedLength >= 0) {
// Performs length check on the underlying data stream.
// For COS encryption client, the underlying data stream here
// refers to the cipher-text data stream (ie not the underlying
// plain-text data stream which in turn may have been wrapped
// with it's own length check input stream.)
LengthCheckInputStream lcis = new // expected
LengthCheckInputStream(// expected
input, // expected
expectedLength, // uploaded
EXCLUDE_SKIPPED_BYTES);
input = lcis;
}
}
if (metadata.getContentMD5() == null && !skipMd5CheckStrategy.skipClientSideValidationPerRequest(uploadObjectRequest)) {
/*
* If the user hasn't set the content MD5, then we don't want to buffer the whole
* stream in memory just to calculate it. Instead, we can calculate it on the fly
* and validate it with the returned ETag from the object upload.
*/
input = md5DigestStream = new MD5DigestCalculatingInputStream(input);
}
populateRequestMetadata(request, metadata);
request.setContent(input);
try {
if (uploadObjectRequest.getPicOperations() != null) {
request.addHeader(Headers.PIC_OPERATIONS, Jackson.toJsonString(uploadObjectRequest.getPicOperations()));
returnedMetadata = invoke(request, new ResponseHeaderHandlerChain<ObjectMetadata>(new Unmarshallers.ImagePersistenceUnmarshaller(), new CosMetadataResponseHandler()));
} else {
returnedMetadata = invoke(request, new CosMetadataResponseHandler());
}
} catch (Throwable t) {
throw Throwables.failure(t);
}
} finally {
CosDataSource.Utils.cleanupDataSource(uploadObjectRequest, file, isOrig, input, log);
}
String contentMd5 = metadata.getContentMD5();
if (md5DigestStream != null) {
contentMd5 = Base64.encodeAsString(md5DigestStream.getMd5Digest());
}
final String etag = returnedMetadata.getETag();
if (contentMd5 != null && uploadMode.equals(UploadMode.PUT_OBJECT) && !skipMd5CheckStrategy.skipClientSideValidationPerPutResponse(returnedMetadata)) {
byte[] clientSideHash = BinaryUtils.fromBase64(contentMd5);
byte[] serverSideHash = null;
try {
serverSideHash = BinaryUtils.fromHex(etag);
} catch (DecoderException e) {
throw new CosClientException("Unable to verify integrity of data upload. " + "Client calculated content hash (contentMD5: " + contentMd5 + " in base 64) didn't match hash (etag: " + etag + " in hex) calculated by COS . " + "You may need to delete the data stored in COS . (metadata.contentMD5: " + metadata.getContentMD5() + ", bucketName: " + bucketName + ", key: " + key + ")");
}
if (!Arrays.equals(clientSideHash, serverSideHash)) {
throw new CosClientException("Unable to verify integrity of data upload. " + "Client calculated content hash (contentMD5: " + contentMd5 + " in base 64) didn't match hash (etag: " + etag + " in hex) calculated by COS . " + "You may need to delete the data stored in COS . (metadata.contentMD5: " + metadata.getContentMD5() + ", bucketName: " + bucketName + ", key: " + key + ")");
}
}
return returnedMetadata;
}
Aggregations