Examples with MD5DigestCalculatingInputStream com.qcloud.cos.internal.MD5DigestCalculatingInputStream used on opensource projects

Example 1 with MD5DigestCalculatingInputStream

use of com.qcloud.cos.internal.MD5DigestCalculatingInputStream in project cos-java-sdk-v5 by tencentyun.

the class COSClient method uploadPart.

@Override
public UploadPartResult uploadPart(UploadPartRequest uploadPartRequest) throws CosClientException, CosServiceException {
    rejectNull(uploadPartRequest, "The request parameter must be specified when uploading a part");
    final File fileOrig = uploadPartRequest.getFile();
    final InputStream isOrig = uploadPartRequest.getInputStream();
    final String bucketName = uploadPartRequest.getBucketName();
    final String key = uploadPartRequest.getKey();
    final String uploadId = uploadPartRequest.getUploadId();
    final int partNumber = uploadPartRequest.getPartNumber();
    final long partSize = uploadPartRequest.getPartSize();
    rejectNull(bucketName, "The bucket name parameter must be specified when uploading a part");
    rejectNull(key, "The key parameter must be specified when uploading a part");
    rejectNull(uploadId, "The upload ID parameter must be specified when uploading a part");
    rejectNull(partNumber, "The part number parameter must be specified when uploading a part");
    rejectNull(partSize, "The part size parameter must be specified when uploading a part");
    rejectNull(clientConfig.getRegion(), "region is null, region in clientConfig must be specified when uploading a part");
    CosHttpRequest<UploadPartRequest> request = createRequest(bucketName, key, uploadPartRequest, HttpMethodName.PUT);
    request.addParameter("uploadId", uploadId);
    request.addParameter("partNumber", Integer.toString(partNumber));
    final ObjectMetadata objectMetadata = uploadPartRequest.getObjectMetadata();
    if (objectMetadata != null)
        populateRequestMetadata(request, objectMetadata);
    addHeaderIfNotNull(request, Headers.CONTENT_MD5, uploadPartRequest.getMd5Digest());
    request.addHeader(Headers.CONTENT_LENGTH, Long.toString(partSize));
    // Populate the SSE-C parameters to the request header
    populateSSE_C(request, uploadPartRequest.getSSECustomerKey());
    // Populate the traffic limit parameter to the request header
    populateTrafficLimit(request, uploadPartRequest.getTrafficLimit());
    InputStream isCurr = isOrig;
    try {
        if (fileOrig == null) {
            if (isOrig == null) {
                throw new IllegalArgumentException("A File or InputStream must be specified when uploading part");
            } else {
                // When isCurr is a FileInputStream, this wrapping enables
                // unlimited mark-and-reset
                isCurr = ReleasableInputStream.wrap(isCurr);
            }
        } else {
            try {
                isCurr = new ResettableInputStream(fileOrig);
            } catch (IOException e) {
                throw new IllegalArgumentException("Failed to open file " + fileOrig, e);
            }
        }
        isCurr = new InputSubstream(isCurr, uploadPartRequest.getFileOffset(), partSize, uploadPartRequest.isLastPart());
        MD5DigestCalculatingInputStream md5DigestStream = null;
        if (uploadPartRequest.getMd5Digest() == null && !skipMd5CheckStrategy.skipClientSideValidationPerRequest(uploadPartRequest)) {
            /*
                 * If the user hasn't set the content MD5, then we don't want to buffer the whole
                 * stream in memory just to calculate it. Instead, we can calculate it on the fly
                 * and validate it with the returned ETag from the object upload.
                 */
            isCurr = md5DigestStream = new MD5DigestCalculatingInputStream(isCurr);
        }
        return doUploadPart(bucketName, key, uploadId, partNumber, partSize, request, isCurr, md5DigestStream);
    } finally {
        CosDataSource.Utils.cleanupDataSource(uploadPartRequest, fileOrig, isOrig, isCurr, log);
    }
}

Example 2 with MD5DigestCalculatingInputStream

use of com.qcloud.cos.internal.MD5DigestCalculatingInputStream in project cos-java-sdk-v5 by tencentyun.

the class COSClient method uploadObjectInternal.

protected <UploadObjectRequest extends PutObjectRequest> ObjectMetadata uploadObjectInternal(UploadMode uploadMode, UploadObjectRequest uploadObjectRequest) throws CosClientException, CosServiceException {
    rejectNull(uploadObjectRequest, "The PutObjectRequest parameter must be specified when uploading an object");
    rejectNull(clientConfig.getRegion(), "region is null, region in clientConfig must be specified when uploading an object");
    final File file = uploadObjectRequest.getFile();
    final InputStream isOrig = uploadObjectRequest.getInputStream();
    final String bucketName = uploadObjectRequest.getBucketName();
    final String key = uploadObjectRequest.getKey();
    ObjectMetadata metadata = uploadObjectRequest.getMetadata();
    InputStream input = isOrig;
    if (metadata == null)
        metadata = new ObjectMetadata();
    rejectNull(bucketName, "The bucket name parameter must be specified when uploading an object");
    rejectNull(key, "The key parameter must be specified when uploading an object");
    // information from it to auto-configure a few options
    if (file == null) {
        // unlimited mark-and-reset
        if (input != null)
            input = ReleasableInputStream.wrap(input);
    } else {
        // Always set the content length, even if it's already set
        metadata.setContentLength(file.length());
        final long maxAllowdSingleFileSize = 5 * 1024L * 1024L * 1024L;
        if (file.length() > maxAllowdSingleFileSize) {
            throw new CosClientException("max size 5GB is allowed by putObject Method, your filesize is " + file.length() + ", please use transferManager to upload big file!");
        }
        final boolean calculateMD5 = metadata.getContentMD5() == null;
        if (calculateMD5 && !skipMd5CheckStrategy.skipServerSideValidation(uploadObjectRequest)) {
            try {
                String contentMd5_b64 = Md5Utils.md5AsBase64(file);
                metadata.setContentMD5(contentMd5_b64);
            } catch (Exception e) {
                throw new CosClientException("Unable to calculate MD5 hash: " + e.getMessage(), e);
            }
        }
        input = ResettableInputStream.newResettableInputStream(file, "Unable to find file to upload");
    }
    final ObjectMetadata returnedMetadata;
    MD5DigestCalculatingInputStream md5DigestStream = null;
    try {
        CosHttpRequest<UploadObjectRequest> request = null;
        if (uploadMode.equals(UploadMode.PUT_OBJECT)) {
            request = createRequest(bucketName, key, uploadObjectRequest, HttpMethodName.PUT);
        } else if (uploadMode.equals(UploadMode.APPEND_OBJECT)) {
            request = createRequest(bucketName, key, uploadObjectRequest, HttpMethodName.POST);
            AppendObjectRequest appendObjectRequest = (AppendObjectRequest) uploadObjectRequest;
            String positionStr = String.valueOf(appendObjectRequest.getPosition());
            request.addParameter("append", null);
            request.addParameter("position", positionStr);
        }
        if (uploadObjectRequest.getAccessControlList() != null) {
            addAclHeaders(request, uploadObjectRequest.getAccessControlList());
        } else if (uploadObjectRequest.getCannedAcl() != null) {
            request.addHeader(Headers.COS_CANNED_ACL, uploadObjectRequest.getCannedAcl().toString());
        }
        if (uploadObjectRequest.getStorageClass() != null) {
            request.addHeader(Headers.STORAGE_CLASS, uploadObjectRequest.getStorageClass());
        }
        if (uploadObjectRequest.getRedirectLocation() != null) {
            request.addHeader(Headers.REDIRECT_LOCATION, uploadObjectRequest.getRedirectLocation());
            if (input == null) {
                input = new ByteArrayInputStream(new byte[0]);
            }
        }
        // Populate the SSE-C parameters to the request header
        populateSSE_C(request, uploadObjectRequest.getSSECustomerKey());
        // Populate the SSE KMS parameters to the request header
        populateSSE_KMS(request, uploadObjectRequest.getSSECOSKeyManagementParams());
        // Populate the traffic limit parameter to the request header
        populateTrafficLimit(request, uploadObjectRequest.getTrafficLimit());
        // Use internal interface to differentiate 0 from unset.
        final Long contentLength = (Long) metadata.getRawMetadataValue(Headers.CONTENT_LENGTH);
        if (contentLength == null) {
            /*
                 * There's nothing we can do except for let the HTTP client buffer the input stream
                 * contents if the caller doesn't tell us how much data to expect in a stream since
                 * we have to explicitly tell how much we're sending before we start sending any of
                 * it.
                 */
            log.warn("No content length specified for stream data.  " + "Stream contents will be buffered in memory and could result in " + "out of memory errors.");
        } else {
            final long expectedLength = contentLength.longValue();
            final long maxAllowdSingleFileSize = 5 * 1024L * 1024L * 1024L;
            if (expectedLength > maxAllowdSingleFileSize) {
                throw new CosClientException("max size 5GB is allowed by putObject Method, your filesize is " + expectedLength + ", please use transferManager to upload big file!");
            }
            if (expectedLength >= 0) {
                // Performs length check on the underlying data stream.
                // For COS encryption client, the underlying data stream here
                // refers to the cipher-text data stream (ie not the underlying
                // plain-text data stream which in turn may have been wrapped
                // with it's own length check input stream.)
                LengthCheckInputStream lcis = new // expected
                LengthCheckInputStream(// expected
                input, // expected
                expectedLength, // uploaded
                EXCLUDE_SKIPPED_BYTES);
                input = lcis;
            }
        }
        if (metadata.getContentMD5() == null && !skipMd5CheckStrategy.skipClientSideValidationPerRequest(uploadObjectRequest)) {
            /*
                 * If the user hasn't set the content MD5, then we don't want to buffer the whole
                 * stream in memory just to calculate it. Instead, we can calculate it on the fly
                 * and validate it with the returned ETag from the object upload.
                 */
            input = md5DigestStream = new MD5DigestCalculatingInputStream(input);
        }
        populateRequestMetadata(request, metadata);
        request.setContent(input);
        try {
            if (uploadObjectRequest.getPicOperations() != null) {
                request.addHeader(Headers.PIC_OPERATIONS, Jackson.toJsonString(uploadObjectRequest.getPicOperations()));
                returnedMetadata = invoke(request, new ResponseHeaderHandlerChain<ObjectMetadata>(new Unmarshallers.ImagePersistenceUnmarshaller(), new CosMetadataResponseHandler()));
            } else {
                returnedMetadata = invoke(request, new CosMetadataResponseHandler());
            }
        } catch (Throwable t) {
            throw Throwables.failure(t);
        }
    } finally {
        CosDataSource.Utils.cleanupDataSource(uploadObjectRequest, file, isOrig, input, log);
    }
    String contentMd5 = metadata.getContentMD5();
    if (md5DigestStream != null) {
        contentMd5 = Base64.encodeAsString(md5DigestStream.getMd5Digest());
    }
    final String etag = returnedMetadata.getETag();
    if (contentMd5 != null && uploadMode.equals(UploadMode.PUT_OBJECT) && !skipMd5CheckStrategy.skipClientSideValidationPerPutResponse(returnedMetadata)) {
        byte[] clientSideHash = BinaryUtils.fromBase64(contentMd5);
        byte[] serverSideHash = null;
        try {
            serverSideHash = BinaryUtils.fromHex(etag);
        } catch (DecoderException e) {
            throw new CosClientException("Unable to verify integrity of data upload.  " + "Client calculated content hash (contentMD5: " + contentMd5 + " in base 64) didn't match hash (etag: " + etag + " in hex) calculated by COS .  " + "You may need to delete the data stored in COS . (metadata.contentMD5: " + metadata.getContentMD5() + ", bucketName: " + bucketName + ", key: " + key + ")");
        }
        if (!Arrays.equals(clientSideHash, serverSideHash)) {
            throw new CosClientException("Unable to verify integrity of data upload.  " + "Client calculated content hash (contentMD5: " + contentMd5 + " in base 64) didn't match hash (etag: " + etag + " in hex) calculated by COS .  " + "You may need to delete the data stored in COS . (metadata.contentMD5: " + metadata.getContentMD5() + ", bucketName: " + bucketName + ", key: " + key + ")");
        }
    }
    return returnedMetadata;
}