Examples with ObjectMetadata com.qcloud.cos.model.ObjectMetadata used on opensource projects

Example 51 with ObjectMetadata

use of com.qcloud.cos.model.ObjectMetadata in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method completeMultipartUpload.

/**
 * complete cos mpu, sometimes return null complete mpu result
 * @param key cos key
 * @param uploadId upload id
 * @param partETagList each part etag list
 * @return result
 * @throws IOException when fail to complete the multipart upload.
 */
public CompleteMultipartUploadResult completeMultipartUpload(String key, String uploadId, List<PartETag> partETagList) throws IOException {
    Collections.sort(partETagList, new Comparator<PartETag>() {

        @Override
        public int compare(PartETag o1, PartETag o2) {
            return o1.getPartNumber() - o2.getPartNumber();
        }
    });
    try {
        ObjectMetadata objectMetadata = new ObjectMetadata();
        if (crc32cEnabled) {
            objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
        }
        CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETagList);
        completeMultipartUploadRequest.setObjectMetadata(objectMetadata);
        return (CompleteMultipartUploadResult) this.callCOSClientWithRetry(completeMultipartUploadRequest);
    } catch (CosServiceException cse) {
        // when first calling with 503 access time out, next retry will 409.
        int statusCode = cse.getStatusCode();
        if (statusCode == 409) {
            // check file whether exist
            FileMetadata fileMetadata = this.queryObjectMetadata(key);
            if (null == fileMetadata) {
                // if file not exist through exception
                handleException(cse, key);
            }
            LOG.warn("Upload the cos key [{}] complete mpu concurrently", key);
        } else {
            // other exception
            String errMsg = String.format("Complete the multipart upload failed. " + "cos service exception, cos key: %s, upload id: %s, " + "exception: %s", key, uploadId, cse.toString());
            handleException(new Exception(errMsg), key);
        }
    } catch (Exception e) {
        String errMsg = String.format("Complete the multipart upload failed. " + "cos key: %s, upload id: %s, " + "exception: %s", key, uploadId, e.toString());
        handleException(new Exception(errMsg), key);
    }
    return null;
}

Example 52 with ObjectMetadata

use of com.qcloud.cos.model.ObjectMetadata in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method uploadPart.

@Override
public PartETag uploadPart(InputStream inputStream, String key, String uploadId, int partNum, long partSize, byte[] md5Hash) throws IOException {
    LOG.debug("Upload the part to the cos key [{}]. upload id: {}, part number: {}, part size: {}", key, uploadId, partNum, partSize);
    ObjectMetadata objectMetadata = new ObjectMetadata();
    if (crc32cEnabled) {
        objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
    }
    UploadPartRequest uploadPartRequest = new UploadPartRequest();
    uploadPartRequest.setBucketName(this.bucketName);
    uploadPartRequest.setUploadId(uploadId);
    uploadPartRequest.setInputStream(inputStream);
    uploadPartRequest.setPartNumber(partNum);
    uploadPartRequest.setPartSize(partSize);
    uploadPartRequest.setObjectMetadata(objectMetadata);
    if (null != md5Hash) {
        uploadPartRequest.setMd5Digest(Base64.encodeAsString(md5Hash));
    }
    uploadPartRequest.setKey(key);
    if (this.trafficLimit >= 0) {
        uploadPartRequest.setTrafficLimit(this.trafficLimit);
    }
    this.setEncryptionMetadata(uploadPartRequest, objectMetadata);
    try {
        UploadPartResult uploadPartResult = (UploadPartResult) callCOSClientWithRetry(uploadPartRequest);
        return uploadPartResult.getPartETag();
    } catch (Exception e) {
        String errMsg = String.format("The current thread:%d, " + "cos key: %s, upload id: %s, part num: %d, " + "exception: %s", Thread.currentThread().getId(), key, uploadId, partNum, e);
        handleException(new Exception(errMsg), key);
    }
    return null;
}

Example 53 with ObjectMetadata

use of com.qcloud.cos.model.ObjectMetadata in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method queryObjectMetadata.

private FileMetadata queryObjectMetadata(String key, CosNResultInfo info) throws IOException {
    LOG.debug("Query Object metadata. cos key: {}.", key);
    GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest(bucketName, key);
    this.setEncryptionMetadata(getObjectMetadataRequest, new ObjectMetadata());
    try {
        ObjectMetadata objectMetadata = (ObjectMetadata) callCOSClientWithRetry(getObjectMetadataRequest);
        long mtime = 0;
        long fileSize;
        if (objectMetadata.getLastModified() != null) {
            mtime = objectMetadata.getLastModified().getTime();
        }
        fileSize = objectMetadata.getContentLength();
        String ETag = objectMetadata.getETag();
        String crc64ecm = objectMetadata.getCrc64Ecma();
        String crc32cm = (String) objectMetadata.getRawMetadataValue(Constants.CRC32C_RESP_HEADER);
        String versionId = objectMetadata.getVersionId();
        Map<String, byte[]> userMetadata = null;
        if (objectMetadata.getUserMetadata() != null) {
            userMetadata = new HashMap<>();
            for (Map.Entry<String, String> userMetadataEntry : objectMetadata.getUserMetadata().entrySet()) {
                if (userMetadataEntry.getKey().startsWith(ensureValidAttributeName(XATTR_PREFIX))) {
                    String xAttrJsonStr = new String(Base64.decode(userMetadataEntry.getValue()), StandardCharsets.UTF_8);
                    CosNXAttr cosNXAttr;
                    try {
                        cosNXAttr = Jackson.fromJsonString(xAttrJsonStr, CosNXAttr.class);
                    } catch (CosClientException e) {
                        LOG.warn("Parse the xAttr failed. name: {}, XAttJsonStr: {}.", userMetadataEntry.getKey(), xAttrJsonStr);
                        // Skip
                        continue;
                    }
                    if (null != cosNXAttr) {
                        userMetadata.put(cosNXAttr.getName(), cosNXAttr.getValue().getBytes(CosNFileSystem.METADATA_ENCODING));
                    }
                }
            }
        }
        boolean isFile = true;
        if (isPosixBucket) {
            if (objectMetadata.isFileModeDir() || key.equals(CosNFileSystem.PATH_DELIMITER)) {
                isFile = false;
            }
        } else {
            isFile = !key.endsWith(CosNFileSystem.PATH_DELIMITER);
        }
        FileMetadata fileMetadata = new FileMetadata(key, fileSize, mtime, isFile, ETag, crc64ecm, crc32cm, versionId, objectMetadata.getStorageClass(), userMetadata);
        // record the last request result info
        if (info != null) {
            info.setRequestID(objectMetadata.getRequestId());
        }
        LOG.debug("Retrieve the file metadata. cos key: {}, ETag:{}, length:{}, crc64ecm: {}.", key, objectMetadata.getETag(), objectMetadata.getContentLength(), objectMetadata.getCrc64Ecma());
        return fileMetadata;
    } catch (CosServiceException e) {
        if (info != null) {
            info.setRequestID(e.getRequestId());
        }
        if (e.getStatusCode() != 404) {
            String errorMsg = String.format("Retrieve the file metadata file failure. " + "cos key: %s, exception: %s", key, e);
            handleException(new Exception(errorMsg), key);
        }
    }
    return null;
}

Example 54 with ObjectMetadata

use of com.qcloud.cos.model.ObjectMetadata in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method storeAttribute.

private void storeAttribute(String key, String attribute, byte[] value, boolean deleted) throws IOException {
    if (deleted) {
        LOG.debug("Delete the extended attribute. cos key: {}, attribute: {}.", key, attribute);
    }
    if (null != value && !deleted) {
        LOG.debug("Store the extended attribute. cos key: {}, attribute: {}, value: {}.", key, attribute, new String(value, StandardCharsets.UTF_8));
    }
    if (null == value && !deleted) {
        throw new IOException("The attribute value to be set can not be null.");
    }
    GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest(bucketName, key);
    this.setEncryptionMetadata(getObjectMetadataRequest, new ObjectMetadata());
    ObjectMetadata objectMetadata = null;
    try {
        objectMetadata = (ObjectMetadata) callCOSClientWithRetry(getObjectMetadataRequest);
    } catch (CosServiceException e) {
        if (e.getStatusCode() != 404) {
            String errorMessage = String.format("Retrieve the file metadata failed. " + "cos key: %s, exception: %s.", key, e);
            handleException(new Exception(errorMessage), key);
        }
    }
    if (null != objectMetadata) {
        Map<String, String> userMetadata = objectMetadata.getUserMetadata();
        if (deleted) {
            if (null != userMetadata) {
                userMetadata.remove(ensureValidAttributeName(attribute));
            } else {
                return;
            }
        } else {
            if (null == userMetadata) {
                userMetadata = new HashMap<>();
            }
            CosNXAttr cosNXAttr = new CosNXAttr();
            cosNXAttr.setName(attribute);
            cosNXAttr.setValue(new String(value, CosNFileSystem.METADATA_ENCODING));
            String xAttrJsonStr = Jackson.toJsonString(cosNXAttr);
            userMetadata.put(ensureValidAttributeName(XATTR_PREFIX + attribute), Base64.encodeAsString(xAttrJsonStr.getBytes(StandardCharsets.UTF_8)));
        }
        objectMetadata.setUserMetadata(userMetadata);
        // 构造原地copy请求来设置用户自定义属性
        if (crc32cEnabled) {
            objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
        }
        CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketName, key, bucketName, key);
        if (null != objectMetadata.getStorageClass()) {
            copyObjectRequest.setStorageClass(objectMetadata.getStorageClass());
        }
        copyObjectRequest.setNewObjectMetadata(objectMetadata);
        copyObjectRequest.setRedirectLocation("Replaced");
        this.setEncryptionMetadata(copyObjectRequest, objectMetadata);
        copyObjectRequest.setSourceEndpointBuilder(this.cosClient.getClientConfig().getEndpointBuilder());
        try {
            callCOSClientWithRetry(copyObjectRequest);
        } catch (Exception e) {
            String errMsg = String.format("Failed to modify the user-defined attributes. " + "cos key: %s, attribute: %s, exception: %s.", key, attribute, e);
            handleException(new Exception(errMsg), key);
        }
    }
}

Example 55 with ObjectMetadata

use of com.qcloud.cos.model.ObjectMetadata in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method storeFileWithRetry.

private void storeFileWithRetry(String key, InputStream inputStream, byte[] md5Hash, long length) throws IOException {
    try {
        ObjectMetadata objectMetadata = new ObjectMetadata();
        if (null != md5Hash) {
            objectMetadata.setContentMD5(Base64.encodeAsString(md5Hash));
        }
        objectMetadata.setContentLength(length);
        if (crc32cEnabled) {
            objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
        }
        PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, inputStream, objectMetadata);
        if (null != this.storageClass) {
            putObjectRequest.setStorageClass(this.storageClass);
        }
        if (this.trafficLimit >= 0) {
            putObjectRequest.setTrafficLimit(this.trafficLimit);
        }
        this.setEncryptionMetadata(putObjectRequest, objectMetadata);
        PutObjectResult putObjectResult = (PutObjectResult) callCOSClientWithRetry(putObjectRequest);
        LOG.debug("Store the file successfully. cos key: {}, ETag: {}.", key, putObjectResult.getETag());
    } catch (CosServiceException cse) {
        // 避免并发上传的问题
        int statusCode = cse.getStatusCode();
        if (statusCode == 409) {
            // Check一下这个文件是否已经存在
            FileMetadata fileMetadata = this.queryObjectMetadata(key);
            if (null == fileMetadata) {
                // 如果文件不存在，则需要抛出异常
                handleException(cse, key);
            }
            LOG.warn("Upload the cos key [{}] concurrently", key);
        } else {
            // 其他错误都要抛出来
            handleException(cse, key);
        }
    } catch (Exception e) {
        String errMsg = String.format("Store the file failed, cos key: %s, exception: %s.", key, e);
        handleException(new Exception(errMsg), key);
    }
}