Examples with MultipartUploadCompleteRequest org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartUploadCompleteRequest used on opensource projects

Example 1 with MultipartUploadCompleteRequest

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartUploadCompleteRequest in project ozone by apache.

the class S3MultipartUploadCompleteRequest method validateAndUpdateCache.

@Override
@SuppressWarnings("methodlength")
public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
    MultipartUploadCompleteRequest multipartUploadCompleteRequest = getOmRequest().getCompleteMultiPartUploadRequest();
    KeyArgs keyArgs = multipartUploadCompleteRequest.getKeyArgs();
    List<OzoneManagerProtocolProtos.Part> partsList = multipartUploadCompleteRequest.getPartsListList();
    Map<String, String> auditMap = buildKeyArgsAuditMap(keyArgs);
    String volumeName = keyArgs.getVolumeName();
    String bucketName = keyArgs.getBucketName();
    final String requestedVolume = volumeName;
    final String requestedBucket = bucketName;
    String keyName = keyArgs.getKeyName();
    String uploadID = keyArgs.getMultipartUploadID();
    String multipartKey = null;
    ozoneManager.getMetrics().incNumCompleteMultipartUploads();
    OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
    boolean acquiredLock = false;
    OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest());
    OMClientResponse omClientResponse = null;
    IOException exception = null;
    Result result = null;
    try {
        keyArgs = resolveBucketLink(ozoneManager, keyArgs, auditMap);
        volumeName = keyArgs.getVolumeName();
        bucketName = keyArgs.getBucketName();
        multipartKey = omMetadataManager.getMultipartKey(volumeName, bucketName, keyName, uploadID);
        // check Acl
        checkKeyAcls(ozoneManager, volumeName, bucketName, keyName, IAccessAuthorizer.ACLType.WRITE, OzoneObj.ResourceType.KEY);
        acquiredLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK, volumeName, bucketName);
        validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
        OmBucketInfo omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName);
        String ozoneKey = omMetadataManager.getOzoneKey(volumeName, bucketName, keyName);
        String dbOzoneKey = getDBOzoneKey(omMetadataManager, volumeName, bucketName, keyName);
        String dbMultipartOpenKey = getDBMultipartOpenKey(volumeName, bucketName, keyName, uploadID, omMetadataManager);
        OmMultipartKeyInfo multipartKeyInfo = omMetadataManager.getMultipartInfoTable().get(multipartKey);
        // Check for directory exists with same name, if it exists throw error.
        checkDirectoryAlreadyExists(ozoneManager, volumeName, bucketName, keyName, omMetadataManager);
        if (multipartKeyInfo == null) {
            throw new OMException(failureMessage(requestedVolume, requestedBucket, keyName), OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR);
        }
        TreeMap<Integer, PartKeyInfo> partKeyInfoMap = multipartKeyInfo.getPartKeyInfoMap();
        if (partsList.size() > 0) {
            if (partKeyInfoMap.size() == 0) {
                LOG.error("Complete MultipartUpload failed for key {} , MPU Key has" + " no parts in OM, parts given to upload are {}", ozoneKey, partsList);
                throw new OMException(failureMessage(requestedVolume, requestedBucket, keyName), OMException.ResultCodes.INVALID_PART);
            }
            // First Check for Invalid Part Order.
            List<Integer> partNumbers = new ArrayList<>();
            int partsListSize = getPartsListSize(requestedVolume, requestedBucket, keyName, ozoneKey, partNumbers, partsList);
            List<OmKeyLocationInfo> partLocationInfos = new ArrayList<>();
            long dataSize = getMultipartDataSize(requestedVolume, requestedBucket, keyName, ozoneKey, partKeyInfoMap, partsListSize, partLocationInfos, partsList, ozoneManager);
            // All parts have same replication information. Here getting from last
            // part.
            OmKeyInfo omKeyInfo = getOmKeyInfo(ozoneManager, trxnLogIndex, keyArgs, volumeName, bucketName, keyName, dbMultipartOpenKey, omMetadataManager, dbOzoneKey, partKeyInfoMap, partLocationInfos, dataSize);
            // Find all unused parts.
            List<OmKeyInfo> unUsedParts = new ArrayList<>();
            for (Map.Entry<Integer, PartKeyInfo> partKeyInfo : partKeyInfoMap.entrySet()) {
                if (!partNumbers.contains(partKeyInfo.getKey())) {
                    unUsedParts.add(OmKeyInfo.getFromProtobuf(partKeyInfo.getValue().getPartKeyInfo()));
                }
            }
            // If bucket versioning is turned on during the update, between key
            // creation and key commit, old versions will be just overwritten and
            // not kept. Bucket versioning will be effective from the first key
            // creation after the knob turned on.
            RepeatedOmKeyInfo oldKeyVersionsToDelete = null;
            OmKeyInfo keyToDelete = omMetadataManager.getKeyTable(getBucketLayout()).get(dbOzoneKey);
            long usedBytesDiff = 0;
            boolean isNamespaceUpdate = false;
            if (keyToDelete != null && !omBucketInfo.getIsVersionEnabled()) {
                oldKeyVersionsToDelete = getOldVersionsToCleanUp(dbOzoneKey, keyToDelete, omMetadataManager, trxnLogIndex, ozoneManager.isRatisEnabled());
                long numCopy = keyToDelete.getReplicationConfig().getRequiredNodes();
                usedBytesDiff -= keyToDelete.getDataSize() * numCopy;
            } else {
                checkBucketQuotaInNamespace(omBucketInfo, 1L);
                omBucketInfo.incrUsedNamespace(1L);
                isNamespaceUpdate = true;
            }
            String dbBucketKey = omMetadataManager.getBucketKey(omBucketInfo.getVolumeName(), omBucketInfo.getBucketName());
            if (usedBytesDiff != 0) {
                omBucketInfo.incrUsedBytes(usedBytesDiff);
            } else if (!isNamespaceUpdate) {
                // If no bucket size and Namespace changed, prevent from updating
                // bucket object.
                omBucketInfo = null;
            }
            updateCache(omMetadataManager, dbBucketKey, omBucketInfo, dbOzoneKey, dbMultipartOpenKey, multipartKey, omKeyInfo, trxnLogIndex);
            if (oldKeyVersionsToDelete != null) {
                OMFileRequest.addDeletedTableCacheEntry(omMetadataManager, dbOzoneKey, oldKeyVersionsToDelete, trxnLogIndex);
            }
            omResponse.setCompleteMultiPartUploadResponse(MultipartUploadCompleteResponse.newBuilder().setVolume(requestedVolume).setBucket(requestedBucket).setKey(keyName).setHash(DigestUtils.sha256Hex(keyName)));
            omClientResponse = getOmClientResponse(multipartKey, omResponse, dbMultipartOpenKey, omKeyInfo, unUsedParts, omBucketInfo, oldKeyVersionsToDelete);
            result = Result.SUCCESS;
        } else {
            throw new OMException(failureMessage(requestedVolume, requestedBucket, keyName) + " because of empty part list", OMException.ResultCodes.INVALID_REQUEST);
        }
    } catch (IOException ex) {
        result = Result.FAILURE;
        exception = ex;
        omClientResponse = getOmClientResponse(omResponse, exception);
    } finally {
        addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, omDoubleBufferHelper);
        if (acquiredLock) {
            omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName, bucketName);
        }
    }
    logResult(ozoneManager, multipartUploadCompleteRequest, partsList, auditMap, volumeName, bucketName, keyName, exception, result);
    return omClientResponse;
}

Example 2 with MultipartUploadCompleteRequest

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartUploadCompleteRequest in project ozone by apache.

the class OzoneManagerProtocolClientSideTranslatorPB method completeMultipartUpload.

@Override
public OmMultipartUploadCompleteInfo completeMultipartUpload(OmKeyArgs omKeyArgs, OmMultipartUploadCompleteList multipartUploadList) throws IOException {
    MultipartUploadCompleteRequest.Builder multipartUploadCompleteRequest = MultipartUploadCompleteRequest.newBuilder();
    KeyArgs.Builder keyArgs = KeyArgs.newBuilder().setVolumeName(omKeyArgs.getVolumeName()).setBucketName(omKeyArgs.getBucketName()).setKeyName(omKeyArgs.getKeyName()).addAllAcls(omKeyArgs.getAcls().stream().map(a -> OzoneAcl.toProtobuf(a)).collect(Collectors.toList())).setMultipartUploadID(omKeyArgs.getMultipartUploadID());
    multipartUploadCompleteRequest.setKeyArgs(keyArgs.build());
    multipartUploadCompleteRequest.addAllPartsList(multipartUploadList.getPartsList());
    OMRequest omRequest = createOMRequest(Type.CompleteMultiPartUpload).setCompleteMultiPartUploadRequest(multipartUploadCompleteRequest.build()).build();
    MultipartUploadCompleteResponse response = handleError(submitRequest(omRequest)).getCompleteMultiPartUploadResponse();
    OmMultipartUploadCompleteInfo info = new OmMultipartUploadCompleteInfo(response.getVolume(), response.getBucket(), response.getKey(), response.getHash());
    return info;
}

Example 3 with MultipartUploadCompleteRequest

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartUploadCompleteRequest in project ozone by apache.

the class OMRequestTestUtils method createCompleteMPURequest.

public static OMRequest createCompleteMPURequest(String volumeName, String bucketName, String keyName, String multipartUploadID, List<OzoneManagerProtocolProtos.Part> partList) {
    KeyArgs.Builder keyArgs = KeyArgs.newBuilder().setVolumeName(volumeName).setKeyName(keyName).setBucketName(bucketName).setMultipartUploadID(multipartUploadID);
    MultipartUploadCompleteRequest multipartUploadCompleteRequest = MultipartUploadCompleteRequest.newBuilder().setKeyArgs(keyArgs).addAllPartsList(partList).build();
    return OMRequest.newBuilder().setClientId(UUID.randomUUID().toString()).setCmdType(OzoneManagerProtocolProtos.Type.CompleteMultiPartUpload).setCompleteMultiPartUploadRequest(multipartUploadCompleteRequest).build();
}

Example 4 with MultipartUploadCompleteRequest

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartUploadCompleteRequest in project ozone by apache.

the class S3MultipartUploadCompleteRequest method preExecute.

@Override
public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
    MultipartUploadCompleteRequest multipartUploadCompleteRequest = getOmRequest().getCompleteMultiPartUploadRequest();
    KeyArgs keyArgs = multipartUploadCompleteRequest.getKeyArgs();
    String keyPath = keyArgs.getKeyName();
    keyPath = validateAndNormalizeKey(ozoneManager.getEnableFileSystemPaths(), keyPath, getBucketLayout());
    return getOmRequest().toBuilder().setCompleteMultiPartUploadRequest(multipartUploadCompleteRequest.toBuilder().setKeyArgs(keyArgs.toBuilder().setModificationTime(Time.now()).setKeyName(keyPath))).setUserInfo(getUserInfo()).build();
}