Examples with CommitKeyRequest org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyRequest used on opensource projects

Example 1 with CommitKeyRequest

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyRequest in project ozone by apache.

the class TestOMKeyCommitRequest method testValidateAndUpdateCacheWithUnknownBlockId.

@Test
public void testValidateAndUpdateCacheWithUnknownBlockId() throws Exception {
    OMRequest modifiedOmRequest = doPreExecute(createCommitKeyRequest());
    OMKeyCommitRequest omKeyCommitRequest = getOmKeyCommitRequest(modifiedOmRequest);
    // Append 3 blocks locations.
    List<OmKeyLocationInfo> allocatedLocationList = getKeyLocation(3).stream().map(OmKeyLocationInfo::getFromProtobuf).collect(Collectors.toList());
    OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, omKeyCommitRequest.getBucketLayout());
    String ozoneKey = addKeyToOpenKeyTable(allocatedLocationList);
    // Key should not be there in key table, as validateAndUpdateCache is
    // still not called.
    OmKeyInfo omKeyInfo = omMetadataManager.getKeyTable(omKeyCommitRequest.getBucketLayout()).get(ozoneKey);
    Assert.assertNull(omKeyInfo);
    OMClientResponse omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L, ozoneManagerDoubleBufferHelper);
    Assert.assertEquals(OzoneManagerProtocolProtos.Status.OK, omClientResponse.getOMResponse().getStatus());
    // Entry should be deleted from openKey Table.
    omKeyInfo = omMetadataManager.getOpenKeyTable(omKeyCommitRequest.getBucketLayout()).get(ozoneKey);
    Assert.assertNull(omKeyInfo);
    // Now entry should be created in key Table.
    omKeyInfo = omMetadataManager.getKeyTable(omKeyCommitRequest.getBucketLayout()).get(ozoneKey);
    Assert.assertNotNull(omKeyInfo);
    // Check modification time
    CommitKeyRequest commitKeyRequest = modifiedOmRequest.getCommitKeyRequest();
    Assert.assertEquals(commitKeyRequest.getKeyArgs().getModificationTime(), omKeyInfo.getModificationTime());
    // Check block location.
    Assert.assertEquals(allocatedLocationList, omKeyInfo.getLatestVersionLocations().getLocationList());
}

Example 2 with CommitKeyRequest

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyRequest in project ozone by apache.

the class OMKeyCommitRequestWithFSO method validateAndUpdateCache.

@Override
@SuppressWarnings("methodlength")
public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
    CommitKeyRequest commitKeyRequest = getOmRequest().getCommitKeyRequest();
    KeyArgs commitKeyArgs = commitKeyRequest.getKeyArgs();
    String volumeName = commitKeyArgs.getVolumeName();
    String bucketName = commitKeyArgs.getBucketName();
    String keyName = commitKeyArgs.getKeyName();
    OMMetrics omMetrics = ozoneManager.getMetrics();
    omMetrics.incNumKeyCommits();
    AuditLogger auditLogger = ozoneManager.getAuditLogger();
    Map<String, String> auditMap = buildKeyArgsAuditMap(commitKeyArgs);
    OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest());
    IOException exception = null;
    OmKeyInfo omKeyInfo = null;
    OmBucketInfo omBucketInfo = null;
    OMClientResponse omClientResponse = null;
    boolean bucketLockAcquired = false;
    Result result;
    OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
    try {
        commitKeyArgs = resolveBucketLink(ozoneManager, commitKeyArgs, auditMap);
        volumeName = commitKeyArgs.getVolumeName();
        bucketName = commitKeyArgs.getBucketName();
        // check Acl
        checkKeyAclsInOpenKeyTable(ozoneManager, volumeName, bucketName, keyName, IAccessAuthorizer.ACLType.WRITE, commitKeyRequest.getClientID());
        String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
        Iterator<Path> pathComponents = Paths.get(keyName).iterator();
        String dbOpenFileKey = null;
        List<OmKeyLocationInfo> locationInfoList = new ArrayList<>();
        for (KeyLocation keyLocation : commitKeyArgs.getKeyLocationsList()) {
            locationInfoList.add(OmKeyLocationInfo.getFromProtobuf(keyLocation));
        }
        bucketLockAcquired = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK, volumeName, bucketName);
        validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
        String fileName = OzoneFSUtils.getFileName(keyName);
        omBucketInfo = omMetadataManager.getBucketTable().get(bucketKey);
        long bucketId = omBucketInfo.getObjectID();
        long parentID = OMFileRequest.getParentID(bucketId, pathComponents, keyName, omMetadataManager, "Cannot create file : " + keyName + " as parent directory doesn't exist");
        String dbFileKey = omMetadataManager.getOzonePathKey(parentID, fileName);
        dbOpenFileKey = omMetadataManager.getOpenFileName(parentID, fileName, commitKeyRequest.getClientID());
        omKeyInfo = OMFileRequest.getOmKeyInfoFromFileTable(true, omMetadataManager, dbOpenFileKey, keyName);
        if (omKeyInfo == null) {
            throw new OMException("Failed to commit key, as " + dbOpenFileKey + "entry is not found in the OpenKey table", KEY_NOT_FOUND);
        }
        omKeyInfo.setDataSize(commitKeyArgs.getDataSize());
        omKeyInfo.setModificationTime(commitKeyArgs.getModificationTime());
        // Update the block length for each block
        omKeyInfo.updateLocationInfoList(locationInfoList, false);
        // Set the UpdateID to current transactionLogIndex
        omKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled());
        // If bucket versioning is turned on during the update, between key
        // creation and key commit, old versions will be just overwritten and
        // not kept. Bucket versioning will be effective from the first key
        // creation after the knob turned on.
        RepeatedOmKeyInfo oldKeyVersionsToDelete = null;
        OmKeyInfo keyToDelete = omMetadataManager.getKeyTable(getBucketLayout()).get(dbFileKey);
        if (keyToDelete != null && !omBucketInfo.getIsVersionEnabled()) {
            oldKeyVersionsToDelete = getOldVersionsToCleanUp(dbFileKey, keyToDelete, omMetadataManager, trxnLogIndex, ozoneManager.isRatisEnabled());
        }
        // Add to cache of open key table and key table.
        OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager, dbFileKey, null, fileName, trxnLogIndex);
        OMFileRequest.addFileTableCacheEntry(omMetadataManager, dbFileKey, omKeyInfo, fileName, trxnLogIndex);
        if (oldKeyVersionsToDelete != null) {
            OMFileRequest.addDeletedTableCacheEntry(omMetadataManager, dbFileKey, oldKeyVersionsToDelete, trxnLogIndex);
        }
        long scmBlockSize = ozoneManager.getScmBlockSize();
        int factor = omKeyInfo.getReplicationConfig().getRequiredNodes();
        // Block was pre-requested and UsedBytes updated when createKey and
        // AllocatedBlock. The space occupied by the Key shall be based on
        // the actual Key size, and the total Block size applied before should
        // be subtracted.
        long correctedSpace = omKeyInfo.getDataSize() * factor - locationInfoList.size() * scmBlockSize * factor;
        // Subtract the size of blocks to be overwritten.
        if (keyToDelete != null) {
            correctedSpace -= keyToDelete.getDataSize() * keyToDelete.getReplicationConfig().getRequiredNodes();
        }
        omBucketInfo.incrUsedBytes(correctedSpace);
        omClientResponse = new OMKeyCommitResponseWithFSO(omResponse.build(), omKeyInfo, dbFileKey, dbOpenFileKey, omBucketInfo.copyObject(), oldKeyVersionsToDelete);
        result = Result.SUCCESS;
    } catch (IOException ex) {
        result = Result.FAILURE;
        exception = ex;
        omClientResponse = new OMKeyCommitResponseWithFSO(createErrorOMResponse(omResponse, exception), getBucketLayout());
    } finally {
        addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, omDoubleBufferHelper);
        if (bucketLockAcquired) {
            omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName, bucketName);
        }
    }
    auditLog(auditLogger, buildAuditMessage(OMAction.COMMIT_KEY, auditMap, exception, getOmRequest().getUserInfo()));
    processResult(commitKeyRequest, volumeName, bucketName, keyName, omMetrics, exception, omKeyInfo, result);
    return omClientResponse;
}

Example 3 with CommitKeyRequest

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyRequest in project ozone by apache.

the class OMKeyCommitRequest method preExecute.

@Override
public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
    CommitKeyRequest commitKeyRequest = getOmRequest().getCommitKeyRequest();
    Preconditions.checkNotNull(commitKeyRequest);
    KeyArgs keyArgs = commitKeyRequest.getKeyArgs();
    // Verify key name
    final boolean checkKeyNameEnabled = ozoneManager.getConfiguration().getBoolean(OMConfigKeys.OZONE_OM_KEYNAME_CHARACTER_CHECK_ENABLED_KEY, OMConfigKeys.OZONE_OM_KEYNAME_CHARACTER_CHECK_ENABLED_DEFAULT);
    if (checkKeyNameEnabled) {
        OmUtils.validateKeyName(StringUtils.removeEnd(keyArgs.getKeyName(), OzoneConsts.FS_FILE_COPYING_TEMP_SUFFIX));
    }
    String keyPath = keyArgs.getKeyName();
    keyPath = validateAndNormalizeKey(ozoneManager.getEnableFileSystemPaths(), keyPath, getBucketLayout());
    KeyArgs.Builder newKeyArgs = keyArgs.toBuilder().setModificationTime(Time.now()).setKeyName(keyPath);
    return getOmRequest().toBuilder().setCommitKeyRequest(commitKeyRequest.toBuilder().setKeyArgs(newKeyArgs)).setUserInfo(getUserInfo()).build();
}

Example 4 with CommitKeyRequest

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyRequest in project ozone by apache.

the class TestOMKeyCommitRequest method createCommitKeyRequest.

/**
 * Create OMRequest which encapsulates CommitKeyRequest.
 */
private OMRequest createCommitKeyRequest() {
    KeyArgs keyArgs = KeyArgs.newBuilder().setDataSize(dataSize).setVolumeName(volumeName).setKeyName(keyName).setBucketName(bucketName).setType(replicationType).setFactor(replicationFactor).addAllKeyLocations(getKeyLocation(5)).build();
    CommitKeyRequest commitKeyRequest = CommitKeyRequest.newBuilder().setKeyArgs(keyArgs).setClientID(clientID).build();
    return OMRequest.newBuilder().setCmdType(OzoneManagerProtocolProtos.Type.CommitKey).setCommitKeyRequest(commitKeyRequest).setClientId(UUID.randomUUID().toString()).build();
}

Example 5 with CommitKeyRequest

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyRequest in project ozone by apache.

the class TestOMKeyCommitRequest method testValidateAndUpdateCacheOnOverwrite.

@Test
public void testValidateAndUpdateCacheOnOverwrite() throws Exception {
    testValidateAndUpdateCache();
    // Become a new client and set next version number
    clientID = Time.now();
    version += 1;
    OMRequest modifiedOmRequest = doPreExecute(createCommitKeyRequest());
    OMKeyCommitRequest omKeyCommitRequest = getOmKeyCommitRequest(modifiedOmRequest);
    KeyArgs keyArgs = modifiedOmRequest.getCommitKeyRequest().getKeyArgs();
    String ozoneKey = getOzonePathKey();
    // Key should be there in key table, as validateAndUpdateCache is called.
    OmKeyInfo omKeyInfo = omMetadataManager.getKeyTable(omKeyCommitRequest.getBucketLayout()).get(ozoneKey);
    Assert.assertNotNull(omKeyInfo);
    // Previously committed version
    Assert.assertEquals(0L, omKeyInfo.getLatestVersionLocations().getVersion());
    // Append new blocks
    List<OmKeyLocationInfo> allocatedLocationList = keyArgs.getKeyLocationsList().stream().map(OmKeyLocationInfo::getFromProtobuf).collect(Collectors.toList());
    addKeyToOpenKeyTable(allocatedLocationList);
    OMClientResponse omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 102L, ozoneManagerDoubleBufferHelper);
    Assert.assertEquals(OzoneManagerProtocolProtos.Status.OK, omClientResponse.getOMResponse().getStatus());
    // New entry should be created in key Table.
    omKeyInfo = omMetadataManager.getKeyTable(omKeyCommitRequest.getBucketLayout()).get(ozoneKey);
    Assert.assertNotNull(omKeyInfo);
    Assert.assertEquals(version, omKeyInfo.getLatestVersionLocations().getVersion());
    // DB keyInfo format
    verifyKeyName(omKeyInfo);
    // Check modification time
    CommitKeyRequest commitKeyRequest = modifiedOmRequest.getCommitKeyRequest();
    Assert.assertEquals(commitKeyRequest.getKeyArgs().getModificationTime(), omKeyInfo.getModificationTime());
    // Check block location.
    List<OmKeyLocationInfo> locationInfoListFromCommitKeyRequest = commitKeyRequest.getKeyArgs().getKeyLocationsList().stream().map(OmKeyLocationInfo::getFromProtobuf).collect(Collectors.toList());
    Assert.assertEquals(locationInfoListFromCommitKeyRequest, omKeyInfo.getLatestVersionLocations().getLocationList());
    Assert.assertEquals(allocatedLocationList, omKeyInfo.getLatestVersionLocations().getLocationList());
    Assert.assertEquals(1, omKeyInfo.getKeyLocationVersions().size());
}