Examples with AllocateBlockRequest org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockRequest used on opensource projects

Example 1 with AllocateBlockRequest

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockRequest in project ozone by apache.

the class TestOMAllocateBlockRequest method createAllocateBlockRequest.

protected OMRequest createAllocateBlockRequest() {
    KeyArgs keyArgs = KeyArgs.newBuilder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setFactor(replicationFactor).setType(replicationType).build();
    AllocateBlockRequest allocateBlockRequest = AllocateBlockRequest.newBuilder().setClientID(clientID).setKeyArgs(keyArgs).build();
    return OMRequest.newBuilder().setCmdType(OzoneManagerProtocolProtos.Type.AllocateBlock).setClientId(UUID.randomUUID().toString()).setAllocateBlockRequest(allocateBlockRequest).build();
}

Example 2 with AllocateBlockRequest

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockRequest in project ozone by apache.

the class TestOMAllocateBlockRequest method doPreExecute.

/**
 * This method calls preExecute and verify the modified request.
 * @param originalOMRequest
 * @return OMRequest - modified request returned from preExecute.
 * @throws Exception
 */
protected OMRequest doPreExecute(OMRequest originalOMRequest) throws Exception {
    OMAllocateBlockRequest omAllocateBlockRequest = getOmAllocateBlockRequest(originalOMRequest);
    OMRequest modifiedOmRequest = omAllocateBlockRequest.preExecute(ozoneManager);
    Assert.assertEquals(originalOMRequest.getCmdType(), modifiedOmRequest.getCmdType());
    Assert.assertEquals(originalOMRequest.getClientId(), modifiedOmRequest.getClientId());
    Assert.assertTrue(modifiedOmRequest.hasAllocateBlockRequest());
    AllocateBlockRequest allocateBlockRequest = modifiedOmRequest.getAllocateBlockRequest();
    // Time should be set
    Assert.assertTrue(allocateBlockRequest.getKeyArgs().getModificationTime() > 0);
    // KeyLocation should be set.
    Assert.assertTrue(allocateBlockRequest.hasKeyLocation());
    Assert.assertEquals(CONTAINER_ID, allocateBlockRequest.getKeyLocation().getBlockID().getContainerBlockID().getContainerID());
    Assert.assertEquals(LOCAL_ID, allocateBlockRequest.getKeyLocation().getBlockID().getContainerBlockID().getLocalID());
    Assert.assertTrue(allocateBlockRequest.getKeyLocation().hasPipeline());
    Assert.assertEquals(allocateBlockRequest.getClientID(), allocateBlockRequest.getClientID());
    return modifiedOmRequest;
}

Example 3 with AllocateBlockRequest

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockRequest in project ozone by apache.

the class OMAllocateBlockRequest method validateAndUpdateCache.

@Override
public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
    OzoneManagerProtocolProtos.AllocateBlockRequest allocateBlockRequest = getOmRequest().getAllocateBlockRequest();
    OzoneManagerProtocolProtos.KeyArgs keyArgs = allocateBlockRequest.getKeyArgs();
    OzoneManagerProtocolProtos.KeyLocation blockLocation = allocateBlockRequest.getKeyLocation();
    Preconditions.checkNotNull(blockLocation);
    String volumeName = keyArgs.getVolumeName();
    String bucketName = keyArgs.getBucketName();
    String keyName = keyArgs.getKeyName();
    long clientID = allocateBlockRequest.getClientID();
    OMMetrics omMetrics = ozoneManager.getMetrics();
    omMetrics.incNumBlockAllocateCalls();
    AuditLogger auditLogger = ozoneManager.getAuditLogger();
    Map<String, String> auditMap = buildKeyArgsAuditMap(keyArgs);
    auditMap.put(OzoneConsts.CLIENT_ID, String.valueOf(clientID));
    OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
    String openKeyName = null;
    OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest());
    OMClientResponse omClientResponse = null;
    OmKeyInfo openKeyInfo = null;
    IOException exception = null;
    OmBucketInfo omBucketInfo = null;
    boolean acquiredLock = false;
    try {
        keyArgs = resolveBucketLink(ozoneManager, keyArgs, auditMap);
        volumeName = keyArgs.getVolumeName();
        bucketName = keyArgs.getBucketName();
        // check Acl
        checkKeyAclsInOpenKeyTable(ozoneManager, volumeName, bucketName, keyName, IAccessAuthorizer.ACLType.WRITE, allocateBlockRequest.getClientID());
        validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
        // Here we don't acquire bucket/volume lock because for a single client
        // allocateBlock is called in serial fashion.
        openKeyName = omMetadataManager.getOpenKey(volumeName, bucketName, keyName, clientID);
        openKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKeyName);
        if (openKeyInfo == null) {
            throw new OMException("Open Key not found " + openKeyName, KEY_NOT_FOUND);
        }
        List<OmKeyLocationInfo> newLocationList = Collections.singletonList(OmKeyLocationInfo.getFromProtobuf(blockLocation));
        acquiredLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK, volumeName, bucketName);
        omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName);
        // check bucket and volume quota
        long preAllocatedSpace = newLocationList.size() * ozoneManager.getScmBlockSize() * openKeyInfo.getReplicationConfig().getRequiredNodes();
        checkBucketQuotaInBytes(omBucketInfo, preAllocatedSpace);
        // Append new block
        openKeyInfo.appendNewBlocks(newLocationList, false);
        // Set modification time.
        openKeyInfo.setModificationTime(keyArgs.getModificationTime());
        // Set the UpdateID to current transactionLogIndex
        openKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled());
        // Add to cache.
        omMetadataManager.getOpenKeyTable(getBucketLayout()).addCacheEntry(new CacheKey<>(openKeyName), new CacheValue<>(Optional.of(openKeyInfo), trxnLogIndex));
        omBucketInfo.incrUsedBytes(preAllocatedSpace);
        omResponse.setAllocateBlockResponse(AllocateBlockResponse.newBuilder().setKeyLocation(blockLocation).build());
        omClientResponse = new OMAllocateBlockResponse(omResponse.build(), openKeyInfo, clientID, omBucketInfo.copyObject(), getBucketLayout());
        LOG.debug("Allocated block for Volume:{}, Bucket:{}, OpenKey:{}", volumeName, bucketName, openKeyName);
    } catch (IOException ex) {
        omMetrics.incNumBlockAllocateCallFails();
        exception = ex;
        omClientResponse = new OMAllocateBlockResponse(createErrorOMResponse(omResponse, exception), getBucketLayout());
        LOG.error("Allocate Block failed. Volume:{}, Bucket:{}, OpenKey:{}. " + "Exception:{}", volumeName, bucketName, openKeyName, exception);
    } finally {
        addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, omDoubleBufferHelper);
        if (acquiredLock) {
            omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName, bucketName);
        }
    }
    auditLog(auditLogger, buildAuditMessage(OMAction.ALLOCATE_BLOCK, auditMap, exception, getOmRequest().getUserInfo()));
    return omClientResponse;
}

Example 4 with AllocateBlockRequest

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockRequest in project ozone by apache.

the class OMAllocateBlockRequest method preExecute.

@Override
public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
    AllocateBlockRequest allocateBlockRequest = getOmRequest().getAllocateBlockRequest();
    Preconditions.checkNotNull(allocateBlockRequest);
    KeyArgs keyArgs = allocateBlockRequest.getKeyArgs();
    String keyPath = keyArgs.getKeyName();
    keyPath = validateAndNormalizeKey(ozoneManager.getEnableFileSystemPaths(), keyPath, getBucketLayout());
    ExcludeList excludeList = new ExcludeList();
    if (allocateBlockRequest.hasExcludeList()) {
        excludeList = ExcludeList.getFromProtoBuf(allocateBlockRequest.getExcludeList());
    }
    // TODO: Here we are allocating block with out any check for key exist in
    // open table or not and also with out any authorization checks.
    // Assumption here is that allocateBlocks with out openKey will be less.
    // There is a chance some one can misuse this api to flood allocateBlock
    // calls. But currently allocateBlock is internally called from
    // BlockOutputStreamEntryPool, so we are fine for now. But if one some
    // one uses direct omclient we might be in trouble.
    // To allocate atleast one block passing requested size and scmBlockSize
    // as same value. When allocating block requested size is same as
    // scmBlockSize.
    List<OmKeyLocationInfo> omKeyLocationInfoList = allocateBlock(ozoneManager.getScmClient(), ozoneManager.getBlockTokenSecretManager(), keyArgs.getType(), keyArgs.getFactor(), excludeList, ozoneManager.getScmBlockSize(), ozoneManager.getScmBlockSize(), ozoneManager.getPreallocateBlocksMax(), ozoneManager.isGrpcBlockTokenEnabled(), ozoneManager.getOMNodeId());
    // Set modification time and normalize key if required.
    KeyArgs.Builder newKeyArgs = keyArgs.toBuilder().setModificationTime(Time.now()).setKeyName(keyPath);
    AllocateBlockRequest.Builder newAllocatedBlockRequest = AllocateBlockRequest.newBuilder().setClientID(allocateBlockRequest.getClientID()).setKeyArgs(newKeyArgs);
    if (allocateBlockRequest.hasExcludeList()) {
        newAllocatedBlockRequest.setExcludeList(allocateBlockRequest.getExcludeList());
    }
    // Add allocated block info.
    newAllocatedBlockRequest.setKeyLocation(omKeyLocationInfoList.get(0).getProtobuf(getOmRequest().getVersion()));
    return getOmRequest().toBuilder().setUserInfo(getUserInfo()).setAllocateBlockRequest(newAllocatedBlockRequest).build();
}

Example 5 with AllocateBlockRequest

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockRequest in project ozone by apache.

the class OMAllocateBlockRequestWithFSO method validateAndUpdateCache.

@Override
public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
    AllocateBlockRequest allocateBlockRequest = getOmRequest().getAllocateBlockRequest();
    KeyArgs keyArgs = allocateBlockRequest.getKeyArgs();
    OzoneManagerProtocolProtos.KeyLocation blockLocation = allocateBlockRequest.getKeyLocation();
    Preconditions.checkNotNull(blockLocation);
    String volumeName = keyArgs.getVolumeName();
    String bucketName = keyArgs.getBucketName();
    String keyName = keyArgs.getKeyName();
    long clientID = allocateBlockRequest.getClientID();
    OMMetrics omMetrics = ozoneManager.getMetrics();
    omMetrics.incNumBlockAllocateCalls();
    AuditLogger auditLogger = ozoneManager.getAuditLogger();
    Map<String, String> auditMap = buildKeyArgsAuditMap(keyArgs);
    auditMap.put(OzoneConsts.CLIENT_ID, String.valueOf(clientID));
    OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
    String openKeyName = null;
    OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest());
    OMClientResponse omClientResponse = null;
    OmKeyInfo openKeyInfo = null;
    IOException exception = null;
    OmBucketInfo omBucketInfo = null;
    boolean acquiredLock = false;
    try {
        keyArgs = resolveBucketLink(ozoneManager, keyArgs, auditMap);
        volumeName = keyArgs.getVolumeName();
        bucketName = keyArgs.getBucketName();
        // check Acl
        checkKeyAclsInOpenKeyTable(ozoneManager, volumeName, bucketName, keyName, IAccessAuthorizer.ACLType.WRITE, allocateBlockRequest.getClientID());
        validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
        // Here we don't acquire bucket/volume lock because for a single client
        // allocateBlock is called in serial fashion. With this approach, it
        // won't make 'fail-fast' during race condition case on delete/rename op,
        // assuming that later it will fail at the key commit operation.
        openKeyName = getOpenKeyName(volumeName, bucketName, keyName, clientID, ozoneManager);
        openKeyInfo = getOpenKeyInfo(omMetadataManager, openKeyName, keyName);
        if (openKeyInfo == null) {
            throw new OMException("Open Key not found " + openKeyName, KEY_NOT_FOUND);
        }
        List<OmKeyLocationInfo> newLocationList = Collections.singletonList(OmKeyLocationInfo.getFromProtobuf(blockLocation));
        acquiredLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK, volumeName, bucketName);
        omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName);
        // check bucket and volume quota
        long preAllocatedSpace = newLocationList.size() * ozoneManager.getScmBlockSize() * openKeyInfo.getReplicationConfig().getRequiredNodes();
        checkBucketQuotaInBytes(omBucketInfo, preAllocatedSpace);
        // Append new block
        openKeyInfo.appendNewBlocks(newLocationList, false);
        // Set modification time.
        openKeyInfo.setModificationTime(keyArgs.getModificationTime());
        // Set the UpdateID to current transactionLogIndex
        openKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled());
        // Add to cache.
        addOpenTableCacheEntry(trxnLogIndex, omMetadataManager, openKeyName, openKeyInfo);
        omBucketInfo.incrUsedBytes(preAllocatedSpace);
        omResponse.setAllocateBlockResponse(AllocateBlockResponse.newBuilder().setKeyLocation(blockLocation).build());
        omClientResponse = getOmClientResponse(clientID, omResponse, openKeyInfo, omBucketInfo.copyObject());
        LOG.debug("Allocated block for Volume:{}, Bucket:{}, OpenKey:{}", volumeName, bucketName, openKeyName);
    } catch (IOException ex) {
        omMetrics.incNumBlockAllocateCallFails();
        exception = ex;
        omClientResponse = new OMAllocateBlockResponseWithFSO(createErrorOMResponse(omResponse, exception), getBucketLayout());
        LOG.error("Allocate Block failed. Volume:{}, Bucket:{}, OpenKey:{}. " + "Exception:{}", volumeName, bucketName, openKeyName, exception);
    } finally {
        addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, omDoubleBufferHelper);
        if (acquiredLock) {
            omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName, bucketName);
        }
    }
    auditLog(auditLogger, buildAuditMessage(OMAction.ALLOCATE_BLOCK, auditMap, exception, getOmRequest().getUserInfo()));
    return omClientResponse;
}