use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateFileRequest in project ozone by apache.
the class TestOMFileCreateRequest method createFileRequest.
/**
* Create OMRequest which encapsulates OMFileCreateRequest.
* @param volumeName
* @param bucketName
* @param keyName
* @param replicationFactor
* @param replicationType
* @return OMRequest
*/
@NotNull
protected OMRequest createFileRequest(String volumeName, String bucketName, String keyName, HddsProtos.ReplicationFactor replicationFactor, HddsProtos.ReplicationType replicationType, boolean overWrite, boolean recursive) {
KeyArgs.Builder keyArgs = KeyArgs.newBuilder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setFactor(replicationFactor).setType(replicationType).setDataSize(dataSize);
CreateFileRequest createFileRequest = CreateFileRequest.newBuilder().setKeyArgs(keyArgs).setIsOverwrite(overWrite).setIsRecursive(recursive).build();
return OMRequest.newBuilder().setCmdType(OzoneManagerProtocolProtos.Type.CreateKey).setClientId(UUID.randomUUID().toString()).setCreateFileRequest(createFileRequest).build();
}
use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateFileRequest in project ozone by apache.
the class OzoneManagerProtocolClientSideTranslatorPB method createFile.
@Override
public OpenKeySession createFile(OmKeyArgs args, boolean overWrite, boolean recursive) throws IOException {
KeyArgs.Builder keyArgsBuilder = KeyArgs.newBuilder().setVolumeName(args.getVolumeName()).setBucketName(args.getBucketName()).setKeyName(args.getKeyName()).setDataSize(args.getDataSize()).addAllAcls(args.getAcls().stream().map(a -> OzoneAcl.toProtobuf(a)).collect(Collectors.toList()));
if (args.getReplicationConfig() != null) {
keyArgsBuilder.setFactor(ReplicationConfig.getLegacyFactor(args.getReplicationConfig()));
keyArgsBuilder.setType(args.getReplicationConfig().getReplicationType());
}
CreateFileRequest createFileRequest = CreateFileRequest.newBuilder().setKeyArgs(keyArgsBuilder.build()).setIsOverwrite(overWrite).setIsRecursive(recursive).build();
OMRequest omRequest = createOMRequest(Type.CreateFile).setCreateFileRequest(createFileRequest).build();
CreateFileResponse resp = handleError(submitRequest(omRequest)).getCreateFileResponse();
return new OpenKeySession(resp.getID(), OmKeyInfo.getFromProtobuf(resp.getKeyInfo()), resp.getOpenVersion());
}
use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateFileRequest in project ozone by apache.
the class OMFileCreateRequestWithFSO method validateAndUpdateCache.
@Override
@SuppressWarnings("methodlength")
public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
CreateFileRequest createFileRequest = getOmRequest().getCreateFileRequest();
KeyArgs keyArgs = createFileRequest.getKeyArgs();
Map<String, String> auditMap = buildKeyArgsAuditMap(keyArgs);
String volumeName = keyArgs.getVolumeName();
String bucketName = keyArgs.getBucketName();
String keyName = keyArgs.getKeyName();
// if isRecursive is true, file would be created even if parent
// directories does not exist.
boolean isRecursive = createFileRequest.getIsRecursive();
if (LOG.isDebugEnabled()) {
LOG.debug("File create for : " + volumeName + "/" + bucketName + "/" + keyName + ":" + isRecursive);
}
// if isOverWrite is true, file would be over written.
boolean isOverWrite = createFileRequest.getIsOverwrite();
OMMetrics omMetrics = ozoneManager.getMetrics();
omMetrics.incNumCreateFile();
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
boolean acquiredLock = false;
OmBucketInfo omBucketInfo = null;
final List<OmKeyLocationInfo> locations = new ArrayList<>();
List<OmDirectoryInfo> missingParentInfos;
int numKeysCreated = 0;
OMClientResponse omClientResponse = null;
OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest());
IOException exception = null;
Result result = null;
try {
keyArgs = resolveBucketLink(ozoneManager, keyArgs, auditMap);
volumeName = keyArgs.getVolumeName();
bucketName = keyArgs.getBucketName();
if (keyName.length() == 0) {
// Check if this is the root of the filesystem.
throw new OMException("Can not write to directory: " + keyName, OMException.ResultCodes.NOT_A_FILE);
}
// check Acl
checkKeyAcls(ozoneManager, volumeName, bucketName, keyName, IAccessAuthorizer.ACLType.CREATE, OzoneObj.ResourceType.KEY);
// acquire lock
acquiredLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK, volumeName, bucketName);
validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
OmKeyInfo dbFileInfo = null;
OMFileRequest.OMPathInfoWithFSO pathInfoFSO = OMFileRequest.verifyDirectoryKeysInPath(omMetadataManager, volumeName, bucketName, keyName, Paths.get(keyName));
if (pathInfoFSO.getDirectoryResult() == OMFileRequest.OMDirectoryResult.FILE_EXISTS) {
String dbFileKey = omMetadataManager.getOzonePathKey(pathInfoFSO.getLastKnownParentId(), pathInfoFSO.getLeafNodeName());
dbFileInfo = OMFileRequest.getOmKeyInfoFromFileTable(false, omMetadataManager, dbFileKey, keyName);
}
// check if the file or directory already existed in OM
checkDirectoryResult(keyName, isOverWrite, pathInfoFSO.getDirectoryResult());
if (!isRecursive) {
checkAllParentsExist(keyArgs, pathInfoFSO);
}
// add all missing parents to dir table
missingParentInfos = OMDirectoryCreateRequestWithFSO.getAllMissingParentDirInfo(ozoneManager, keyArgs, pathInfoFSO, trxnLogIndex);
// total number of keys created.
numKeysCreated = missingParentInfos.size();
// do open key
OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().get(omMetadataManager.getBucketKey(volumeName, bucketName));
OmKeyInfo omFileInfo = prepareFileInfo(omMetadataManager, keyArgs, dbFileInfo, keyArgs.getDataSize(), locations, getFileEncryptionInfo(keyArgs), ozoneManager.getPrefixManager(), bucketInfo, pathInfoFSO, trxnLogIndex, pathInfoFSO.getLeafNodeObjectId(), ozoneManager.isRatisEnabled());
long openVersion = omFileInfo.getLatestVersionLocations().getVersion();
long clientID = createFileRequest.getClientID();
String dbOpenFileName = omMetadataManager.getOpenFileName(pathInfoFSO.getLastKnownParentId(), pathInfoFSO.getLeafNodeName(), clientID);
// Append new blocks
List<OmKeyLocationInfo> newLocationList = keyArgs.getKeyLocationsList().stream().map(OmKeyLocationInfo::getFromProtobuf).collect(Collectors.toList());
omFileInfo.appendNewBlocks(newLocationList, false);
omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName);
// check bucket and volume quota
long preAllocatedSpace = newLocationList.size() * ozoneManager.getScmBlockSize() * omFileInfo.getReplicationConfig().getRequiredNodes();
checkBucketQuotaInBytes(omBucketInfo, preAllocatedSpace);
checkBucketQuotaInNamespace(omBucketInfo, 1L);
// Add to cache entry can be done outside of lock for this openKey.
// Even if bucket gets deleted, when commitKey we shall identify if
// bucket gets deleted.
OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager, dbOpenFileName, omFileInfo, pathInfoFSO.getLeafNodeName(), trxnLogIndex);
// Add cache entries for the prefix directories.
// Skip adding for the file key itself, until Key Commit.
OMFileRequest.addDirectoryTableCacheEntries(omMetadataManager, Optional.absent(), Optional.of(missingParentInfos), trxnLogIndex);
omBucketInfo.incrUsedBytes(preAllocatedSpace);
// Update namespace quota
omBucketInfo.incrUsedNamespace(1L);
// Prepare response. Sets user given full key name in the 'keyName'
// attribute in response object.
int clientVersion = getOmRequest().getVersion();
omResponse.setCreateFileResponse(CreateFileResponse.newBuilder().setKeyInfo(omFileInfo.getNetworkProtobuf(keyName, clientVersion, keyArgs.getLatestVersionLocation())).setID(clientID).setOpenVersion(openVersion).build()).setCmdType(Type.CreateFile);
omClientResponse = new OMFileCreateResponseWithFSO(omResponse.build(), omFileInfo, missingParentInfos, clientID, omBucketInfo.copyObject());
result = Result.SUCCESS;
} catch (IOException ex) {
result = Result.FAILURE;
exception = ex;
omMetrics.incNumCreateFileFails();
omResponse.setCmdType(Type.CreateFile);
omClientResponse = new OMFileCreateResponseWithFSO(createErrorOMResponse(omResponse, exception), getBucketLayout());
} finally {
addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, omDoubleBufferHelper);
if (acquiredLock) {
omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName, bucketName);
}
}
// Audit Log outside the lock
auditLog(ozoneManager.getAuditLogger(), buildAuditMessage(OMAction.CREATE_FILE, auditMap, exception, getOmRequest().getUserInfo()));
switch(result) {
case SUCCESS:
omMetrics.incNumKeys(numKeysCreated);
LOG.debug("File created. Volume:{}, Bucket:{}, Key:{}", volumeName, bucketName, keyName);
break;
case FAILURE:
LOG.error("File create failed. Volume:{}, Bucket:{}, Key:{}.", volumeName, bucketName, keyName, exception);
break;
default:
LOG.error("Unrecognized Result for OMFileCreateRequest: {}", createFileRequest);
}
return omClientResponse;
}
use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateFileRequest in project ozone by apache.
the class OMFileCreateRequest method validateAndUpdateCache.
@Override
@SuppressWarnings("methodlength")
public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
CreateFileRequest createFileRequest = getOmRequest().getCreateFileRequest();
KeyArgs keyArgs = createFileRequest.getKeyArgs();
Map<String, String> auditMap = buildKeyArgsAuditMap(keyArgs);
String volumeName = keyArgs.getVolumeName();
String bucketName = keyArgs.getBucketName();
String keyName = keyArgs.getKeyName();
int numMissingParents = 0;
// if isRecursive is true, file would be created even if parent
// directories does not exist.
boolean isRecursive = createFileRequest.getIsRecursive();
if (LOG.isDebugEnabled()) {
LOG.debug("File create for : " + volumeName + "/" + bucketName + "/" + keyName + ":" + isRecursive);
}
// if isOverWrite is true, file would be over written.
boolean isOverWrite = createFileRequest.getIsOverwrite();
OMMetrics omMetrics = ozoneManager.getMetrics();
omMetrics.incNumCreateFile();
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
boolean acquiredLock = false;
OmKeyInfo omKeyInfo = null;
OmBucketInfo omBucketInfo = null;
final List<OmKeyLocationInfo> locations = new ArrayList<>();
List<OmKeyInfo> missingParentInfos;
OMClientResponse omClientResponse = null;
OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest());
IOException exception = null;
Result result = null;
try {
keyArgs = resolveBucketLink(ozoneManager, keyArgs, auditMap);
volumeName = keyArgs.getVolumeName();
bucketName = keyArgs.getBucketName();
// check Acl
checkKeyAcls(ozoneManager, volumeName, bucketName, keyName, IAccessAuthorizer.ACLType.CREATE, OzoneObj.ResourceType.KEY);
// acquire lock
acquiredLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK, volumeName, bucketName);
validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
if (keyName.length() == 0) {
// Check if this is the root of the filesystem.
throw new OMException("Can not write to directory: " + keyName, OMException.ResultCodes.NOT_A_FILE);
}
String ozoneKey = omMetadataManager.getOzoneKey(volumeName, bucketName, keyName);
OmKeyInfo dbKeyInfo = omMetadataManager.getKeyTable(getBucketLayout()).getIfExist(ozoneKey);
OMFileRequest.OMPathInfo pathInfo = OMFileRequest.verifyFilesInPath(omMetadataManager, volumeName, bucketName, keyName, Paths.get(keyName));
OMFileRequest.OMDirectoryResult omDirectoryResult = pathInfo.getDirectoryResult();
List<OzoneAcl> inheritAcls = pathInfo.getAcls();
// Check if a file or directory exists with same key name.
checkDirectoryResult(keyName, isOverWrite, omDirectoryResult);
if (!isRecursive) {
checkAllParentsExist(keyArgs, pathInfo);
}
// do open key
OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().get(omMetadataManager.getBucketKey(volumeName, bucketName));
omKeyInfo = prepareKeyInfo(omMetadataManager, keyArgs, dbKeyInfo, keyArgs.getDataSize(), locations, getFileEncryptionInfo(keyArgs), ozoneManager.getPrefixManager(), bucketInfo, trxnLogIndex, ozoneManager.getObjectIdFromTxId(trxnLogIndex), ozoneManager.isRatisEnabled());
long openVersion = omKeyInfo.getLatestVersionLocations().getVersion();
long clientID = createFileRequest.getClientID();
String dbOpenKeyName = omMetadataManager.getOpenKey(volumeName, bucketName, keyName, clientID);
missingParentInfos = OMDirectoryCreateRequest.getAllParentInfo(ozoneManager, keyArgs, pathInfo.getMissingParents(), inheritAcls, trxnLogIndex);
// Append new blocks
List<OmKeyLocationInfo> newLocationList = keyArgs.getKeyLocationsList().stream().map(OmKeyLocationInfo::getFromProtobuf).collect(Collectors.toList());
omKeyInfo.appendNewBlocks(newLocationList, false);
omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName);
// check bucket and volume quota
long preAllocatedSpace = newLocationList.size() * ozoneManager.getScmBlockSize() * omKeyInfo.getReplicationConfig().getRequiredNodes();
checkBucketQuotaInBytes(omBucketInfo, preAllocatedSpace);
checkBucketQuotaInNamespace(omBucketInfo, 1L);
// Add to cache entry can be done outside of lock for this openKey.
// Even if bucket gets deleted, when commitKey we shall identify if
// bucket gets deleted.
omMetadataManager.getOpenKeyTable(getBucketLayout()).addCacheEntry(new CacheKey<>(dbOpenKeyName), new CacheValue<>(Optional.of(omKeyInfo), trxnLogIndex));
// Add cache entries for the prefix directories.
// Skip adding for the file key itself, until Key Commit.
OMFileRequest.addKeyTableCacheEntries(omMetadataManager, volumeName, bucketName, Optional.absent(), Optional.of(missingParentInfos), trxnLogIndex);
omBucketInfo.incrUsedBytes(preAllocatedSpace);
// Update namespace quota
omBucketInfo.incrUsedNamespace(1L);
numMissingParents = missingParentInfos.size();
// Prepare response
omResponse.setCreateFileResponse(CreateFileResponse.newBuilder().setKeyInfo(omKeyInfo.getNetworkProtobuf(getOmRequest().getVersion(), keyArgs.getLatestVersionLocation())).setID(clientID).setOpenVersion(openVersion).build()).setCmdType(CreateFile);
omClientResponse = new OMFileCreateResponse(omResponse.build(), omKeyInfo, missingParentInfos, clientID, omBucketInfo.copyObject());
result = Result.SUCCESS;
} catch (IOException ex) {
result = Result.FAILURE;
exception = ex;
omMetrics.incNumCreateFileFails();
omResponse.setCmdType(CreateFile);
omClientResponse = new OMFileCreateResponse(createErrorOMResponse(omResponse, exception), getBucketLayout());
} finally {
addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, omDoubleBufferHelper);
if (acquiredLock) {
omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName, bucketName);
}
}
// Audit Log outside the lock
auditLog(ozoneManager.getAuditLogger(), buildAuditMessage(OMAction.CREATE_FILE, auditMap, exception, getOmRequest().getUserInfo()));
switch(result) {
case SUCCESS:
// Missing directories are created immediately, counting that here.
// The metric for the file is incremented as part of the file commit.
omMetrics.incNumKeys(numMissingParents);
LOG.debug("File created. Volume:{}, Bucket:{}, Key:{}", volumeName, bucketName, keyName);
break;
case FAILURE:
LOG.error("File create failed. Volume:{}, Bucket:{}, Key{}.", volumeName, bucketName, keyName, exception);
break;
default:
LOG.error("Unrecognized Result for OMFileCreateRequest: {}", createFileRequest);
}
return omClientResponse;
}
use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateFileRequest in project ozone by apache.
the class OMFileCreateRequest method preExecute.
@Override
public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
CreateFileRequest createFileRequest = getOmRequest().getCreateFileRequest();
Preconditions.checkNotNull(createFileRequest);
KeyArgs keyArgs = createFileRequest.getKeyArgs();
// Verify key name
final boolean checkKeyNameEnabled = ozoneManager.getConfiguration().getBoolean(OMConfigKeys.OZONE_OM_KEYNAME_CHARACTER_CHECK_ENABLED_KEY, OMConfigKeys.OZONE_OM_KEYNAME_CHARACTER_CHECK_ENABLED_DEFAULT);
if (checkKeyNameEnabled) {
OmUtils.validateKeyName(StringUtils.removeEnd(keyArgs.getKeyName(), OzoneConsts.FS_FILE_COPYING_TEMP_SUFFIX));
}
if (keyArgs.getKeyName().length() == 0) {
// checking volume/bucket exists.
return getOmRequest().toBuilder().setUserInfo(getUserInfo()).build();
}
long scmBlockSize = ozoneManager.getScmBlockSize();
// NOTE size of a key is not a hard limit on anything, it is a value that
// client should expect, in terms of current size of key. If client sets
// a value, then this value is used, otherwise, we allocate a single
// block which is the current size, if read by the client.
final long requestedSize = keyArgs.getDataSize() > 0 ? keyArgs.getDataSize() : scmBlockSize;
boolean useRatis = ozoneManager.shouldUseRatis();
HddsProtos.ReplicationFactor factor = keyArgs.getFactor();
if (factor == null) {
factor = useRatis ? HddsProtos.ReplicationFactor.THREE : HddsProtos.ReplicationFactor.ONE;
}
HddsProtos.ReplicationType type = keyArgs.getType();
if (type == null) {
type = useRatis ? HddsProtos.ReplicationType.RATIS : HddsProtos.ReplicationType.STAND_ALONE;
}
// TODO: Here we are allocating block with out any check for
// bucket/key/volume or not and also with out any authorization checks.
List<OmKeyLocationInfo> omKeyLocationInfoList = allocateBlock(ozoneManager.getScmClient(), ozoneManager.getBlockTokenSecretManager(), type, factor, new ExcludeList(), requestedSize, scmBlockSize, ozoneManager.getPreallocateBlocksMax(), ozoneManager.isGrpcBlockTokenEnabled(), ozoneManager.getOMNodeId());
KeyArgs.Builder newKeyArgs = keyArgs.toBuilder().setModificationTime(Time.now()).setType(type).setFactor(factor).setDataSize(requestedSize);
newKeyArgs.addAllKeyLocations(omKeyLocationInfoList.stream().map(info -> info.getProtobuf(getOmRequest().getVersion())).collect(Collectors.toList()));
generateRequiredEncryptionInfo(keyArgs, newKeyArgs, ozoneManager);
CreateFileRequest.Builder newCreateFileRequest = createFileRequest.toBuilder().setKeyArgs(newKeyArgs).setClientID(UniqueId.next());
return getOmRequest().toBuilder().setCreateFileRequest(newCreateFileRequest).setUserInfo(getUserInfo()).build();
}
Aggregations