Search in sources :

Example 16 with OmKeyArgs

use of org.apache.hadoop.ozone.om.helpers.OmKeyArgs in project ozone by apache.

the class TestStorageContainerManagerHelper method createKeys.

public Map<String, OmKeyInfo> createKeys(int numOfKeys, int keySize) throws Exception {
    Map<String, OmKeyInfo> keyLocationMap = Maps.newHashMap();
    OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
    // Write 20 keys in bucketName.
    Set<String> keyNames = Sets.newHashSet();
    for (int i = 0; i < numOfKeys; i++) {
        String keyName = RandomStringUtils.randomAlphabetic(5) + i;
        keyNames.add(keyName);
        TestDataUtil.createKey(bucket, keyName, RandomStringUtils.randomAlphabetic(5));
    }
    for (String key : keyNames) {
        OmKeyArgs arg = new OmKeyArgs.Builder().setVolumeName(bucket.getVolumeName()).setBucketName(bucket.getName()).setKeyName(key).setRefreshPipeline(true).build();
        OmKeyInfo location = cluster.getOzoneManager().lookupKey(arg);
        keyLocationMap.put(key, location);
    }
    return keyLocationMap;
}
Also used : OzoneBucket(org.apache.hadoop.ozone.client.OzoneBucket) OmKeyInfo(org.apache.hadoop.ozone.om.helpers.OmKeyInfo) OmKeyArgs(org.apache.hadoop.ozone.om.helpers.OmKeyArgs)

Example 17 with OmKeyArgs

use of org.apache.hadoop.ozone.om.helpers.OmKeyArgs in project ozone by apache.

the class RpcClient method completeMultipartUpload.

@Override
public OmMultipartUploadCompleteInfo completeMultipartUpload(String volumeName, String bucketName, String keyName, String uploadID, Map<Integer, String> partsMap) throws IOException {
    verifyVolumeName(volumeName);
    verifyBucketName(bucketName);
    HddsClientUtils.checkNotNull(keyName, uploadID);
    OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setMultipartUploadID(uploadID).setAcls(getAclList()).build();
    OmMultipartUploadCompleteList omMultipartUploadCompleteList = new OmMultipartUploadCompleteList(partsMap);
    OmMultipartUploadCompleteInfo omMultipartUploadCompleteInfo = ozoneManagerClient.completeMultipartUpload(keyArgs, omMultipartUploadCompleteList);
    return omMultipartUploadCompleteInfo;
}
Also used : OmMultipartUploadCompleteInfo(org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo) OmMultipartUploadCompleteList(org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteList) CacheBuilder(com.google.common.cache.CacheBuilder) ThreadFactoryBuilder(com.google.common.util.concurrent.ThreadFactoryBuilder) OmKeyArgs(org.apache.hadoop.ozone.om.helpers.OmKeyArgs)

Example 18 with OmKeyArgs

use of org.apache.hadoop.ozone.om.helpers.OmKeyArgs in project ozone by apache.

the class RpcClient method createMultipartKey.

@Override
public OzoneOutputStream createMultipartKey(String volumeName, String bucketName, String keyName, long size, int partNumber, String uploadID) throws IOException {
    verifyVolumeName(volumeName);
    verifyBucketName(bucketName);
    if (checkKeyNameEnabled) {
        HddsClientUtils.verifyKeyName(keyName);
    }
    HddsClientUtils.checkNotNull(keyName, uploadID);
    Preconditions.checkArgument(partNumber > 0 && partNumber <= 10000, "Part " + "number should be greater than zero and less than or equal to 10000");
    Preconditions.checkArgument(size >= 0, "size should be greater than or " + "equal to zero");
    String requestId = UUID.randomUUID().toString();
    OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setDataSize(size).setIsMultipartKey(true).setMultipartUploadID(uploadID).setMultipartUploadPartNumber(partNumber).setAcls(getAclList()).build();
    OpenKeySession openKey = ozoneManagerClient.openKey(keyArgs);
    KeyOutputStream keyOutputStream = createKeyOutputStream(openKey, requestId).setMultipartNumber(partNumber).setMultipartUploadID(uploadID).setIsMultipartKey(true).build();
    keyOutputStream.addPreallocateBlocks(openKey.getKeyInfo().getLatestVersionLocations(), openKey.getOpenVersion());
    FileEncryptionInfo feInfo = openKey.getKeyInfo().getFileEncryptionInfo();
    if (feInfo != null) {
        KeyProvider.KeyVersion decrypted = getDEK(feInfo);
        final CryptoOutputStream cryptoOut = new CryptoOutputStream(keyOutputStream, OzoneKMSUtil.getCryptoCodec(conf, feInfo), decrypted.getMaterial(), feInfo.getIV());
        return new OzoneOutputStream(cryptoOut);
    } else {
        return new OzoneOutputStream(keyOutputStream);
    }
}
Also used : KeyProvider(org.apache.hadoop.crypto.key.KeyProvider) CryptoOutputStream(org.apache.hadoop.crypto.CryptoOutputStream) OzoneOutputStream(org.apache.hadoop.ozone.client.io.OzoneOutputStream) OpenKeySession(org.apache.hadoop.ozone.om.helpers.OpenKeySession) ECKeyOutputStream(org.apache.hadoop.ozone.client.io.ECKeyOutputStream) KeyOutputStream(org.apache.hadoop.ozone.client.io.KeyOutputStream) FileEncryptionInfo(org.apache.hadoop.fs.FileEncryptionInfo) OmKeyArgs(org.apache.hadoop.ozone.om.helpers.OmKeyArgs)

Example 19 with OmKeyArgs

use of org.apache.hadoop.ozone.om.helpers.OmKeyArgs in project ozone by apache.

the class RpcClient method abortMultipartUpload.

@Override
public void abortMultipartUpload(String volumeName, String bucketName, String keyName, String uploadID) throws IOException {
    verifyVolumeName(volumeName);
    verifyBucketName(bucketName);
    HddsClientUtils.checkNotNull(keyName, uploadID);
    OmKeyArgs omKeyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setMultipartUploadID(uploadID).build();
    ozoneManagerClient.abortMultipartUpload(omKeyArgs);
}
Also used : OmKeyArgs(org.apache.hadoop.ozone.om.helpers.OmKeyArgs)

Example 20 with OmKeyArgs

use of org.apache.hadoop.ozone.om.helpers.OmKeyArgs in project ozone by apache.

the class RpcClient method getKey.

@Override
public OzoneInputStream getKey(String volumeName, String bucketName, String keyName) throws IOException {
    verifyVolumeName(volumeName);
    verifyBucketName(bucketName);
    Preconditions.checkNotNull(keyName);
    OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setRefreshPipeline(true).setSortDatanodesInPipeline(topologyAwareReadEnabled).setLatestVersionLocation(getLatestVersionLocation).build();
    OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs);
    return getInputStreamWithRetryFunction(keyInfo);
}
Also used : OmKeyInfo(org.apache.hadoop.ozone.om.helpers.OmKeyInfo) RepeatedOmKeyInfo(org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo) OmKeyArgs(org.apache.hadoop.ozone.om.helpers.OmKeyArgs)

Aggregations

OmKeyArgs (org.apache.hadoop.ozone.om.helpers.OmKeyArgs)112 OmKeyInfo (org.apache.hadoop.ozone.om.helpers.OmKeyInfo)46 Test (org.junit.Test)36 OmKeyLocationInfo (org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo)35 OzoneOutputStream (org.apache.hadoop.ozone.client.io.OzoneOutputStream)28 OpenKeySession (org.apache.hadoop.ozone.om.helpers.OpenKeySession)24 ContainerInfo (org.apache.hadoop.hdds.scm.container.ContainerInfo)21 KeyArgs (org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs)19 ArrayList (java.util.ArrayList)18 DatanodeDetails (org.apache.hadoop.hdds.protocol.DatanodeDetails)18 Pipeline (org.apache.hadoop.hdds.scm.pipeline.Pipeline)18 OzoneFileStatus (org.apache.hadoop.ozone.om.helpers.OzoneFileStatus)18 KeyOutputStream (org.apache.hadoop.ozone.client.io.KeyOutputStream)17 DeleteKeyArgs (org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyArgs)15 OMRequest (org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest)15 RepeatedOmKeyInfo (org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo)14 IOException (java.io.IOException)12 Test (org.junit.jupiter.api.Test)12 OMException (org.apache.hadoop.ozone.om.exceptions.OMException)10 BlockID (org.apache.hadoop.hdds.client.BlockID)9