Search in sources :

Example 1 with KeyValue

use of org.apache.hadoop.hdds.utils.db.Table.KeyValue in project ozone by apache.

the class TestTypedRDBTableStore method forEachAndIterator.

@Test
public void forEachAndIterator() throws Exception {
    final int iterCount = 100;
    try (Table<String, String> testTable = createTypedTable("Sixth")) {
        for (int x = 0; x < iterCount; x++) {
            String key = RandomStringUtils.random(10);
            String value = RandomStringUtils.random(10);
            testTable.put(key, value);
        }
        int localCount = 0;
        try (TableIterator<String, ? extends KeyValue<String, String>> iter = testTable.iterator()) {
            while (iter.hasNext()) {
                Table.KeyValue keyValue = iter.next();
                localCount++;
            }
            Assert.assertEquals(iterCount, localCount);
            iter.seekToFirst();
            iter.forEachRemaining(TestTypedRDBTableStore::consume);
            Assert.assertEquals(iterCount, count);
        }
    }
}
Also used : KeyValue(org.apache.hadoop.hdds.utils.db.Table.KeyValue) Test(org.junit.Test)

Example 2 with KeyValue

use of org.apache.hadoop.hdds.utils.db.Table.KeyValue in project ozone by apache.

the class OmMetadataManagerImpl method getPendingDeletionKeys.

@Override
public List<BlockGroup> getPendingDeletionKeys(final int keyCount) throws IOException {
    List<BlockGroup> keyBlocksList = Lists.newArrayList();
    try (TableIterator<String, ? extends KeyValue<String, RepeatedOmKeyInfo>> keyIter = getDeletedTable().iterator()) {
        int currentCount = 0;
        while (keyIter.hasNext() && currentCount < keyCount) {
            KeyValue<String, RepeatedOmKeyInfo> kv = keyIter.next();
            if (kv != null) {
                RepeatedOmKeyInfo infoList = kv.getValue();
                // Get block keys as a list.
                for (OmKeyInfo info : infoList.getOmKeyInfoList()) {
                    OmKeyLocationInfoGroup latest = info.getLatestVersionLocations();
                    List<BlockID> item = latest.getLocationList().stream().map(b -> new BlockID(b.getContainerID(), b.getLocalID())).collect(Collectors.toList());
                    BlockGroup keyBlocks = BlockGroup.newBuilder().setKeyName(kv.getKey()).addAllBlockIDs(item).build();
                    keyBlocksList.add(keyBlocks);
                    currentCount++;
                }
            }
        }
    }
    return keyBlocksList;
}
Also used : TypedTable(org.apache.hadoop.hdds.utils.db.TypedTable) OmVolumeArgsCodec(org.apache.hadoop.ozone.om.codec.OmVolumeArgsCodec) LoggerFactory(org.slf4j.LoggerFactory) OzoneFSUtils(org.apache.hadoop.ozone.om.helpers.OzoneFSUtils) OmDirectoryInfoCodec(org.apache.hadoop.ozone.om.codec.OmDirectoryInfoCodec) DBStore(org.apache.hadoop.hdds.utils.db.DBStore) OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT(org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT) StringUtils(org.apache.commons.lang3.StringUtils) RepeatedOmKeyInfoCodec(org.apache.hadoop.ozone.om.codec.RepeatedOmKeyInfoCodec) OM_KEY_PREFIX(org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX) BucketLayout(org.apache.hadoop.ozone.om.helpers.BucketLayout) Duration(java.time.Duration) Map(java.util.Map) CacheValue(org.apache.hadoop.hdds.utils.db.cache.CacheValue) OzoneManagerLock(org.apache.hadoop.ozone.om.lock.OzoneManagerLock) OzoneTokenIdentifier(org.apache.hadoop.ozone.security.OzoneTokenIdentifier) OmMultipartUpload(org.apache.hadoop.ozone.om.helpers.OmMultipartUpload) DBStoreBuilder(org.apache.hadoop.hdds.utils.db.DBStoreBuilder) OmBucketInfo(org.apache.hadoop.ozone.om.helpers.OmBucketInfo) TokenIdentifierCodec(org.apache.hadoop.ozone.om.codec.TokenIdentifierCodec) Set(java.util.Set) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) List(java.util.List) S3SecretValueCodec(org.apache.hadoop.ozone.om.codec.S3SecretValueCodec) OMException(org.apache.hadoop.ozone.om.exceptions.OMException) RocksDBConfiguration(org.apache.hadoop.hdds.utils.db.RocksDBConfiguration) OM_DB_NAME(org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME) KeyValue(org.apache.hadoop.hdds.utils.db.Table.KeyValue) TransactionInfo(org.apache.hadoop.hdds.utils.TransactionInfo) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) OmPrefixInfo(org.apache.hadoop.ozone.om.helpers.OmPrefixInfo) BlockID(org.apache.hadoop.hdds.client.BlockID) HashMap(java.util.HashMap) OmUtils(org.apache.hadoop.ozone.OmUtils) TreeSet(java.util.TreeSet) ResultCodes(org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes) StringUtil(org.eclipse.jetty.util.StringUtil) ArrayList(java.util.ArrayList) Strings(com.google.common.base.Strings) UserVolumeInfoCodec(org.apache.hadoop.ozone.om.codec.UserVolumeInfoCodec) Lists(com.google.common.collect.Lists) OmKeyInfo(org.apache.hadoop.ozone.om.helpers.OmKeyInfo) S3SecretValue(org.apache.hadoop.ozone.om.helpers.S3SecretValue) DB_TRANSIENT_MARKER(org.apache.hadoop.ozone.OzoneConsts.DB_TRANSIENT_MARKER) OmDirectoryInfo(org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) CacheType(org.apache.hadoop.hdds.utils.db.cache.TableCache.CacheType) BlockGroup(org.apache.hadoop.ozone.common.BlockGroup) IOException(java.io.IOException) OzoneConsts(org.apache.hadoop.ozone.OzoneConsts) OmKeyInfoCodec(org.apache.hadoop.ozone.om.codec.OmKeyInfoCodec) File(java.io.File) OmPrefixInfoCodec(org.apache.hadoop.ozone.om.codec.OmPrefixInfoCodec) OmVolumeArgs(org.apache.hadoop.ozone.om.helpers.OmVolumeArgs) TransactionInfoCodec(org.apache.hadoop.hdds.utils.TransactionInfoCodec) ChronoUnit(java.time.temporal.ChronoUnit) TreeMap(java.util.TreeMap) RepeatedOmKeyInfo(org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo) Paths(java.nio.file.Paths) OmBucketInfoCodec(org.apache.hadoop.ozone.om.codec.OmBucketInfoCodec) Table(org.apache.hadoop.hdds.utils.db.Table) CacheKey(org.apache.hadoop.hdds.utils.db.cache.CacheKey) OmKeyLocationInfoGroup(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup) ExitUtils(org.apache.ratis.util.ExitUtils) OmMultipartKeyInfoCodec(org.apache.hadoop.ozone.om.codec.OmMultipartKeyInfoCodec) OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS(org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS) PersistedUserVolumeInfo(org.apache.hadoop.ozone.storage.proto.OzoneManagerStorageProtos.PersistedUserVolumeInfo) VisibleForTesting(com.google.common.annotations.VisibleForTesting) TableIterator(org.apache.hadoop.hdds.utils.db.TableIterator) OmMultipartKeyInfo(org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo) OmKeyLocationInfoGroup(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup) OmKeyInfo(org.apache.hadoop.ozone.om.helpers.OmKeyInfo) RepeatedOmKeyInfo(org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo) BlockID(org.apache.hadoop.hdds.client.BlockID) BlockGroup(org.apache.hadoop.ozone.common.BlockGroup) RepeatedOmKeyInfo(org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo)

Aggregations

KeyValue (org.apache.hadoop.hdds.utils.db.Table.KeyValue)2 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Strings (com.google.common.base.Strings)1 Lists (com.google.common.collect.Lists)1 File (java.io.File)1 IOException (java.io.IOException)1 Paths (java.nio.file.Paths)1 Duration (java.time.Duration)1 Instant (java.time.Instant)1 ChronoUnit (java.time.temporal.ChronoUnit)1 ArrayList (java.util.ArrayList)1 HashMap (java.util.HashMap)1 Iterator (java.util.Iterator)1 List (java.util.List)1 Map (java.util.Map)1 Set (java.util.Set)1 TreeMap (java.util.TreeMap)1 TreeSet (java.util.TreeSet)1 Collectors (java.util.stream.Collectors)1 StringUtils (org.apache.commons.lang3.StringUtils)1