Examples with TableIterator org.apache.hadoop.hdds.utils.db.TableIterator used on opensource projects

Example 1 with TableIterator

use of org.apache.hadoop.hdds.utils.db.TableIterator in project ozone by apache.

the class TestReconWithOzoneManagerHA method testReconGetsSnapshotFromLeader.

@Test
public void testReconGetsSnapshotFromLeader() throws Exception {
    AtomicReference<OzoneManager> ozoneManager = new AtomicReference<>();
    // Wait for OM leader election to finish
    GenericTestUtils.waitFor(() -> {
        OzoneManager om = cluster.getOMLeader();
        ozoneManager.set(om);
        return om != null;
    }, 100, 120000);
    Assert.assertNotNull("Timed out waiting OM leader election to finish: " + "no leader or more than one leader.", ozoneManager);
    Assert.assertTrue("Should have gotten the leader!", ozoneManager.get().isLeaderReady());
    OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) cluster.getReconServer().getOzoneManagerServiceProvider();
    String hostname = ozoneManager.get().getHttpServer().getHttpAddress().getHostName();
    String expectedUrl = "http://" + (hostname.equals("0.0.0.0") ? "localhost" : hostname) + ":" + ozoneManager.get().getHttpServer().getHttpAddress().getPort() + OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
    String snapshotUrl = impl.getOzoneManagerSnapshotUrl();
    Assert.assertEquals("OM Snapshot should be requested from the leader.", expectedUrl, snapshotUrl);
    // Write some data
    String keyPrefix = "ratis";
    OzoneOutputStream key = objectStore.getVolume(VOL_NAME).getBucket(VOL_NAME).createKey(keyPrefix, 1024, ReplicationType.RATIS, ReplicationFactor.ONE, new HashMap<>());
    key.write(keyPrefix.getBytes(UTF_8));
    key.flush();
    key.close();
    // Sync data to Recon
    impl.syncDataFromOM();
    ReconContainerMetadataManager reconContainerMetadataManager = cluster.getReconServer().getReconContainerMetadataManager();
    TableIterator iterator = reconContainerMetadataManager.getContainerTableIterator();
    String reconKeyPrefix = null;
    while (iterator.hasNext()) {
        Table.KeyValue<ContainerKeyPrefix, Integer> keyValue = (Table.KeyValue<ContainerKeyPrefix, Integer>) iterator.next();
        reconKeyPrefix = keyValue.getKey().getKeyPrefix();
    }
    Assert.assertEquals("Container data should be synced to recon.", String.format("/%s/%s/%s", VOL_NAME, VOL_NAME, keyPrefix), reconKeyPrefix);
}

Example 2 with TableIterator

use of org.apache.hadoop.hdds.utils.db.TableIterator in project ozone by apache.

the class KeyManagerImpl method listStatus.

/**
 * List the status for a file or a directory and its contents.
 *
 * @param args       Key args
 * @param recursive  For a directory if true all the descendants of a
 *                   particular directory are listed
 * @param startKey   Key from which listing needs to start. If startKey exists
 *                   its status is included in the final list.
 * @param numEntries Number of entries to list from the start key
 * @param clientAddress a hint to key manager, order the datanode in returned
 *                      pipeline by distance between client and datanode.
 * @return list of file status
 */
@Override
@SuppressWarnings("methodlength")
public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive, String startKey, long numEntries, String clientAddress) throws IOException {
    Preconditions.checkNotNull(args, "Key args can not be null");
    String volName = args.getVolumeName();
    String buckName = args.getBucketName();
    List<OzoneFileStatus> fileStatusList = new ArrayList<>();
    if (numEntries <= 0) {
        return fileStatusList;
    }
    if (isBucketFSOptimized(volName, buckName)) {
        return listStatusFSO(args, recursive, startKey, numEntries, clientAddress);
    }
    String volumeName = args.getVolumeName();
    String bucketName = args.getBucketName();
    String keyName = args.getKeyName();
    // A map sorted by OmKey to combine results from TableCache and DB.
    TreeMap<String, OzoneFileStatus> cacheKeyMap = new TreeMap<>();
    if (Strings.isNullOrEmpty(startKey)) {
        OzoneFileStatus fileStatus = getFileStatus(args, clientAddress);
        if (fileStatus.isFile()) {
            return Collections.singletonList(fileStatus);
        }
        // keyName is a directory
        startKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName);
    }
    // Note: eliminating the case where startCacheKey could end with '//'
    String keyArgs = OzoneFSUtils.addTrailingSlashIfNeeded(metadataManager.getOzoneKey(volumeName, bucketName, keyName));
    metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName, bucketName);
    Table keyTable = metadataManager.getKeyTable(getBucketLayout(metadataManager, volName, buckName));
    TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> iterator;
    try {
        Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>> cacheIter = keyTable.cacheIterator();
        String startCacheKey = OZONE_URI_DELIMITER + volumeName + OZONE_URI_DELIMITER + bucketName + OZONE_URI_DELIMITER + ((startKey.equals(OZONE_URI_DELIMITER)) ? "" : startKey);
        // First, find key in TableCache
        listStatusFindKeyInTableCache(cacheIter, keyArgs, startCacheKey, recursive, cacheKeyMap);
        iterator = keyTable.iterator();
    } finally {
        metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, bucketName);
    }
    // Then, find key in DB
    String seekKeyInDb = metadataManager.getOzoneKey(volumeName, bucketName, startKey);
    Table.KeyValue<String, OmKeyInfo> entry = iterator.seek(seekKeyInDb);
    int countEntries = 0;
    if (iterator.hasNext()) {
        if (entry.getKey().equals(keyArgs)) {
            // Skip the key itself, since we are listing inside the directory
            iterator.next();
        }
        // Iterate through seek results
        while (iterator.hasNext() && numEntries - countEntries > 0) {
            entry = iterator.next();
            String entryInDb = entry.getKey();
            OmKeyInfo omKeyInfo = entry.getValue();
            if (entryInDb.startsWith(keyArgs)) {
                String entryKeyName = omKeyInfo.getKeyName();
                if (recursive) {
                    if (!isKeyDeleted(entryInDb, keyTable)) {
                        cacheKeyMap.put(entryInDb, new OzoneFileStatus(omKeyInfo, scmBlockSize, !OzoneFSUtils.isFile(entryKeyName)));
                        countEntries++;
                    }
                } else {
                    // get the child of the directory to list from the entry. For
                    // example if directory to list is /a and entry is /a/b/c where
                    // c is a file. The immediate child is b which is a directory. c
                    // should not be listed as child of a.
                    String immediateChild = OzoneFSUtils.getImmediateChild(entryKeyName, keyName);
                    boolean isFile = OzoneFSUtils.isFile(immediateChild);
                    if (isFile) {
                        if (!isKeyDeleted(entryInDb, keyTable)) {
                            cacheKeyMap.put(entryInDb, new OzoneFileStatus(omKeyInfo, scmBlockSize, !isFile));
                            countEntries++;
                        }
                    } else {
                        // if entry is a directory
                        if (!isKeyDeleted(entryInDb, keyTable)) {
                            if (!entryKeyName.equals(immediateChild)) {
                                OmKeyInfo fakeDirEntry = createDirectoryKey(omKeyInfo.getVolumeName(), omKeyInfo.getBucketName(), immediateChild, omKeyInfo.getAcls());
                                cacheKeyMap.put(entryInDb, new OzoneFileStatus(fakeDirEntry, scmBlockSize, true));
                            } else {
                                // If entryKeyName matches dir name, we have the info
                                cacheKeyMap.put(entryInDb, new OzoneFileStatus(omKeyInfo, 0, true));
                            }
                            countEntries++;
                        }
                        // skip the other descendants of this child directory.
                        iterator.seek(getNextGreaterString(volumeName, bucketName, immediateChild));
                    }
                }
            } else {
                break;
            }
        }
    }
    countEntries = 0;
    // Convert results in cacheKeyMap to List
    for (OzoneFileStatus fileStatus : cacheKeyMap.values()) {
        // No need to check if a key is deleted or not here, this is handled
        // when adding entries to cacheKeyMap from DB.
        fileStatusList.add(fileStatus);
        countEntries++;
        if (countEntries >= numEntries) {
            break;
        }
    }
    // Clean up temp map and set
    cacheKeyMap.clear();
    List<OmKeyInfo> keyInfoList = new ArrayList<>(fileStatusList.size());
    fileStatusList.stream().map(s -> s.getKeyInfo()).forEach(keyInfoList::add);
    if (args.getLatestVersionLocation()) {
        slimLocationVersion(keyInfoList.toArray(new OmKeyInfo[0]));
    }
    refreshPipeline(keyInfoList);
    if (args.getSortDatanodes()) {
        sortDatanodes(clientAddress, keyInfoList.toArray(new OmKeyInfo[0]));
    }
    return fileStatusList;
}

Example 3 with TableIterator

use of org.apache.hadoop.hdds.utils.db.TableIterator in project ozone by apache.

the class OmMetadataManagerImpl method getPendingDeletionKeys.

@Override
public List<BlockGroup> getPendingDeletionKeys(final int keyCount) throws IOException {
    List<BlockGroup> keyBlocksList = Lists.newArrayList();
    try (TableIterator<String, ? extends KeyValue<String, RepeatedOmKeyInfo>> keyIter = getDeletedTable().iterator()) {
        int currentCount = 0;
        while (keyIter.hasNext() && currentCount < keyCount) {
            KeyValue<String, RepeatedOmKeyInfo> kv = keyIter.next();
            if (kv != null) {
                RepeatedOmKeyInfo infoList = kv.getValue();
                // Get block keys as a list.
                for (OmKeyInfo info : infoList.getOmKeyInfoList()) {
                    OmKeyLocationInfoGroup latest = info.getLatestVersionLocations();
                    List<BlockID> item = latest.getLocationList().stream().map(b -> new BlockID(b.getContainerID(), b.getLocalID())).collect(Collectors.toList());
                    BlockGroup keyBlocks = BlockGroup.newBuilder().setKeyName(kv.getKey()).addAllBlockIDs(item).build();
                    keyBlocksList.add(keyBlocks);
                    currentCount++;
                }
            }
        }
    }
    return keyBlocksList;
}

Example 4 with TableIterator

use of org.apache.hadoop.hdds.utils.db.TableIterator in project ozone by apache.

the class TableCountTask method reprocess.

/**
 * Iterate the rows of each table in OM snapshot DB and calculate the
 * counts for each table.
 *
 * @param omMetadataManager OM Metadata instance.
 * @return Pair
 */
@Override
public Pair<String, Boolean> reprocess(OMMetadataManager omMetadataManager) {
    for (String tableName : getTaskTables()) {
        Table table = omMetadataManager.getTable(tableName);
        try (TableIterator keyIter = table.iterator()) {
            long count = getCount(keyIter);
            ReconUtils.upsertGlobalStatsTable(sqlConfiguration, globalStatsDao, getRowKeyFromTable(tableName), count);
        } catch (IOException ioEx) {
            LOG.error("Unable to populate Table Count in Recon DB.", ioEx);
            return new ImmutablePair<>(getTaskName(), false);
        }
    }
    LOG.info("Completed a 'reprocess' run of TableCountTask.");
    return new ImmutablePair<>(getTaskName(), true);
}