Examples with Pair com.github.ambry.utils.Pair used on opensource projects

Example 6 with Pair

use of com.github.ambry.utils.Pair in project ambry by linkedin.

the class StoreCopier method copy.

/**
 * Copies data starting from {@code startToken} until all the data is copied.
 * @param startToken the {@link FindToken} to start copying from. It is expected that start token does not cause
 *                   the copier to attempt to copy blobs that have already been copied. If that happens, the boolean
 *                   in the return value will be {@code true}.
 * @return a {@link Pair} of the {@link FindToken} until which data has been copied and a {@link Boolean} indicating
 * whether the source had problems that were skipped over - like duplicates ({@code true} indicates that there were).
 * @throws Exception if there is any exception during processing
 */
public Pair<FindToken, Boolean> copy(FindToken startToken) throws Exception {
    boolean sourceHasProblems = false;
    FindToken lastToken;
    FindToken token = startToken;
    do {
        lastToken = token;
        FindInfo findInfo = src.findEntriesSince(lastToken, fetchSizeInBytes, null, null);
        List<MessageInfo> messageInfos = findInfo.getMessageEntries();
        for (Transformer transformer : transformers) {
            transformer.warmup(messageInfos);
        }
        for (MessageInfo messageInfo : messageInfos) {
            logger.trace("Processing {} - isDeleted: {}, isExpired {}", messageInfo.getStoreKey(), messageInfo.isDeleted(), messageInfo.isExpired());
            if (!messageInfo.isExpired() && !messageInfo.isDeleted()) {
                if (tgt.findMissingKeys(Collections.singletonList(messageInfo.getStoreKey())).size() == 1) {
                    StoreInfo storeInfo = src.get(Collections.singletonList(messageInfo.getStoreKey()), EnumSet.allOf(StoreGetOptions.class));
                    MessageReadSet readSet = storeInfo.getMessageReadSet();
                    if (readSet.sizeInBytes(0) > Integer.MAX_VALUE) {
                        throw new IllegalStateException("Cannot copy blobs whose size > Integer.MAX_VALUE");
                    }
                    int size = (int) readSet.sizeInBytes(0);
                    byte[] buf = new byte[size];
                    readSet.writeTo(0, new ByteBufferChannel(ByteBuffer.wrap(buf)), 0, size);
                    Message message = new Message(storeInfo.getMessageReadSetInfo().get(0), new ByteArrayInputStream(buf));
                    for (Transformer transformer : transformers) {
                        TransformationOutput tfmOutput = transformer.transform(message);
                        if (tfmOutput.getException() != null) {
                            throw tfmOutput.getException();
                        } else {
                            message = tfmOutput.getMsg();
                        }
                        if (message == null) {
                            break;
                        }
                    }
                    if (message == null) {
                        logger.trace("Dropping {} because the transformers did not return a message", messageInfo.getStoreKey());
                        continue;
                    }
                    MessageFormatWriteSet writeSet = new MessageFormatWriteSet(message.getStream(), Collections.singletonList(message.getMessageInfo()), false);
                    tgt.put(writeSet);
                    MessageInfo tgtMsgInfo = message.getMessageInfo();
                    if (tgtMsgInfo.isTtlUpdated()) {
                        MessageInfo updateMsgInfo = new MessageInfo(tgtMsgInfo.getStoreKey(), 0, false, true, tgtMsgInfo.getExpirationTimeInMs(), tgtMsgInfo.getAccountId(), tgtMsgInfo.getContainerId(), tgtMsgInfo.getOperationTimeMs());
                        tgt.updateTtl(Collections.singletonList(updateMsgInfo));
                    }
                    logger.trace("Copied {} as {}", messageInfo.getStoreKey(), tgtMsgInfo.getStoreKey());
                } else if (!messageInfo.isTtlUpdated()) {
                    logger.warn("Found a duplicate entry for {} while copying data", messageInfo.getStoreKey());
                    sourceHasProblems = true;
                }
            }
        }
        token = findInfo.getFindToken();
        double percentBytesRead = src.isEmpty() ? 100.0 : token.getBytesRead() * 100.0 / src.getSizeInBytes();
        logger.info("[{}] [{}] {}% copied", Thread.currentThread().getName(), storeId, df.format(percentBytesRead));
    } while (!token.equals(lastToken));
    return new Pair<>(token, sourceHasProblems);
}

Example 7 with Pair

use of com.github.ambry.utils.Pair in project ambry by linkedin.

the class IndexTest method findDeletedEntriesSinceToJournalBasedTest.

/**
 * Tests all cases of {@link PersistentIndex#findDeletedEntriesSince(FindToken, long, long)} that result in an journal
 * based {@link StoreFindToken} being returned.
 * 1. Uninitialized -> Journal
 * 2. Index -> Journal
 * 3. Journal -> Journal
 * 4. No movement.
 * @throws StoreException
 */
private void findDeletedEntriesSinceToJournalBasedTest() throws StoreException {
    IndexSegment segmentOfToken = state.index.getIndexSegments().lastEntry().getValue();
    StoreFindToken absoluteEndToken = new StoreFindToken(state.logOrder.lastKey(), state.sessionId, state.incarnationId, false, segmentOfToken.getResetKey(), segmentOfToken.getResetKeyType(), segmentOfToken.getResetKeyLifeVersion());
    // ------------------
    // 1. Uninitialized -> Journal
    doFindDeletedEntriesSinceTest(new StoreFindToken(), Long.MAX_VALUE, state.deletedKeys, absoluteEndToken);
    // ------------------
    // 2. Index -> Journal
    Offset secondIndexSegmentStartOffset = state.referenceIndex.higherKey(state.referenceIndex.firstKey());
    // second index segment contains the first delete entry
    StoreKey firstDeletedKey = getDeletedKeyFromIndexSegment(secondIndexSegmentStartOffset);
    StoreFindToken startToken = new StoreFindToken(firstDeletedKey, secondIndexSegmentStartOffset, state.sessionId, state.incarnationId, null, null, UNINITIALIZED_RESET_KEY_VERSION);
    Set<MockId> expectedKeys = new HashSet<>(state.deletedKeys);
    expectedKeys.remove(firstDeletedKey);
    doFindDeletedEntriesSinceTest(startToken, Long.MAX_VALUE, expectedKeys, absoluteEndToken);
    // ------------------
    // 3. Journal -> Journal
    // a. Token no longer in journal
    startToken = new StoreFindToken(state.getExpectedValue((MockId) firstDeletedKey, false).getOffset(), state.sessionId, state.incarnationId, false, null, null, UNINITIALIZED_RESET_KEY_VERSION);
    doFindDeletedEntriesSinceTest(startToken, Long.MAX_VALUE, state.deletedKeys, absoluteEndToken);
    // b. Token still in journal
    startToken = new StoreFindToken(state.index.journal.getFirstOffset(), state.sessionId, state.incarnationId, false, null, null, UNINITIALIZED_RESET_KEY_VERSION);
    expectedKeys.clear();
    for (Map.Entry<Offset, Pair<MockId, CuratedLogIndexState.LogEntry>> entry : state.logOrder.tailMap(startToken.getOffset(), false).entrySet()) {
        if (entry.getValue().getSecond().indexValue.isDelete() && state.getExpectedValue(entry.getValue().getFirst(), EnumSet.of(PersistentIndex.IndexEntryType.UNDELETE), null) == null) {
            expectedKeys.add(entry.getValue().getFirst());
        }
    }
    doFindDeletedEntriesSinceTest(startToken, Long.MAX_VALUE, expectedKeys, absoluteEndToken);
    // ------------------
    // 4. Journal no change
    doFindDeletedEntriesSinceTest(absoluteEndToken, Long.MAX_VALUE, Collections.emptySet(), absoluteEndToken);
}

Example 8 with Pair

use of com.github.ambry.utils.Pair in project ambry by linkedin.

the class BlobStoreStatsTest method getContainerStorageStats.

/**
 * Go over the referenceIndex to collect valid data size information per container. The result is used for
 * verification purposes.
 * @param deleteReferenceTimeInMs the reference time in ms until which deletes are relevant
 * @param expiryReferenceTimeInMs the reference time in ms until which expirations are relevant
 * @param deleteTombstoneStats a hashmap that tracks stats related delete tombstones in log segments.
 * @return a nested {@link Map} of serviceId to containerId to valid data size
 */
private Map<Short, Map<Short, ContainerStorageStats>> getContainerStorageStats(long deleteReferenceTimeInMs, long expiryReferenceTimeInMs, Map<String, Pair<AtomicLong, AtomicLong>> deleteTombstoneStats) {
    Map<Short, Map<Short, ContainerStorageStats>> containerStorageStats = new HashMap<>();
    Map<Short, Map<Short, Long>> validSizeMap = new HashMap<>();
    Map<Short, Map<Short, Long>> physicalSizeMap = new HashMap<>();
    Map<Short, Map<Short, Set<StoreKey>>> storeKeyMap = new HashMap<>();
    Pair<Set<MockId>, Set<MockId>> expiredDeletes = new Pair<>(new HashSet<>(), new HashSet<>());
    for (Offset indSegStartOffset : state.referenceIndex.keySet()) {
        state.getValidIndexEntriesForIndexSegment(indSegStartOffset, deleteReferenceTimeInMs, expiryReferenceTimeInMs, null, deleteTombstoneStats, expiredDeletes, true, (entry, isValid) -> {
            IndexValue indexValue = entry.getValue();
            if (indexValue.isPut() && isValid) {
                StatsUtils.updateNestedMapHelper(validSizeMap, indexValue.getAccountId(), indexValue.getContainerId(), indexValue.getSize());
            }
            StatsUtils.updateNestedMapHelper(physicalSizeMap, indexValue.getAccountId(), indexValue.getContainerId(), indexValue.getSize());
            storeKeyMap.computeIfAbsent(indexValue.getAccountId(), k -> new HashMap<>()).computeIfAbsent(indexValue.getContainerId(), k -> new HashSet<>()).add(entry.getKey());
        });
    }
    for (short accountId : validSizeMap.keySet()) {
        for (short containerId : validSizeMap.get(accountId).keySet()) {
            containerStorageStats.computeIfAbsent(accountId, k -> new HashMap<>()).put(containerId, new ContainerStorageStats(containerId, validSizeMap.get(accountId).get(containerId), physicalSizeMap.get(accountId).get(containerId), storeKeyMap.get(accountId).get(containerId).size()));
        }
    }
    return containerStorageStats;
}

Example 9 with Pair

use of com.github.ambry.utils.Pair in project ambry by linkedin.

the class BlobStoreStatsTest method verifyContainerStorageStatsAndGetTotalValidSize.

/**
 * Verify the correctness of valid data size information per container returned by BlobStoreStats and return the
 * total valid data size of all containers.
 * @param blobStoreStats the {@link BlobStoreStats} to be verified
 * @param referenceTimeInMs the reference time in ms until which deletes and expiration are relevant
 * @return the total valid data size of all containers (from all serviceIds)
 */
private long verifyContainerStorageStatsAndGetTotalValidSize(BlobStoreStats blobStoreStats, long referenceTimeInMs) throws StoreException {
    Map<String, Pair<AtomicLong, AtomicLong>> deleteTombstoneStats = generateDeleteTombstoneStats();
    Map<Short, Map<Short, ContainerStorageStats>> actualContainerStorageStatsMap = blobStoreStats.getContainerStorageStats(referenceTimeInMs);
    Map<Short, Map<Short, ContainerStorageStats>> expectedContainerStorageStatsMap = getContainerStorageStats(referenceTimeInMs, state.time.milliseconds(), deleteTombstoneStats);
    long totalValidSize = 0L;
    for (Map.Entry<Short, Map<Short, ContainerStorageStats>> expectedContainerStorageStatsEntry : expectedContainerStorageStatsMap.entrySet()) {
        short accountId = expectedContainerStorageStatsEntry.getKey();
        assertTrue("Expected accountId: " + accountId + " not found", actualContainerStorageStatsMap.containsKey(accountId));
        Map<Short, ContainerStorageStats> innerMap = expectedContainerStorageStatsEntry.getValue();
        for (Map.Entry<Short, ContainerStorageStats> innerEntry : innerMap.entrySet()) {
            short containerId = innerEntry.getKey();
            assertTrue("Expected containerId: " + containerId + " not found in accountId: " + accountId, actualContainerStorageStatsMap.get(accountId).containsKey(containerId));
            ContainerStorageStats expectedContainerStorageStats = innerEntry.getValue();
            ContainerStorageStats actualContainerStorageStats = actualContainerStorageStatsMap.get(accountId).get(containerId);
            assertEquals("Storage stats mismatch for accountId: " + accountId + " containerId: " + containerId, expectedContainerStorageStats, actualContainerStorageStats);
            totalValidSize += expectedContainerStorageStats.getLogicalStorageUsage();
        }
        if (innerMap.size() != actualContainerStorageStatsMap.get(accountId).size()) {
            // make sure all the new items have value 0
            for (Map.Entry<Short, ContainerStorageStats> actualContainerEntry : actualContainerStorageStatsMap.get(accountId).entrySet()) {
                if (!innerMap.containsKey(actualContainerEntry.getKey())) {
                    assertEquals("Expecting 0 value for account id " + accountId + " and container " + actualContainerEntry.getKey(), 0, actualContainerEntry.getValue().getLogicalStorageUsage());
                }
            }
        }
        actualContainerStorageStatsMap.remove(accountId);
    }
    for (Map.Entry<Short, Map<Short, ContainerStorageStats>> actualContainerValidSizeEntry : actualContainerStorageStatsMap.entrySet()) {
        if (actualContainerValidSizeEntry.getValue().size() != 0) {
            for (Map.Entry<Short, ContainerStorageStats> mapEntry : actualContainerValidSizeEntry.getValue().entrySet()) {
                assertEquals("Additional values found in actual container valid size map for service " + actualContainerValidSizeEntry.getKey(), 0, mapEntry.getValue().getLogicalStorageUsage());
            }
        }
    }
    // verify delete tombstone stats
    verifyDeleteTombstoneStats(blobStoreStats, deleteTombstoneStats);
    return totalValidSize;
}

Example 10 with Pair

use of com.github.ambry.utils.Pair in project ambry by linkedin.

the class CompactionPolicyFactoryTest method testCompactionPolicyFactory.

/**
 * Tests {@link CompactionPolicyFactory}
 * @throws Exception
 */
@Test
public void testCompactionPolicyFactory() throws Exception {
    List<Pair<String, String>> validCompactionPolicyInfos = new ArrayList<>();
    validCompactionPolicyInfos.add(new Pair<>("com.github.ambry.store.StatsBasedCompactionPolicyFactory", "com.github.ambry.store.StatsBasedCompactionPolicy"));
    validCompactionPolicyInfos.add(new Pair<>("com.github.ambry.store.CompactAllPolicyFactory", "com.github.ambry.store.CompactAllPolicy"));
    for (Pair<String, String> validCompactionPolicyInfo : validCompactionPolicyInfos) {
        Properties properties = new Properties();
        properties.setProperty("store.compaction.policy.factory", validCompactionPolicyInfo.getFirst());
        StoreConfig config = new StoreConfig(new VerifiableProperties(properties));
        Time time = new MockTime();
        CompactionPolicyFactory compactionPolicyFactory = Utils.getObj(config.storeCompactionPolicyFactory, config, time);
        Assert.assertEquals("Did not receive expected CompactionPolicy instance", validCompactionPolicyInfo.getFirst(), compactionPolicyFactory.getClass().getCanonicalName());
        CompactionPolicy compactionPolicy = compactionPolicyFactory.getCompactionPolicy();
        Assert.assertEquals("Did not receive expected CompactionPolicy instance", validCompactionPolicyInfo.getSecond(), compactionPolicy.getClass().getCanonicalName());
    }
}