Examples with FindToken com.github.ambry.replication.FindToken used on opensource projects

Example 1 with FindToken

use of com.github.ambry.replication.FindToken in project ambry by linkedin.

the class HardDeleteVerifier method getOffsetFromCleanupToken.

private long getOffsetFromCleanupToken(File cleanupTokenFile) throws Exception {
    long parsedTokenValue = -1;
    if (cleanupTokenFile.exists()) {
        CrcInputStream crcStream = new CrcInputStream(new FileInputStream(cleanupTokenFile));
        DataInputStream stream = new DataInputStream(crcStream);
        try {
            // The format of the cleanup token is documented in PersistentIndex.persistCleanupToken()
            short version = stream.readShort();
            if (version != HARD_DELETE_TOKEN_V0) {
                throw new IllegalStateException("Unknown version encountered while parsing cleanup token");
            }
            StoreKeyFactory storeKeyFactory = Utils.getObj("com.github.ambry.commons.BlobIdFactory", map);
            FindTokenFactory factory = Utils.getObj("com.github.ambry.store.StoreFindTokenFactory", storeKeyFactory);
            FindToken startToken = factory.getFindToken(stream);
            // read past the end token.
            factory.getFindToken(stream);
            ByteBuffer bytebufferToken = ByteBuffer.wrap(startToken.toBytes());
            short tokenVersion = bytebufferToken.getShort();
            if (tokenVersion != 0) {
                throw new IllegalArgumentException("token version: " + tokenVersion + " is unknown");
            }
            int sessionIdsize = bytebufferToken.getInt();
            bytebufferToken.position(bytebufferToken.position() + sessionIdsize);
            parsedTokenValue = bytebufferToken.getLong();
            if (parsedTokenValue == -1) {
                /* Index based token, get index start offset */
                parsedTokenValue = bytebufferToken.getLong();
            }
            /* Just read the remaining fields and verify that the crc matches. We don't really need the fields for this
           test */
            int num = stream.readInt();
            List<StoreKey> storeKeyList = new ArrayList<StoreKey>(num);
            for (int i = 0; i < num; i++) {
                // Read BlobReadOptions
                short blobReadOptionsVersion = stream.readShort();
                if (blobReadOptionsVersion != 0) {
                    throw new IllegalStateException("Unknown blobReadOptionsVersion: " + blobReadOptionsVersion);
                }
                long offset = stream.readLong();
                long sz = stream.readLong();
                long ttl = stream.readLong();
                StoreKey key = storeKeyFactory.getStoreKey(stream);
                storeKeyList.add(key);
            }
            for (int i = 0; i < num; i++) {
                int length = stream.readInt();
                short headerVersion = stream.readShort();
                short userMetadataVersion = stream.readShort();
                int userMetadataSize = stream.readInt();
                short blobRecordVersion = stream.readShort();
                long blobStreamSize = stream.readLong();
                StoreKey key = storeKeyFactory.getStoreKey(stream);
                if (!storeKeyList.get(i).equals(key)) {
                    throw new IllegalStateException("Parsed key mismatch");
                }
            }
            long crc = crcStream.getValue();
            if (crc != stream.readLong()) {
                throw new IllegalStateException("Crc mismatch while reading cleanup token");
            }
        } finally {
            stream.close();
        }
    } else {
        throw new IllegalStateException("No cleanup token");
    }
    return parsedTokenValue;
}

Example 2 with FindToken

use of com.github.ambry.replication.FindToken in project ambry by linkedin.

the class StoreCopier method copy.

/**
 * Copies data starting from {@code startToken} until all the data is copied.
 * @param startToken the {@link FindToken} to start copying from. It is expected that start token does not cause
 *                   the copier to attempt to copy blobs that have already been copied. If that happens, the boolean
 *                   in the return value will be {@code true}.
 * @return a {@link Pair} of the {@link FindToken} until which data has been copied and a {@link Boolean} indicating
 * whether the source had problems that were skipped over - like duplicates ({@code true} indicates that there were).
 * @throws Exception if there is any exception during processing
 */
public Pair<FindToken, Boolean> copy(FindToken startToken) throws Exception {
    boolean sourceHasProblems = false;
    FindToken lastToken;
    FindToken token = startToken;
    do {
        lastToken = token;
        FindInfo findInfo = src.findEntriesSince(lastToken, fetchSizeInBytes, null, null);
        List<MessageInfo> messageInfos = findInfo.getMessageEntries();
        for (Transformer transformer : transformers) {
            transformer.warmup(messageInfos);
        }
        for (MessageInfo messageInfo : messageInfos) {
            logger.trace("Processing {} - isDeleted: {}, isExpired {}", messageInfo.getStoreKey(), messageInfo.isDeleted(), messageInfo.isExpired());
            if (!messageInfo.isExpired() && !messageInfo.isDeleted()) {
                if (tgt.findMissingKeys(Collections.singletonList(messageInfo.getStoreKey())).size() == 1) {
                    StoreInfo storeInfo = src.get(Collections.singletonList(messageInfo.getStoreKey()), EnumSet.allOf(StoreGetOptions.class));
                    MessageReadSet readSet = storeInfo.getMessageReadSet();
                    if (readSet.sizeInBytes(0) > Integer.MAX_VALUE) {
                        throw new IllegalStateException("Cannot copy blobs whose size > Integer.MAX_VALUE");
                    }
                    int size = (int) readSet.sizeInBytes(0);
                    byte[] buf = new byte[size];
                    readSet.writeTo(0, new ByteBufferChannel(ByteBuffer.wrap(buf)), 0, size);
                    Message message = new Message(storeInfo.getMessageReadSetInfo().get(0), new ByteArrayInputStream(buf));
                    for (Transformer transformer : transformers) {
                        TransformationOutput tfmOutput = transformer.transform(message);
                        if (tfmOutput.getException() != null) {
                            throw tfmOutput.getException();
                        } else {
                            message = tfmOutput.getMsg();
                        }
                        if (message == null) {
                            break;
                        }
                    }
                    if (message == null) {
                        logger.trace("Dropping {} because the transformers did not return a message", messageInfo.getStoreKey());
                        continue;
                    }
                    MessageFormatWriteSet writeSet = new MessageFormatWriteSet(message.getStream(), Collections.singletonList(message.getMessageInfo()), false);
                    tgt.put(writeSet);
                    MessageInfo tgtMsgInfo = message.getMessageInfo();
                    if (tgtMsgInfo.isTtlUpdated()) {
                        MessageInfo updateMsgInfo = new MessageInfo(tgtMsgInfo.getStoreKey(), 0, false, true, tgtMsgInfo.getExpirationTimeInMs(), tgtMsgInfo.getAccountId(), tgtMsgInfo.getContainerId(), tgtMsgInfo.getOperationTimeMs());
                        tgt.updateTtl(Collections.singletonList(updateMsgInfo));
                    }
                    logger.trace("Copied {} as {}", messageInfo.getStoreKey(), tgtMsgInfo.getStoreKey());
                } else if (!messageInfo.isTtlUpdated()) {
                    logger.warn("Found a duplicate entry for {} while copying data", messageInfo.getStoreKey());
                    sourceHasProblems = true;
                }
            }
        }
        token = findInfo.getFindToken();
        double percentBytesRead = src.isEmpty() ? 100.0 : token.getBytesRead() * 100.0 / src.getSizeInBytes();
        logger.info("[{}] [{}] {}% copied", Thread.currentThread().getName(), storeId, df.format(percentBytesRead));
    } while (!token.equals(lastToken));
    return new Pair<>(token, sourceHasProblems);
}

Example 3 with FindToken

use of com.github.ambry.replication.FindToken in project ambry by linkedin.

the class CloudBlobStoreTest method testFindEntriesSince.

/**
 * Test the CloudBlobStore findEntriesSince method.
 */
@Test
public void testFindEntriesSince() throws Exception {
    setupCloudStore(false, true, defaultCacheLimit, true);
    long maxTotalSize = 1000000;
    // 1) start with empty token, call find, return some data
    long blobSize = 200000;
    int numBlobsFound = 5;
    CosmosChangeFeedFindToken cosmosChangeFeedFindToken = new CosmosChangeFeedFindToken(blobSize * numBlobsFound, "start", "end", 0, numBlobsFound, UUID.randomUUID().toString());
    // create a list of 10 blobs with total size less than maxSize, and return it as part of query ChangeFeed
    when(dest.findEntriesSince(anyString(), any(CosmosChangeFeedFindToken.class), anyLong())).thenReturn(new FindResult(Collections.emptyList(), cosmosChangeFeedFindToken));
    CosmosChangeFeedFindToken startToken = new CosmosChangeFeedFindToken();
    // remote node host name and replica path are not really used by cloud store, it's fine to keep them null
    FindInfo findInfo = store.findEntriesSince(startToken, maxTotalSize, null, null);
    CosmosChangeFeedFindToken outputToken = (CosmosChangeFeedFindToken) findInfo.getFindToken();
    assertEquals(blobSize * numBlobsFound, outputToken.getBytesRead());
    assertEquals(numBlobsFound, outputToken.getTotalItems());
    assertEquals(0, outputToken.getIndex());
    // 2) call find with new token, return more data including lastBlob, verify token updated
    cosmosChangeFeedFindToken = new CosmosChangeFeedFindToken(blobSize * 2 * numBlobsFound, "start2", "end2", 0, numBlobsFound, UUID.randomUUID().toString());
    when(dest.findEntriesSince(anyString(), any(CosmosChangeFeedFindToken.class), anyLong())).thenReturn(new FindResult(Collections.emptyList(), cosmosChangeFeedFindToken));
    findInfo = store.findEntriesSince(outputToken, maxTotalSize, null, null);
    outputToken = (CosmosChangeFeedFindToken) findInfo.getFindToken();
    assertEquals(blobSize * 2 * numBlobsFound, outputToken.getBytesRead());
    assertEquals(numBlobsFound, outputToken.getTotalItems());
    assertEquals(0, outputToken.getIndex());
    // 3) call find with new token, no more data, verify token unchanged
    when(dest.findEntriesSince(anyString(), any(CosmosChangeFeedFindToken.class), anyLong())).thenReturn(new FindResult(Collections.emptyList(), outputToken));
    findInfo = store.findEntriesSince(outputToken, maxTotalSize, null, null);
    assertTrue(findInfo.getMessageEntries().isEmpty());
    FindToken finalToken = findInfo.getFindToken();
    assertEquals(outputToken, finalToken);
}

Example 4 with FindToken

use of com.github.ambry.replication.FindToken in project ambry by linkedin.

the class IndexTest method findEntriesSinceIncarnationIdTest.

/**
 * Tests behaviour of {@link PersistentIndex#findEntriesSince(FindToken, long)} relating to incarnationId
 * @throws StoreException
 */
@Test
public void findEntriesSinceIncarnationIdTest() throws StoreException {
    Offset lastRecordOffset = state.index.journal.getLastOffset();
    state.appendToLog(2 * CuratedLogIndexState.PUT_RECORD_SIZE);
    // will be recovered
    FileSpan firstRecordFileSpan = state.log.getFileSpanForMessage(state.index.getCurrentEndOffset(), CuratedLogIndexState.PUT_RECORD_SIZE);
    // will not be recovered
    FileSpan secondRecordFileSpan = state.log.getFileSpanForMessage(firstRecordFileSpan.getEndOffset(), CuratedLogIndexState.PUT_RECORD_SIZE);
    UUID oldSessionId = state.sessionId;
    UUID oldIncarnationId = state.incarnationId;
    final MockId newId = state.getUniqueId();
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    long operationTimeMs = state.time.milliseconds();
    // add to allKeys() so that doFindEntriesSinceTest() works correctly.
    IndexValue putValue = new IndexValue(CuratedLogIndexState.PUT_RECORD_SIZE, firstRecordFileSpan.getStartOffset(), Utils.Infinite_Time, operationTimeMs, accountId, containerId);
    state.allKeys.computeIfAbsent(newId, k -> new TreeSet<>()).add(putValue);
    state.recovery = (read, startOffset, endOffset, factory) -> Collections.singletonList(new MessageInfo(newId, CuratedLogIndexState.PUT_RECORD_SIZE, accountId, containerId, operationTimeMs));
    // change in incarnationId
    state.incarnationId = UUID.randomUUID();
    state.reloadIndex(true, true);
    long bytesRead = state.index.getAbsolutePositionInLogForOffset(firstRecordFileSpan.getEndOffset());
    // create a token that will be past the index end offset on startup after recovery with old incarnationId
    StoreFindToken startToken = new StoreFindToken(secondRecordFileSpan.getEndOffset(), oldSessionId, oldIncarnationId, false, null, null, UNINITIALIZED_RESET_KEY_VERSION);
    // token should get reset internally, all keys should be returned and the returned token should be pointing to
    // start offset of firstRecordFileSpan.
    IndexSegment segmentOfToken = state.index.getIndexSegments().floorEntry(firstRecordFileSpan.getStartOffset()).getValue();
    StoreFindToken expectedEndToken = new StoreFindToken(firstRecordFileSpan.getStartOffset(), state.sessionId, state.incarnationId, false, segmentOfToken.getResetKey(), segmentOfToken.getResetKeyType(), segmentOfToken.getResetKeyLifeVersion());
    expectedEndToken.setBytesRead(bytesRead);
    doFindEntriesSinceTest(startToken, Long.MAX_VALUE, state.allKeys.keySet(), expectedEndToken);
    // create a token that is not past the index end offset on startup after recovery with old incarnationId.
    // token should get reset internally, all keys should be returned and the returned token should be be pointing to
    // start offset of firstRecordFileSpan.
    startToken = new StoreFindToken(lastRecordOffset, oldSessionId, oldIncarnationId, false, null, null, UNINITIALIZED_RESET_KEY_VERSION);
    expectedEndToken = new StoreFindToken(firstRecordFileSpan.getStartOffset(), state.sessionId, state.incarnationId, false, segmentOfToken.getResetKey(), segmentOfToken.getResetKeyType(), segmentOfToken.getResetKeyLifeVersion());
    expectedEndToken.setBytesRead(bytesRead);
    doFindEntriesSinceTest(startToken, Long.MAX_VALUE, state.allKeys.keySet(), expectedEndToken);
}

Example 5 with FindToken

use of com.github.ambry.replication.FindToken in project ambry by linkedin.

the class IndexTest method findEntriesSinceOnRestartTest.

/**
 * Tests behaviour of {@link PersistentIndex#findEntriesSince(FindToken, long)} on crash-restart of index and some
 * recovery. Specifically tests cases where tokens have been handed out before the "crash" failure.
 * @throws IOException
 * @throws StoreException
 */
@Test
public void findEntriesSinceOnRestartTest() throws IOException, StoreException {
    Offset lastRecordOffset = state.index.journal.getLastOffset();
    state.appendToLog(2 * CuratedLogIndexState.PUT_RECORD_SIZE);
    // this record will be recovered.
    FileSpan firstRecordFileSpan = state.log.getFileSpanForMessage(state.index.getCurrentEndOffset(), CuratedLogIndexState.PUT_RECORD_SIZE);
    // this record will not be recovered.
    FileSpan secondRecordFileSpan = state.log.getFileSpanForMessage(firstRecordFileSpan.getEndOffset(), CuratedLogIndexState.PUT_RECORD_SIZE);
    // if there is no bad shutdown but the store token is past the index end offset, it is an error state
    StoreFindToken startToken = new StoreFindToken(secondRecordFileSpan.getStartOffset(), new UUID(1, 1), state.incarnationId, false, null, null, UNINITIALIZED_RESET_KEY_VERSION);
    doFindEntriesSinceFailureTest(startToken, StoreErrorCodes.Unknown_Error);
    UUID oldSessionId = state.sessionId;
    final MockId newId = state.getUniqueId();
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    long operationTimeMs = state.time.milliseconds();
    // add to allKeys() so that doFindEntriesSinceTest() works correctly.
    IndexValue putValue = new IndexValue(CuratedLogIndexState.PUT_RECORD_SIZE, firstRecordFileSpan.getStartOffset(), Utils.Infinite_Time, operationTimeMs, accountId, containerId);
    state.allKeys.computeIfAbsent(newId, k -> new TreeSet<>()).add(putValue);
    state.recovery = (read, startOffset, endOffset, factory) -> Collections.singletonList(new MessageInfo(newId, CuratedLogIndexState.PUT_RECORD_SIZE, accountId, containerId, operationTimeMs));
    state.reloadIndex(true, true);
    // If there is no incarnationId in the incoming token, for backwards compatibility purposes we consider it as valid
    // and proceed with session id validation and so on.
    UUID[] incarnationIds = new UUID[] { state.incarnationId, null };
    for (UUID incarnationIdToTest : incarnationIds) {
        long bytesRead = state.index.getAbsolutePositionInLogForOffset(firstRecordFileSpan.getEndOffset());
        // create a token that will be past the index end offset on startup after recovery.
        if (incarnationIdToTest == null) {
            startToken = getTokenWithNullIncarnationId(new StoreFindToken(secondRecordFileSpan.getEndOffset(), oldSessionId, state.incarnationId, false, null, null, UNINITIALIZED_RESET_KEY_VERSION));
            assertNull("IncarnationId is expected to be null ", startToken.getIncarnationId());
        } else {
            startToken = new StoreFindToken(secondRecordFileSpan.getEndOffset(), oldSessionId, incarnationIdToTest, false, null, null, UNINITIALIZED_RESET_KEY_VERSION);
        }
        // token should get reset internally, no keys should be returned and the returned token should be correct (offset
        // in it will be the current log end offset = firstRecordFileSpan.getEndOffset()). The returned token should have correct reset key info.
        IndexSegment segmentOfToken = state.index.getIndexSegments().floorEntry(firstRecordFileSpan.getEndOffset()).getValue();
        StoreFindToken expectedEndToken = new StoreFindToken(firstRecordFileSpan.getEndOffset(), state.sessionId, state.incarnationId, true, segmentOfToken.getResetKey(), segmentOfToken.getResetKeyType(), segmentOfToken.getResetKeyLifeVersion());
        expectedEndToken.setBytesRead(bytesRead);
        doFindEntriesSinceTest(startToken, Long.MAX_VALUE, Collections.emptySet(), expectedEndToken);
        // create a token that is not past the index end offset on startup after recovery. Should work as expected
        if (incarnationIdToTest == null) {
            startToken = getTokenWithNullIncarnationId(new StoreFindToken(lastRecordOffset, oldSessionId, state.incarnationId, false, null, null, UNINITIALIZED_RESET_KEY_VERSION));
            assertNull("IncarnationId is expected to be null ", startToken.getIncarnationId());
        } else {
            startToken = new StoreFindToken(lastRecordOffset, oldSessionId, incarnationIdToTest, false, null, null, UNINITIALIZED_RESET_KEY_VERSION);
        }
        segmentOfToken = state.index.getIndexSegments().floorEntry(firstRecordFileSpan.getStartOffset()).getValue();
        expectedEndToken = new StoreFindToken(firstRecordFileSpan.getStartOffset(), state.sessionId, state.incarnationId, false, segmentOfToken.getResetKey(), segmentOfToken.getResetKeyType(), segmentOfToken.getResetKeyLifeVersion());
        expectedEndToken.setBytesRead(bytesRead);
        doFindEntriesSinceTest(startToken, Long.MAX_VALUE, Collections.singleton(newId), expectedEndToken);
    }
}