Examples with CrcInputStream com.github.ambry.utils.CrcInputStream used on opensource projects

Example 1 with CrcInputStream

use of com.github.ambry.utils.CrcInputStream in project ambry by linkedin.

the class ReplicationManager method readFromFileAndPersistIfNecessary.

/**
 * Reads the replica tokens from the file and populates the Remote replica info
 * and persists the token file if necessary.
 * @param mountPath The mount path where the replica tokens are stored
 * @throws ReplicationException
 * @throws IOException
 */
private void readFromFileAndPersistIfNecessary(String mountPath) throws ReplicationException, IOException {
    logger.info("Reading replica tokens for mount path {}", mountPath);
    long readStartTimeMs = SystemTime.getInstance().milliseconds();
    File replicaTokenFile = new File(mountPath, replicaTokenFileName);
    boolean tokenWasReset = false;
    if (replicaTokenFile.exists()) {
        CrcInputStream crcStream = new CrcInputStream(new FileInputStream(replicaTokenFile));
        DataInputStream stream = new DataInputStream(crcStream);
        try {
            short version = stream.readShort();
            switch(version) {
                case 0:
                    while (stream.available() > Crc_Size) {
                        // read partition id
                        PartitionId partitionId = clusterMap.getPartitionIdFromStream(stream);
                        // read remote node host name
                        String hostname = Utils.readIntString(stream);
                        // read remote replica path
                        String replicaPath = Utils.readIntString(stream);
                        // read remote port
                        int port = stream.readInt();
                        // read total bytes read from local store
                        long totalBytesReadFromLocalStore = stream.readLong();
                        // read replica token
                        FindToken token = factory.getFindToken(stream);
                        // update token
                        PartitionInfo partitionInfo = partitionsToReplicate.get(partitionId);
                        if (partitionInfo != null) {
                            boolean updatedToken = false;
                            for (RemoteReplicaInfo remoteReplicaInfo : partitionInfo.getRemoteReplicaInfos()) {
                                if (remoteReplicaInfo.getReplicaId().getDataNodeId().getHostname().equalsIgnoreCase(hostname) && remoteReplicaInfo.getReplicaId().getDataNodeId().getPort() == port && remoteReplicaInfo.getReplicaId().getReplicaPath().equals(replicaPath)) {
                                    logger.info("Read token for partition {} remote host {} port {} token {}", partitionId, hostname, port, token);
                                    if (partitionInfo.getStore().getSizeInBytes() > 0) {
                                        remoteReplicaInfo.initializeTokens(token);
                                        remoteReplicaInfo.setTotalBytesReadFromLocalStore(totalBytesReadFromLocalStore);
                                    } else {
                                        // if the local replica is empty, it could have been newly created. In this case, the offset in
                                        // every peer replica which the local replica lags from should be set to 0, so that the local
                                        // replica starts fetching from the beginning of the peer. The totalBytes the peer read from the
                                        // local replica should also be set to 0. During initialization these values are already set to 0,
                                        // so we let them be.
                                        tokenWasReset = true;
                                        logTokenReset(partitionId, hostname, port, token);
                                    }
                                    updatedToken = true;
                                    break;
                                }
                            }
                            if (!updatedToken) {
                                logger.warn("Persisted remote replica host {} and port {} not present in new cluster ", hostname, port);
                            }
                        } else {
                            // If this partition was not found in partitionsToReplicate, it means that the local store corresponding
                            // to this partition could not be started. In such a case, the tokens for its remote replicas should be
                            // reset.
                            tokenWasReset = true;
                            logTokenReset(partitionId, hostname, port, token);
                        }
                    }
                    long crc = crcStream.getValue();
                    if (crc != stream.readLong()) {
                        throw new ReplicationException("Crc check does not match for replica token file for mount path " + mountPath);
                    }
                    break;
                default:
                    throw new ReplicationException("Invalid version in replica token file for mount path " + mountPath);
            }
        } catch (IOException e) {
            throw new ReplicationException("IO error while reading from replica token file " + e);
        } finally {
            stream.close();
            replicationMetrics.remoteReplicaTokensRestoreTime.update(SystemTime.getInstance().milliseconds() - readStartTimeMs);
        }
    }
    if (tokenWasReset) {
        // We must ensure that the the token file is persisted if any of the tokens in the file got reset. We need to do
        // this before an associated store takes any writes, to avoid the case where a store takes writes and persists it,
        // before the replica token file is persisted after the reset.
        persistor.write(mountPath, false);
    }
}

Example 2 with CrcInputStream

use of com.github.ambry.utils.CrcInputStream in project ambry by linkedin.

the class PutMessageFormatInputStream method createStreamWithMessageHeaderV2.

/**
 * Helper method to create a stream with encryption key record. This will be the standard once all nodes in a cluster
 * understand reading messages with encryption key record.
 */
private void createStreamWithMessageHeaderV2(StoreKey key, ByteBuffer blobEncryptionKey, BlobProperties blobProperties, ByteBuffer userMetadata, InputStream blobStream, long streamSize, BlobType blobType) throws MessageFormatException {
    int headerSize = MessageFormatRecord.MessageHeader_Format_V2.getHeaderSize();
    int blobEncryptionKeySize = blobEncryptionKey == null ? 0 : MessageFormatRecord.BlobEncryptionKey_Format_V1.getBlobEncryptionKeyRecordSize(blobEncryptionKey);
    int blobPropertiesRecordSize = MessageFormatRecord.BlobProperties_Format_V1.getBlobPropertiesRecordSize(blobProperties);
    int userMetadataSize = MessageFormatRecord.UserMetadata_Format_V1.getUserMetadataSize(userMetadata);
    long blobSize = MessageFormatRecord.Blob_Format_V2.getBlobRecordSize(streamSize);
    buffer = ByteBuffer.allocate(headerSize + key.sizeInBytes() + blobEncryptionKeySize + blobPropertiesRecordSize + userMetadataSize + (int) (blobSize - streamSize - MessageFormatRecord.Crc_Size));
    long totalSize = blobEncryptionKeySize + blobPropertiesRecordSize + userMetadataSize + blobSize;
    int blobEncryptionKeyRecordRelativeOffset = blobEncryptionKey == null ? MessageFormatRecord.Message_Header_Invalid_Relative_Offset : headerSize + key.sizeInBytes();
    int blobPropertiesRecordRelativeOffset = blobEncryptionKey == null ? headerSize + key.sizeInBytes() : blobEncryptionKeyRecordRelativeOffset + blobEncryptionKeySize;
    int deleteRecordRelativeOffset = MessageFormatRecord.Message_Header_Invalid_Relative_Offset;
    int userMetadataRecordRelativeOffset = blobPropertiesRecordRelativeOffset + blobPropertiesRecordSize;
    int blobRecordRelativeOffset = userMetadataRecordRelativeOffset + userMetadataSize;
    MessageFormatRecord.MessageHeader_Format_V2.serializeHeader(buffer, totalSize, blobEncryptionKeyRecordRelativeOffset, blobPropertiesRecordRelativeOffset, deleteRecordRelativeOffset, userMetadataRecordRelativeOffset, blobRecordRelativeOffset);
    buffer.put(key.toBytes());
    if (blobEncryptionKey != null) {
        MessageFormatRecord.BlobEncryptionKey_Format_V1.serializeBlobEncryptionKeyRecord(buffer, blobEncryptionKey);
    }
    MessageFormatRecord.BlobProperties_Format_V1.serializeBlobPropertiesRecord(buffer, blobProperties);
    MessageFormatRecord.UserMetadata_Format_V1.serializeUserMetadataRecord(buffer, userMetadata);
    int bufferBlobStart = buffer.position();
    MessageFormatRecord.Blob_Format_V2.serializePartialBlobRecord(buffer, streamSize, blobType);
    Crc32 crc = new Crc32();
    crc.update(buffer.array(), bufferBlobStart, buffer.position() - bufferBlobStart);
    stream = new CrcInputStream(crc, blobStream);
    streamLength = streamSize;
    messageLength = buffer.capacity() + streamLength + MessageFormatRecord.Crc_Size;
    buffer.flip();
}

Example 3 with CrcInputStream

use of com.github.ambry.utils.CrcInputStream in project ambry by linkedin.

the class ServerTestUtil method checkReplicaTokens.

/**
 * Repeatedly check the replication token file until a certain offset value on all nodes on a certain
 * partition is found.  Fail if {@code numTries} is exceeded or a token offset larger than the target
 * is found.
 * @param clusterMap the cluster map that contains the data node to inspect
 * @param dataNodeId the data node to inspect
 * @param targetOffset the token offset to look for in the {@code targetPartition}
 * @param targetPartition the name of the partition to look for the {@code targetOffset}
 * @throws Exception
 */
private static void checkReplicaTokens(MockClusterMap clusterMap, DataNodeId dataNodeId, long targetOffset, String targetPartition) throws Exception {
    List<String> mountPaths = ((MockDataNodeId) dataNodeId).getMountPaths();
    // we should have an entry for each partition - remote replica pair
    Set<String> completeSetToCheck = new HashSet<>();
    List<ReplicaId> replicaIds = clusterMap.getReplicaIds(dataNodeId);
    int numRemoteNodes = 0;
    for (ReplicaId replicaId : replicaIds) {
        List<? extends ReplicaId> peerReplicas = replicaId.getPeerReplicaIds();
        if (replicaId.getPartitionId().isEqual(targetPartition)) {
            numRemoteNodes = peerReplicas.size();
        }
        for (ReplicaId peerReplica : peerReplicas) {
            completeSetToCheck.add(replicaId.getPartitionId().toString() + peerReplica.getDataNodeId().getHostname() + peerReplica.getDataNodeId().getPort());
        }
    }
    StoreKeyFactory storeKeyFactory = Utils.getObj("com.github.ambry.commons.BlobIdFactory", clusterMap);
    FindTokenFactory factory = Utils.getObj("com.github.ambry.store.StoreFindTokenFactory", storeKeyFactory);
    int numTries = 4;
    boolean foundTarget = false;
    while (!foundTarget && numTries > 0) {
        Thread.sleep(5000);
        numTries--;
        Set<String> setToCheck = new HashSet<String>(completeSetToCheck);
        int numFound = 0;
        for (String mountPath : mountPaths) {
            File replicaTokenFile = new File(mountPath, "replicaTokens");
            if (replicaTokenFile.exists()) {
                CrcInputStream crcStream = new CrcInputStream(new FileInputStream(replicaTokenFile));
                DataInputStream dataInputStream = new DataInputStream(crcStream);
                try {
                    short version = dataInputStream.readShort();
                    assertEquals(1, version);
                    while (dataInputStream.available() > 8) {
                        // read partition id
                        PartitionId partitionId = clusterMap.getPartitionIdFromStream(dataInputStream);
                        // read remote node host name
                        String hostname = Utils.readIntString(dataInputStream);
                        // read remote replica path
                        Utils.readIntString(dataInputStream);
                        // read remote port
                        int port = dataInputStream.readInt();
                        assertTrue(setToCheck.contains(partitionId.toString() + hostname + port));
                        setToCheck.remove(partitionId.toString() + hostname + port);
                        // read total bytes read from local store
                        dataInputStream.readLong();
                        // read replica type
                        ReplicaType replicaType = ReplicaType.values()[dataInputStream.readShort()];
                        // read replica token
                        StoreFindToken token = (StoreFindToken) factory.getFindToken(dataInputStream);
                        System.out.println("partitionId " + partitionId + " hostname " + hostname + " port " + port + " token " + token);
                        Offset endTokenOffset = token.getOffset();
                        long parsedToken = endTokenOffset == null ? -1 : endTokenOffset.getOffset();
                        System.out.println("The parsed token is " + parsedToken);
                        if (partitionId.isEqual(targetPartition)) {
                            assertFalse("Parsed offset: " + parsedToken + " must not be larger than target value: " + targetOffset, parsedToken > targetOffset);
                            if (parsedToken == targetOffset) {
                                numFound++;
                            }
                        } else {
                            assertEquals("Tokens should remain at -1 offsets on unmodified partitions", -1, parsedToken);
                        }
                    }
                    long crc = crcStream.getValue();
                    assertEquals(crc, dataInputStream.readLong());
                } catch (IOException e) {
                    fail();
                } finally {
                    dataInputStream.close();
                }
            }
        }
        if (numFound == numRemoteNodes) {
            foundTarget = true;
        }
    }
    if (!foundTarget) {
        fail("Could not find target token offset: " + targetOffset);
    }
}

Example 4 with CrcInputStream

use of com.github.ambry.utils.CrcInputStream in project ambry by linkedin.

the class MockIndexSegment method loadBloomFileFailureTest.

/**
 * Test failure on loading bloom filter file and verify that bloom filter is rebuilt.
 * @throws Exception
 */
@Test
public void loadBloomFileFailureTest() throws Exception {
    assumeTrue(formatVersion > PersistentIndex.VERSION_1);
    LogSegmentName logSegmentName1 = LogSegmentName.fromPositionAndGeneration(0, 0);
    int indexValueSize = PersistentIndex.CURRENT_VERSION >= PersistentIndex.VERSION_3 ? IndexValue.INDEX_VALUE_SIZE_IN_BYTES_V3_V4 : IndexValue.INDEX_VALUE_SIZE_IN_BYTES_V1_V2;
    IndexSegment indexSegment1 = new IndexSegment(tempDir.getAbsolutePath(), new Offset(logSegmentName1, 0), STORE_KEY_FACTORY, KEY_SIZE + indexValueSize, indexValueSize, config, metrics, time);
    Random random = new Random();
    short accountId1 = getRandomShort(random);
    short containerId1 = getRandomShort(random);
    MockId id1 = new MockId(TestUtils.getRandomString(CUSTOM_ID_SIZE), accountId1, containerId1);
    IndexValue value1 = IndexValueTest.getIndexValue(1000, new Offset(logSegmentName1, 0), Utils.Infinite_Time, time.milliseconds(), accountId1, containerId1, (short) 0, formatVersion);
    indexSegment1.addEntry(new IndexEntry(id1, value1), new Offset(logSegmentName1, 1000));
    indexSegment1.writeIndexSegmentToFile(new Offset(logSegmentName1, 1000));
    indexSegment1.seal();
    File bloomFile = new File(tempDir, generateIndexSegmentFilenamePrefix(new Offset(logSegmentName1, 0)) + BLOOM_FILE_NAME_SUFFIX);
    assertTrue("The bloom file should exist", bloomFile.exists());
    CrcInputStream crcBloom = new CrcInputStream(new FileInputStream(bloomFile));
    IFilter bloomFilter;
    try (DataInputStream stream = new DataInputStream(crcBloom)) {
        bloomFilter = FilterFactory.deserialize(stream, config.storeBloomFilterMaximumPageCount);
        long crcValue = crcBloom.getValue();
        assertEquals("Crc mismatch", crcValue, stream.readLong());
    }
    // induce crc mismatch during serialization
    CrcOutputStream crcStream = new CrcOutputStream(new FileOutputStream(bloomFile));
    DataOutputStream stream = new DataOutputStream(crcStream);
    FilterFactory.serialize(bloomFilter, stream);
    long crcValue = crcStream.getValue() + 1;
    stream.writeLong(crcValue);
    stream.close();
    // load from file, which should trigger bloom filter rebuild as crc value doesn't match
    new IndexSegment(indexSegment1.getFile(), true, STORE_KEY_FACTORY, config, metrics, null, time);
    assertEquals("Bloom filter rebuild count mismatch", 1, metrics.bloomRebuildOnLoadFailureCount.getCount());
}

Example 5 with CrcInputStream

use of com.github.ambry.utils.CrcInputStream in project ambry by linkedin.

the class DeserializedBlob method deserializeUpdateRecord.

public static UpdateRecord deserializeUpdateRecord(InputStream stream) throws IOException, MessageFormatException {
    CrcInputStream crcStream = new CrcInputStream(stream);
    DataInputStream inputStream = new DataInputStream(crcStream);
    short version = inputStream.readShort();
    switch(version) {
        case Update_Version_V1:
            return Update_Format_V1.deserialize(crcStream);
        case Update_Version_V2:
            return Update_Format_V2.deserialize(crcStream);
        case Update_Version_V3:
            return Update_Format_V3.deserialize(crcStream);
        default:
            throw new MessageFormatException("update record version not supported: " + version, MessageFormatErrorCodes.Unknown_Format_Version);
    }
}