Examples with CrcInputStream com.github.ambry.utils.CrcInputStream used on opensource projects

Example 6 with CrcInputStream

use of com.github.ambry.utils.CrcInputStream in project ambry by linkedin.

the class DeserializedBlob method deserializeAndGetUserMetadataWithVersion.

static DeserializedUserMetadata deserializeAndGetUserMetadataWithVersion(InputStream stream) throws IOException, MessageFormatException {
    CrcInputStream crcStream = new CrcInputStream(stream);
    DataInputStream inputStream = new DataInputStream(crcStream);
    short version = inputStream.readShort();
    switch(version) {
        case UserMetadata_Version_V1:
            return new DeserializedUserMetadata(UserMetadata_Version_V1, UserMetadata_Format_V1.deserializeUserMetadataRecord(crcStream));
        default:
            throw new MessageFormatException("metadata version not supported", MessageFormatErrorCodes.Unknown_Format_Version);
    }
}

Example 7 with CrcInputStream

use of com.github.ambry.utils.CrcInputStream in project ambry by linkedin.

the class PutMessageFormatInputStream method createStreamWithMessageHeaderV1.

/**
 * Helper method to create a stream without encryption key record. This is the default currently, but once all nodes
 * once all nodes in a cluster understand reading messages with encryption key record, and writing in the new format
 * is enabled, this method can be removed.
 */
private void createStreamWithMessageHeaderV1(StoreKey key, BlobProperties blobProperties, ByteBuffer userMetadata, InputStream blobStream, long streamSize, BlobType blobType) throws MessageFormatException {
    int headerSize = MessageFormatRecord.MessageHeader_Format_V1.getHeaderSize();
    int blobPropertiesRecordSize = MessageFormatRecord.BlobProperties_Format_V1.getBlobPropertiesRecordSize(blobProperties);
    int userMetadataSize = MessageFormatRecord.UserMetadata_Format_V1.getUserMetadataSize(userMetadata);
    long blobSize = MessageFormatRecord.Blob_Format_V2.getBlobRecordSize(streamSize);
    buffer = ByteBuffer.allocate(headerSize + key.sizeInBytes() + blobPropertiesRecordSize + userMetadataSize + (int) (blobSize - streamSize - MessageFormatRecord.Crc_Size));
    MessageFormatRecord.MessageHeader_Format_V1.serializeHeader(buffer, blobPropertiesRecordSize + userMetadataSize + blobSize, headerSize + key.sizeInBytes(), MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerSize + key.sizeInBytes() + blobPropertiesRecordSize, headerSize + key.sizeInBytes() + blobPropertiesRecordSize + userMetadataSize);
    buffer.put(key.toBytes());
    MessageFormatRecord.BlobProperties_Format_V1.serializeBlobPropertiesRecord(buffer, blobProperties);
    MessageFormatRecord.UserMetadata_Format_V1.serializeUserMetadataRecord(buffer, userMetadata);
    int bufferBlobStart = buffer.position();
    MessageFormatRecord.Blob_Format_V2.serializePartialBlobRecord(buffer, streamSize, blobType);
    Crc32 crc = new Crc32();
    crc.update(buffer.array(), bufferBlobStart, buffer.position() - bufferBlobStart);
    stream = new CrcInputStream(crc, blobStream);
    streamLength = streamSize;
    messageLength = buffer.capacity() + streamLength + MessageFormatRecord.Crc_Size;
    buffer.flip();
}

Example 8 with CrcInputStream

use of com.github.ambry.utils.CrcInputStream in project ambry by linkedin.

the class PutMessageFormatInputStream method createStreamWithMessageHeader.

/**
 * Helper method to create a stream with encryption key record. This will be the standard once all nodes in a cluster
 * understand reading messages with encryption key record.
 */
private void createStreamWithMessageHeader(StoreKey key, ByteBuffer blobEncryptionKey, BlobProperties blobProperties, ByteBuffer userMetadata, InputStream blobStream, long streamSize, BlobType blobType, short lifeVersion) throws MessageFormatException {
    int headerSize = MessageFormatRecord.getHeaderSizeForVersion(MessageFormatRecord.headerVersionToUse);
    int blobEncryptionKeySize = blobEncryptionKey == null ? 0 : MessageFormatRecord.BlobEncryptionKey_Format_V1.getBlobEncryptionKeyRecordSize(blobEncryptionKey);
    int blobPropertiesRecordSize = MessageFormatRecord.BlobProperties_Format_V1.getBlobPropertiesRecordSize(blobProperties);
    int userMetadataSize = MessageFormatRecord.UserMetadata_Format_V1.getUserMetadataSize(userMetadata);
    long blobSize = MessageFormatRecord.Blob_Format_V2.getBlobRecordSize(streamSize);
    buffer = ByteBuffer.allocate(headerSize + key.sizeInBytes() + blobEncryptionKeySize + blobPropertiesRecordSize + userMetadataSize + (int) (blobSize - streamSize - MessageFormatRecord.Crc_Size));
    long totalSize = blobEncryptionKeySize + blobPropertiesRecordSize + userMetadataSize + blobSize;
    int blobEncryptionKeyRecordRelativeOffset = blobEncryptionKey == null ? MessageFormatRecord.Message_Header_Invalid_Relative_Offset : headerSize + key.sizeInBytes();
    int blobPropertiesRecordRelativeOffset = blobEncryptionKey == null ? headerSize + key.sizeInBytes() : blobEncryptionKeyRecordRelativeOffset + blobEncryptionKeySize;
    int updateRecordRelativeOffset = MessageFormatRecord.Message_Header_Invalid_Relative_Offset;
    int userMetadataRecordRelativeOffset = blobPropertiesRecordRelativeOffset + blobPropertiesRecordSize;
    int blobRecordRelativeOffset = userMetadataRecordRelativeOffset + userMetadataSize;
    if (MessageFormatRecord.headerVersionToUse == MessageFormatRecord.Message_Header_Version_V2) {
        MessageFormatRecord.MessageHeader_Format_V2.serializeHeader(buffer, totalSize, blobEncryptionKeyRecordRelativeOffset, blobPropertiesRecordRelativeOffset, updateRecordRelativeOffset, userMetadataRecordRelativeOffset, blobRecordRelativeOffset);
    } else {
        MessageFormatRecord.MessageHeader_Format_V3.serializeHeader(buffer, lifeVersion, totalSize, blobEncryptionKeyRecordRelativeOffset, blobPropertiesRecordRelativeOffset, updateRecordRelativeOffset, userMetadataRecordRelativeOffset, blobRecordRelativeOffset);
    }
    buffer.put(key.toBytes());
    if (blobEncryptionKey != null) {
        MessageFormatRecord.BlobEncryptionKey_Format_V1.serializeBlobEncryptionKeyRecord(buffer, blobEncryptionKey);
    }
    MessageFormatRecord.BlobProperties_Format_V1.serializeBlobPropertiesRecord(buffer, blobProperties);
    MessageFormatRecord.UserMetadata_Format_V1.serializeUserMetadataRecord(buffer, userMetadata);
    int bufferBlobStart = buffer.position();
    MessageFormatRecord.Blob_Format_V2.serializePartialBlobRecord(buffer, streamSize, blobType);
    Crc32 crc = new Crc32();
    crc.update(buffer.array(), bufferBlobStart, buffer.position() - bufferBlobStart);
    stream = new CrcInputStream(crc, blobStream);
    streamLength = streamSize;
    messageLength = buffer.capacity() + streamLength + MessageFormatRecord.Crc_Size;
    buffer.flip();
}

Example 9 with CrcInputStream

use of com.github.ambry.utils.CrcInputStream in project ambry by linkedin.

the class ServerTestUtil method checkReplicaTokens.

/**
 * Repeatedly check the replication token file until a certain offset value on all nodes on a certain
 * partition is found.  Fail if {@code numTries} is exceeded or a token offset larger than the target
 * is found.
 * @param clusterMap the cluster map that contains the data node to inspect
 * @param dataNodeId the data node to inspect
 * @param targetOffset the token offset to look for in the {@code targetPartition}
 * @param targetPartition the name of the partition to look for the {@code targetOffset}
 * @throws Exception
 */
private static void checkReplicaTokens(MockClusterMap clusterMap, DataNodeId dataNodeId, long targetOffset, String targetPartition) throws Exception {
    List<String> mountPaths = ((MockDataNodeId) dataNodeId).getMountPaths();
    // we should have an entry for each partition - remote replica pair
    Set<String> completeSetToCheck = new HashSet<>();
    List<ReplicaId> replicaIds = clusterMap.getReplicaIds(dataNodeId);
    int numRemoteNodes = 0;
    for (ReplicaId replicaId : replicaIds) {
        List<? extends ReplicaId> peerReplicas = replicaId.getPeerReplicaIds();
        if (replicaId.getPartitionId().isEqual(targetPartition)) {
            numRemoteNodes = peerReplicas.size();
        }
        for (ReplicaId peerReplica : peerReplicas) {
            completeSetToCheck.add(replicaId.getPartitionId().toString() + peerReplica.getDataNodeId().getHostname() + peerReplica.getDataNodeId().getPort());
        }
    }
    StoreKeyFactory storeKeyFactory = Utils.getObj("com.github.ambry.commons.BlobIdFactory", clusterMap);
    FindTokenFactory factory = Utils.getObj("com.github.ambry.store.StoreFindTokenFactory", storeKeyFactory);
    int numTries = 4;
    boolean foundTarget = false;
    while (!foundTarget && numTries > 0) {
        Thread.sleep(5000);
        numTries--;
        Set<String> setToCheck = new HashSet<String>(completeSetToCheck);
        int numFound = 0;
        for (String mountPath : mountPaths) {
            File replicaTokenFile = new File(mountPath, "replicaTokens");
            if (replicaTokenFile.exists()) {
                CrcInputStream crcStream = new CrcInputStream(new FileInputStream(replicaTokenFile));
                DataInputStream dataInputStream = new DataInputStream(crcStream);
                try {
                    short version = dataInputStream.readShort();
                    assertEquals(1, version);
                    while (dataInputStream.available() > 8) {
                        // read partition id
                        PartitionId partitionId = clusterMap.getPartitionIdFromStream(dataInputStream);
                        // read remote node host name
                        String hostname = Utils.readIntString(dataInputStream);
                        // read remote replica path
                        Utils.readIntString(dataInputStream);
                        // read remote port
                        int port = dataInputStream.readInt();
                        assertTrue(setToCheck.contains(partitionId.toString() + hostname + port));
                        setToCheck.remove(partitionId.toString() + hostname + port);
                        // read total bytes read from local store
                        dataInputStream.readLong();
                        // read replica type
                        ReplicaType replicaType = ReplicaType.values()[dataInputStream.readShort()];
                        // read replica token
                        StoreFindToken token = (StoreFindToken) factory.getFindToken(dataInputStream);
                        System.out.println("partitionId " + partitionId + " hostname " + hostname + " port " + port + " token " + token);
                        Offset endTokenOffset = token.getOffset();
                        long parsedToken = endTokenOffset == null ? -1 : endTokenOffset.getOffset();
                        System.out.println("The parsed token is " + parsedToken);
                        if (partitionId.isEqual(targetPartition)) {
                            assertFalse("Parsed offset: " + parsedToken + " must not be larger than target value: " + targetOffset, parsedToken > targetOffset);
                            if (parsedToken == targetOffset) {
                                numFound++;
                            }
                        } else {
                            assertEquals("Tokens should remain at -1 offsets on unmodified partitions", -1, parsedToken);
                        }
                    }
                    long crc = crcStream.getValue();
                    assertEquals(crc, dataInputStream.readLong());
                } catch (IOException e) {
                    fail();
                } finally {
                    dataInputStream.close();
                }
            }
        }
        if (numFound == numRemoteNodes) {
            foundTarget = true;
        }
    }
    if (!foundTarget) {
        fail("Could not find target token offset: " + targetOffset);
    }
}

Example 10 with CrcInputStream

use of com.github.ambry.utils.CrcInputStream in project ambry by linkedin.

the class IndexSegment method readFromFile.

/**
 * Reads the index segment from file into an in memory representation
 * @param fileToRead The file to read the index segment from
 * @param journal The journal to use.
 * @throws StoreException
 * @throws FileNotFoundException
 */
private void readFromFile(File fileToRead, Journal journal) throws StoreException, FileNotFoundException {
    logger.info("IndexSegment : {} reading index from file", indexFile.getAbsolutePath());
    index.clear();
    CrcInputStream crcStream = new CrcInputStream(new FileInputStream(fileToRead));
    try (DataInputStream stream = new DataInputStream(crcStream)) {
        setVersion(stream.readShort());
        switch(getVersion()) {
            case PersistentIndex.VERSION_0:
            case PersistentIndex.VERSION_1:
                int keySize = stream.readInt();
                valueSize = stream.readInt();
                persistedEntrySize = keySize + valueSize;
                indexSizeExcludingEntries = VERSION_FIELD_LENGTH + KEY_OR_ENTRY_SIZE_FIELD_LENGTH + VALUE_SIZE_FIELD_LENGTH + LOG_END_OFFSET_FIELD_LENGTH + CRC_FIELD_LENGTH;
                break;
            case PersistentIndex.VERSION_2:
            case PersistentIndex.VERSION_3:
            case PersistentIndex.VERSION_4:
                persistedEntrySize = stream.readInt();
                valueSize = stream.readInt();
                indexSizeExcludingEntries = VERSION_FIELD_LENGTH + KEY_OR_ENTRY_SIZE_FIELD_LENGTH + VALUE_SIZE_FIELD_LENGTH + LOG_END_OFFSET_FIELD_LENGTH + CRC_FIELD_LENGTH;
                break;
            default:
                throw new StoreException("IndexSegment : " + indexFile.getAbsolutePath() + " invalid version in index file ", StoreErrorCodes.Index_Version_Error);
        }
        long logEndOffset = stream.readLong();
        if (getVersion() == PersistentIndex.VERSION_0) {
            lastModifiedTimeSec.set(indexFile.lastModified() / 1000);
        } else {
            lastModifiedTimeSec.set(stream.readLong());
            StoreKey storeKey = factory.getStoreKey(stream);
            PersistentIndex.IndexEntryType resetKeyType = PersistentIndex.IndexEntryType.values()[stream.readShort()];
            short resetKeyLifeVersion = UNINITIALIZED_RESET_KEY_VERSION;
            indexSizeExcludingEntries += LAST_MODIFIED_TIME_FIELD_LENGTH + storeKey.sizeInBytes() + RESET_KEY_TYPE_FIELD_LENGTH;
            if (getVersion() >= PersistentIndex.VERSION_4) {
                resetKeyLifeVersion = stream.readShort();
                indexSizeExcludingEntries += RESET_KEY_VERSION_FIELD_LENGTH;
            }
            resetKeyInfo = new ResetKeyInfo(storeKey, resetKeyType, resetKeyLifeVersion);
        }
        firstKeyRelativeOffset = indexSizeExcludingEntries - CRC_FIELD_LENGTH;
        logger.trace("IndexSegment : {} reading log end offset {} from file", indexFile.getAbsolutePath(), logEndOffset);
        long maxEndOffset = Long.MIN_VALUE;
        byte[] padding = new byte[persistedEntrySize - valueSize];
        while (stream.available() > CRC_FIELD_LENGTH) {
            StoreKey key = factory.getStoreKey(stream);
            byte[] value = new byte[valueSize];
            stream.readFully(value);
            if (getVersion() >= PersistentIndex.VERSION_2) {
                stream.readFully(padding, 0, persistedEntrySize - (key.sizeInBytes() + valueSize));
            }
            IndexValue blobValue = new IndexValue(startOffset.getName(), ByteBuffer.wrap(value), getVersion());
            long offsetInLogSegment = blobValue.getOffset().getOffset();
            // ignore entries that have offsets outside the log end offset that this index represents
            if (offsetInLogSegment + blobValue.getSize() <= logEndOffset) {
                boolean isPresent = index.containsKey(key);
                index.computeIfAbsent(key, k -> new ConcurrentSkipListSet<>()).add(blobValue);
                logger.trace("IndexSegment : {} putting key {} in index offset {} size {}", indexFile.getAbsolutePath(), key, blobValue.getOffset(), blobValue.getSize());
                // regenerate the bloom filter for index segments that are not sealed
                if (!isPresent) {
                    bloomFilter.add(getStoreKeyBytes(key));
                }
                // add to the journal
                long oMsgOff = blobValue.getOriginalMessageOffset();
                if (oMsgOff != IndexValue.UNKNOWN_ORIGINAL_MESSAGE_OFFSET && offsetInLogSegment != oMsgOff && oMsgOff >= startOffset.getOffset() && journal.getKeyAtOffset(new Offset(startOffset.getName(), oMsgOff)) == null) {
                    // we add an entry for the original message offset if it is within the same index segment and
                    // an entry is not already in the journal
                    journal.addEntry(new Offset(startOffset.getName(), blobValue.getOriginalMessageOffset()), key);
                }
                journal.addEntry(blobValue.getOffset(), key);
                // sizeWritten is only used for in-memory segments, and for those the padding does not come into picture.
                sizeWritten.addAndGet(key.sizeInBytes() + valueSize);
                numberOfItems.incrementAndGet();
                if (offsetInLogSegment + blobValue.getSize() > maxEndOffset) {
                    maxEndOffset = offsetInLogSegment + blobValue.getSize();
                }
            } else {
                logger.info("IndexSegment : {} ignoring index entry outside the log end offset that was not synced logEndOffset " + "{} key {} entryOffset {} entrySize {} entryDeleteState {}", indexFile.getAbsolutePath(), logEndOffset, key, blobValue.getOffset(), blobValue.getSize(), blobValue.isDelete());
            }
        }
        endOffset.set(new Offset(startOffset.getName(), maxEndOffset));
        logger.trace("IndexSegment : {} setting end offset for index {}", indexFile.getAbsolutePath(), maxEndOffset);
        long crc = crcStream.getValue();
        if (crc != stream.readLong()) {
            // reset structures
            persistedEntrySize = ENTRY_SIZE_INVALID_VALUE;
            valueSize = VALUE_SIZE_INVALID_VALUE;
            endOffset.set(startOffset);
            index.clear();
            bloomFilter.clear();
            throw new StoreException("IndexSegment : " + indexFile.getAbsolutePath() + " crc check does not match", StoreErrorCodes.Index_Creation_Failure);
        }
    } catch (IOException e) {
        StoreErrorCodes errorCode = StoreException.resolveErrorCode(e);
        throw new StoreException("IndexSegment : " + indexFile.getAbsolutePath() + " encountered " + errorCode.toString() + " while reading from file ", e, errorCode);
    }
}