Examples with CrcOutputStream com.github.ambry.utils.CrcOutputStream used on opensource projects

Example 6 with CrcOutputStream

use of com.github.ambry.utils.CrcOutputStream in project ambry by linkedin.

the class IndexSegment method persistBloomFilter.

/**
 * Persist the bloom filter.
 * @throws StoreException
 */
private void persistBloomFilter() throws StoreException {
    try {
        CrcOutputStream crcStream = new CrcOutputStream(new FileOutputStream(bloomFile));
        DataOutputStream stream = new DataOutputStream(crcStream);
        FilterFactory.serialize(bloomFilter, stream);
        long crcValue = crcStream.getValue();
        stream.writeLong(crcValue);
        stream.close();
        if (config.storeSetFilePermissionEnabled) {
            Files.setPosixFilePermissions(bloomFile.toPath(), config.storeDataFilePermission);
        }
    } catch (IOException e) {
        metrics.bloomPersistFailureCount.inc();
        StoreErrorCodes errorCode = StoreException.resolveErrorCode(e);
        throw new StoreException(errorCode.toString() + " while trying to persist bloom filter", e, errorCode);
    }
}

Example 7 with CrcOutputStream

use of com.github.ambry.utils.CrcOutputStream in project ambry by linkedin.

the class HardDeleter method persistCleanupToken.

private void persistCleanupToken() throws IOException, StoreException {
    /* The cleanup token format is as follows:
           --
           token_version

           Cleanup_Token_Version_V0
            startTokenForRecovery
            endTokenForRecovery
            numBlobsInRange
            --
            blob1_blobReadOptions {version, offset, sz, ttl, key}
            blob2_blobReadOptions
            ....
            blobN_blobReadOptions
            --
            length_of_blob1_messageStoreRecoveryInfo
            blob1_messageStoreRecoveryInfo {headerVersion, userMetadataVersion, userMetadataSize, blobRecordVersion, blobStreamSize}
            length_of_blob2_messageStoreRecoveryInfo
            blob2_messageStoreRecoveryInfo
            ....
            length_of_blobN_messageStoreRecoveryInfo
            blobN_messageStoreRecoveryInfo

           Cleanup_Token_Version_V1
            startTokenForRecovery
            endTokenForRecovery
            pause flag
            numBlobsInRange
            --
            blob1_blobReadOptions {version, offset, sz, ttl, key}
            blob2_blobReadOptions
            ....
            blobN_blobReadOptions
            --
            length_of_blob1_messageStoreRecoveryInfo
            blob1_messageStoreRecoveryInfo {headerVersion, userMetadataVersion, userMetadataSize, blobRecordVersion, blobStreamSize}
            length_of_blob2_messageStoreRecoveryInfo
            blob2_messageStoreRecoveryInfo
            ....
            length_of_blobN_messageStoreRecoveryInfo
            blobN_messageStoreRecoveryInfo

           --
           crc
           ---
         */
    if (endToken == null) {
        return;
    }
    final Timer.Context context = metrics.cleanupTokenFlushTime.time();
    File tempFile = new File(dataDir, Cleanup_Token_Filename + ".tmp");
    File actual = new File(dataDir, Cleanup_Token_Filename);
    FileOutputStream fileStream = new FileOutputStream(tempFile);
    CrcOutputStream crc = new CrcOutputStream(fileStream);
    DataOutputStream writer = new DataOutputStream(crc);
    try {
        // write the current version
        writer.writeShort(Cleanup_Token_Version_V1);
        writer.write(startTokenSafeToPersist.toBytes());
        writer.write(endToken.toBytes());
        writer.writeByte(isPaused() ? (byte) 1 : (byte) 0);
        writer.write(hardDeleteRecoveryRange.toBytes());
        long crcValue = crc.getValue();
        writer.writeLong(crcValue);
        fileStream.getChannel().force(true);
        tempFile.renameTo(actual);
        if (config.storeSetFilePermissionEnabled) {
            Files.setPosixFilePermissions(actual.toPath(), config.storeDataFilePermission);
        }
    } catch (IOException e) {
        StoreErrorCodes errorCode = StoreException.resolveErrorCode(e);
        throw new StoreException(errorCode.toString() + " while persisting cleanup tokens to disk " + tempFile.getAbsoluteFile(), errorCode);
    } finally {
        writer.close();
        context.stop();
    }
    logger.debug("Completed writing cleanup tokens to file {}", actual.getAbsolutePath());
}

Example 8 with CrcOutputStream

use of com.github.ambry.utils.CrcOutputStream in project ambry by linkedin.

the class IndexSegment method writeIndexSegmentToFile.

/**
 * Writes the index to a persistent file.
 *
 * Those that are written in version 4 has the following format:
 *
 *  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 * | version | entrysize | valuesize | fileendpointer |  last modified time(in secs) | Reset key | Reset key type | Reset key version ...
 * |(2 bytes)|(4 bytes)  | (4 bytes) |    (8 bytes)   |     (4 bytes)                | (m bytes) |   ( 2 bytes)   |  (2 bytes)        ...
 *  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 *         - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 *     ...   key 1    | value 1          | Padding    | ...  | key n     | value n           | Padding    | crc      |
 *     ...   (m bytes)| (valuesize bytes)| (var size) |      | (p bytes) | (valuesize bytes) | (var size) | (8 bytes)|
 *         - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 *          |<- - - - - (entrysize bytes) - - - - - ->|      |<- - - - - - (entrysize bytes)- - - - - - ->|
 *
 *  version            - the index format version
 *  entrysize          - the total size of an entry (key + value + padding) in this index segment
 *  valuesize          - the size of the value in this index segment
 *  fileendpointer     - the log end pointer that pertains to the index being persisted
 *  last modified time - the last modified time of the index segment in secs
 *  reset key          - the reset key(StoreKey) of the index segment
 *  reset key type     - the reset key index entry type(PUT/DELETE/TTLUpdate/UNDELETE)
 *  reset key version  - the life version of reset key
 *  key n / value n    - the key and value entries contained in this index segment
 *  crc                - the crc of the index segment content
 *
 * Those that are written in version 2 and 3 have the following format:
 *
 *  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 * | version | entrysize | valuesize | fileendpointer |  last modified time(in secs) | Reset key | Reset key type  ...
 * |(2 bytes)|(4 bytes)  | (4 bytes) |    (8 bytes)   |     (4 bytes)                | (m bytes) |   ( 2 bytes)    ...
 *  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 *         - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 *     ...   key 1    | value 1          | Padding    | ...  | key n     | value n           | Padding    | crc      |
 *     ...   (m bytes)| (valuesize bytes)| (var size) |      | (p bytes) | (valuesize bytes) | (var size) | (8 bytes)|
 *         - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 *          |<- - - - - (entrysize bytes) - - - - - ->|      |<- - - - - - (entrysize bytes)- - - - - - ->|
 *
 *  version            - the index format version
 *  entrysize          - the total size of an entry (key + value + padding) in this index segment
 *  valuesize          - the size of the value in this index segment
 *  fileendpointer     - the log end pointer that pertains to the index being persisted
 *  last modified time - the last modified time of the index segment in secs
 *  reset key          - the reset key(StoreKey) of the index segment
 *  reset key type     - the reset key index entry type(PUT/DELETE/TTLUpdate/UNDELETE)
 *  key n / value n    - the key and value entries contained in this index segment
 *  crc                - the crc of the index segment content
 *
 * Those that were written in version 1 have the following format
 *  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 * | version | keysize | valuesize | fileendpointer |  last modified time(in secs) | Reset key | Reset key type
 * |(2 bytes)|(4 bytes)| (4 bytes) |    (8 bytes)   |     (4 bytes)                | (m bytes) |   ( 2 bytes)
 *  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 *                      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 *                  ...  key 1          | value 1          |  ...  |   key n         | value n           | crc       |
 *                  ...  (keysize bytes)| (valuesize bytes)|       | (keysize bytes) | (valuesize bytes) | (8 bytes) |
 *                      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 *  version            - the index format version
 *  keysize            - the size of the key in this index segment
 *  valuesize          - the size of the value in this index segment
 *  fileendpointer     - the log end pointer that pertains to the index being persisted
 *  last modified time - the last modified time of the index segment in secs
 *  reset key          - the reset key(StoreKey) of the index segment
 *  reset key type     - the reset key index entry type(PUT/DELETE)
 *  key n / value n    - the key and value entries contained in this index segment
 *  crc                - the crc of the index segment content
 *
 * Those that were written in version 0 have the following format
 *  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 * | version | keysize | valuesize | fileendpointer |   key 1        | value 1          |  ...  |   key n          ...
 * |(2 bytes)|(4 bytes)| (4 bytes) |    (8 bytes)   | (keysize bytes)| (valuesize bytes)|       | (keysize bytes)  ...
 *  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 *                                                          - - - - - - - - - - - - - - - -
 *                                                      ...  value n            | crc      |
 *                                                      ...  (valuesize  bytes) | (8 bytes)|
 *                                                          - - - - - - - - - - - - - - - -
 *
 *  version         - the index format version
 *  keysize         - the size of the key in this index segment
 *  valuesize       - the size of the value in this index segment
 *  fileendpointer  - the log end pointer that pertains to the index being persisted
 *  key n / value n - the key and value entries contained in this index segment
 *  crc             - the crc of the index segment content
 *
 * @param safeEndPoint the end point (that is relevant to this segment) until which the log has been flushed.
 * @throws FileNotFoundException
 * @throws StoreException
 */
void writeIndexSegmentToFile(Offset safeEndPoint) throws FileNotFoundException, StoreException {
    if (sealed.get()) {
        logger.trace("Cannot persist sealed index segment : {}", indexFile.getAbsolutePath());
        return;
    }
    if (safeEndPoint.compareTo(startOffset) <= 0) {
        return;
    }
    if (!safeEndPoint.equals(prevSafeEndPoint)) {
        if (safeEndPoint.compareTo(getEndOffset()) > 0) {
            throw new StoreException("SafeEndOffSet " + safeEndPoint + " is greater than current end offset for current " + "index segment " + getEndOffset(), StoreErrorCodes.Illegal_Index_Operation);
        }
        File temp = new File(getFile().getAbsolutePath() + ".tmp");
        FileOutputStream fileStream = new FileOutputStream(temp);
        CrcOutputStream crc = new CrcOutputStream(fileStream);
        rwLock.readLock().lock();
        try (DataOutputStream writer = new DataOutputStream(crc)) {
            writer.writeShort(getVersion());
            if (getVersion() >= PersistentIndex.VERSION_2) {
                writer.writeInt(getPersistedEntrySize());
            } else {
                // write the key size
                writer.writeInt(getPersistedEntrySize() - getValueSize());
            }
            writer.writeInt(getValueSize());
            writer.writeLong(safeEndPoint.getOffset());
            if (getVersion() != PersistentIndex.VERSION_0) {
                // write last modified time and reset key in case of version != 0
                writer.writeLong(lastModifiedTimeSec.get());
                writer.write(resetKeyInfo.getResetKey().toBytes());
                writer.writeShort(resetKeyInfo.getResetKeyType().ordinal());
            }
            if (getVersion() >= PersistentIndex.VERSION_4) {
                // write reset key life version
                writer.writeShort(resetKeyInfo.getResetKeyLifeVersion());
            }
            byte[] maxPaddingBytes = null;
            if (getVersion() >= PersistentIndex.VERSION_2) {
                maxPaddingBytes = new byte[persistedEntrySize - valueSize];
            }
            if (index != null) {
                for (Map.Entry<StoreKey, ConcurrentSkipListSet<IndexValue>> entry : index.entrySet()) {
                    for (IndexValue value : entry.getValue()) {
                        if (value.getOffset().getOffset() + value.getSize() <= safeEndPoint.getOffset()) {
                            writer.write(entry.getKey().toBytes());
                            writer.write(value.getBytes().array());
                            if (getVersion() >= PersistentIndex.VERSION_2) {
                                // Add padding if necessary
                                writer.write(maxPaddingBytes, 0, persistedEntrySize - (entry.getKey().sizeInBytes() + valueSize));
                            }
                            logger.trace("IndexSegment : {} writing key - {} value - offset {} size {} fileEndOffset {}", getFile().getAbsolutePath(), entry.getKey(), value.getOffset(), value.getSize(), safeEndPoint);
                        }
                    }
                }
            }
            prevSafeEndPoint = safeEndPoint;
            long crcValue = crc.getValue();
            writer.writeLong(crcValue);
            // flush and overwrite old file
            fileStream.getChannel().force(true);
            // swap temp file with the original file
            temp.renameTo(getFile());
            if (config.storeSetFilePermissionEnabled) {
                Files.setPosixFilePermissions(getFile().toPath(), config.storeDataFilePermission);
            }
        } catch (IOException e) {
            StoreErrorCodes errorCode = StoreException.resolveErrorCode(e);
            throw new StoreException("IndexSegment : " + indexFile.getAbsolutePath() + " encountered " + errorCode.toString() + " while persisting index to disk", e, errorCode);
        } finally {
            rwLock.readLock().unlock();
        }
        logger.trace("IndexSegment : {} completed writing index to file", indexFile.getAbsolutePath());
    }
}

Example 9 with CrcOutputStream

use of com.github.ambry.utils.CrcOutputStream in project ambry by linkedin.

the class CompactionLog method flush.

/**
 * Flushes all changes to the file backing this compaction log.
 */
private void flush() {
    /*
        Description of serialized format
        Version 0:
          version
          startTime
          index of current cycle's log
          size of cycle log list
          cycleLog1 (see CycleLog#toBytes())
          cycleLog2
          ...
          crc
         Version 1:
          version
          startTime
          byte to indicate whether startOffsetOfLastIndexSegmentForDeleteCheck is present (1) or not (0)
          startOffsetOfLastIndexSegmentForDeleteCheck if not null
          index of current cycle's log
          size of cycle log list
          cycleLog1 (see CycleLog#toBytes())
          cycleLog2
          ...
          crc
       */
    File tempFile = new File(file.getAbsolutePath() + ".tmp");
    try (FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) {
        CrcOutputStream crcOutputStream = new CrcOutputStream(fileOutputStream);
        DataOutputStream stream = new DataOutputStream(crcOutputStream);
        stream.writeShort(CURRENT_VERSION);
        stream.writeLong(startTime);
        if (startOffsetOfLastIndexSegmentForDeleteCheck == null) {
            stream.writeByte(0);
        } else {
            stream.writeByte(1);
            stream.write(startOffsetOfLastIndexSegmentForDeleteCheck.toBytes());
        }
        stream.writeInt(currentIdx);
        stream.writeInt(cycleLogs.size());
        for (CycleLog cycleLog : cycleLogs) {
            stream.write(cycleLog.toBytes());
        }
        stream.writeLong(crcOutputStream.getValue());
        fileOutputStream.getChannel().force(true);
    } catch (IOException e) {
        throw new IllegalStateException(e);
    }
    if (!tempFile.renameTo(file)) {
        throw new IllegalStateException("Newly written compaction log could not be saved");
    }
}

Example 10 with CrcOutputStream

use of com.github.ambry.utils.CrcOutputStream in project ambry by linkedin.

the class StoreDescriptorTest method createStoreFile.

/**
 * Creates a new StoreDescriptor file with the given byte array as content
 * @param storeDescriptorFile the store descriptor file
 * @param toBytes content that needs to go into the file
 * @throws IOException
 */
private void createStoreFile(File storeDescriptorFile, byte[] toBytes) throws IOException {
    CrcOutputStream crcOutputStream = new CrcOutputStream(new FileOutputStream(storeDescriptorFile));
    DataOutputStream dataOutputStream = new DataOutputStream(crcOutputStream);
    dataOutputStream.write(toBytes);
    dataOutputStream.writeLong(crcOutputStream.getValue());
    dataOutputStream.close();
}