Examples with Metadata org.corfudb.format.Types.Metadata used on opensource projects

Example 1 with Metadata

use of org.corfudb.format.Types.Metadata in project CorfuDB by CorfuDB.

the class StreamLogFilesTest method testStreamLogDataCorruption.

@Test
public void testStreamLogDataCorruption() throws Exception {
    // This test manipulates a log file directly and manipulates
    // log records by overwriting some parts of the record simulating
    // different data corruption scenarios
    String logDir = getContext().getServerConfig().get("--log-path") + File.separator + "log";
    StreamLog log = new StreamLogFiles(getContext(), false);
    ByteBuf b = Unpooled.buffer();
    byte[] streamEntry = "Payload".getBytes();
    Serializers.CORFU.serialize(streamEntry, b);
    // Write to two segments
    long address0 = 0;
    long address1 = StreamLogFiles.RECORDS_PER_LOG_FILE + 1L;
    log.append(address0, new LogData(DataType.DATA, b));
    log.append(address1, new LogData(DataType.DATA, b));
    assertThat(log.read(address0).getPayload(null)).isEqualTo(streamEntry);
    log.close();
    final int OVERWRITE_DELIMITER = 0xFFFF;
    final int OVERWRITE_BYTES = 4;
    // Overwrite 2 bytes of the checksum and 2 bytes of the entry's address
    String logFilePath1 = logDir + File.separator + 0 + ".log";
    String logFilePath2 = logDir + File.separator + 1 + ".log";
    RandomAccessFile file1 = new RandomAccessFile(logFilePath1, "rw");
    RandomAccessFile file2 = new RandomAccessFile(logFilePath2, "rw");
    ByteBuffer metaDataBuf = ByteBuffer.allocate(METADATA_SIZE);
    file1.getChannel().read(metaDataBuf);
    metaDataBuf.flip();
    Metadata metadata = Metadata.parseFrom(metaDataBuf.array());
    final int offset1 = METADATA_SIZE + metadata.getLength();
    final int offset2 = METADATA_SIZE + metadata.getLength() + Short.BYTES + OVERWRITE_BYTES;
    // Corrupt delimiter in the first segment
    file1.seek(offset1);
    file1.writeShort(0);
    file1.close();
    assertThatThrownBy(() -> new StreamLogFiles(getContext(), false).read(0L)).isInstanceOf(DataCorruptionException.class);
    // Corrupt metadata in the second segment
    file2.seek(offset2);
    file2.writeInt(OVERWRITE_DELIMITER);
    file2.close();
    assertThatThrownBy(() -> new StreamLogFiles(getContext(), false)).isInstanceOf(DataCorruptionException.class);
}

Example 2 with Metadata

use of org.corfudb.format.Types.Metadata in project CorfuDB by CorfuDB.

the class StreamLogFiles method verifyLogs.

private void verifyLogs() {
    String[] extension = { "log" };
    File dir = new File(logDir);
    if (dir.exists()) {
        Collection<File> files = FileUtils.listFiles(dir, extension, true);
        for (File file : files) {
            try {
                FileInputStream fIn = new FileInputStream(file);
                FileChannel fc = fIn.getChannel();
                ByteBuffer metadataBuf = ByteBuffer.allocate(METADATA_SIZE);
                fc.read(metadataBuf);
                metadataBuf.flip();
                Metadata metadata = Metadata.parseFrom(metadataBuf.array());
                ByteBuffer headerBuf = ByteBuffer.allocate(metadata.getLength());
                fc.read(headerBuf);
                headerBuf.flip();
                LogHeader header = LogHeader.parseFrom(headerBuf.array());
                fc.close();
                fIn.close();
                if (metadata.getChecksum() != getChecksum(header.toByteArray())) {
                    log.error("Checksum mismatch detected while trying to read header for logfile {}", file);
                    throw new DataCorruptionException();
                }
                if (header.getVersion() != VERSION) {
                    String msg = String.format("Log version {} for {} should match the logunit log version {}", header.getVersion(), file.getAbsoluteFile(), VERSION);
                    throw new RuntimeException(msg);
                }
                if (!noVerify && !header.getVerifyChecksum()) {
                    String msg = String.format("Log file {} not generated with checksums, can't verify!", file.getAbsoluteFile());
                    throw new RuntimeException(msg);
                }
            } catch (IOException e) {
                throw new RuntimeException(e.getMessage(), e);
            }
        }
    }
}

Example 3 with Metadata

use of org.corfudb.format.Types.Metadata in project CorfuDB by CorfuDB.

the class StreamLogFiles method readAddressSpace.

/**
     * Reads an address space from a log file into a SegmentHandle
     *
     * @param sh
     */
private void readAddressSpace(SegmentHandle sh) throws IOException {
    long logFileSize;
    try (MultiReadWriteLock.AutoCloseableLock ignored = segmentLocks.acquireReadLock(sh.getSegment())) {
        logFileSize = sh.logChannel.size();
    }
    FileChannel fc = getChannel(sh.fileName, true);
    if (fc == null) {
        log.trace("Can't read address space, {} doesn't exist", sh.fileName);
        return;
    }
    // Skip the header
    ByteBuffer headerMetadataBuf = ByteBuffer.allocate(METADATA_SIZE);
    fc.read(headerMetadataBuf);
    headerMetadataBuf.flip();
    Metadata headerMetadata = Metadata.parseFrom(headerMetadataBuf.array());
    fc.position(fc.position() + headerMetadata.getLength());
    long channelOffset = fc.position();
    ByteBuffer o = ByteBuffer.allocate((int) logFileSize - (int) fc.position());
    fc.read(o);
    fc.close();
    o.flip();
    while (o.hasRemaining()) {
        short magic = o.getShort();
        channelOffset += Short.BYTES;
        if (magic != RECORD_DELIMITER) {
            log.error("Expected a delimiter but found something else while trying to read file {}", sh.fileName);
            throw new DataCorruptionException();
        }
        byte[] metadataBuf = new byte[METADATA_SIZE];
        o.get(metadataBuf);
        channelOffset += METADATA_SIZE;
        try {
            Metadata metadata = Metadata.parseFrom(metadataBuf);
            byte[] logEntryBuf = new byte[metadata.getLength()];
            o.get(logEntryBuf);
            LogEntry entry = LogEntry.parseFrom(logEntryBuf);
            if (!noVerify) {
                if (metadata.getChecksum() != getChecksum(entry.toByteArray())) {
                    log.error("Checksum mismatch detected while trying to read file {}", sh.fileName);
                    throw new DataCorruptionException();
                }
            }
            sh.knownAddresses.put(entry.getGlobalAddress(), new AddressMetaData(metadata.getChecksum(), metadata.getLength(), channelOffset));
            channelOffset += metadata.getLength();
        } catch (InvalidProtocolBufferException e) {
            throw new DataCorruptionException();
        }
    }
}

Example 4 with Metadata

use of org.corfudb.format.Types.Metadata in project CorfuDB by CorfuDB.

the class logReader method processHeader.

final logHeader processHeader() throws IOException {
    fileChannelIn.position(0);
    ByteBuffer mdBuffer = ByteBuffer.allocate(metadataSize);
    int r = fileChannelIn.read(mdBuffer);
    mdBuffer.flip();
    if (fileChannelOut != null) {
        fileChannelOut.write(mdBuffer);
    }
    if (r > 0) {
        logHeader header = new logHeader();
        Metadata md = Metadata.parseFrom(mdBuffer.array());
        int logHeaderSize = md.getLength();
        header.setChecksum(md.getChecksum());
        header.setLength(md.getLength());
        ByteBuffer lhBuffer = ByteBuffer.allocate(logHeaderSize);
        r = fileChannelIn.read(lhBuffer);
        lhBuffer.flip();
        if (fileChannelOut != null) {
            fileChannelOut.write(lhBuffer);
        }
        if (r > 0) {
            LogHeader lh = LogHeader.parseFrom(lhBuffer.array());
            header.setVersion(lh.getVersion());
            header.setVerifyChecksum(lh.getVerifyChecksum());
        }
        return header;
    }
    return new logHeader();
}

Example 5 with Metadata

use of org.corfudb.format.Types.Metadata in project CorfuDB by CorfuDB.

the class StreamLogFiles method getByteBufferWithMetaData.

private static ByteBuffer getByteBufferWithMetaData(AbstractMessage message) {
    Metadata metadata = getMetadata(message);
    ByteBuffer buf = ByteBuffer.allocate(metadata.getSerializedSize() + message.getSerializedSize());
    buf.put(metadata.toByteArray());
    buf.put(message.toByteArray());
    buf.flip();
    return buf;
}