Examples with Metadata org.corfudb.format.Types.Metadata used on opensource projects

Example 6 with Metadata

use of org.corfudb.format.Types.Metadata in project CorfuDB by CorfuDB.

the class logReader method processRecord.

final LogEntryExtended processRecord() throws IOException {
    ByteBuffer commaBuffer = ByteBuffer.allocate(2);
    int bytesRead = fileChannelIn.read(commaBuffer);
    commaBuffer.flip();
    Short delim = commaBuffer.getShort();
    commaBuffer.flip();
    if (delim != StreamLogFiles.RECORD_DELIMITER) {
        System.out.println("Incorrect delimiter");
    }
    ByteBuffer mdBuffer = ByteBuffer.allocate(metadataSize);
    bytesRead += fileChannelIn.read(mdBuffer);
    mdBuffer.flip();
    Metadata md = Metadata.parseFrom(mdBuffer.array());
    ByteBuffer recordBuffer = ByteBuffer.allocate(md.getLength());
    bytesRead += fileChannelIn.read(recordBuffer);
    recordBuffer.flip();
    int cksum = StreamLogFiles.getChecksum(recordBuffer.array());
    if (cksum != md.getChecksum()) {
        System.out.println("Checksum ERROR");
    }
    LogEntry leNew = processRecordBody(recordBuffer);
    return new LogEntryExtended(leNew, bytesRead, cksum);
}

Example 7 with Metadata

use of org.corfudb.format.Types.Metadata in project CorfuDB by CorfuDB.

the class StreamLogFiles method writeRecord.

/**
     * Write a log entry record to a file.
     *
     * @param fh      The file handle to use.
     * @param address The address of the entry.
     * @param entry   The LogData to append.
     * @return Returns metadata for the written record
     */
private AddressMetaData writeRecord(SegmentHandle fh, long address, LogData entry) throws IOException {
    LogEntry logEntry = getLogEntry(address, entry);
    Metadata metadata = getMetadata(logEntry);
    ByteBuffer record = getByteBuffer(metadata, logEntry);
    ByteBuffer recordBuf = ByteBuffer.allocate(// Delimiter
    Short.BYTES + record.capacity());
    recordBuf.putShort(RECORD_DELIMITER);
    recordBuf.put(record.array());
    recordBuf.flip();
    long channelOffset;
    try (MultiReadWriteLock.AutoCloseableLock ignored = segmentLocks.acquireWriteLock(fh.getSegment())) {
        channelOffset = fh.logChannel.position() + Short.BYTES + METADATA_SIZE;
        fh.logChannel.write(recordBuf);
        channelsToSync.add(fh.logChannel);
        syncTailSegment(address);
    }
    return new AddressMetaData(metadata.getChecksum(), metadata.getLength(), channelOffset);
}

Example 8 with Metadata

use of org.corfudb.format.Types.Metadata in project CorfuDB by CorfuDB.

the class StreamLogFiles method getCompactedEntries.

private CompactedEntry getCompactedEntries(String filePath, Set<Long> pendingTrim) throws IOException {
    FileChannel fc = getChannel(filePath, true);
    // Skip the header
    ByteBuffer headerMetadataBuf = ByteBuffer.allocate(METADATA_SIZE);
    fc.read(headerMetadataBuf);
    headerMetadataBuf.flip();
    Metadata headerMetadata = Metadata.parseFrom(headerMetadataBuf.array());
    ByteBuffer headerBuf = ByteBuffer.allocate(headerMetadata.getLength());
    fc.read(headerBuf);
    headerBuf.flip();
    LogHeader header = LogHeader.parseFrom(headerBuf.array());
    ByteBuffer o = ByteBuffer.allocate((int) fc.size() - (int) fc.position());
    fc.read(o);
    fc.close();
    o.flip();
    LinkedHashMap<Long, LogEntry> compacted = new LinkedHashMap<>();
    while (o.hasRemaining()) {
        //Skip delimiter
        o.getShort();
        byte[] metadataBuf = new byte[METADATA_SIZE];
        o.get(metadataBuf);
        try {
            Metadata metadata = Metadata.parseFrom(metadataBuf);
            byte[] logEntryBuf = new byte[metadata.getLength()];
            o.get(logEntryBuf);
            LogEntry entry = LogEntry.parseFrom(logEntryBuf);
            if (!noVerify) {
                if (metadata.getChecksum() != getChecksum(entry.toByteArray())) {
                    log.error("Checksum mismatch detected while trying to read address {}", entry.getGlobalAddress());
                    throw new DataCorruptionException();
                }
            }
            if (!pendingTrim.contains(entry.getGlobalAddress())) {
                compacted.put(entry.getGlobalAddress(), entry);
            }
        } catch (InvalidProtocolBufferException e) {
            throw new DataCorruptionException();
        }
    }
    return new CompactedEntry(header, compacted.values());
}

Example 9 with Metadata

use of org.corfudb.format.Types.Metadata in project CorfuDB by CorfuDB.

the class logReader method writeBuffer.

final void writeBuffer(final LogEntry le, final FileChannel fcOut) throws IOException {
    byte[] b1 = le.toByteArray();
    int cksum = StreamLogFiles.getChecksum(b1);
    ByteBuffer recordBuffer = ByteBuffer.allocate(b1.length);
    recordBuffer.put(b1);
    recordBuffer.flip();
    Metadata mdNew = Metadata.newBuilder().setLength(b1.length).setChecksum(cksum).build();
    byte[] b2 = mdNew.toByteArray();
    ByteBuffer mdBuff = ByteBuffer.allocate(metadataSize);
    mdBuff.put(b2);
    mdBuff.flip();
    writeRecord(fcOut, mdBuff, recordBuffer);
}

Example 10 with Metadata

use of org.corfudb.format.Types.Metadata in project CorfuDB by CorfuDB.

the class logReader method init.

public final boolean init(final String[] args) {
    Docopt parser = new Docopt(USAGE);
    parser.withExit(false);
    Map<String, Object> opts = parser.parse(args);
    op = new Operation(Operation.OperationType.REPORT);
    String logFileName = new String();
    boolean useOutputFile = false;
    if (opts.get("<log_file>") != null) {
        logFileName = (String) opts.get("<log_file>");
        System.out.println("Log file: " + logFileName);
    } else {
        System.out.print(USAGE);
        return false;
    }
    int startAddr = 0;
    int finalAddr = -1;
    if (opts.get("--from") != null) {
        startAddr = Integer.parseInt((String) opts.get("--from"));
    }
    if (opts.get("--to") != null) {
        finalAddr = Integer.parseInt((String) opts.get("--to"));
    }
    if ((Boolean) opts.get("display")) {
        Boolean showBinary = (Boolean) opts.get("--show_binary");
        System.out.format("display from %d to %d show_binary=%s\n", startAddr, finalAddr, showBinary.toString());
        if (showBinary) {
            op = new Operation(Operation.OperationType.DISPLAY, startAddr, finalAddr);
        } else {
            op = new Operation(Operation.OperationType.DISPLAY_ALL, startAddr, finalAddr);
        }
    } else if ((Boolean) opts.get("erase")) {
        System.out.format("erase from %d to %d\n", startAddr, finalAddr);
        op = new Operation(Operation.OperationType.ERASE_RANGE, startAddr, finalAddr);
        useOutputFile = true;
    }
    Metadata md = Metadata.newBuilder().setChecksum(NON_ZERO).setLength(// size is arbitrary but cannot be 0 (default)
    NON_ZERO).build();
    metadataSize = md.getSerializedSize();
    File fIn = new File(logFileName);
    File fOut = useOutputFile ? new File(logFileName + ".modified") : null;
    if (fIn.canRead()) {
        try {
            fileStreamIn = new FileInputStream(fIn);
            fileChannelIn = fileStreamIn.getChannel();
            if (useOutputFile) {
                fileStreamOut = new FileOutputStream(fOut);
                fileChannelOut = fileStreamOut.getChannel();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return true;
    }
    return false;
}