Examples with Checksum java.util.zip.Checksum used on opensource projects

Example 26 with Checksum

use of java.util.zip.Checksum in project eiger by wlloyd.

the class CommitLogTest method testRecoveryWithBadSizeChecksum.

@Test
public void testRecoveryWithBadSizeChecksum() throws Exception {
    Checksum checksum = new CRC32();
    checksum.update(100);
    testRecoveryWithBadSizeArgument(100, 100, ~checksum.getValue());
}

Example 27 with Checksum

use of java.util.zip.Checksum in project eiger by wlloyd.

the class CommitLogTest method testRecoveryWithBadSizeArgument.

protected void testRecoveryWithBadSizeArgument(int size, int dataSize) throws Exception {
    Checksum checksum = new CRC32();
    checksum.update(size);
    testRecoveryWithBadSizeArgument(size, dataSize, checksum.getValue());
}

Example 28 with Checksum

use of java.util.zip.Checksum in project eiger by wlloyd.

the class CommitLog method recover.

/**
     * Perform recovery on a list of commit log files.
     *
     * @param clogs   the list of commit log files to replay
     * @return the number of mutations replayed
     */
public int recover(File[] clogs) throws IOException {
    final Set<Table> tablesRecovered = new HashSet<Table>();
    List<Future<?>> futures = new ArrayList<Future<?>>();
    byte[] bytes = new byte[4096];
    Map<Integer, AtomicInteger> invalidMutations = new HashMap<Integer, AtomicInteger>();
    // count the number of replayed mutation. We don't really care about atomicity, but we need it to be a reference.
    final AtomicInteger replayedCount = new AtomicInteger();
    // compute per-CF and global replay positions
    final Map<Integer, ReplayPosition> cfPositions = new HashMap<Integer, ReplayPosition>();
    for (ColumnFamilyStore cfs : ColumnFamilyStore.all()) {
        // it's important to call RP.gRP per-cf, before aggregating all the positions w/ the Ordering.min call
        // below: gRP will return NONE if there are no flushed sstables, which is important to have in the
        // list (otherwise we'll just start replay from the first flush position that we do have, which is not correct).
        ReplayPosition rp = ReplayPosition.getReplayPosition(cfs.getSSTables());
        cfPositions.put(cfs.metadata.cfId, rp);
    }
    final ReplayPosition globalPosition = Ordering.from(ReplayPosition.comparator).min(cfPositions.values());
    Checksum checksum = new CRC32();
    for (final File file : clogs) {
        logger.info("Replaying " + file.getPath());
        final long segment = CommitLogSegment.idFromFilename(file.getName());
        RandomAccessReader reader = RandomAccessReader.open(new File(file.getAbsolutePath()), true);
        assert reader.length() <= Integer.MAX_VALUE;
        try {
            int replayPosition;
            if (globalPosition.segment < segment)
                replayPosition = 0;
            else if (globalPosition.segment == segment)
                replayPosition = globalPosition.position;
            else
                replayPosition = (int) reader.length();
            if (replayPosition < 0 || replayPosition >= reader.length()) {
                // replayPosition > reader.length() can happen if some data gets flushed before it is written to the commitlog
                // (see https://issues.apache.org/jira/browse/CASSANDRA-2285)
                logger.debug("skipping replay of fully-flushed {}", file);
                continue;
            }
            reader.seek(replayPosition);
            if (logger.isDebugEnabled())
                logger.debug("Replaying " + file + " starting at " + reader.getFilePointer());
            /* read the logs populate RowMutation and apply */
            while (!reader.isEOF()) {
                if (logger.isDebugEnabled())
                    logger.debug("Reading mutation at " + reader.getFilePointer());
                long claimedCRC32;
                int serializedSize;
                try {
                    // any of the reads may hit EOF
                    serializedSize = reader.readInt();
                    if (serializedSize == CommitLog.END_OF_SEGMENT_MARKER) {
                        logger.debug("Encountered end of segment marker at " + reader.getFilePointer());
                        break;
                    }
                    // This prevents CRC by being fooled by special-case garbage in the file; see CASSANDRA-2128
                    if (serializedSize < 10)
                        break;
                    long claimedSizeChecksum = reader.readLong();
                    checksum.reset();
                    checksum.update(serializedSize);
                    if (checksum.getValue() != claimedSizeChecksum)
                        // entry wasn't synced correctly/fully.  that's ok.
                        break;
                    if (serializedSize > bytes.length)
                        bytes = new byte[(int) (1.2 * serializedSize)];
                    reader.readFully(bytes, 0, serializedSize);
                    claimedCRC32 = reader.readLong();
                } catch (EOFException eof) {
                    // last CL entry didn't get completely written.  that's ok.
                    break;
                }
                checksum.update(bytes, 0, serializedSize);
                if (claimedCRC32 != checksum.getValue()) {
                    // but just in case there is no harm in trying them (since we still read on an entry boundary)
                    continue;
                }
                /* deserialize the commit log entry */
                FastByteArrayInputStream bufIn = new FastByteArrayInputStream(bytes, 0, serializedSize);
                RowMutation rm = null;
                try {
                    // assuming version here. We've gone to lengths to make sure what gets written to the CL is in
                    // the current version.  so do make sure the CL is drained prior to upgrading a node.
                    rm = RowMutation.serializer().deserialize(new DataInputStream(bufIn), MessagingService.version_, IColumnSerializer.Flag.LOCAL);
                } catch (UnserializableColumnFamilyException ex) {
                    AtomicInteger i = invalidMutations.get(ex.cfId);
                    if (i == null) {
                        i = new AtomicInteger(1);
                        invalidMutations.put(ex.cfId, i);
                    } else
                        i.incrementAndGet();
                    continue;
                }
                if (logger.isDebugEnabled())
                    logger.debug(String.format("replaying mutation for %s.%s: %s", rm.getTable(), ByteBufferUtil.bytesToHex(rm.key()), "{" + StringUtils.join(rm.getColumnFamilies().iterator(), ", ") + "}"));
                final long entryLocation = reader.getFilePointer();
                final RowMutation frm = rm;
                Runnable runnable = new WrappedRunnable() {

                    public void runMayThrow() throws IOException {
                        if (Schema.instance.getKSMetaData(frm.getTable()) == null)
                            return;
                        final Table table = Table.open(frm.getTable());
                        RowMutation newRm = new RowMutation(frm.getTable(), frm.key());
                        // thing based on the cfid instead.
                        for (ColumnFamily columnFamily : frm.getColumnFamilies()) {
                            if (Schema.instance.getCF(columnFamily.id()) == null)
                                // null means the cf has been dropped
                                continue;
                            ReplayPosition rp = cfPositions.get(columnFamily.id());
                            // segment, if we are after the replay position
                            if (segment > rp.segment || (segment == rp.segment && entryLocation > rp.position)) {
                                newRm.add(columnFamily);
                                replayedCount.incrementAndGet();
                            }
                        }
                        if (!newRm.isEmpty()) {
                            Table.open(newRm.getTable()).apply(newRm, false);
                            tablesRecovered.add(table);
                        }
                    }
                };
                futures.add(StageManager.getStage(Stage.MUTATION).submit(runnable));
                if (futures.size() > MAX_OUTSTANDING_REPLAY_COUNT) {
                    FBUtilities.waitOnFutures(futures);
                    futures.clear();
                }
            }
        } finally {
            FileUtils.closeQuietly(reader);
            logger.info("Finished reading " + file);
        }
    }
    for (Map.Entry<Integer, AtomicInteger> entry : invalidMutations.entrySet()) logger.info(String.format("Skipped %d mutations from unknown (probably removed) CF with id %d", entry.getValue().intValue(), entry.getKey()));
    // wait for all the writes to finish on the mutation stage
    FBUtilities.waitOnFutures(futures);
    logger.debug("Finished waiting on mutations from recovery");
    // flush replayed tables
    futures.clear();
    for (Table table : tablesRecovered) futures.addAll(table.flush());
    FBUtilities.waitOnFutures(futures);
    return replayedCount.get();
}

Example 29 with Checksum

use of java.util.zip.Checksum in project voltdb by VoltDB.

the class LogFormatter method main.

/**
     * @param args
     */
public static void main(String[] args) throws Exception {
    if (args.length != 1) {
        System.err.println("USAGE: LogFormatter log_file");
        System.exit(2);
    }
    FileInputStream fis = new FileInputStream(args[0]);
    BinaryInputArchive logStream = BinaryInputArchive.getArchive(fis);
    FileHeader fhdr = new FileHeader();
    fhdr.deserialize(logStream, "fileheader");
    if (fhdr.getMagic() != FileTxnLog.TXNLOG_MAGIC) {
        System.err.println("Invalid magic number for " + args[0]);
        System.exit(2);
    }
    System.out.println("ZooKeeper Transactional Log File with dbid " + fhdr.getDbid() + " txnlog format version " + fhdr.getVersion());
    int count = 0;
    while (true) {
        long crcValue;
        byte[] bytes;
        try {
            crcValue = logStream.readLong("crcvalue");
            bytes = logStream.readBuffer("txnEntry");
        } catch (EOFException e) {
            System.out.println("EOF reached after " + count + " txns.");
            return;
        }
        if (bytes.length == 0) {
            // Since we preallocate, we define EOF to be an
            // empty transaction
            System.out.println("EOF reached after " + count + " txns.");
            return;
        }
        Checksum crc = new Adler32();
        crc.update(bytes, 0, bytes.length);
        if (crcValue != crc.getValue()) {
            throw new IOException("CRC doesn't match " + crcValue + " vs " + crc.getValue());
        }
        InputArchive iab = BinaryInputArchive.getArchive(new ByteArrayInputStream(bytes));
        TxnHeader hdr = new TxnHeader();
        SerializeUtils.deserializeTxn(iab, hdr);
        System.out.println(DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.LONG).format(new Date(hdr.getTime())) + " session 0x" + Long.toHexString(hdr.getClientId()) + " cxid 0x" + Long.toHexString(hdr.getCxid()) + " zxid 0x" + Long.toHexString(hdr.getZxid()) + " " + TraceFormatter.op2String(hdr.getType()));
        if (logStream.readByte("EOR") != 'B') {
            LOG.error("Last transaction was partial.");
            throw new EOFException("Last transaction was partial.");
        }
        count++;
    }
}

Example 30 with Checksum

use of java.util.zip.Checksum in project voltdb by VoltDB.

the class FileTxnLog method append.

/**
     * append an entry to the transaction log
     * @param hdr the header of the transaction
     * @param txn the transaction part of the entry
     * returns true iff something appended, otw false
     */
public synchronized boolean append(TxnHeader hdr, Record txn) throws IOException {
    if (hdr != null) {
        if (hdr.getZxid() <= lastZxidSeen) {
            LOG.warn("Current zxid " + hdr.getZxid() + " is <= " + lastZxidSeen + " for " + hdr.getType());
        }
        if (logStream == null) {
            if (LOG.isInfoEnabled()) {
                LOG.info("Creating new log file: log." + Long.toHexString(hdr.getZxid()));
            }
            logFileWrite = new File(logDir, ("log." + Long.toHexString(hdr.getZxid())));
            fos = new FileOutputStream(logFileWrite);
            logStream = new BufferedOutputStream(fos);
            oa = BinaryOutputArchive.getArchive(logStream);
            FileHeader fhdr = new FileHeader(TXNLOG_MAGIC, VERSION, dbId);
            fhdr.serialize(oa, "fileheader");
            currentSize = fos.getChannel().position();
            streamsToFlush.add(fos);
        }
        padFile(fos);
        byte[] buf = Util.marshallTxnEntry(hdr, txn);
        if (buf == null || buf.length == 0) {
            throw new IOException("Faulty serialization for header " + "and txn");
        }
        Checksum crc = makeChecksumAlgorithm();
        crc.update(buf, 0, buf.length);
        oa.writeLong(crc.getValue(), "txnEntryCRC");
        Util.writeTxnBytes(oa, buf);
        return true;
    }
    return false;
}