Examples with RandomAccessReader org.apache.cassandra.io.util.RandomAccessReader used on opensource projects

Example 11 with RandomAccessReader

use of org.apache.cassandra.io.util.RandomAccessReader in project eiger by wlloyd.

the class CompactionManager method scrubOne.

private void scrubOne(ColumnFamilyStore cfs, SSTableReader sstable) throws IOException {
    logger.info("Scrubbing " + sstable);
    CompactionController controller = new CompactionController(cfs, Collections.singletonList(sstable), getDefaultGcBefore(cfs), true);
    boolean isCommutative = cfs.metadata.getDefaultValidator().isCommutative();
    // Calculate the expected compacted filesize
    File compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(sstable.onDiskLength());
    if (compactionFileLocation == null)
        throw new IOException("disk full");
    int expectedBloomFilterSize = Math.max(DatabaseDescriptor.getIndexInterval(), (int) (SSTableReader.getApproximateKeyCount(Arrays.asList(sstable))));
    // loop through each row, deserializing to check for damage.
    // we'll also loop through the index at the same time, using the position from the index to recover if the
    // row header (key or data size) is corrupt. (This means our position in the index file will be one row
    // "ahead" of the data file.)
    final RandomAccessReader dataFile = sstable.openDataReader(true);
    RandomAccessReader indexFile = RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX)), true);
    ScrubInfo scrubInfo = new ScrubInfo(dataFile, sstable);
    executor.beginCompaction(scrubInfo);
    SSTableWriter writer = null;
    SSTableReader newSstable = null;
    int goodRows = 0, badRows = 0, emptyRows = 0;
    try {
        ByteBuffer nextIndexKey = ByteBufferUtil.readWithShortLength(indexFile);
        {
            // throw away variable so we don't have a side effect in the assert
            long firstRowPositionFromIndex = indexFile.readLong();
            assert firstRowPositionFromIndex == 0 : firstRowPositionFromIndex;
        }
        // TODO errors when creating the writer may leave empty temp files.
        writer = maybeCreateWriter(cfs, compactionFileLocation, expectedBloomFilterSize, null, Collections.singletonList(sstable));
        while (!dataFile.isEOF()) {
            if (scrubInfo.isStopped())
                throw new CompactionInterruptedException(scrubInfo.getCompactionInfo());
            long rowStart = dataFile.getFilePointer();
            if (logger.isDebugEnabled())
                logger.debug("Reading row at " + rowStart);
            DecoratedKey key = null;
            long dataSize = -1;
            try {
                key = SSTableReader.decodeKey(sstable.partitioner, sstable.descriptor, ByteBufferUtil.readWithShortLength(dataFile));
                dataSize = sstable.descriptor.hasIntRowSize ? dataFile.readInt() : dataFile.readLong();
                if (logger.isDebugEnabled())
                    logger.debug(String.format("row %s is %s bytes", ByteBufferUtil.bytesToHex(key.key), dataSize));
            } catch (Throwable th) {
                throwIfFatal(th);
            // check for null key below
            }
            ByteBuffer currentIndexKey = nextIndexKey;
            long nextRowPositionFromIndex;
            try {
                nextIndexKey = indexFile.isEOF() ? null : ByteBufferUtil.readWithShortLength(indexFile);
                nextRowPositionFromIndex = indexFile.isEOF() ? dataFile.length() : indexFile.readLong();
            } catch (Throwable th) {
                logger.warn("Error reading index file", th);
                nextIndexKey = null;
                nextRowPositionFromIndex = dataFile.length();
            }
            long dataStart = dataFile.getFilePointer();
            long dataStartFromIndex = currentIndexKey == null ? -1 : rowStart + 2 + currentIndexKey.remaining() + (sstable.descriptor.hasIntRowSize ? 4 : 8);
            long dataSizeFromIndex = nextRowPositionFromIndex - dataStartFromIndex;
            assert currentIndexKey != null || indexFile.isEOF();
            if (logger.isDebugEnabled() && currentIndexKey != null)
                logger.debug(String.format("Index doublecheck: row %s is %s bytes", ByteBufferUtil.bytesToHex(currentIndexKey), dataSizeFromIndex));
            writer.mark();
            try {
                if (key == null)
                    throw new IOError(new IOException("Unable to read row key from data file"));
                if (dataSize > dataFile.length())
                    throw new IOError(new IOException("Impossible row size " + dataSize));
                SSTableIdentityIterator row = new SSTableIdentityIterator(sstable, dataFile, key, dataStart, dataSize, true);
                AbstractCompactedRow compactedRow = controller.getCompactedRow(row);
                if (compactedRow.isEmpty()) {
                    emptyRows++;
                } else {
                    writer.append(compactedRow);
                    goodRows++;
                }
                if (!key.key.equals(currentIndexKey) || dataStart != dataStartFromIndex)
                    logger.warn("Index file contained a different key or row size; using key from data file");
            } catch (Throwable th) {
                throwIfFatal(th);
                logger.warn("Non-fatal error reading row (stacktrace follows)", th);
                writer.resetAndTruncate();
                if (currentIndexKey != null && (key == null || !key.key.equals(currentIndexKey) || dataStart != dataStartFromIndex || dataSize != dataSizeFromIndex)) {
                    logger.info(String.format("Retrying from row index; data is %s bytes starting at %s", dataSizeFromIndex, dataStartFromIndex));
                    key = SSTableReader.decodeKey(sstable.partitioner, sstable.descriptor, currentIndexKey);
                    try {
                        SSTableIdentityIterator row = new SSTableIdentityIterator(sstable, dataFile, key, dataStartFromIndex, dataSizeFromIndex, true);
                        AbstractCompactedRow compactedRow = controller.getCompactedRow(row);
                        if (compactedRow.isEmpty()) {
                            emptyRows++;
                        } else {
                            writer.append(compactedRow);
                            goodRows++;
                        }
                    } catch (Throwable th2) {
                        throwIfFatal(th2);
                        // Skipping rows is dangerous for counters (see CASSANDRA-2759)
                        if (isCommutative)
                            throw new IOError(th2);
                        logger.warn("Retry failed too.  Skipping to next row (retry's stacktrace follows)", th2);
                        writer.resetAndTruncate();
                        dataFile.seek(nextRowPositionFromIndex);
                        badRows++;
                    }
                } else {
                    // Skipping rows is dangerous for counters (see CASSANDRA-2759)
                    if (isCommutative)
                        throw new IOError(th);
                    logger.warn("Row at " + dataStart + " is unreadable; skipping to next");
                    if (currentIndexKey != null)
                        dataFile.seek(nextRowPositionFromIndex);
                    badRows++;
                }
            }
        }
        if (writer.getFilePointer() > 0)
            newSstable = writer.closeAndOpenReader(sstable.maxDataAge);
    } catch (Exception e) {
        if (writer != null)
            writer.abort();
        throw FBUtilities.unchecked(e);
    } finally {
        FileUtils.closeQuietly(dataFile);
        FileUtils.closeQuietly(indexFile);
        executor.finishCompaction(scrubInfo);
    }
    if (newSstable == null) {
        cfs.markCompacted(Arrays.asList(sstable));
        if (badRows > 0)
            logger.warn("No valid rows found while scrubbing " + sstable + "; it is marked for deletion now. If you want to attempt manual recovery, you can find a copy in the pre-scrub snapshot");
        else
            logger.info("Scrub of " + sstable + " complete; looks like all " + emptyRows + " rows were tombstoned");
    } else {
        cfs.replaceCompactedSSTables(Arrays.asList(sstable), Arrays.asList(newSstable));
        logger.info("Scrub of " + sstable + " complete: " + goodRows + " rows in new sstable and " + emptyRows + " empty (tombstoned) rows dropped");
        if (badRows > 0)
            logger.warn("Unable to recover " + badRows + " rows that were skipped.  You can attempt manual recovery from the pre-scrub snapshot.  You can also run nodetool repair to transfer the data from a healthy replica, if any");
    }
}

Example 12 with RandomAccessReader

use of org.apache.cassandra.io.util.RandomAccessReader in project eiger by wlloyd.

the class FileStreamTask method stream.

/**
     * Stream file by it's sections specified by this.header
     * @throws IOException on any I/O error
     */
private void stream() throws IOException {
    ByteBuffer HeaderBuffer = MessagingService.instance().constructStreamHeader(header, false, Gossiper.instance.getVersion(to));
    // write header (this should not be compressed for compatibility with other messages)
    output.write(ByteBufferUtil.getArray(HeaderBuffer));
    if (header.file == null)
        return;
    // TODO just use a raw RandomAccessFile since we're managing our own buffer here
    RandomAccessReader file = // try to skip kernel page cache if possible
    (header.file.sstable.compression) ? CompressedRandomAccessReader.open(header.file.getFilename(), header.file.sstable.getCompressionMetadata(), true) : RandomAccessReader.open(new File(header.file.getFilename()), true);
    // setting up data compression stream
    compressedoutput = new LZFOutputStream(output);
    MessagingService.instance().incrementActiveStreamsOutbound();
    try {
        // stream each of the required sections of the file
        for (Pair<Long, Long> section : header.file.sections) {
            // seek to the beginning of the section
            file.seek(section.left);
            // length of the section to stream
            long length = section.right - section.left;
            // tracks write progress
            long bytesTransferred = 0;
            while (bytesTransferred < length) {
                long lastWrite = write(file, length, bytesTransferred);
                bytesTransferred += lastWrite;
                // store streaming progress
                header.file.progress += lastWrite;
            }
            // make sure that current section is send
            compressedoutput.flush();
            if (logger.isDebugEnabled())
                logger.debug("Bytes transferred " + bytesTransferred + "/" + header.file.size);
        }
        // receive reply confirmation
        receiveReply();
    } finally {
        MessagingService.instance().decrementActiveStreamsOutbound();
        // no matter what happens close file
        FileUtils.closeQuietly(file);
    }
}

Example 13 with RandomAccessReader

use of org.apache.cassandra.io.util.RandomAccessReader in project eiger by wlloyd.

the class SSTableTest method verifySingle.

private void verifySingle(SSTableReader sstable, ByteBuffer bytes, ByteBuffer key) throws IOException {
    RandomAccessReader file = sstable.openDataReader(false);
    file.seek(sstable.getPosition(sstable.partitioner.decorateKey(key), SSTableReader.Operator.EQ));
    assert key.equals(ByteBufferUtil.readWithShortLength(file));
    int size = (int) SSTableReader.readRowSize(file, sstable.descriptor);
    byte[] bytes2 = new byte[size];
    file.readFully(bytes2);
    assert ByteBuffer.wrap(bytes2).equals(bytes);
}

Example 14 with RandomAccessReader

use of org.apache.cassandra.io.util.RandomAccessReader in project eiger by wlloyd.

the class TableTest method testGetSliceFromLarge.

@Test
public void testGetSliceFromLarge() throws Throwable {
    // tests slicing against 1000 columns in an sstable
    Table table = Table.open("Keyspace1");
    ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
    DecoratedKey key = Util.dk("row3");
    RowMutation rm = new RowMutation("Keyspace1", key.key);
    ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
    for (int i = 1000; i < 2000; i++) cf.addColumn(column("col" + i, ("v" + i), 1L));
    rm.add(cf);
    rm.apply();
    cfStore.forceBlockingFlush();
    validateSliceLarge(cfStore);
    // compact so we have a big row with more than the minimum index count
    if (cfStore.getSSTables().size() > 1) {
        CompactionManager.instance.performMaximal(cfStore);
    }
    // verify that we do indeed have multiple index entries
    SSTableReader sstable = cfStore.getSSTables().iterator().next();
    long position = sstable.getPosition(key, SSTableReader.Operator.EQ);
    RandomAccessReader file = sstable.openDataReader(false);
    file.seek(position);
    assert ByteBufferUtil.readWithShortLength(file).equals(key.key);
    SSTableReader.readRowSize(file, sstable.descriptor);
    IndexHelper.skipBloomFilter(file);
    ArrayList<IndexHelper.IndexInfo> indexes = IndexHelper.deserializeIndex(file);
    assert indexes.size() > 2;
    validateSliceLarge(cfStore);
}

Example 15 with RandomAccessReader

use of org.apache.cassandra.io.util.RandomAccessReader in project cassandra by apache.

the class TokenTreeTest method testSerializedSize.

public void testSerializedSize(final TokenTreeBuilder builder) throws Exception {
    builder.finish();
    final File treeFile = File.createTempFile("token-tree-size-test", "tt");
    treeFile.deleteOnExit();
    try (SequentialWriter writer = new SequentialWriter(treeFile, DEFAULT_OPT)) {
        builder.write(writer);
        writer.sync();
    }
    final RandomAccessReader reader = RandomAccessReader.open(treeFile);
    Assert.assertEquals((int) reader.bytesRemaining(), builder.serializedSize());
    reader.close();
}