Examples with StatsMetadata org.apache.cassandra.io.sstable.metadata.StatsMetadata used on opensource projects

Example 1 with StatsMetadata

use of org.apache.cassandra.io.sstable.metadata.StatsMetadata in project cassandra by apache.

the class Upgrader method upgrade.

public void upgrade(boolean keepOriginals) {
    outputHandler.output("Upgrading " + sstable);
    int nowInSec = FBUtilities.nowInSeconds();
    try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, keepOriginals, CompactionTask.getMaxDataAge(transaction.originals()));
        AbstractCompactionStrategy.ScannerList scanners = strategyManager.getScanners(transaction.originals());
        CompactionIterator iter = new CompactionIterator(transaction.opType(), scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID())) {
        StatsMetadata metadata = sstable.getSSTableMetadata();
        writer.switchWriter(createCompactionWriter(metadata.repairedAt, metadata.pendingRepair));
        while (iter.hasNext()) writer.append(iter.next());
        writer.finish();
        outputHandler.output("Upgrade of " + sstable + " complete.");
    } catch (Exception e) {
        Throwables.propagate(e);
    } finally {
        controller.close();
    }
}

Example 2 with StatsMetadata

use of org.apache.cassandra.io.sstable.metadata.StatsMetadata in project cassandra by apache.

the class Scrubber method scrub.

public void scrub() {
    List<SSTableReader> finished = new ArrayList<>();
    boolean completed = false;
    outputHandler.output(String.format("Scrubbing %s (%s)", sstable, FBUtilities.prettyPrintMemory(dataFile.length())));
    try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge)) {
        nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null;
        if (indexAvailable()) {
            // throw away variable so we don't have a side effect in the assert
            long firstRowPositionFromIndex = rowIndexEntrySerializer.deserializePositionAndSkip(indexFile);
            assert firstRowPositionFromIndex == 0 : firstRowPositionFromIndex;
        }
        StatsMetadata metadata = sstable.getSSTableMetadata();
        writer.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, sstable, transaction));
        DecoratedKey prevKey = null;
        while (!dataFile.isEOF()) {
            if (scrubInfo.isStopRequested())
                throw new CompactionInterruptedException(scrubInfo.getCompactionInfo());
            long rowStart = dataFile.getFilePointer();
            outputHandler.debug("Reading row at " + rowStart);
            DecoratedKey key = null;
            try {
                key = sstable.decorateKey(ByteBufferUtil.readWithShortLength(dataFile));
            } catch (Throwable th) {
                throwIfFatal(th);
            // check for null key below
            }
            updateIndexKey();
            long dataStart = dataFile.getFilePointer();
            long dataStartFromIndex = -1;
            long dataSizeFromIndex = -1;
            if (currentIndexKey != null) {
                dataStartFromIndex = currentRowPositionFromIndex + 2 + currentIndexKey.remaining();
                dataSizeFromIndex = nextRowPositionFromIndex - dataStartFromIndex;
            }
            // avoid an NPE if key is null
            String keyName = key == null ? "(unreadable key)" : ByteBufferUtil.bytesToHex(key.getKey());
            outputHandler.debug(String.format("row %s is %s", keyName, FBUtilities.prettyPrintMemory(dataSizeFromIndex)));
            assert currentIndexKey != null || !indexAvailable();
            try {
                if (key == null)
                    throw new IOError(new IOException("Unable to read row key from data file"));
                if (currentIndexKey != null && !key.getKey().equals(currentIndexKey)) {
                    throw new IOError(new IOException(String.format("Key from data file (%s) does not match key from index file (%s)", //ByteBufferUtil.bytesToHex(key.getKey()), ByteBufferUtil.bytesToHex(currentIndexKey))));
                    "_too big_", ByteBufferUtil.bytesToHex(currentIndexKey))));
                }
                if (indexFile != null && dataSizeFromIndex > dataFile.length())
                    throw new IOError(new IOException("Impossible row size (greater than file length): " + dataSizeFromIndex));
                if (indexFile != null && dataStart != dataStartFromIndex)
                    outputHandler.warn(String.format("Data file row position %d differs from index file row position %d", dataStart, dataStartFromIndex));
                if (tryAppend(prevKey, key, writer))
                    prevKey = key;
            } catch (Throwable th) {
                throwIfFatal(th);
                outputHandler.warn("Error reading row (stacktrace follows):", th);
                if (currentIndexKey != null && (key == null || !key.getKey().equals(currentIndexKey) || dataStart != dataStartFromIndex)) {
                    outputHandler.output(String.format("Retrying from row index; data is %s bytes starting at %s", dataSizeFromIndex, dataStartFromIndex));
                    key = sstable.decorateKey(currentIndexKey);
                    try {
                        dataFile.seek(dataStartFromIndex);
                        if (tryAppend(prevKey, key, writer))
                            prevKey = key;
                    } catch (Throwable th2) {
                        throwIfFatal(th2);
                        throwIfCannotContinue(key, th2);
                        outputHandler.warn("Retry failed too. Skipping to next row (retry's stacktrace follows)", th2);
                        badRows++;
                        seekToNextRow();
                    }
                } else {
                    throwIfCannotContinue(key, th);
                    outputHandler.warn("Row starting at position " + dataStart + " is unreadable; skipping to next");
                    badRows++;
                    if (currentIndexKey != null)
                        seekToNextRow();
                }
            }
        }
        if (!outOfOrder.isEmpty()) {
            // out of order rows, but no bad rows found - we can keep our repairedAt time
            long repairedAt = badRows > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : metadata.repairedAt;
            SSTableReader newInOrderSstable;
            try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, metadata.pendingRepair, sstable, transaction)) {
                for (Partition partition : outOfOrder) inOrderWriter.append(partition.unfilteredIterator());
                newInOrderSstable = inOrderWriter.finish(-1, sstable.maxDataAge, true);
            }
            transaction.update(newInOrderSstable, false);
            finished.add(newInOrderSstable);
            outputHandler.warn(String.format("%d out of order rows found while scrubbing %s; Those have been written (in order) to a new sstable (%s)", outOfOrder.size(), sstable, newInOrderSstable));
        }
        // finish obsoletes the old sstable
        finished.addAll(writer.setRepairedAt(badRows > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt).finish());
        completed = true;
    } catch (IOException e) {
        throw Throwables.propagate(e);
    } finally {
        if (transaction.isOffline())
            finished.forEach(sstable -> sstable.selfRef().release());
    }
    if (completed) {
        if (badRows > 0)
            outputHandler.warn("No valid rows found while scrubbing " + sstable + "; it is marked for deletion now. If you want to attempt manual recovery, you can find a copy in the pre-scrub snapshot");
        else
            outputHandler.output("Scrub of " + sstable + " complete; looks like all " + emptyRows + " rows were tombstoned");
    } else {
        outputHandler.output("Scrub of " + sstable + " complete: " + goodRows + " rows in new sstable and " + emptyRows + " empty (tombstoned) rows dropped");
        if (badRows > 0)
            outputHandler.warn("Unable to recover " + badRows + " rows that were skipped.  You can attempt manual recovery from the pre-scrub snapshot.  You can also run nodetool repair to transfer the data from a healthy replica, if any");
    }
}

Example 3 with StatsMetadata

use of org.apache.cassandra.io.sstable.metadata.StatsMetadata in project cassandra by apache.

the class UnfilteredRowIteratorWithLowerBound method getMetadataLowerBound.

/**
     * @return a global lower bound made from the clustering values stored in the sstable metadata, note that
     * this currently does not correctly compare tombstone bounds, especially ranges.
     */
private ClusteringBound getMetadataLowerBound() {
    if (!canUseMetadataLowerBound())
        return null;
    final StatsMetadata m = sstable.getSSTableMetadata();
    List<ByteBuffer> vals = filter.isReversed() ? m.maxClusteringValues : m.minClusteringValues;
    assert vals.size() <= metadata().comparator.size() : String.format("Unexpected number of clustering values %d, expected %d or fewer for %s", vals.size(), metadata().comparator.size(), sstable.getFilename());
    return ClusteringBound.inclusiveOpen(filter.isReversed(), vals.toArray(new ByteBuffer[vals.size()]));
}

Example 4 with StatsMetadata

use of org.apache.cassandra.io.sstable.metadata.StatsMetadata in project cassandra by apache.

the class BigTableWriter method openFinal.

@SuppressWarnings("resource")
private SSTableReader openFinal(SSTableReader.OpenReason openReason) {
    if (maxDataAge < 0)
        maxDataAge = System.currentTimeMillis();
    StatsMetadata stats = statsMetadata();
    // finalize in-memory state for the reader
    IndexSummary indexSummary = iwriter.summary.build(metadata().partitioner);
    long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length();
    int dataBufferSize = optimizationStrategy.bufferSize(stats.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile()));
    int indexBufferSize = optimizationStrategy.bufferSize(indexFileLength / indexSummary.size());
    FileHandle ifile = iwriter.builder.bufferSize(indexBufferSize).complete();
    if (compression)
        dbuilder.withCompressionMetadata(((CompressedSequentialWriter) dataFile).open(0));
    FileHandle dfile = dbuilder.bufferSize(dataBufferSize).complete();
    invalidateCacheAtBoundary(dfile);
    SSTableReader sstable = SSTableReader.internalOpen(descriptor, components, metadata, ifile, dfile, indexSummary, iwriter.bf.sharedCopy(), maxDataAge, stats, openReason, header);
    sstable.first = getMinimalKey(first);
    sstable.last = getMinimalKey(last);
    return sstable;
}

Example 5 with StatsMetadata

use of org.apache.cassandra.io.sstable.metadata.StatsMetadata in project cassandra by apache.

the class SSTableLevelResetter method main.

/**
     * @param args a list of sstables whose metadata we are changing
     */
public static void main(String[] args) {
    PrintStream out = System.out;
    if (args.length == 0) {
        out.println("This command should be run with Cassandra stopped!");
        out.println("Usage: sstablelevelreset <keyspace> <table>");
        System.exit(1);
    }
    if (!args[0].equals("--really-reset") || args.length != 3) {
        out.println("This command should be run with Cassandra stopped, otherwise you will get very strange behavior");
        out.println("Verify that Cassandra is not running and then execute the command like this:");
        out.println("Usage: sstablelevelreset --really-reset <keyspace> <table>");
        System.exit(1);
    }
    Util.initDatabaseDescriptor();
    // So we have to explicitly call System.exit.
    try {
        // load keyspace descriptions.
        Schema.instance.loadFromDisk(false);
        String keyspaceName = args[1];
        String columnfamily = args[2];
        // validate columnfamily
        if (Schema.instance.getTableMetadataRef(keyspaceName, columnfamily) == null) {
            System.err.println("ColumnFamily not found: " + keyspaceName + "/" + columnfamily);
            System.exit(1);
        }
        Keyspace keyspace = Keyspace.openWithoutSSTables(keyspaceName);
        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnfamily);
        boolean foundSSTable = false;
        for (Map.Entry<Descriptor, Set<Component>> sstable : cfs.getDirectories().sstableLister(Directories.OnTxnErr.THROW).list().entrySet()) {
            if (sstable.getValue().contains(Component.STATS)) {
                foundSSTable = true;
                Descriptor descriptor = sstable.getKey();
                StatsMetadata metadata = (StatsMetadata) descriptor.getMetadataSerializer().deserialize(descriptor, MetadataType.STATS);
                if (metadata.sstableLevel > 0) {
                    out.println("Changing level from " + metadata.sstableLevel + " to 0 on " + descriptor.filenameFor(Component.DATA));
                    descriptor.getMetadataSerializer().mutateLevel(descriptor, 0);
                } else {
                    out.println("Skipped " + descriptor.filenameFor(Component.DATA) + " since it is already on level 0");
                }
            }
        }
        if (!foundSSTable) {
            out.println("Found no sstables, did you give the correct keyspace/table?");
        }
    } catch (Throwable t) {
        JVMStabilityInspector.inspectThrowable(t);
        t.printStackTrace();
        System.exit(1);
    }
    System.exit(0);
}