Search in sources :

Example 1 with KeyIterator

use of org.apache.cassandra.io.sstable.KeyIterator in project cassandra by apache.

the class SASIIndexBuilder method build.

public void build() {
    AbstractType<?> keyValidator = cfs.metadata().partitionKeyType;
    for (Map.Entry<SSTableReader, Map<ColumnMetadata, ColumnIndex>> e : sstables.entrySet()) {
        SSTableReader sstable = e.getKey();
        Map<ColumnMetadata, ColumnIndex> indexes = e.getValue();
        try (RandomAccessReader dataFile = sstable.openDataReader()) {
            PerSSTableIndexWriter indexWriter = SASIIndex.newWriter(keyValidator, sstable.descriptor, indexes, OperationType.COMPACTION);
            long previousKeyPosition = 0;
            try (KeyIterator keys = new KeyIterator(sstable.descriptor, cfs.metadata())) {
                while (keys.hasNext()) {
                    if (isStopRequested())
                        throw new CompactionInterruptedException(getCompactionInfo());
                    final DecoratedKey key = keys.next();
                    final long keyPosition = keys.getKeyPosition();
                    indexWriter.startPartition(key, keyPosition);
                    try {
                        RowIndexEntry indexEntry = sstable.getPosition(key, SSTableReader.Operator.EQ);
                        dataFile.seek(indexEntry.position);
                        // key
                        ByteBufferUtil.readWithShortLength(dataFile);
                        try (SSTableIdentityIterator partition = SSTableIdentityIterator.create(sstable, dataFile, key)) {
                            // if the row has statics attached, it has to be indexed separately
                            if (cfs.metadata().hasStaticColumns())
                                indexWriter.nextUnfilteredCluster(partition.staticRow());
                            while (partition.hasNext()) indexWriter.nextUnfilteredCluster(partition.next());
                        }
                    } catch (IOException ex) {
                        throw new FSReadError(ex, sstable.getFilename());
                    }
                    bytesProcessed += keyPosition - previousKeyPosition;
                    previousKeyPosition = keyPosition;
                }
                completeSSTable(indexWriter, sstable, indexes.values());
            }
        }
    }
}
Also used : ColumnMetadata(org.apache.cassandra.schema.ColumnMetadata) SSTableIdentityIterator(org.apache.cassandra.io.sstable.SSTableIdentityIterator) KeyIterator(org.apache.cassandra.io.sstable.KeyIterator) CompactionInterruptedException(org.apache.cassandra.db.compaction.CompactionInterruptedException) DecoratedKey(org.apache.cassandra.db.DecoratedKey) IOException(java.io.IOException) RowIndexEntry(org.apache.cassandra.db.RowIndexEntry) PerSSTableIndexWriter(org.apache.cassandra.index.sasi.disk.PerSSTableIndexWriter) SSTableReader(org.apache.cassandra.io.sstable.format.SSTableReader) ColumnIndex(org.apache.cassandra.index.sasi.conf.ColumnIndex) RandomAccessReader(org.apache.cassandra.io.util.RandomAccessReader) FSReadError(org.apache.cassandra.io.FSReadError)

Example 2 with KeyIterator

use of org.apache.cassandra.io.sstable.KeyIterator in project cassandra by apache.

the class SSTableExport method main.

/**
     * Given arguments specifying an SSTable, and optionally an output file, export the contents of the SSTable to JSON.
     *
     * @param args
     *            command lines arguments
     * @throws ConfigurationException
     *             on configuration failure (wrong params given)
     */
public static void main(String[] args) throws ConfigurationException {
    CommandLineParser parser = new PosixParser();
    try {
        cmd = parser.parse(options, args);
    } catch (ParseException e1) {
        System.err.println(e1.getMessage());
        printUsage();
        System.exit(1);
    }
    if (cmd.getArgs().length != 1) {
        System.err.println("You must supply exactly one sstable");
        printUsage();
        System.exit(1);
    }
    String[] keys = cmd.getOptionValues(KEY_OPTION);
    HashSet<String> excludes = new HashSet<>(Arrays.asList(cmd.getOptionValues(EXCLUDE_KEY_OPTION) == null ? new String[0] : cmd.getOptionValues(EXCLUDE_KEY_OPTION)));
    String ssTableFileName = new File(cmd.getArgs()[0]).getAbsolutePath();
    if (!new File(ssTableFileName).exists()) {
        System.err.println("Cannot find file " + ssTableFileName);
        System.exit(1);
    }
    Descriptor desc = Descriptor.fromFilename(ssTableFileName);
    try {
        TableMetadata metadata = metadataFromSSTable(desc);
        if (cmd.hasOption(ENUMERATE_KEYS_OPTION)) {
            try (KeyIterator iter = new KeyIterator(desc, metadata)) {
                JsonTransformer.keysToJson(null, iterToStream(iter), cmd.hasOption(RAW_TIMESTAMPS), metadata, System.out);
            }
        } else {
            SSTableReader sstable = SSTableReader.openNoValidation(desc, TableMetadataRef.forOfflineTools(metadata));
            IPartitioner partitioner = sstable.getPartitioner();
            final ISSTableScanner currentScanner;
            if ((keys != null) && (keys.length > 0)) {
                List<AbstractBounds<PartitionPosition>> bounds = Arrays.stream(keys).filter(key -> !excludes.contains(key)).map(metadata.partitionKeyType::fromString).map(partitioner::decorateKey).sorted().map(DecoratedKey::getToken).map(token -> new Bounds<>(token.minKeyBound(), token.maxKeyBound())).collect(Collectors.toList());
                currentScanner = sstable.getScanner(bounds.iterator());
            } else {
                currentScanner = sstable.getScanner();
            }
            Stream<UnfilteredRowIterator> partitions = iterToStream(currentScanner).filter(i -> excludes.isEmpty() || !excludes.contains(metadata.partitionKeyType.getString(i.partitionKey().getKey())));
            if (cmd.hasOption(DEBUG_OUTPUT_OPTION)) {
                AtomicLong position = new AtomicLong();
                partitions.forEach(partition -> {
                    position.set(currentScanner.getCurrentPosition());
                    if (!partition.partitionLevelDeletion().isLive()) {
                        System.out.println("[" + metadata.partitionKeyType.getString(partition.partitionKey().getKey()) + "]@" + position.get() + " " + partition.partitionLevelDeletion());
                    }
                    if (!partition.staticRow().isEmpty()) {
                        System.out.println("[" + metadata.partitionKeyType.getString(partition.partitionKey().getKey()) + "]@" + position.get() + " " + partition.staticRow().toString(metadata, true));
                    }
                    partition.forEachRemaining(row -> {
                        System.out.println("[" + metadata.partitionKeyType.getString(partition.partitionKey().getKey()) + "]@" + position.get() + " " + row.toString(metadata, false, true));
                        position.set(currentScanner.getCurrentPosition());
                    });
                });
            } else {
                JsonTransformer.toJson(currentScanner, partitions, cmd.hasOption(RAW_TIMESTAMPS), metadata, System.out);
            }
        }
    } catch (IOException e) {
        // throwing exception outside main with broken pipe causes windows cmd to hang
        e.printStackTrace(System.err);
    }
    System.exit(0);
}
Also used : ISSTableScanner(org.apache.cassandra.io.sstable.ISSTableScanner) java.util(java.util) org.apache.commons.cli(org.apache.commons.cli) SSTableReader(org.apache.cassandra.io.sstable.format.SSTableReader) KeyIterator(org.apache.cassandra.io.sstable.KeyIterator) UTF8Type(org.apache.cassandra.db.marshal.UTF8Type) DecoratedKey(org.apache.cassandra.db.DecoratedKey) ConfigurationException(org.apache.cassandra.exceptions.ConfigurationException) UnfilteredRowIterator(org.apache.cassandra.db.rows.UnfilteredRowIterator) MetadataComponent(org.apache.cassandra.io.sstable.metadata.MetadataComponent) Descriptor(org.apache.cassandra.io.sstable.Descriptor) StreamSupport(java.util.stream.StreamSupport) DatabaseDescriptor(org.apache.cassandra.config.DatabaseDescriptor) SerializationHeader(org.apache.cassandra.db.SerializationHeader) FBUtilities(org.apache.cassandra.utils.FBUtilities) MetadataType(org.apache.cassandra.io.sstable.metadata.MetadataType) ISSTableScanner(org.apache.cassandra.io.sstable.ISSTableScanner) IOException(java.io.IOException) org.apache.cassandra.dht(org.apache.cassandra.dht) Collectors(java.util.stream.Collectors) File(java.io.File) AtomicLong(java.util.concurrent.atomic.AtomicLong) Stream(java.util.stream.Stream) PartitionPosition(org.apache.cassandra.db.PartitionPosition) ColumnIdentifier(org.apache.cassandra.cql3.ColumnIdentifier) TableMetadataRef(org.apache.cassandra.schema.TableMetadataRef) TableMetadata(org.apache.cassandra.schema.TableMetadata) UnfilteredRowIterator(org.apache.cassandra.db.rows.UnfilteredRowIterator) SSTableReader(org.apache.cassandra.io.sstable.format.SSTableReader) TableMetadata(org.apache.cassandra.schema.TableMetadata) KeyIterator(org.apache.cassandra.io.sstable.KeyIterator) DecoratedKey(org.apache.cassandra.db.DecoratedKey) IOException(java.io.IOException) AtomicLong(java.util.concurrent.atomic.AtomicLong) Descriptor(org.apache.cassandra.io.sstable.Descriptor) DatabaseDescriptor(org.apache.cassandra.config.DatabaseDescriptor) File(java.io.File)

Aggregations

IOException (java.io.IOException)2 DecoratedKey (org.apache.cassandra.db.DecoratedKey)2 KeyIterator (org.apache.cassandra.io.sstable.KeyIterator)2 SSTableReader (org.apache.cassandra.io.sstable.format.SSTableReader)2 File (java.io.File)1 java.util (java.util)1 AtomicLong (java.util.concurrent.atomic.AtomicLong)1 Collectors (java.util.stream.Collectors)1 Stream (java.util.stream.Stream)1 StreamSupport (java.util.stream.StreamSupport)1 DatabaseDescriptor (org.apache.cassandra.config.DatabaseDescriptor)1 ColumnIdentifier (org.apache.cassandra.cql3.ColumnIdentifier)1 PartitionPosition (org.apache.cassandra.db.PartitionPosition)1 RowIndexEntry (org.apache.cassandra.db.RowIndexEntry)1 SerializationHeader (org.apache.cassandra.db.SerializationHeader)1 CompactionInterruptedException (org.apache.cassandra.db.compaction.CompactionInterruptedException)1 UTF8Type (org.apache.cassandra.db.marshal.UTF8Type)1 UnfilteredRowIterator (org.apache.cassandra.db.rows.UnfilteredRowIterator)1 org.apache.cassandra.dht (org.apache.cassandra.dht)1 ConfigurationException (org.apache.cassandra.exceptions.ConfigurationException)1