Examples with ColumnFamilyStore org.apache.cassandra.db.ColumnFamilyStore used on opensource projects

Example 6 with ColumnFamilyStore

use of org.apache.cassandra.db.ColumnFamilyStore in project cassandra by apache.

the class RepairMessageVerbHandler method doVerb.

public void doVerb(final MessageIn<RepairMessage> message, final int id) {
    // TODO add cancel/interrupt message
    RepairJobDesc desc = message.payload.desc;
    try {
        switch(message.payload.messageType) {
            case PREPARE_MESSAGE:
                PrepareMessage prepareMessage = (PrepareMessage) message.payload;
                logger.debug("Preparing, {}", prepareMessage);
                List<ColumnFamilyStore> columnFamilyStores = new ArrayList<>(prepareMessage.tableIds.size());
                for (TableId tableId : prepareMessage.tableIds) {
                    ColumnFamilyStore columnFamilyStore = ColumnFamilyStore.getIfExists(tableId);
                    if (columnFamilyStore == null) {
                        logErrorAndSendFailureResponse(String.format("Table with id %s was dropped during prepare phase of repair", tableId), message.from, id);
                        return;
                    }
                    columnFamilyStores.add(columnFamilyStore);
                }
                ActiveRepairService.instance.registerParentRepairSession(prepareMessage.parentRepairSession, message.from, columnFamilyStores, prepareMessage.ranges, prepareMessage.isIncremental, prepareMessage.timestamp, prepareMessage.isGlobal);
                MessagingService.instance().sendReply(new MessageOut(MessagingService.Verb.INTERNAL_RESPONSE), id, message.from);
                break;
            case SNAPSHOT:
                logger.debug("Snapshotting {}", desc);
                final ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(desc.keyspace, desc.columnFamily);
                if (cfs == null) {
                    logErrorAndSendFailureResponse(String.format("Table %s.%s was dropped during snapshot phase of repair", desc.keyspace, desc.columnFamily), message.from, id);
                    return;
                }
                ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(desc.parentSessionId);
                if (prs.isGlobal) {
                    prs.maybeSnapshot(cfs.metadata.id, desc.parentSessionId);
                } else {
                    cfs.snapshot(desc.sessionId.toString(), new Predicate<SSTableReader>() {

                        public boolean apply(SSTableReader sstable) {
                            return sstable != null && // exclude SSTables from 2i
                            !sstable.metadata().isIndex() && new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(desc.ranges);
                        }
                    }, true, //ephemeral snapshot, if repair fails, it will be cleaned next startup
                    false);
                }
                logger.debug("Enqueuing response to snapshot request {} to {}", desc.sessionId, message.from);
                MessagingService.instance().sendReply(new MessageOut(MessagingService.Verb.INTERNAL_RESPONSE), id, message.from);
                break;
            case VALIDATION_REQUEST:
                ValidationRequest validationRequest = (ValidationRequest) message.payload;
                logger.debug("Validating {}", validationRequest);
                // trigger read-only compaction
                ColumnFamilyStore store = ColumnFamilyStore.getIfExists(desc.keyspace, desc.columnFamily);
                if (store == null) {
                    logger.error("Table {}.{} was dropped during snapshot phase of repair", desc.keyspace, desc.columnFamily);
                    MessagingService.instance().sendOneWay(new ValidationComplete(desc).createMessage(), message.from);
                    return;
                }
                ActiveRepairService.instance.consistent.local.maybeSetRepairing(desc.parentSessionId);
                Validator validator = new Validator(desc, message.from, validationRequest.gcBefore, isConsistent(desc.parentSessionId));
                CompactionManager.instance.submitValidation(store, validator);
                break;
            case SYNC_REQUEST:
                // forwarded sync request
                SyncRequest request = (SyncRequest) message.payload;
                logger.debug("Syncing {}", request);
                long repairedAt = ActiveRepairService.UNREPAIRED_SSTABLE;
                if (desc.parentSessionId != null && ActiveRepairService.instance.getParentRepairSession(desc.parentSessionId) != null)
                    repairedAt = ActiveRepairService.instance.getParentRepairSession(desc.parentSessionId).getRepairedAt();
                StreamingRepairTask task = new StreamingRepairTask(desc, request, repairedAt, isConsistent(desc.parentSessionId));
                task.run();
                break;
            case CLEANUP:
                logger.debug("cleaning up repair");
                CleanupMessage cleanup = (CleanupMessage) message.payload;
                ActiveRepairService.instance.removeParentRepairSession(cleanup.parentRepairSession);
                MessagingService.instance().sendReply(new MessageOut(MessagingService.Verb.INTERNAL_RESPONSE), id, message.from);
                break;
            case CONSISTENT_REQUEST:
                ActiveRepairService.instance.consistent.local.handlePrepareMessage(message.from, (PrepareConsistentRequest) message.payload);
                break;
            case CONSISTENT_RESPONSE:
                ActiveRepairService.instance.consistent.coordinated.handlePrepareResponse((PrepareConsistentResponse) message.payload);
                break;
            case FINALIZE_PROPOSE:
                ActiveRepairService.instance.consistent.local.handleFinalizeProposeMessage(message.from, (FinalizePropose) message.payload);
                break;
            case FINALIZE_PROMISE:
                ActiveRepairService.instance.consistent.coordinated.handleFinalizePromiseMessage((FinalizePromise) message.payload);
                break;
            case FINALIZE_COMMIT:
                ActiveRepairService.instance.consistent.local.handleFinalizeCommitMessage(message.from, (FinalizeCommit) message.payload);
                break;
            case FAILED_SESSION:
                FailSession failure = (FailSession) message.payload;
                ActiveRepairService.instance.consistent.coordinated.handleFailSessionMessage(failure);
                ActiveRepairService.instance.consistent.local.handleFailSessionMessage(message.from, failure);
                break;
            case STATUS_REQUEST:
                ActiveRepairService.instance.consistent.local.handleStatusRequest(message.from, (StatusRequest) message.payload);
                break;
            case STATUS_RESPONSE:
                ActiveRepairService.instance.consistent.local.handleStatusResponse(message.from, (StatusResponse) message.payload);
                break;
            default:
                ActiveRepairService.instance.handleMessage(message.from, message.payload);
                break;
        }
    } catch (Exception e) {
        logger.error("Got error, removing parent repair session");
        if (desc != null && desc.parentSessionId != null)
            ActiveRepairService.instance.removeParentRepairSession(desc.parentSessionId);
        throw new RuntimeException(e);
    }
}

Example 7 with ColumnFamilyStore

use of org.apache.cassandra.db.ColumnFamilyStore in project cassandra by apache.

the class RepairRunnable method runMayThrow.

protected void runMayThrow() throws Exception {
    final TraceState traceState;
    final UUID parentSession = UUIDGen.getTimeUUID();
    final String tag = "repair:" + cmd;
    final AtomicInteger progress = new AtomicInteger();
    // get valid column families, calculate neighbors, validation, prepare for repair + number of ranges to repair
    final int totalProgress = 4 + options.getRanges().size();
    String[] columnFamilies = options.getColumnFamilies().toArray(new String[options.getColumnFamilies().size()]);
    Iterable<ColumnFamilyStore> validColumnFamilies;
    try {
        validColumnFamilies = storageService.getValidColumnFamilies(false, false, keyspace, columnFamilies);
        progress.incrementAndGet();
    } catch (IllegalArgumentException e) {
        logger.error("Repair failed:", e);
        fireErrorAndComplete(tag, progress.get(), totalProgress, e.getMessage());
        return;
    }
    final long startTime = System.currentTimeMillis();
    String message = String.format("Starting repair command #%d (%s), repairing keyspace %s with %s", cmd, parentSession, keyspace, options);
    logger.info(message);
    if (options.isTraced()) {
        StringBuilder cfsb = new StringBuilder();
        for (ColumnFamilyStore cfs : validColumnFamilies) cfsb.append(", ").append(cfs.keyspace.getName()).append(".").append(cfs.name);
        UUID sessionId = Tracing.instance.newSession(Tracing.TraceType.REPAIR);
        traceState = Tracing.instance.begin("repair", ImmutableMap.of("keyspace", keyspace, "columnFamilies", cfsb.substring(2)));
        message = message + " tracing with " + sessionId;
        fireProgressEvent(tag, new ProgressEvent(ProgressEventType.START, 0, 100, message));
        Tracing.traceRepair(message);
        traceState.enableActivityNotification(tag);
        for (ProgressListener listener : listeners) traceState.addProgressListener(listener);
        Thread queryThread = createQueryThread(cmd, sessionId);
        queryThread.setName("RepairTracePolling");
        queryThread.start();
    } else {
        fireProgressEvent(tag, new ProgressEvent(ProgressEventType.START, 0, 100, message));
        traceState = null;
    }
    final Set<InetAddress> allNeighbors = new HashSet<>();
    List<Pair<Set<InetAddress>, ? extends Collection<Range<Token>>>> commonRanges = new ArrayList<>();
    //pre-calculate output of getLocalRanges and pass it to getNeighbors to increase performance and prevent
    //calculation multiple times
    Collection<Range<Token>> keyspaceLocalRanges = storageService.getLocalRanges(keyspace);
    try {
        for (Range<Token> range : options.getRanges()) {
            Set<InetAddress> neighbors = ActiveRepairService.getNeighbors(keyspace, keyspaceLocalRanges, range, options.getDataCenters(), options.getHosts());
            addRangeToNeighbors(commonRanges, range, neighbors);
            allNeighbors.addAll(neighbors);
        }
        progress.incrementAndGet();
    } catch (IllegalArgumentException e) {
        logger.error("Repair failed:", e);
        fireErrorAndComplete(tag, progress.get(), totalProgress, e.getMessage());
        return;
    }
    // Validate columnfamilies
    List<ColumnFamilyStore> columnFamilyStores = new ArrayList<>();
    try {
        Iterables.addAll(columnFamilyStores, validColumnFamilies);
        progress.incrementAndGet();
    } catch (IllegalArgumentException e) {
        fireErrorAndComplete(tag, progress.get(), totalProgress, e.getMessage());
        return;
    }
    String[] cfnames = new String[columnFamilyStores.size()];
    for (int i = 0; i < columnFamilyStores.size(); i++) {
        cfnames[i] = columnFamilyStores.get(i).name;
    }
    SystemDistributedKeyspace.startParentRepair(parentSession, keyspace, cfnames, options);
    long repairedAt;
    try {
        ActiveRepairService.instance.prepareForRepair(parentSession, FBUtilities.getBroadcastAddress(), allNeighbors, options, columnFamilyStores);
        repairedAt = ActiveRepairService.instance.getParentRepairSession(parentSession).getRepairedAt();
        progress.incrementAndGet();
    } catch (Throwable t) {
        SystemDistributedKeyspace.failParentRepair(parentSession, t);
        fireErrorAndComplete(tag, progress.get(), totalProgress, t.getMessage());
        return;
    }
    if (options.isIncremental()) {
        consistentRepair(parentSession, repairedAt, startTime, traceState, allNeighbors, commonRanges, cfnames);
    } else {
        normalRepair(parentSession, startTime, traceState, allNeighbors, commonRanges, cfnames);
    }
}

Example 8 with ColumnFamilyStore

use of org.apache.cassandra.db.ColumnFamilyStore in project cassandra by apache.

the class StreamSession method getSSTableSectionsForRanges.

@VisibleForTesting
public static List<SSTableStreamingSections> getSSTableSectionsForRanges(Collection<Range<Token>> ranges, Collection<ColumnFamilyStore> stores, long overriddenRepairedAt, UUID pendingRepair) {
    Refs<SSTableReader> refs = new Refs<>();
    try {
        for (ColumnFamilyStore cfStore : stores) {
            final List<Range<PartitionPosition>> keyRanges = new ArrayList<>(ranges.size());
            for (Range<Token> range : ranges) keyRanges.add(Range.makeRowRange(range));
            refs.addAll(cfStore.selectAndReference(view -> {
                Set<SSTableReader> sstables = Sets.newHashSet();
                SSTableIntervalTree intervalTree = SSTableIntervalTree.build(view.select(SSTableSet.CANONICAL));
                Predicate<SSTableReader> predicate;
                if (pendingRepair == ActiveRepairService.NO_PENDING_REPAIR) {
                    predicate = Predicates.alwaysTrue();
                } else {
                    predicate = s -> s.isPendingRepair() && s.getSSTableMetadata().pendingRepair.equals(pendingRepair);
                }
                for (Range<PartitionPosition> keyRange : keyRanges) {
                    for (SSTableReader sstable : Iterables.filter(View.sstablesInBounds(keyRange.left, keyRange.right, intervalTree), predicate)) {
                        sstables.add(sstable);
                    }
                }
                if (logger.isDebugEnabled())
                    logger.debug("ViewFilter for {}/{} sstables", sstables.size(), Iterables.size(view.select(SSTableSet.CANONICAL)));
                return sstables;
            }).refs);
        }
        List<SSTableStreamingSections> sections = new ArrayList<>(refs.size());
        for (SSTableReader sstable : refs) {
            long repairedAt = overriddenRepairedAt;
            if (overriddenRepairedAt == ActiveRepairService.UNREPAIRED_SSTABLE)
                repairedAt = sstable.getSSTableMetadata().repairedAt;
            sections.add(new SSTableStreamingSections(refs.get(sstable), sstable.getPositionsForRanges(ranges), sstable.estimatedKeysForRanges(ranges), repairedAt));
        }
        return sections;
    } catch (Throwable t) {
        refs.release();
        throw t;
    }
}

Example 9 with ColumnFamilyStore

use of org.apache.cassandra.db.ColumnFamilyStore in project cassandra by apache.

the class CompressedStreamReader method read.

/**
     * @return SSTable transferred
     * @throws java.io.IOException if reading the remote sstable fails. Will throw an RTE if local write fails.
     */
@Override
// channel needs to remain open, streams on top of it can't be closed
@SuppressWarnings("resource")
public SSTableMultiWriter read(ReadableByteChannel channel) throws IOException {
    long totalSize = totalSize();
    ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
    if (cfs == null) {
        // schema was dropped during streaming
        throw new IOException("CF " + tableId + " was dropped during streaming");
    }
    logger.debug("[Stream #{}] Start receiving file #{} from {}, repairedAt = {}, size = {}, ks = '{}', pendingRepair = '{}', table = '{}'.", session.planId(), fileSeqNum, session.peer, repairedAt, totalSize, cfs.keyspace.getName(), session.getPendingRepair(), cfs.getTableName());
    CompressedInputStream cis = new CompressedInputStream(Channels.newInputStream(channel), compressionInfo, ChecksumType.CRC32, cfs::getCrcCheckChance);
    TrackedInputStream in = new TrackedInputStream(cis);
    StreamDeserializer deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()));
    SSTableMultiWriter writer = null;
    try {
        writer = createWriter(cfs, totalSize, repairedAt, session.getPendingRepair(), format);
        String filename = writer.getFilename();
        int sectionIdx = 0;
        for (Pair<Long, Long> section : sections) {
            assert cis.getTotalCompressedBytesRead() <= totalSize;
            long sectionLength = section.right - section.left;
            logger.trace("[Stream #{}] Reading section {} with length {} from stream.", session.planId(), sectionIdx++, sectionLength);
            // skip to beginning of section inside chunk
            cis.position(section.left);
            in.reset(0);
            while (in.getBytesRead() < sectionLength) {
                writePartition(deserializer, writer);
                // when compressed, report total bytes of compressed chunks read since remoteFile.size is the sum of chunks transferred
                session.progress(filename, ProgressInfo.Direction.IN, cis.getTotalCompressedBytesRead(), totalSize);
            }
        }
        logger.debug("[Stream #{}] Finished receiving file #{} from {} readBytes = {}, totalSize = {}", session.planId(), fileSeqNum, session.peer, FBUtilities.prettyPrintMemory(cis.getTotalCompressedBytesRead()), FBUtilities.prettyPrintMemory(totalSize));
        return writer;
    } catch (Throwable e) {
        logger.warn("[Stream {}] Error while reading partition {} from stream on ks='{}' and table='{}'.", session.planId(), deserializer.partitionKey(), cfs.keyspace.getName(), cfs.getTableName());
        if (writer != null) {
            writer.abort(e);
        }
        if (extractIOExceptionCause(e).isPresent())
            throw e;
        throw Throwables.propagate(e);
    }
}

Example 10 with ColumnFamilyStore

use of org.apache.cassandra.db.ColumnFamilyStore in project cassandra by apache.

the class SSTableExpiredBlockers method main.

public static void main(String[] args) {
    PrintStream out = System.out;
    if (args.length < 2) {
        out.println("Usage: sstableexpiredblockers <keyspace> <table>");
        System.exit(1);
    }
    Util.initDatabaseDescriptor();
    String keyspace = args[args.length - 2];
    String columnfamily = args[args.length - 1];
    Schema.instance.loadFromDisk(false);
    TableMetadata metadata = Schema.instance.validateTable(keyspace, columnfamily);
    Keyspace ks = Keyspace.openWithoutSSTables(keyspace);
    ColumnFamilyStore cfs = ks.getColumnFamilyStore(columnfamily);
    Directories.SSTableLister lister = cfs.getDirectories().sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true);
    Set<SSTableReader> sstables = new HashSet<>();
    for (Map.Entry<Descriptor, Set<Component>> sstable : lister.list().entrySet()) {
        if (sstable.getKey() != null) {
            try {
                SSTableReader reader = SSTableReader.open(sstable.getKey());
                sstables.add(reader);
            } catch (Throwable t) {
                out.println("Couldn't open sstable: " + sstable.getKey().filenameFor(Component.DATA) + " (" + t.getMessage() + ")");
            }
        }
    }
    if (sstables.isEmpty()) {
        out.println("No sstables for " + keyspace + "." + columnfamily);
        System.exit(1);
    }
    int gcBefore = (int) (System.currentTimeMillis() / 1000) - metadata.params.gcGraceSeconds;
    Multimap<SSTableReader, SSTableReader> blockers = checkForExpiredSSTableBlockers(sstables, gcBefore);
    for (SSTableReader blocker : blockers.keySet()) {
        out.println(String.format("%s blocks %d expired sstables from getting dropped: %s%n", formatForExpiryTracing(Collections.singleton(blocker)), blockers.get(blocker).size(), formatForExpiryTracing(blockers.get(blocker))));
    }
    System.exit(0);
}