Examples with Pair org.apache.cassandra.utils.Pair used on opensource projects

Example 1 with Pair

use of org.apache.cassandra.utils.Pair in project cassandra by apache.

the class AutoSavingCache method loadSaved.

public int loadSaved() {
    int count = 0;
    long start = System.nanoTime();
    // modern format, allows both key and value (so key cache load can be purely sequential)
    File dataPath = getCacheDataPath(CURRENT_VERSION);
    File crcPath = getCacheCrcPath(CURRENT_VERSION);
    if (dataPath.exists() && crcPath.exists()) {
        DataInputStreamPlus in = null;
        try {
            logger.info("reading saved cache {}", dataPath);
            in = new DataInputStreamPlus(new LengthAvailableInputStream(new BufferedInputStream(streamFactory.getInputStream(dataPath, crcPath)), dataPath.length()));
            //Check the schema has not changed since CFs are looked up by name which is ambiguous
            UUID schemaVersion = new UUID(in.readLong(), in.readLong());
            if (!schemaVersion.equals(Schema.instance.getVersion()))
                throw new RuntimeException("Cache schema version " + schemaVersion + " does not match current schema version " + Schema.instance.getVersion());
            ArrayDeque<Future<Pair<K, V>>> futures = new ArrayDeque<Future<Pair<K, V>>>();
            while (in.available() > 0) {
                //tableId and indexName are serialized by the serializers in CacheService
                //That is delegated there because there are serializer specific conditions
                //where a cache key is skipped and not written
                TableId tableId = TableId.deserialize(in);
                String indexName = in.readUTF();
                if (indexName.isEmpty())
                    indexName = null;
                ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(tableId);
                if (indexName != null && cfs != null)
                    cfs = cfs.indexManager.getIndexByName(indexName).getBackingTable().orElse(null);
                Future<Pair<K, V>> entryFuture = cacheLoader.deserialize(in, cfs);
                // Key cache entry can return null, if the SSTable doesn't exist.
                if (entryFuture == null)
                    continue;
                futures.offer(entryFuture);
                count++;
                /*
                     * Kind of unwise to accrue an unbounded number of pending futures
                     * So now there is this loop to keep a bounded number pending.
                     */
                do {
                    while (futures.peek() != null && futures.peek().isDone()) {
                        Future<Pair<K, V>> future = futures.poll();
                        Pair<K, V> entry = future.get();
                        if (entry != null && entry.right != null)
                            put(entry.left, entry.right);
                    }
                    if (futures.size() > 1000)
                        Thread.yield();
                } while (futures.size() > 1000);
            }
            Future<Pair<K, V>> future = null;
            while ((future = futures.poll()) != null) {
                Pair<K, V> entry = future.get();
                if (entry != null && entry.right != null)
                    put(entry.left, entry.right);
            }
        } catch (CorruptFileException e) {
            JVMStabilityInspector.inspectThrowable(e);
            logger.warn(String.format("Non-fatal checksum error reading saved cache %s", dataPath.getAbsolutePath()), e);
        } catch (Throwable t) {
            JVMStabilityInspector.inspectThrowable(t);
            logger.info(String.format("Harmless error reading saved cache %s", dataPath.getAbsolutePath()), t);
        } finally {
            FileUtils.closeQuietly(in);
        }
    }
    if (logger.isTraceEnabled())
        logger.trace("completed reading ({} ms; {} keys) saved cache {}", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start), count, dataPath);
    return count;
}

Example 2 with Pair

use of org.apache.cassandra.utils.Pair in project cassandra by apache.

the class Directories method getSnapshotDetails.

/**
     *
     * @return  Return a map of all snapshots to space being used
     * The pair for a snapshot has size on disk and true size.
     */
public Map<String, Pair<Long, Long>> getSnapshotDetails() {
    final Map<String, Pair<Long, Long>> snapshotSpaceMap = new HashMap<>();
    for (File snapshot : listSnapshots()) {
        final long sizeOnDisk = FileUtils.folderSize(snapshot);
        final long trueSize = getTrueAllocatedSizeIn(snapshot);
        Pair<Long, Long> spaceUsed = snapshotSpaceMap.get(snapshot.getName());
        if (spaceUsed == null)
            spaceUsed = Pair.create(sizeOnDisk, trueSize);
        else
            spaceUsed = Pair.create(spaceUsed.left + sizeOnDisk, spaceUsed.right + trueSize);
        snapshotSpaceMap.put(snapshot.getName(), spaceUsed);
    }
    return snapshotSpaceMap;
}

Example 3 with Pair

use of org.apache.cassandra.utils.Pair in project cassandra by apache.

the class PendingRepairManager method replaceSSTables.

synchronized void replaceSSTables(Set<SSTableReader> removed, Set<SSTableReader> added) {
    if (removed.isEmpty() && added.isEmpty())
        return;
    // left=removed, right=added
    Map<UUID, Pair<Set<SSTableReader>, Set<SSTableReader>>> groups = new HashMap<>();
    for (SSTableReader sstable : removed) {
        UUID sessionID = sstable.getSSTableMetadata().pendingRepair;
        if (!groups.containsKey(sessionID)) {
            groups.put(sessionID, Pair.create(new HashSet<>(), new HashSet<>()));
        }
        groups.get(sessionID).left.add(sstable);
    }
    for (SSTableReader sstable : added) {
        UUID sessionID = sstable.getSSTableMetadata().pendingRepair;
        if (!groups.containsKey(sessionID)) {
            groups.put(sessionID, Pair.create(new HashSet<>(), new HashSet<>()));
        }
        groups.get(sessionID).right.add(sstable);
    }
    for (Map.Entry<UUID, Pair<Set<SSTableReader>, Set<SSTableReader>>> entry : groups.entrySet()) {
        AbstractCompactionStrategy strategy = getOrCreate(entry.getKey());
        Set<SSTableReader> groupRemoved = entry.getValue().left;
        Set<SSTableReader> groupAdded = entry.getValue().right;
        if (!groupRemoved.isEmpty())
            strategy.replaceSSTables(groupRemoved, groupAdded);
        else
            strategy.addSSTables(groupAdded);
    }
}

Example 4 with Pair

use of org.apache.cassandra.utils.Pair in project cassandra by apache.

the class RepairRunnable method runMayThrow.

protected void runMayThrow() throws Exception {
    final TraceState traceState;
    final UUID parentSession = UUIDGen.getTimeUUID();
    final String tag = "repair:" + cmd;
    final AtomicInteger progress = new AtomicInteger();
    // get valid column families, calculate neighbors, validation, prepare for repair + number of ranges to repair
    final int totalProgress = 4 + options.getRanges().size();
    String[] columnFamilies = options.getColumnFamilies().toArray(new String[options.getColumnFamilies().size()]);
    Iterable<ColumnFamilyStore> validColumnFamilies;
    try {
        validColumnFamilies = storageService.getValidColumnFamilies(false, false, keyspace, columnFamilies);
        progress.incrementAndGet();
    } catch (IllegalArgumentException e) {
        logger.error("Repair failed:", e);
        fireErrorAndComplete(tag, progress.get(), totalProgress, e.getMessage());
        return;
    }
    final long startTime = System.currentTimeMillis();
    String message = String.format("Starting repair command #%d (%s), repairing keyspace %s with %s", cmd, parentSession, keyspace, options);
    logger.info(message);
    if (options.isTraced()) {
        StringBuilder cfsb = new StringBuilder();
        for (ColumnFamilyStore cfs : validColumnFamilies) cfsb.append(", ").append(cfs.keyspace.getName()).append(".").append(cfs.name);
        UUID sessionId = Tracing.instance.newSession(Tracing.TraceType.REPAIR);
        traceState = Tracing.instance.begin("repair", ImmutableMap.of("keyspace", keyspace, "columnFamilies", cfsb.substring(2)));
        message = message + " tracing with " + sessionId;
        fireProgressEvent(tag, new ProgressEvent(ProgressEventType.START, 0, 100, message));
        Tracing.traceRepair(message);
        traceState.enableActivityNotification(tag);
        for (ProgressListener listener : listeners) traceState.addProgressListener(listener);
        Thread queryThread = createQueryThread(cmd, sessionId);
        queryThread.setName("RepairTracePolling");
        queryThread.start();
    } else {
        fireProgressEvent(tag, new ProgressEvent(ProgressEventType.START, 0, 100, message));
        traceState = null;
    }
    final Set<InetAddress> allNeighbors = new HashSet<>();
    List<Pair<Set<InetAddress>, ? extends Collection<Range<Token>>>> commonRanges = new ArrayList<>();
    //pre-calculate output of getLocalRanges and pass it to getNeighbors to increase performance and prevent
    //calculation multiple times
    Collection<Range<Token>> keyspaceLocalRanges = storageService.getLocalRanges(keyspace);
    try {
        for (Range<Token> range : options.getRanges()) {
            Set<InetAddress> neighbors = ActiveRepairService.getNeighbors(keyspace, keyspaceLocalRanges, range, options.getDataCenters(), options.getHosts());
            addRangeToNeighbors(commonRanges, range, neighbors);
            allNeighbors.addAll(neighbors);
        }
        progress.incrementAndGet();
    } catch (IllegalArgumentException e) {
        logger.error("Repair failed:", e);
        fireErrorAndComplete(tag, progress.get(), totalProgress, e.getMessage());
        return;
    }
    // Validate columnfamilies
    List<ColumnFamilyStore> columnFamilyStores = new ArrayList<>();
    try {
        Iterables.addAll(columnFamilyStores, validColumnFamilies);
        progress.incrementAndGet();
    } catch (IllegalArgumentException e) {
        fireErrorAndComplete(tag, progress.get(), totalProgress, e.getMessage());
        return;
    }
    String[] cfnames = new String[columnFamilyStores.size()];
    for (int i = 0; i < columnFamilyStores.size(); i++) {
        cfnames[i] = columnFamilyStores.get(i).name;
    }
    SystemDistributedKeyspace.startParentRepair(parentSession, keyspace, cfnames, options);
    long repairedAt;
    try {
        ActiveRepairService.instance.prepareForRepair(parentSession, FBUtilities.getBroadcastAddress(), allNeighbors, options, columnFamilyStores);
        repairedAt = ActiveRepairService.instance.getParentRepairSession(parentSession).getRepairedAt();
        progress.incrementAndGet();
    } catch (Throwable t) {
        SystemDistributedKeyspace.failParentRepair(parentSession, t);
        fireErrorAndComplete(tag, progress.get(), totalProgress, t.getMessage());
        return;
    }
    if (options.isIncremental()) {
        consistentRepair(parentSession, repairedAt, startTime, traceState, allNeighbors, commonRanges, cfnames);
    } else {
        normalRepair(parentSession, startTime, traceState, allNeighbors, commonRanges, cfnames);
    }
}

Example 5 with Pair

use of org.apache.cassandra.utils.Pair in project cassandra by apache.

the class CompressedStreamWriter method write.

@Override
public void write(DataOutputStreamPlus out) throws IOException {
    long totalSize = totalSize();
    logger.debug("[Stream #{}] Start streaming file {} to {}, repairedAt = {}, totalSize = {}", session.planId(), sstable.getFilename(), session.peer, sstable.getSSTableMetadata().repairedAt, totalSize);
    try (ChannelProxy fc = sstable.getDataChannel().sharedCopy()) {
        long progress = 0L;
        // calculate chunks to transfer. we want to send continuous chunks altogether.
        List<Pair<Long, Long>> sections = getTransferSections(compressionInfo.chunks);
        int sectionIdx = 0;
        // stream each of the required sections of the file
        for (final Pair<Long, Long> section : sections) {
            // length of the section to stream
            long length = section.right - section.left;
            logger.trace("[Stream #{}] Writing section {} with length {} to stream.", session.planId(), sectionIdx++, length);
            // tracks write progress
            long bytesTransferred = 0;
            while (bytesTransferred < length) {
                final long bytesTransferredFinal = bytesTransferred;
                final int toTransfer = (int) Math.min(CHUNK_SIZE, length - bytesTransferred);
                limiter.acquire(toTransfer);
                long lastWrite = out.applyToChannel((wbc) -> fc.transferTo(section.left + bytesTransferredFinal, toTransfer, wbc));
                bytesTransferred += lastWrite;
                progress += lastWrite;
                session.progress(sstable.descriptor.filenameFor(Component.DATA), ProgressInfo.Direction.OUT, progress, totalSize);
            }
        }
        logger.debug("[Stream #{}] Finished streaming file {} to {}, bytesTransferred = {}, totalSize = {}", session.planId(), sstable.getFilename(), session.peer, FBUtilities.prettyPrintMemory(progress), FBUtilities.prettyPrintMemory(totalSize));
    }
}