Examples with Keyspace org.apache.cassandra.db.Keyspace used on opensource projects

Example 26 with Keyspace

use of org.apache.cassandra.db.Keyspace in project cassandra by apache.

the class RangeStreamer method fetchAsync.

public StreamResultFuture fetchAsync() {
    toFetch.forEach((keyspace, sources) -> {
        logger.debug("Keyspace {} Sources {}", keyspace, sources);
        sources.asMap().forEach((source, fetchReplicas) -> {
            // filter out already streamed ranges
            SystemKeyspace.AvailableRanges available = stateStore.getAvailableRanges(keyspace, metadata.partitioner);
            Predicate<FetchReplica> isAvailable = fetch -> {
                boolean isInFull = available.full.contains(fetch.local.range());
                boolean isInTrans = available.trans.contains(fetch.local.range());
                if (!isInFull && !isInTrans)
                    // Range is unavailable
                    return false;
                if (fetch.local.isFull())
                    // For full, pick only replicas with matching transientness
                    return isInFull == fetch.remote.isFull();
                // Any transient or full will do
                return true;
            };
            List<FetchReplica> remaining = fetchReplicas.stream().filter(not(isAvailable)).collect(Collectors.toList());
            if (remaining.size() < available.full.size() + available.trans.size()) {
                List<FetchReplica> skipped = fetchReplicas.stream().filter(isAvailable).collect(Collectors.toList());
                logger.info("Some ranges of {} are already available. Skipping streaming those ranges. Skipping {}. Fully available {} Transiently available {}", fetchReplicas, skipped, available.full, available.trans);
            }
            if (logger.isTraceEnabled())
                logger.trace("{}ing from {} ranges {}", description, source, StringUtils.join(remaining, ", "));
            InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort();
            RangesAtEndpoint full = remaining.stream().filter(pair -> pair.remote.isFull()).map(pair -> pair.local).collect(RangesAtEndpoint.collector(self));
            RangesAtEndpoint transientReplicas = remaining.stream().filter(pair -> pair.remote.isTransient()).map(pair -> pair.local).collect(RangesAtEndpoint.collector(self));
            logger.debug("Source and our replicas {}", fetchReplicas);
            logger.debug("Source {} Keyspace {}  streaming full {} transient {}", source, keyspace, full, transientReplicas);
            /* Send messages to respective folks to stream data over to me */
            streamPlan.requestRanges(source, keyspace, full, transientReplicas);
        });
    });
    return streamPlan.execute();
}

Example 27 with Keyspace

use of org.apache.cassandra.db.Keyspace in project cassandra by apache.

the class RangeStreamer method calculateRangesToFetchWithPreferredEndpoints.

/**
 * Get a map of all ranges and the source that will be cleaned up once this bootstrapped node is added for the given ranges.
 * For each range, the list should only contain a single source. This allows us to consistently migrate data without violating
 * consistency.
 */
public static EndpointsByReplica calculateRangesToFetchWithPreferredEndpoints(BiFunction<InetAddressAndPort, EndpointsForRange, EndpointsForRange> snitchGetSortedListByProximity, AbstractReplicationStrategy strat, ReplicaCollection<?> fetchRanges, boolean useStrictConsistency, TokenMetadata tmdBefore, TokenMetadata tmdAfter, String keyspace, Collection<SourceFilter> sourceFilters) {
    EndpointsByRange rangeAddresses = strat.getRangeAddresses(tmdBefore);
    InetAddressAndPort localAddress = FBUtilities.getBroadcastAddressAndPort();
    logger.debug("Keyspace: {}", keyspace);
    logger.debug("To fetch RN: {}", fetchRanges);
    logger.debug("Fetch ranges: {}", rangeAddresses);
    Predicate<Replica> testSourceFilters = and(sourceFilters);
    Function<EndpointsForRange, EndpointsForRange> sorted = endpoints -> snitchGetSortedListByProximity.apply(localAddress, endpoints);
    // This list of replicas is just candidates. With strict consistency it's going to be a narrow list.
    EndpointsByReplica.Builder rangesToFetchWithPreferredEndpoints = new EndpointsByReplica.Builder();
    for (Replica toFetch : fetchRanges) {
        // Replica that is sufficient to provide the data we need
        // With strict consistency and transient replication we may end up with multiple types
        // so this isn't used with strict consistency
        Predicate<Replica> isSufficient = r -> toFetch.isTransient() || r.isFull();
        logger.debug("To fetch {}", toFetch);
        for (Range<Token> range : rangeAddresses.keySet()) {
            if (!range.contains(toFetch.range()))
                continue;
            final EndpointsForRange oldEndpoints = sorted.apply(rangeAddresses.get(range));
            // Ultimately we populate this with whatever is going to be fetched from to satisfy toFetch
            // It could be multiple endpoints and we must fetch from all of them if they are there
            // With transient replication and strict consistency this is to get the full data from a full replica and
            // transient data from the transient replica losing data
            EndpointsForRange sources;
            // Due to CASSANDRA-5953 we can have a higher RF than we have endpoints.
            // So we need to be careful to only be strict when endpoints == RF
            boolean isStrictConsistencyApplicable = useStrictConsistency && (oldEndpoints.size() == strat.getReplicationFactor().allReplicas);
            if (isStrictConsistencyApplicable) {
                EndpointsForRange strictEndpoints;
                // Start with two sets of who replicates the range before and who replicates it after
                EndpointsForRange newEndpoints = strat.calculateNaturalReplicas(toFetch.range().right, tmdAfter);
                logger.debug("Old endpoints {}", oldEndpoints);
                logger.debug("New endpoints {}", newEndpoints);
                // Remove new endpoints from old endpoints based on address
                strictEndpoints = oldEndpoints.without(newEndpoints.endpoints());
                if (strictEndpoints.size() > 1)
                    throw new AssertionError("Expected <= 1 endpoint but found " + strictEndpoints);
                // required for strict consistency
                if (!all(strictEndpoints, testSourceFilters))
                    throw new IllegalStateException("Necessary replicas for strict consistency were removed by source filters: " + buildErrorMessage(sourceFilters, strictEndpoints));
                // So it's an error if we don't find what we need.
                if (strictEndpoints.isEmpty() && toFetch.isTransient())
                    throw new AssertionError("If there are no endpoints to fetch from then we must be transitioning from transient to full for range " + toFetch);
                if (!any(strictEndpoints, isSufficient)) {
                    // need an additional replica; include all our filters, to ensure we include a matching node
                    Optional<Replica> fullReplica = Iterables.<Replica>tryFind(oldEndpoints, and(isSufficient, testSourceFilters)).toJavaUtil();
                    if (fullReplica.isPresent())
                        strictEndpoints = Endpoints.concat(strictEndpoints, EndpointsForRange.of(fullReplica.get()));
                    else
                        throw new IllegalStateException("Couldn't find any matching sufficient replica out of " + buildErrorMessage(sourceFilters, oldEndpoints));
                }
                sources = strictEndpoints;
            } else {
                // Without strict consistency we have given up on correctness so no point in fetching from
                // a random full + transient replica since it's also likely to lose data
                // Also apply testSourceFilters that were given to us so we can safely select a single source
                sources = sorted.apply(oldEndpoints.filter(and(isSufficient, testSourceFilters)));
                // Limit it to just the first possible source, we don't need more than one and downstream
                // will fetch from every source we supply
                sources = sources.size() > 0 ? sources.subList(0, 1) : sources;
            }
            // storing range and preferred endpoint set
            rangesToFetchWithPreferredEndpoints.putAll(toFetch, sources, Conflict.NONE);
            logger.debug("Endpoints to fetch for {} are {}", toFetch, sources);
        }
        EndpointsForRange addressList = rangesToFetchWithPreferredEndpoints.getIfPresent(toFetch);
        if (addressList == null)
            throw new IllegalStateException("Failed to find endpoints to fetch " + toFetch);
        /*
              * When we move forwards (shrink our bucket) we are the one losing a range and no one else loses
              * from that action (we also don't gain). When we move backwards there are two people losing a range. One is a full replica
              * and the other is a transient replica. So we must need fetch from two places in that case for the full range we gain.
              * For a transient range we only need to fetch from one.
              */
        if (useStrictConsistency && addressList.size() > 1 && (addressList.filter(Replica::isFull).size() > 1 || addressList.filter(Replica::isTransient).size() > 1))
            throw new IllegalStateException(String.format("Multiple strict sources found for %s, sources: %s", toFetch, addressList));
        // We must have enough stuff to fetch from
        if (!any(addressList, isSufficient)) {
            if (strat.getReplicationFactor().allReplicas == 1) {
                if (useStrictConsistency) {
                    logger.warn("A node required to move the data consistently is down");
                    throw new IllegalStateException("Unable to find sufficient sources for streaming range " + toFetch + " in keyspace " + keyspace + " with RF=1. " + "Ensure this keyspace contains replicas in the source datacenter.");
                } else
                    logger.warn("Unable to find sufficient sources for streaming range {} in keyspace {} with RF=1. " + "Keyspace might be missing data.", toFetch, keyspace);
            } else {
                if (useStrictConsistency)
                    logger.warn("A node required to move the data consistently is down");
                throw new IllegalStateException("Unable to find sufficient sources for streaming range " + toFetch + " in keyspace " + keyspace);
            }
        }
    }
    return rangesToFetchWithPreferredEndpoints.build();
}

Example 28 with Keyspace

use of org.apache.cassandra.db.Keyspace in project cassandra by apache.

the class TableMetricTables method buildMetadata.

/**
 *  Identify the type of Metric it is (gauge, counter etc) abd create the TableMetadata. The column name
 *  and type for a counter/gauge is formatted differently based on the units (bytes/time) so allowed to
 *  be set.
 */
private static TableMetadata buildMetadata(String keyspace, String table, Function<TableMetrics, ? extends Metric> func, String colName, AbstractType colType, String suffix) {
    TableMetadata.Builder metadata = TableMetadata.builder(keyspace, table).kind(TableMetadata.Kind.VIRTUAL).addPartitionKeyColumn(KEYSPACE_NAME, UTF8Type.instance).addPartitionKeyColumn(TABLE_NAME, UTF8Type.instance).partitioner(PARTITIONER);
    // get a table from system keyspace and get metric from it for determining type of metric
    Keyspace system = Keyspace.system().iterator().next();
    Metric test = func.apply(system.getColumnFamilyStores().iterator().next().metric);
    if (test instanceof Counting) {
        metadata.addRegularColumn(colName, colType);
        // if it has a Histogram include some information about distribution
        if (test instanceof Sampling) {
            metadata.addRegularColumn(P50 + suffix, DoubleType.instance).addRegularColumn(P99 + suffix, DoubleType.instance).addRegularColumn(MAX + suffix, DoubleType.instance);
        }
        if (test instanceof Metered) {
            metadata.addRegularColumn(RATE, DoubleType.instance);
        }
    } else if (test instanceof Gauge) {
        metadata.addRegularColumn(colName, colType);
    }
    return metadata.build();
}

Example 29 with Keyspace

use of org.apache.cassandra.db.Keyspace in project cassandra by apache.

the class BootStrapper method allocateTokens.

static Collection<Token> allocateTokens(final TokenMetadata metadata, InetAddressAndPort address, String allocationKeyspace, int numTokens, long schemaWaitDelay) {
    StorageService.instance.waitForSchema(schemaWaitDelay);
    if (!FBUtilities.getBroadcastAddressAndPort().equals(InetAddressAndPort.getLoopbackAddress()))
        Gossiper.waitToSettle();
    Keyspace ks = Keyspace.open(allocationKeyspace);
    if (ks == null)
        throw new ConfigurationException("Problem opening token allocation keyspace " + allocationKeyspace);
    AbstractReplicationStrategy rs = ks.getReplicationStrategy();
    Collection<Token> tokens = TokenAllocation.allocateTokens(metadata, rs, address, numTokens);
    BootstrapDiagnostics.tokensAllocated(address, metadata, allocationKeyspace, numTokens, tokens);
    return tokens;
}

Example 30 with Keyspace

use of org.apache.cassandra.db.Keyspace in project cassandra by apache.

the class UpdateTest method isMemtableEmpty.

/**
 * Checks if the memtable is empty or not
 * @return {@code true} if the memtable is empty, {@code false} otherwise.
 */
private boolean isMemtableEmpty() {
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(currentTable());
    return cfs.metric.allMemtablesLiveDataSize.getValue() == 0;
}