Examples with Range org.apache.commons.lang3.Range used on opensource projects

Example 21 with Range

use of org.apache.commons.lang3.Range in project presto by prestodb.

the class ColumnCardinalityCache method getColumnCardinality.

/**
 * Gets the column cardinality for all of the given range values. May reach out to the
 * metrics table in Accumulo to retrieve new cache elements.
 *
 * @param schema Table schema
 * @param table Table name
 * @param auths Scan authorizations
 * @param family Accumulo column family
 * @param qualifier Accumulo column qualifier
 * @param colValues All range values to summarize for the cardinality
 * @return The cardinality of the column
 */
public long getColumnCardinality(String schema, String table, Authorizations auths, String family, String qualifier, Collection<Range> colValues) throws ExecutionException {
    LOG.debug("Getting cardinality for %s:%s", family, qualifier);
    // Collect all exact Accumulo Ranges, i.e. single value entries vs. a full scan
    Collection<CacheKey> exactRanges = colValues.stream().filter(ColumnCardinalityCache::isExact).map(range -> new CacheKey(schema, table, family, qualifier, range, auths)).collect(Collectors.toList());
    LOG.debug("Column values contain %s exact ranges of %s", exactRanges.size(), colValues.size());
    // Sum the cardinalities for the exact-value Ranges
    // This is where the reach-out to Accumulo occurs for all Ranges that have not
    // previously been fetched
    long sum = cache.getAll(exactRanges).values().stream().mapToLong(Long::longValue).sum();
    // then there is at least one non-exact range
    if (exactRanges.size() != colValues.size()) {
        // for each range in the column value
        for (Range range : colValues) {
            // if this range is not exact
            if (!isExact(range)) {
                // Then get the value for this range using the single-value cache lookup
                sum += cache.get(new CacheKey(schema, table, family, qualifier, range, auths));
            }
        }
    }
    return sum;
}

Example 22 with Range

use of org.apache.commons.lang3.Range in project cassandra by apache.

the class StorageService method restoreReplicaCount.

/**
 * Called when an endpoint is removed from the ring. This function checks
 * whether this node becomes responsible for new ranges as a
 * consequence and streams data if needed.
 *
 * This is rather ineffective, but it does not matter so much
 * since this is called very seldom
 *
 * @param endpoint the node that left
 */
private void restoreReplicaCount(InetAddressAndPort endpoint, final InetAddressAndPort notifyEndpoint) {
    Map<String, Multimap<InetAddressAndPort, FetchReplica>> replicasToFetch = new HashMap<>();
    InetAddressAndPort myAddress = FBUtilities.getBroadcastAddressAndPort();
    for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces()) {
        logger.debug("Restoring replica count for keyspace {}", keyspaceName);
        EndpointsByReplica changedReplicas = getChangedReplicasForLeaving(keyspaceName, endpoint, tokenMetadata, Keyspace.open(keyspaceName).getReplicationStrategy());
        Set<LeavingReplica> myNewReplicas = new HashSet<>();
        for (Map.Entry<Replica, Replica> entry : changedReplicas.flattenEntries()) {
            Replica replica = entry.getValue();
            if (replica.endpoint().equals(myAddress)) {
                // Maybe we don't technically need to fetch transient data from somewhere
                // but it's probably not a lot and it probably makes things a hair more resilient to people
                // not running repair when they should.
                myNewReplicas.add(new LeavingReplica(entry.getKey(), entry.getValue()));
            }
        }
        logger.debug("Changed replicas for leaving {}, myNewReplicas {}", changedReplicas, myNewReplicas);
        replicasToFetch.put(keyspaceName, getNewSourceReplicas(keyspaceName, myNewReplicas));
    }
    StreamPlan stream = new StreamPlan(StreamOperation.RESTORE_REPLICA_COUNT);
    replicasToFetch.forEach((keyspaceName, sources) -> {
        logger.debug("Requesting keyspace {} sources", keyspaceName);
        sources.asMap().forEach((sourceAddress, fetchReplicas) -> {
            logger.debug("Source and our replicas are {}", fetchReplicas);
            // Remember whether this node is providing the full or transient replicas for this range. We are going
            // to pass streaming the local instance of Replica for the range which doesn't tell us anything about the source
            // By encoding it as two separate sets we retain this information about the source.
            RangesAtEndpoint full = fetchReplicas.stream().filter(f -> f.remote.isFull()).map(f -> f.local).collect(RangesAtEndpoint.collector(myAddress));
            RangesAtEndpoint transientReplicas = fetchReplicas.stream().filter(f -> f.remote.isTransient()).map(f -> f.local).collect(RangesAtEndpoint.collector(myAddress));
            if (logger.isDebugEnabled())
                logger.debug("Requesting from {} full replicas {} transient replicas {}", sourceAddress, StringUtils.join(full, ", "), StringUtils.join(transientReplicas, ", "));
            stream.requestRanges(sourceAddress, keyspaceName, full, transientReplicas);
        });
    });
    StreamResultFuture future = stream.execute();
    future.addCallback(new FutureCallback<StreamState>() {

        public void onSuccess(StreamState finalState) {
            sendReplicationNotification(notifyEndpoint);
        }

        public void onFailure(Throwable t) {
            logger.warn("Streaming to restore replica count failed", t);
            // We still want to send the notification
            sendReplicationNotification(notifyEndpoint);
        }
    });
}

Example 23 with Range

use of org.apache.commons.lang3.Range in project cassandra by apache.

the class StorageService method getChangedReplicasForLeaving.

/**
 * This is used in three contexts, graceful decomission, and restoreReplicaCount/removeNode.
 * Graceful decomission should never lose data and it's going to be important that transient data
 * is streamed to at least one other node from this one for each range.
 *
 * For ranges this node replicates its removal should cause a new replica to be selected either as transient or full
 * for every range. So I believe the current code doesn't have to do anything special because it will engage in streaming
 * for every range it replicates to at least one other node and that should propagate the transient data that was here.
 * When I graphed this out on paper the result of removal looked correct and there are no issues such as
 * this node needing to create a full replica for a range it transiently replicates because what is created is just another
 * transient replica to replace this node.
 * @param keyspaceName
 * @param endpoint
 * @return
 */
// needs to be modified to accept either a keyspace or ARS.
static EndpointsByReplica getChangedReplicasForLeaving(String keyspaceName, InetAddressAndPort endpoint, TokenMetadata tokenMetadata, AbstractReplicationStrategy strat) {
    // First get all ranges the leaving endpoint is responsible for
    RangesAtEndpoint replicas = strat.getAddressReplicas(endpoint);
    if (logger.isDebugEnabled())
        logger.debug("Node {} replicas [{}]", endpoint, StringUtils.join(replicas, ", "));
    Map<Replica, EndpointsForRange> currentReplicaEndpoints = Maps.newHashMapWithExpectedSize(replicas.size());
    // Find (for each range) all nodes that store replicas for these ranges as well
    // don't do this in the loop! #7758
    TokenMetadata metadata = tokenMetadata.cloneOnlyTokenMap();
    for (Replica replica : replicas) currentReplicaEndpoints.put(replica, strat.calculateNaturalReplicas(replica.range().right, metadata));
    TokenMetadata temp = tokenMetadata.cloneAfterAllLeft();
    // command was used), it is still present in temp and must be removed.
    if (temp.isMember(endpoint))
        temp.removeEndpoint(endpoint);
    EndpointsByReplica.Builder changedRanges = new EndpointsByReplica.Builder();
    // range.
    for (Replica replica : replicas) {
        EndpointsForRange newReplicaEndpoints = strat.calculateNaturalReplicas(replica.range().right, temp);
        newReplicaEndpoints = newReplicaEndpoints.filter(newReplica -> {
            Optional<Replica> currentReplicaOptional = tryFind(currentReplicaEndpoints.get(replica), currentReplica -> newReplica.endpoint().equals(currentReplica.endpoint())).toJavaUtil();
            // If it is newly replicating then yes we must do something to get the data there
            if (!currentReplicaOptional.isPresent())
                return true;
            Replica currentReplica = currentReplicaOptional.get();
            // transient -> transient and full -> full don't require any action
            if (currentReplica.isTransient() && newReplica.isFull())
                return true;
            return false;
        });
        if (logger.isDebugEnabled())
            if (newReplicaEndpoints.isEmpty())
                logger.debug("Replica {} already in all replicas", replica);
            else
                logger.debug("Replica {} will be responsibility of {}", replica, StringUtils.join(newReplicaEndpoints, ", "));
        changedRanges.putAll(replica, newReplicaEndpoints, Conflict.NONE);
    }
    return changedRanges.build();
}

Example 24 with Range

use of org.apache.commons.lang3.Range in project cassandra by apache.

the class SelectStatement method asCQL.

private String asCQL(QueryOptions options) {
    ColumnFilter columnFilter = selection.newSelectors(options).getColumnFilter();
    StringBuilder sb = new StringBuilder();
    sb.append("SELECT ").append(queriedColumns().toCQLString());
    sb.append(" FROM ").append(table.keyspace).append('.').append(table.name);
    if (restrictions.isKeyRange() || restrictions.usesSecondaryIndexing()) {
        // partition range
        ClusteringIndexFilter clusteringIndexFilter = makeClusteringIndexFilter(options, columnFilter);
        if (clusteringIndexFilter == null)
            return "EMPTY";
        RowFilter rowFilter = getRowFilter(options);
        // The LIMIT provided by the user is the number of CQL row he wants returned.
        // We want to have getRangeSlice to count the number of columns, not the number of keys.
        AbstractBounds<PartitionPosition> keyBounds = restrictions.getPartitionKeyBounds(options);
        if (keyBounds == null)
            return "EMPTY";
        DataRange dataRange = new DataRange(keyBounds, clusteringIndexFilter);
        if (!dataRange.isUnrestricted(table) || !rowFilter.isEmpty()) {
            sb.append(" WHERE ");
            // We put the row filter first because the data range can end by "ORDER BY"
            if (!rowFilter.isEmpty()) {
                sb.append(rowFilter);
                if (!dataRange.isUnrestricted(table))
                    sb.append(" AND ");
            }
            if (!dataRange.isUnrestricted(table))
                sb.append(dataRange.toCQLString(table, rowFilter));
        }
    } else {
        // single partition
        Collection<ByteBuffer> keys = restrictions.getPartitionKeys(options);
        if (keys.isEmpty())
            return "EMPTY";
        ClusteringIndexFilter filter = makeClusteringIndexFilter(options, columnFilter);
        if (filter == null)
            return "EMPTY";
        sb.append(" WHERE ");
        boolean compoundPk = table.partitionKeyColumns().size() > 1;
        if (compoundPk)
            sb.append('(');
        sb.append(ColumnMetadata.toCQLString(table.partitionKeyColumns()));
        if (compoundPk)
            sb.append(')');
        if (keys.size() == 1) {
            sb.append(" = ");
            if (compoundPk)
                sb.append('(');
            DataRange.appendKeyString(sb, table.partitionKeyType, Iterables.getOnlyElement(keys));
            if (compoundPk)
                sb.append(')');
        } else {
            sb.append(" IN (");
            boolean first = true;
            for (ByteBuffer key : keys) {
                if (!first)
                    sb.append(", ");
                if (compoundPk)
                    sb.append('(');
                DataRange.appendKeyString(sb, table.partitionKeyType, key);
                if (compoundPk)
                    sb.append(')');
                first = false;
            }
            sb.append(')');
        }
        RowFilter rowFilter = getRowFilter(options);
        if (!rowFilter.isEmpty())
            sb.append(" AND ").append(rowFilter);
        String filterString = filter.toCQLString(table, rowFilter);
        if (!filterString.isEmpty())
            sb.append(" AND ").append(filterString);
    }
    DataLimits limits = getDataLimits(getLimit(options), getPerPartitionLimit(options), options.getPageSize());
    if (limits != DataLimits.NONE)
        sb.append(' ').append(limits);
    return sb.toString();
}

Example 25 with Range

use of org.apache.commons.lang3.Range in project cassandra by apache.

the class RangeStreamer method fetchAsync.

public StreamResultFuture fetchAsync() {
    toFetch.forEach((keyspace, sources) -> {
        logger.debug("Keyspace {} Sources {}", keyspace, sources);
        sources.asMap().forEach((source, fetchReplicas) -> {
            // filter out already streamed ranges
            SystemKeyspace.AvailableRanges available = stateStore.getAvailableRanges(keyspace, metadata.partitioner);
            Predicate<FetchReplica> isAvailable = fetch -> {
                boolean isInFull = available.full.contains(fetch.local.range());
                boolean isInTrans = available.trans.contains(fetch.local.range());
                if (!isInFull && !isInTrans)
                    // Range is unavailable
                    return false;
                if (fetch.local.isFull())
                    // For full, pick only replicas with matching transientness
                    return isInFull == fetch.remote.isFull();
                // Any transient or full will do
                return true;
            };
            List<FetchReplica> remaining = fetchReplicas.stream().filter(not(isAvailable)).collect(Collectors.toList());
            if (remaining.size() < available.full.size() + available.trans.size()) {
                List<FetchReplica> skipped = fetchReplicas.stream().filter(isAvailable).collect(Collectors.toList());
                logger.info("Some ranges of {} are already available. Skipping streaming those ranges. Skipping {}. Fully available {} Transiently available {}", fetchReplicas, skipped, available.full, available.trans);
            }
            if (logger.isTraceEnabled())
                logger.trace("{}ing from {} ranges {}", description, source, StringUtils.join(remaining, ", "));
            InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort();
            RangesAtEndpoint full = remaining.stream().filter(pair -> pair.remote.isFull()).map(pair -> pair.local).collect(RangesAtEndpoint.collector(self));
            RangesAtEndpoint transientReplicas = remaining.stream().filter(pair -> pair.remote.isTransient()).map(pair -> pair.local).collect(RangesAtEndpoint.collector(self));
            logger.debug("Source and our replicas {}", fetchReplicas);
            logger.debug("Source {} Keyspace {}  streaming full {} transient {}", source, keyspace, full, transientReplicas);
            /* Send messages to respective folks to stream data over to me */
            streamPlan.requestRanges(source, keyspace, full, transientReplicas);
        });
    });
    return streamPlan.execute();
}