Examples with EndpointAffinity org.apache.drill.exec.physical.EndpointAffinity used on opensource projects

Example 6 with EndpointAffinity

use of org.apache.drill.exec.physical.EndpointAffinity in project drill by axbaretto.

the class AffinityCreator method getAffinityMap.

public static <T extends CompleteWork> List<EndpointAffinity> getAffinityMap(List<T> work) {
    Stopwatch watch = Stopwatch.createStarted();
    long totalBytes = 0;
    for (CompleteWork entry : work) {
        totalBytes += entry.getTotalBytes();
    }
    ObjectFloatHashMap<DrillbitEndpoint> affinities = new ObjectFloatHashMap<DrillbitEndpoint>();
    for (CompleteWork entry : work) {
        for (ObjectLongCursor<DrillbitEndpoint> cursor : entry.getByteMap()) {
            long bytes = cursor.value;
            float affinity = (float) bytes / (float) totalBytes;
            affinities.putOrAdd(cursor.key, affinity, affinity);
        }
    }
    List<EndpointAffinity> affinityList = Lists.newLinkedList();
    for (ObjectFloatCursor<DrillbitEndpoint> d : affinities) {
        logger.debug("Endpoint {} has affinity {}", d.key.getAddress(), d.value);
        affinityList.add(new EndpointAffinity(d.key, d.value));
    }
    logger.debug("Took {} ms to get operator affinity", watch.elapsed(TimeUnit.MILLISECONDS));
    return affinityList;
}

Example 7 with EndpointAffinity

use of org.apache.drill.exec.physical.EndpointAffinity in project drill by apache.

the class HiveScan method getOperatorAffinity.

@Override
public List<EndpointAffinity> getOperatorAffinity() {
    final Map<String, DrillbitEndpoint> endpointMap = new HashMap<>();
    for (final DrillbitEndpoint endpoint : hiveStoragePlugin.getContext().getBits()) {
        endpointMap.put(endpoint.getAddress(), endpoint);
        logger.debug("Endpoint address: {}", endpoint.getAddress());
    }
    final Map<DrillbitEndpoint, EndpointAffinity> affinityMap = new HashMap<>();
    try {
        long totalSize = 0;
        final List<LogicalInputSplit> inputSplits = getInputSplits();
        for (final LogicalInputSplit split : inputSplits) {
            totalSize += Math.max(1, split.getLength());
        }
        for (final LogicalInputSplit split : inputSplits) {
            final float affinity = ((float) Math.max(1, split.getLength())) / totalSize;
            for (final String loc : split.getLocations()) {
                logger.debug("Split location: {}", loc);
                final DrillbitEndpoint endpoint = endpointMap.get(loc);
                if (endpoint != null) {
                    if (affinityMap.containsKey(endpoint)) {
                        affinityMap.get(endpoint).addAffinity(affinity);
                    } else {
                        affinityMap.put(endpoint, new EndpointAffinity(endpoint, affinity));
                    }
                }
            }
        }
    } catch (final IOException e) {
        throw new DrillRuntimeException(e);
    }
    return new ArrayList<>(affinityMap.values());
}

Example 8 with EndpointAffinity

use of org.apache.drill.exec.physical.EndpointAffinity in project drill by apache.

the class AbstractExchange method getDefaultAffinityMap.

/**
 * Get a default endpoint affinity map where affinity of a Drillbit is proportional to the number of its occurrences
 * in given endpoint list.
 *
 * @param fragmentEndpoints Drillbit endpoint assignments of fragments.
 * @return List of EndpointAffinity objects for each Drillbit endpoint given <i>fragmentEndpoints</i>.
 */
protected static List<EndpointAffinity> getDefaultAffinityMap(List<DrillbitEndpoint> fragmentEndpoints) {
    Map<DrillbitEndpoint, EndpointAffinity> affinityMap = Maps.newHashMap();
    final double affinityPerOccurrence = 1.0d / fragmentEndpoints.size();
    for (DrillbitEndpoint sender : fragmentEndpoints) {
        if (affinityMap.containsKey(sender)) {
            affinityMap.get(sender).addAffinity(affinityPerOccurrence);
        } else {
            affinityMap.put(sender, new EndpointAffinity(sender, affinityPerOccurrence));
        }
    }
    return new ArrayList<>(affinityMap.values());
}

Example 9 with EndpointAffinity

use of org.apache.drill.exec.physical.EndpointAffinity in project drill by apache.

the class AffinityCreator method getAffinityMap.

public static <T extends CompleteWork> List<EndpointAffinity> getAffinityMap(List<T> work) {
    Stopwatch watch = logger.isDebugEnabled() ? Stopwatch.createStarted() : null;
    long totalBytes = work.stream().mapToLong(CompleteWork::getTotalBytes).sum();
    ObjectFloatHashMap<DrillbitEndpoint> affinities = new ObjectFloatHashMap<>();
    for (CompleteWork entry : work) {
        for (ObjectLongCursor<DrillbitEndpoint> cursor : entry.getByteMap()) {
            long bytes = cursor.value;
            float affinity = totalBytes == 0 ? 0.0F : (float) bytes / (float) totalBytes;
            affinities.putOrAdd(cursor.key, affinity, affinity);
        }
    }
    List<EndpointAffinity> affinityList = new LinkedList<>();
    for (ObjectFloatCursor<DrillbitEndpoint> d : affinities) {
        logger.debug("Endpoint {} has affinity {}", d.key.getAddress(), d.value);
        affinityList.add(new EndpointAffinity(d.key, d.value));
    }
    if (watch != null) {
        logger.debug("Took {} ms to get operator affinity", watch.elapsed(TimeUnit.MILLISECONDS));
    }
    return affinityList;
}

Example 10 with EndpointAffinity

use of org.apache.drill.exec.physical.EndpointAffinity in project drill by apache.

the class MongoGroupScan method getOperatorAffinity.

@Override
public List<EndpointAffinity> getOperatorAffinity() {
    watch.reset();
    watch.start();
    Map<String, DrillbitEndpoint> endpointMap = Maps.newHashMap();
    for (DrillbitEndpoint endpoint : storagePlugin.getContext().getBits()) {
        endpointMap.put(endpoint.getAddress(), endpoint);
        logger.debug("Endpoint address: {}", endpoint.getAddress());
    }
    Map<DrillbitEndpoint, EndpointAffinity> affinityMap = Maps.newHashMap();
    // multiple replicas for each chunk.
    for (Set<ServerAddress> addressList : chunksMapping.values()) {
        // meets affinity.
        for (ServerAddress address : addressList) {
            DrillbitEndpoint ep = endpointMap.get(address.getHost());
            if (ep != null) {
                EndpointAffinity affinity = affinityMap.get(ep);
                if (affinity == null) {
                    affinityMap.put(ep, new EndpointAffinity(ep, 1));
                } else {
                    affinity.addAffinity(1);
                }
                break;
            }
        }
    }
    logger.debug("Took {} µs to get operator affinity", watch.elapsed(TimeUnit.NANOSECONDS) / 1000);
    logger.debug("Affined drillbits : " + affinityMap.values());
    return Lists.newArrayList(affinityMap.values());
}