Examples with EndpointAffinity org.apache.drill.exec.physical.EndpointAffinity used on opensource projects

Example 1 with EndpointAffinity

use of org.apache.drill.exec.physical.EndpointAffinity in project drill by axbaretto.

the class HBaseGroupScan method getOperatorAffinity.

@Override
public List<EndpointAffinity> getOperatorAffinity() {
    watch.reset();
    watch.start();
    Map<String, DrillbitEndpoint> endpointMap = new HashMap<String, DrillbitEndpoint>();
    for (DrillbitEndpoint ep : storagePlugin.getContext().getBits()) {
        endpointMap.put(ep.getAddress(), ep);
    }
    Map<DrillbitEndpoint, EndpointAffinity> affinityMap = new HashMap<DrillbitEndpoint, EndpointAffinity>();
    for (ServerName sn : regionsToScan.values()) {
        DrillbitEndpoint ep = endpointMap.get(sn.getHostname());
        if (ep != null) {
            EndpointAffinity affinity = affinityMap.get(ep);
            if (affinity == null) {
                affinityMap.put(ep, new EndpointAffinity(ep, 1));
            } else {
                affinity.addAffinity(1);
            }
        }
    }
    logger.debug("Took {} µs to get operator affinity", watch.elapsed(TimeUnit.NANOSECONDS) / 1000);
    return Lists.newArrayList(affinityMap.values());
}

Example 2 with EndpointAffinity

use of org.apache.drill.exec.physical.EndpointAffinity in project drill by axbaretto.

the class MapRDBGroupScan method getOperatorAffinity.

@Override
public List<EndpointAffinity> getOperatorAffinity() {
    watch.reset();
    watch.start();
    Map<String, DrillbitEndpoint> endpointMap = new HashMap<String, DrillbitEndpoint>();
    for (DrillbitEndpoint ep : formatPlugin.getContext().getBits()) {
        endpointMap.put(ep.getAddress(), ep);
    }
    Map<DrillbitEndpoint, EndpointAffinity> affinityMap = new HashMap<DrillbitEndpoint, EndpointAffinity>();
    for (String serverName : regionsToScan.values()) {
        DrillbitEndpoint ep = endpointMap.get(serverName);
        if (ep != null) {
            EndpointAffinity affinity = affinityMap.get(ep);
            if (affinity == null) {
                affinityMap.put(ep, new EndpointAffinity(ep, 1));
            } else {
                affinity.addAffinity(1);
            }
        }
    }
    logger.debug("Took {} µs to get operator affinity", watch.elapsed(TimeUnit.NANOSECONDS) / 1000);
    return Lists.newArrayList(affinityMap.values());
}

Example 3 with EndpointAffinity

use of org.apache.drill.exec.physical.EndpointAffinity in project drill by axbaretto.

the class HiveScan method getOperatorAffinity.

@Override
public List<EndpointAffinity> getOperatorAffinity() {
    final Map<String, DrillbitEndpoint> endpointMap = new HashMap<>();
    for (final DrillbitEndpoint endpoint : hiveStoragePlugin.getContext().getBits()) {
        endpointMap.put(endpoint.getAddress(), endpoint);
        logger.debug("endpoing address: {}", endpoint.getAddress());
    }
    final Map<DrillbitEndpoint, EndpointAffinity> affinityMap = new HashMap<>();
    try {
        long totalSize = 0;
        final List<LogicalInputSplit> inputSplits = getInputSplits();
        for (final LogicalInputSplit split : inputSplits) {
            totalSize += Math.max(1, split.getLength());
        }
        for (final LogicalInputSplit split : inputSplits) {
            final float affinity = ((float) Math.max(1, split.getLength())) / totalSize;
            for (final String loc : split.getLocations()) {
                logger.debug("split location: {}", loc);
                final DrillbitEndpoint endpoint = endpointMap.get(loc);
                if (endpoint != null) {
                    if (affinityMap.containsKey(endpoint)) {
                        affinityMap.get(endpoint).addAffinity(affinity);
                    } else {
                        affinityMap.put(endpoint, new EndpointAffinity(endpoint, affinity));
                    }
                }
            }
        }
    } catch (final IOException e) {
        throw new DrillRuntimeException(e);
    }
    for (final DrillbitEndpoint ep : affinityMap.keySet()) {
        Preconditions.checkNotNull(ep);
    }
    for (final EndpointAffinity a : affinityMap.values()) {
        Preconditions.checkNotNull(a.getEndpoint());
    }
    return Lists.newArrayList(affinityMap.values());
}

Example 4 with EndpointAffinity

use of org.apache.drill.exec.physical.EndpointAffinity in project drill by axbaretto.

the class EasyGroupScan method createMappings.

private void createMappings(List<EndpointAffinity> affinities) {
    List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    for (EndpointAffinity e : affinities) {
        endpoints.add(e.getEndpoint());
    }
    this.applyAssignments(endpoints);
}

Example 5 with EndpointAffinity

use of org.apache.drill.exec.physical.EndpointAffinity in project drill by axbaretto.

the class SystemTableScan method getOperatorAffinity.

/**
 * If distributed, the scan needs to happen on every node. Since width is enforced, the number of fragments equals
 * number of Drillbits. And here we set, each endpoint as mandatory assignment required to ensure every
 * Drillbit executes a fragment.
 * @return the Drillbit endpoint affinities
 */
@Override
public List<EndpointAffinity> getOperatorAffinity() {
    if (table.isDistributed()) {
        final List<EndpointAffinity> affinities = Lists.newArrayList();
        final Collection<DrillbitEndpoint> bits = plugin.getContext().getBits();
        final double affinityPerNode = 1d / bits.size();
        for (final DrillbitEndpoint endpoint : bits) {
            affinities.add(new EndpointAffinity(endpoint, affinityPerNode, true, /* maxWidth = */
            1));
        }
        return affinities;
    } else {
        return Collections.emptyList();
    }
}