Examples with DrillbitEndpoint org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint used on opensource projects

Example 61 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class SystemTableScan method getOperatorAffinity.

/**
   * If distributed, the scan needs to happen on every node. Since width is enforced, the number of fragments equals
   * number of Drillbits. And here we set, each endpoint as mandatory assignment required to ensure every
   * Drillbit executes a fragment.
   * @return the Drillbit endpoint affinities
   */
@Override
public List<EndpointAffinity> getOperatorAffinity() {
    if (table.isDistributed()) {
        final List<EndpointAffinity> affinities = Lists.newArrayList();
        final Collection<DrillbitEndpoint> bits = plugin.getContext().getBits();
        final double affinityPerNode = 1d / bits.size();
        for (final DrillbitEndpoint endpoint : bits) {
            affinities.add(new EndpointAffinity(endpoint, affinityPerNode, true, /* maxWidth = */
            1));
        }
        return affinities;
    } else {
        return Collections.emptyList();
    }
}

Example 62 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class AssignmentCreator method getEndpointIterators.

/**
   * Groups minor fragments together by corresponding endpoint, and creates an iterator that can be used to evenly
   * distribute work assigned to a given endpoint to all corresponding minor fragments evenly
   *
   * @return
   */
private Map<DrillbitEndpoint, FragIteratorWrapper> getEndpointIterators() {
    Stopwatch watch = Stopwatch.createStarted();
    Map<DrillbitEndpoint, FragIteratorWrapper> map = Maps.newLinkedHashMap();
    Map<DrillbitEndpoint, List<Integer>> mmap = Maps.newLinkedHashMap();
    for (int i = 0; i < incomingEndpoints.size(); i++) {
        DrillbitEndpoint endpoint = incomingEndpoints.get(i);
        List<Integer> intList = mmap.get(incomingEndpoints.get(i));
        if (intList == null) {
            intList = Lists.newArrayList();
        }
        intList.add(Integer.valueOf(i));
        mmap.put(endpoint, intList);
    }
    int totalMaxCount = 0;
    for (DrillbitEndpoint endpoint : mmap.keySet()) {
        FragIteratorWrapper wrapper = new FragIteratorWrapper();
        wrapper.iter = Iterators.cycle(mmap.get(endpoint));
        // To distribute the load among nodes equally, limit the maxCount per node.
        int maxCount = (int) ((double) mmap.get(endpoint).size() / incomingEndpoints.size() * units.size());
        wrapper.maxCount = Math.min(maxWork * mmap.get(endpoint).size(), maxCount);
        totalMaxCount += wrapper.maxCount;
        wrapper.minCount = Math.max(maxWork - 1, 1) * mmap.get(endpoint).size();
        map.put(endpoint, wrapper);
    }
    // Take care of leftovers.
    while (totalMaxCount < units.size()) {
        for (Entry<DrillbitEndpoint, FragIteratorWrapper> entry : map.entrySet()) {
            FragIteratorWrapper iteratorWrapper = entry.getValue();
            iteratorWrapper.maxCountLeftOver++;
            totalMaxCount++;
            if (totalMaxCount == units.size()) {
                break;
            }
        }
    }
    return map;
}

Example 63 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class AssignmentCreator method getWorkList.

/**
   * Builds the list of WorkEndpointListPairs, which pair a work unit with a list of endpoints sorted by affinity
   * @return the list of WorkEndpointListPairs
   */
private LinkedList<WorkEndpointListPair<T>> getWorkList() {
    Stopwatch watch = Stopwatch.createStarted();
    LinkedList<WorkEndpointListPair<T>> workList = Lists.newLinkedList();
    for (T work : units) {
        List<Map.Entry<DrillbitEndpoint, Long>> entries = Lists.newArrayList();
        for (ObjectLongCursor<DrillbitEndpoint> cursor : work.getByteMap()) {
            final DrillbitEndpoint ep = cursor.key;
            final Long val = cursor.value;
            Map.Entry<DrillbitEndpoint, Long> entry = new Entry<DrillbitEndpoint, Long>() {

                @Override
                public DrillbitEndpoint getKey() {
                    return ep;
                }

                @Override
                public Long getValue() {
                    return val;
                }

                @Override
                public Long setValue(Long value) {
                    throw new UnsupportedOperationException();
                }
            };
            entries.add(entry);
        }
        Collections.sort(entries, comparator);
        List<DrillbitEndpoint> sortedEndpoints = Lists.newArrayList();
        for (Entry<DrillbitEndpoint, Long> entry : entries) {
            sortedEndpoints.add(entry.getKey());
        }
        workList.add(new WorkEndpointListPair<T>(work, sortedEndpoints));
    }
    return workList;
}

Example 64 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class BlockMapBuilder method buildEndpointMap.

/**
   * Builds a mapping of Drillbit endpoints to hostnames
   */
private static ImmutableMap<String, DrillbitEndpoint> buildEndpointMap(Collection<DrillbitEndpoint> endpoints) {
    Stopwatch watch = Stopwatch.createStarted();
    HashMap<String, DrillbitEndpoint> endpointMap = Maps.newHashMap();
    for (DrillbitEndpoint d : endpoints) {
        String hostName = d.getAddress();
        endpointMap.put(hostName, d);
    }
    watch.stop();
    logger.debug("Took {} ms to build endpoint map", watch.elapsed(TimeUnit.MILLISECONDS));
    return ImmutableMap.copyOf(endpointMap);
}

Example 65 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class DrillbitIterator method next.

@Override
public Object next() {
    DrillbitEndpoint ep = endpoints.next();
    DrillbitInstance i = new DrillbitInstance();
    i.current = ep.equals(current);
    i.hostname = ep.getAddress();
    i.user_port = ep.getUserPort();
    i.control_port = ep.getControlPort();
    i.data_port = ep.getDataPort();
    i.version = ep.getVersion();
    return i;
}