Examples with DrillbitEndpoint org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint used on opensource projects

Example 46 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class DataConnectionCreator method start.

public DrillbitEndpoint start(DrillbitEndpoint partialEndpoint, boolean allowPortHunting) {
    server = new DataServer(config);
    int port = partialEndpoint.getControlPort() + 1;
    if (config.getBootstrapContext().getConfig().hasPath(ExecConstants.INITIAL_DATA_PORT)) {
        port = config.getBootstrapContext().getConfig().getInt(ExecConstants.INITIAL_DATA_PORT);
    }
    port = server.bind(port, allowPortHunting);
    return partialEndpoint.toBuilder().setDataPort(port).build();
}

Example 47 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class ControllerImpl method start.

@Override
public DrillbitEndpoint start(DrillbitEndpoint partialEndpoint, final boolean allowPortHunting) {
    server = new ControlServer(config, connectionRegistry);
    int port = config.getBootstrapContext().getConfig().getInt(ExecConstants.INITIAL_BIT_PORT);
    port = server.bind(port, allowPortHunting);
    DrillbitEndpoint completeEndpoint = partialEndpoint.toBuilder().setControlPort(port).build();
    connectionRegistry.setLocalEndpoint(completeEndpoint);
    handlerRegistry.setEndpoint(completeEndpoint);
    return completeEndpoint;
}

Example 48 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class QueryManager method addFragmentStatusTracker.

void addFragmentStatusTracker(final PlanFragment fragment, final boolean isRoot) {
    final DrillbitEndpoint assignment = fragment.getAssignment();
    NodeTracker tracker = nodeMap.get(assignment);
    if (tracker == null) {
        tracker = new NodeTracker(assignment);
        nodeMap.put(assignment, tracker);
    }
    tracker.addFragment();
    addFragment(new FragmentData(fragment.getHandle(), assignment, isRoot));
}

Example 49 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class MapRDBGroupScan method getOperatorAffinity.

@Override
public List<EndpointAffinity> getOperatorAffinity() {
    watch.reset();
    watch.start();
    Map<String, DrillbitEndpoint> endpointMap = new HashMap<String, DrillbitEndpoint>();
    for (DrillbitEndpoint ep : formatPlugin.getContext().getBits()) {
        endpointMap.put(ep.getAddress(), ep);
    }
    Map<DrillbitEndpoint, EndpointAffinity> affinityMap = new HashMap<DrillbitEndpoint, EndpointAffinity>();
    for (String serverName : regionsToScan.values()) {
        DrillbitEndpoint ep = endpointMap.get(serverName);
        if (ep != null) {
            EndpointAffinity affinity = affinityMap.get(ep);
            if (affinity == null) {
                affinityMap.put(ep, new EndpointAffinity(ep, 1));
            } else {
                affinity.addAffinity(1);
            }
        }
    }
    logger.debug("Took {} µs to get operator affinity", watch.elapsed(TimeUnit.NANOSECONDS) / 1000);
    return Lists.newArrayList(affinityMap.values());
}

Example 50 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class MapRDBGroupScan method applyAssignments.

/**
   *
   * @param incomingEndpoints
   */
@Override
public void applyAssignments(List<DrillbitEndpoint> incomingEndpoints) {
    watch.reset();
    watch.start();
    final int numSlots = incomingEndpoints.size();
    Preconditions.checkArgument(numSlots <= regionsToScan.size(), String.format("Incoming endpoints %d is greater than number of scan regions %d", numSlots, regionsToScan.size()));
    /*
     * Minimum/Maximum number of assignment per slot
     */
    final int minPerEndpointSlot = (int) Math.floor((double) regionsToScan.size() / numSlots);
    final int maxPerEndpointSlot = (int) Math.ceil((double) regionsToScan.size() / numSlots);
    /*
     * initialize (endpoint index => HBaseSubScanSpec list) map
     */
    endpointFragmentMapping = Maps.newHashMapWithExpectedSize(numSlots);
    /*
     * another map with endpoint (hostname => corresponding index list) in 'incomingEndpoints' list
     */
    Map<String, Queue<Integer>> endpointHostIndexListMap = Maps.newHashMap();
    /*
     * Initialize these two maps
     */
    for (int i = 0; i < numSlots; ++i) {
        endpointFragmentMapping.put(i, new ArrayList<MapRDBSubScanSpec>(maxPerEndpointSlot));
        String hostname = incomingEndpoints.get(i).getAddress();
        Queue<Integer> hostIndexQueue = endpointHostIndexListMap.get(hostname);
        if (hostIndexQueue == null) {
            hostIndexQueue = Lists.newLinkedList();
            endpointHostIndexListMap.put(hostname, hostIndexQueue);
        }
        hostIndexQueue.add(i);
    }
    Set<Entry<TabletFragmentInfo, String>> regionsToAssignSet = Sets.newHashSet(regionsToScan.entrySet());
    /*
     * First, we assign regions which are hosted on region servers running on drillbit endpoints
     */
    for (Iterator<Entry<TabletFragmentInfo, String>> regionsIterator = regionsToAssignSet.iterator(); regionsIterator.hasNext(); ) /*nothing*/
    {
        Entry<TabletFragmentInfo, String> regionEntry = regionsIterator.next();
        /*
       * Test if there is a drillbit endpoint which is also an HBase RegionServer that hosts the current HBase region
       */
        Queue<Integer> endpointIndexlist = endpointHostIndexListMap.get(regionEntry.getValue());
        if (endpointIndexlist != null) {
            Integer slotIndex = endpointIndexlist.poll();
            List<MapRDBSubScanSpec> endpointSlotScanList = endpointFragmentMapping.get(slotIndex);
            endpointSlotScanList.add(getSubScanSpec(regionEntry.getKey()));
            // add to the tail of the slot list, to add more later in round robin fashion
            endpointIndexlist.offer(slotIndex);
            // this region has been assigned
            regionsIterator.remove();
        }
    }
    /*
     * Build priority queues of slots, with ones which has tasks lesser than 'minPerEndpointSlot' and another which have more.
     */
    PriorityQueue<List<MapRDBSubScanSpec>> minHeap = new PriorityQueue<List<MapRDBSubScanSpec>>(numSlots, LIST_SIZE_COMPARATOR);
    PriorityQueue<List<MapRDBSubScanSpec>> maxHeap = new PriorityQueue<List<MapRDBSubScanSpec>>(numSlots, LIST_SIZE_COMPARATOR_REV);
    for (List<MapRDBSubScanSpec> listOfScan : endpointFragmentMapping.values()) {
        if (listOfScan.size() <= minPerEndpointSlot) {
            minHeap.offer(listOfScan);
        } else if (listOfScan.size() > minPerEndpointSlot) {
            maxHeap.offer(listOfScan);
        }
    }
    /*
     * Now, let's process any regions which remain unassigned and assign them to slots with minimum number of assignments.
     */
    if (regionsToAssignSet.size() > 0) {
        for (Entry<TabletFragmentInfo, String> regionEntry : regionsToAssignSet) {
            List<MapRDBSubScanSpec> smallestList = minHeap.poll();
            smallestList.add(getSubScanSpec(regionEntry.getKey()));
            if (smallestList.size() < maxPerEndpointSlot) {
                minHeap.offer(smallestList);
            }
        }
    }
    /*
     * While there are slots with lesser than 'minPerEndpointSlot' unit work, balance from those with more.
     */
    while (minHeap.peek() != null && minHeap.peek().size() < minPerEndpointSlot) {
        List<MapRDBSubScanSpec> smallestList = (List<MapRDBSubScanSpec>) minHeap.poll();
        List<MapRDBSubScanSpec> largestList = (List<MapRDBSubScanSpec>) maxHeap.poll();
        smallestList.add(largestList.remove(largestList.size() - 1));
        if (largestList.size() > minPerEndpointSlot) {
            maxHeap.offer(largestList);
        }
        if (smallestList.size() < minPerEndpointSlot) {
            minHeap.offer(smallestList);
        }
    }
    /* no slot should be empty at this point */
    assert (minHeap.peek() == null || minHeap.peek().size() > 0) : String.format("Unable to assign tasks to some endpoints.\nEndpoints: {}.\nAssignment Map: {}.", incomingEndpoints, endpointFragmentMapping.toString());
    logger.debug("Built assignment map in {} µs.\nEndpoints: {}.\nAssignment Map: {}", watch.elapsed(TimeUnit.NANOSECONDS) / 1000, incomingEndpoints, endpointFragmentMapping.toString());
}