Examples with DrillbitEndpoint org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint used on opensource projects

Example 36 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class AssignmentCreator method getMappings.

/**
   * Does the work of creating the mappings for this AssignmentCreator
   * @return the minor fragment id to work units mapping
   */
private ListMultimap<Integer, T> getMappings() {
    Stopwatch watch = Stopwatch.createStarted();
    maxWork = (int) Math.ceil(units.size() / ((float) incomingEndpoints.size()));
    LinkedList<WorkEndpointListPair<T>> workList = getWorkList();
    LinkedList<WorkEndpointListPair<T>> unassignedWorkList;
    Map<DrillbitEndpoint, FragIteratorWrapper> endpointIterators = getEndpointIterators();
    // Assign upto maxCount per node based on locality.
    unassignedWorkList = assign(workList, endpointIterators, false);
    // Assign upto minCount per node in a round robin fashion.
    assignLeftovers(unassignedWorkList, endpointIterators, true);
    // Assign upto maxCount + leftovers per node based on locality.
    unassignedWorkList = assign(unassignedWorkList, endpointIterators, true);
    // Assign upto maxCount + leftovers per node in a round robin fashion.
    assignLeftovers(unassignedWorkList, endpointIterators, false);
    if (unassignedWorkList.size() != 0) {
        throw new DrillRuntimeException("There are still unassigned work units");
    }
    logger.debug("Took {} ms to assign {} work units to {} fragments", watch.elapsed(TimeUnit.MILLISECONDS), units.size(), incomingEndpoints.size());
    return mappings;
}

Example 37 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class BlockMapBuilder method getEndpointByteMap.

/**
   * For a given FileWork, calculate how many bytes are available on each on drillbit endpoint
   *
   * @param work the FileWork to calculate endpoint bytes for
   * @throws IOException
   */
public EndpointByteMap getEndpointByteMap(FileWork work) throws IOException {
    Stopwatch watch = Stopwatch.createStarted();
    Path fileName = new Path(work.getPath());
    ImmutableRangeMap<Long, BlockLocation> blockMap = getBlockMap(fileName);
    EndpointByteMapImpl endpointByteMap = new EndpointByteMapImpl();
    long start = work.getStart();
    long end = start + work.getLength();
    Range<Long> rowGroupRange = Range.closedOpen(start, end);
    // Find submap of ranges that intersect with the rowGroup
    ImmutableRangeMap<Long, BlockLocation> subRangeMap = blockMap.subRangeMap(rowGroupRange);
    // Iterate through each block in this submap and get the host for the block location
    for (Map.Entry<Range<Long>, BlockLocation> block : subRangeMap.asMapOfRanges().entrySet()) {
        String[] hosts;
        Range<Long> blockRange = block.getKey();
        try {
            hosts = block.getValue().getHosts();
        } catch (IOException ioe) {
            throw new RuntimeException("Failed to get hosts for block location", ioe);
        }
        Range<Long> intersection = rowGroupRange.intersection(blockRange);
        long bytes = intersection.upperEndpoint() - intersection.lowerEndpoint();
        // For each host in the current block location, add the intersecting bytes to the corresponding endpoint
        for (String host : hosts) {
            DrillbitEndpoint endpoint = getDrillBitEndpoint(host);
            if (endpoint != null) {
                endpointByteMap.add(endpoint, bytes);
            } else {
                logger.info("Failure finding Drillbit running on host {}.  Skipping affinity to that host.", host);
            }
        }
    }
    logger.debug("FileWork group ({},{}) max bytes {}", work.getPath(), work.getStart(), endpointByteMap.getMaxBytes());
    logger.debug("Took {} ms to set endpoint bytes", watch.stop().elapsed(TimeUnit.MILLISECONDS));
    return endpointByteMap;
}

Example 38 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class MemoryIterator method next.

@Override
public Object next() {
    if (!beforeFirst) {
        throw new IllegalStateException();
    }
    beforeFirst = false;
    final MemoryInfo memoryInfo = new MemoryInfo();
    final DrillbitEndpoint endpoint = context.getIdentity();
    memoryInfo.hostname = endpoint.getAddress();
    memoryInfo.user_port = endpoint.getUserPort();
    final MemoryUsage heapMemoryUsage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
    memoryInfo.heap_current = heapMemoryUsage.getUsed();
    memoryInfo.heap_max = heapMemoryUsage.getMax();
    BufferPoolMXBean directBean = getDirectBean();
    memoryInfo.jvm_direct_current = directBean.getMemoryUsed();
    memoryInfo.direct_current = context.getDrillbitContext().getAllocator().getAllocatedMemory();
    memoryInfo.direct_max = DrillConfig.getMaxDirectMemory();
    return memoryInfo;
}

Example 39 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class ThreadsIterator method next.

@Override
public Object next() {
    if (!beforeFirst) {
        throw new IllegalStateException();
    }
    beforeFirst = false;
    final ThreadsInfo threadsInfo = new ThreadsInfo();
    final DrillbitEndpoint endpoint = context.getIdentity();
    threadsInfo.hostname = endpoint.getAddress();
    threadsInfo.user_port = endpoint.getUserPort();
    final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
    threadsInfo.total_threads = threadMXBean.getPeakThreadCount();
    threadsInfo.busy_threads = threadMXBean.getThreadCount();
    return threadsInfo;
}

Example 40 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class TestAssignment method testBalanceAcrossNodes.

@Test
public void testBalanceAcrossNodes() throws Exception {
    int numChunks = widthPerNode * numEndPoints + 100;
    List<CompleteFileWork> chunks = generateChunks(numChunks);
    Iterator<DrillbitEndpoint> incomingEndpointsIterator = Iterators.cycle(endpoints);
    List<DrillbitEndpoint> incomingEndpoints = Lists.newArrayList();
    List<Integer> expectedAssignments = Lists.newArrayList();
    List<Integer> actualAssignments = Lists.newArrayList();
    final int width = widthPerNode * numEndPoints;
    for (int i = 0; i < width; i++) {
        incomingEndpoints.add(incomingEndpointsIterator.next());
    }
    // Calculate expected assignments for each node.
    final int numAssignmentsPerNode = numChunks / numEndPoints;
    int leftOver = numChunks - numAssignmentsPerNode * numEndPoints;
    for (int i = 0; i < numEndPoints; i++) {
        int additional = leftOver > 0 ? 1 : 0;
        expectedAssignments.add(numAssignmentsPerNode + additional);
        if (leftOver > 0) {
            leftOver--;
        }
    }
    ListMultimap<Integer, CompleteFileWork> mappings = AssignmentCreator.getMappings(incomingEndpoints, chunks);
    System.out.println(mappings.keySet().size());
    // Verify that all fragments have chunks assigned.
    for (int i = 0; i < width; i++) {
        Assert.assertTrue("no mapping for entry " + i, mappings.get(i) != null && mappings.get(i).size() > 0);
    }
    // Compute actual assignments for each node.
    for (int i = 0; i < numEndPoints; i++) {
        int numAssignments = 0;
        int index = i;
        while (index < numEndPoints * widthPerNode) {
            numAssignments += mappings.get(index).size();
            index += numEndPoints;
        }
        actualAssignments.add(numAssignments);
    }
    for (int i = 0; i < numEndPoints; i++) {
        Assert.assertTrue(actualAssignments.get(i) == expectedAssignments.get(i));
    }
}