Examples with CompleteFileWork org.apache.drill.exec.store.schedule.CompleteFileWork used on opensource projects

Example 1 with CompleteFileWork

use of org.apache.drill.exec.store.schedule.CompleteFileWork in project drill by apache.

the class TestAssignment method testBalanceAcrossNodes.

@Test
public void testBalanceAcrossNodes() throws Exception {
    int numChunks = widthPerNode * numEndPoints + 100;
    List<CompleteFileWork> chunks = generateChunks(numChunks);
    Iterator<DrillbitEndpoint> incomingEndpointsIterator = Iterators.cycle(endpoints);
    List<DrillbitEndpoint> incomingEndpoints = Lists.newArrayList();
    List<Integer> expectedAssignments = Lists.newArrayList();
    List<Integer> actualAssignments = Lists.newArrayList();
    final int width = widthPerNode * numEndPoints;
    for (int i = 0; i < width; i++) {
        incomingEndpoints.add(incomingEndpointsIterator.next());
    }
    // Calculate expected assignments for each node.
    final int numAssignmentsPerNode = numChunks / numEndPoints;
    int leftOver = numChunks - numAssignmentsPerNode * numEndPoints;
    for (int i = 0; i < numEndPoints; i++) {
        int additional = leftOver > 0 ? 1 : 0;
        expectedAssignments.add(numAssignmentsPerNode + additional);
        if (leftOver > 0) {
            leftOver--;
        }
    }
    ListMultimap<Integer, CompleteFileWork> mappings = AssignmentCreator.getMappings(incomingEndpoints, chunks);
    System.out.println(mappings.keySet().size());
    // Verify that all fragments have chunks assigned.
    for (int i = 0; i < width; i++) {
        Assert.assertTrue("no mapping for entry " + i, mappings.get(i) != null && mappings.get(i).size() > 0);
    }
    // Compute actual assignments for each node.
    for (int i = 0; i < numEndPoints; i++) {
        int numAssignments = 0;
        int index = i;
        while (index < numEndPoints * widthPerNode) {
            numAssignments += mappings.get(index).size();
            index += numEndPoints;
        }
        actualAssignments.add(numAssignments);
    }
    for (int i = 0; i < numEndPoints; i++) {
        Assert.assertTrue(actualAssignments.get(i) == expectedAssignments.get(i));
    }
}

Example 2 with CompleteFileWork

use of org.apache.drill.exec.store.schedule.CompleteFileWork in project drill by apache.

the class TextFormatPlugin method getScanStats.

@Override
protected ScanStats getScanStats(final PlannerSettings settings, final EasyGroupScan scan) {
    long data = 0;
    for (final CompleteFileWork work : scan.getWorkIterable()) {
        data += work.getTotalBytes();
    }
    final double estimatedRowSize = settings.getOptions().getOption(ExecConstants.TEXT_ESTIMATED_ROW_SIZE);
    final double estRowCount = data / estimatedRowSize;
    return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, (long) estRowCount, 1, data);
}

Example 3 with CompleteFileWork

use of org.apache.drill.exec.store.schedule.CompleteFileWork in project drill by apache.

the class EasyFormatPlugin method getScanStats.

protected ScanStats getScanStats(final PlannerSettings settings, final EasyGroupScan scan) {
    long data = 0;
    for (final CompleteFileWork work : scan.getWorkIterable()) {
        data += work.getTotalBytes();
    }
    final long estRowCount = data / 1024;
    return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, estRowCount, 1, data);
}

Example 4 with CompleteFileWork

use of org.apache.drill.exec.store.schedule.CompleteFileWork in project drill by apache.

the class TestAssignment method manyFiles.

@Test
public void manyFiles() throws Exception {
    List<CompleteFileWork> chunks = generateChunks(1000);
    Iterator<DrillbitEndpoint> incomingEndpointsIterator = Iterators.cycle(endpoints);
    List<DrillbitEndpoint> incomingEndpoints = Lists.newArrayList();
    final int width = widthPerNode * numEndPoints;
    for (int i = 0; i < width; i++) {
        incomingEndpoints.add(incomingEndpointsIterator.next());
    }
    ListMultimap<Integer, CompleteFileWork> mappings = AssignmentCreator.getMappings(incomingEndpoints, chunks);
    System.out.println(mappings.keySet().size());
    for (int i = 0; i < width; i++) {
        Assert.assertTrue("no mapping for entry " + i, mappings.get(i) != null && mappings.get(i).size() > 0);
    }
}

Example 5 with CompleteFileWork

use of org.apache.drill.exec.store.schedule.CompleteFileWork in project drill by apache.

the class TestAssignment method generateChunks.

private List<CompleteFileWork> generateChunks(int chunks) {
    List<CompleteFileWork> chunkList = Lists.newArrayList();
    for (int i = 0; i < chunks; i++) {
        CompleteFileWork chunk = new CompleteFileWork(createByteMap(), 0, FILE_SIZE, "file" + i);
        chunkList.add(chunk);
    }
    return chunkList;
}