Examples with EndpointAffinity org.apache.drill.exec.physical.EndpointAffinity used on opensource projects

Example 16 with EndpointAffinity

use of org.apache.drill.exec.physical.EndpointAffinity in project drill by axbaretto.

the class TestHardAffinityFragmentParallelizer method simpleCase2.

@Test
public void simpleCase2() throws Exception {
    // Set the slice target to 1
    final Wrapper wrapper = newWrapper(200, 1, 20, Collections.singletonList(new EndpointAffinity(N1_EP1, 1.0, true, MAX_VALUE)));
    INSTANCE.parallelizeFragment(wrapper, newParameters(1, 5, 20), null);
    // Expect the fragment parallelization to be 5:
    // 1. the cost (200) is above the threshold (SLICE_TARGET_DEFAULT) (which gives 200/1=200 width) and
    // 2. Max width per node is 5 (limits the width 200 to 5)
    assertEquals(5, wrapper.getWidth());
    final List<DrillbitEndpoint> assignedEps = wrapper.getAssignedEndpoints();
    assertEquals(5, assignedEps.size());
    for (DrillbitEndpoint ep : assignedEps) {
        assertEquals(N1_EP1, ep);
    }
}

Example 17 with EndpointAffinity

use of org.apache.drill.exec.physical.EndpointAffinity in project drill by axbaretto.

the class TestHardAffinityFragmentParallelizer method multiNodeCluster2.

@Test
public void multiNodeCluster2() throws Exception {
    final Wrapper wrapper = newWrapper(200, 1, 20, ImmutableList.of(new EndpointAffinity(N1_EP2, 0.15, true, MAX_VALUE), new EndpointAffinity(N2_EP2, 0.15, true, MAX_VALUE), new EndpointAffinity(N3_EP1, 0.10, true, MAX_VALUE), new EndpointAffinity(N4_EP2, 0.20, true, MAX_VALUE), new EndpointAffinity(N1_EP1, 0.20, true, MAX_VALUE)));
    INSTANCE.parallelizeFragment(wrapper, newParameters(1, 5, 20), null);
    // Expect the fragment parallelization to be 20 because:
    // 1. the cost (200) is above the threshold (SLICE_TARGET_DEFAULT) (which gives 200/1=200 width) and
    // 2. Number of mandatory node assignments are 5 (current width 200 satisfies the requirement)
    // 3. max fragment width is 20 which limits the width
    assertEquals(20, wrapper.getWidth());
    final List<DrillbitEndpoint> assignedEps = wrapper.getAssignedEndpoints();
    assertEquals(20, assignedEps.size());
    final HashMultiset<DrillbitEndpoint> counts = HashMultiset.create();
    for (final DrillbitEndpoint ep : assignedEps) {
        counts.add(ep);
    }
    // Each node gets at max 5.
    assertTrue(counts.count(N1_EP2) <= 5);
    assertTrue(counts.count(N2_EP2) <= 5);
    assertTrue(counts.count(N3_EP1) <= 5);
    assertTrue(counts.count(N4_EP2) <= 5);
    assertTrue(counts.count(N1_EP1) <= 5);
}

Example 18 with EndpointAffinity

use of org.apache.drill.exec.physical.EndpointAffinity in project drill by axbaretto.

the class MongoGroupScan method getOperatorAffinity.

@Override
public List<EndpointAffinity> getOperatorAffinity() {
    watch.reset();
    watch.start();
    Map<String, DrillbitEndpoint> endpointMap = Maps.newHashMap();
    for (DrillbitEndpoint endpoint : storagePlugin.getContext().getBits()) {
        endpointMap.put(endpoint.getAddress(), endpoint);
        logger.debug("Endpoint address: {}", endpoint.getAddress());
    }
    Map<DrillbitEndpoint, EndpointAffinity> affinityMap = Maps.newHashMap();
    // multiple replicas for each chunk.
    for (Set<ServerAddress> addressList : chunksMapping.values()) {
        // meets affinity.
        for (ServerAddress address : addressList) {
            DrillbitEndpoint ep = endpointMap.get(address.getHost());
            if (ep != null) {
                EndpointAffinity affinity = affinityMap.get(ep);
                if (affinity == null) {
                    affinityMap.put(ep, new EndpointAffinity(ep, 1));
                } else {
                    affinity.addAffinity(1);
                }
                break;
            }
        }
    }
    logger.debug("Took {} µs to get operator affinity", watch.elapsed(TimeUnit.NANOSECONDS) / 1000);
    logger.debug("Affined drillbits : " + affinityMap.values());
    return Lists.newArrayList(affinityMap.values());
}

Example 19 with EndpointAffinity

use of org.apache.drill.exec.physical.EndpointAffinity in project drill by axbaretto.

the class AbstractDeMuxExchange method getSenderParallelizationInfo.

@Override
public ParallelizationInfo getSenderParallelizationInfo(List<DrillbitEndpoint> receiverFragmentEndpoints) {
    Preconditions.checkArgument(receiverFragmentEndpoints != null && receiverFragmentEndpoints.size() > 0, "Receiver fragment endpoint list should not be empty");
    // We want to run one demux sender per Drillbit endpoint.
    // Identify the number of unique Drillbit endpoints in receiver fragment endpoints.
    List<DrillbitEndpoint> drillbitEndpoints = ImmutableSet.copyOf(receiverFragmentEndpoints).asList();
    List<EndpointAffinity> affinities = Lists.newArrayList();
    for (DrillbitEndpoint ep : drillbitEndpoints) {
        affinities.add(new EndpointAffinity(ep, Double.POSITIVE_INFINITY));
    }
    return ParallelizationInfo.create(affinities.size(), affinities.size(), affinities);
}

Example 20 with EndpointAffinity

use of org.apache.drill.exec.physical.EndpointAffinity in project drill by axbaretto.

the class AbstractMuxExchange method getReceiverParallelizationInfo.

@Override
public ParallelizationInfo getReceiverParallelizationInfo(List<DrillbitEndpoint> senderFragmentEndpoints) {
    Preconditions.checkArgument(senderFragmentEndpoints != null && senderFragmentEndpoints.size() > 0, "Sender fragment endpoint list should not be empty");
    // We want to run one mux receiver per Drillbit endpoint.
    // Identify the number of unique Drillbit endpoints in sender fragment endpoints.
    List<DrillbitEndpoint> drillbitEndpoints = ImmutableSet.copyOf(senderFragmentEndpoints).asList();
    List<EndpointAffinity> affinities = Lists.newArrayList();
    for (DrillbitEndpoint ep : drillbitEndpoints) {
        affinities.add(new EndpointAffinity(ep, Double.POSITIVE_INFINITY));
    }
    return ParallelizationInfo.create(affinities.size(), affinities.size(), affinities);
}