Examples with DrillbitEndpoint org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint used on opensource projects

Example 41 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class HBaseGroupScan method getOperatorAffinity.

@Override
public List<EndpointAffinity> getOperatorAffinity() {
    watch.reset();
    watch.start();
    Map<String, DrillbitEndpoint> endpointMap = new HashMap<String, DrillbitEndpoint>();
    for (DrillbitEndpoint ep : storagePlugin.getContext().getBits()) {
        endpointMap.put(ep.getAddress(), ep);
    }
    Map<DrillbitEndpoint, EndpointAffinity> affinityMap = new HashMap<DrillbitEndpoint, EndpointAffinity>();
    for (ServerName sn : regionsToScan.values()) {
        DrillbitEndpoint ep = endpointMap.get(sn.getHostname());
        if (ep != null) {
            EndpointAffinity affinity = affinityMap.get(ep);
            if (affinity == null) {
                affinityMap.put(ep, new EndpointAffinity(ep, 1));
            } else {
                affinity.addAffinity(1);
            }
        }
    }
    logger.debug("Took {} µs to get operator affinity", watch.elapsed(TimeUnit.NANOSECONDS) / 1000);
    return Lists.newArrayList(affinityMap.values());
}

Example 42 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class TestHBaseRegionScanAssignments method testHBaseGroupScanAssignmentNoAfinity.

@Test
public void testHBaseGroupScanAssignmentNoAfinity() throws Exception {
    NavigableMap<HRegionInfo, ServerName> regionsToScan = Maps.newTreeMap();
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[0], splits[1]), SERVER_X);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[1], splits[2]), SERVER_X);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[2], splits[3]), SERVER_X);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[3], splits[4]), SERVER_X);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[4], splits[5]), SERVER_X);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[5], splits[6]), SERVER_X);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[6], splits[7]), SERVER_X);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[7], splits[0]), SERVER_X);
    final List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_A).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_B).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_C).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_D).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_E).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_F).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_G).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_H).setControlPort(1234).build());
    HBaseGroupScan scan = new HBaseGroupScan();
    scan.setRegionsToScan(regionsToScan);
    scan.setHBaseScanSpec(new HBaseScanSpec(TABLE_NAME_STR, splits[0], splits[0], null));
    scan.applyAssignments(endpoints);
    int i = 0;
    // 'A'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'B'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'C'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'D'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'E'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'F'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'G'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'H'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    testParallelizationWidth(scan, i);
}

Example 43 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class TestHBaseRegionScanAssignments method testHBaseGroupScanAssignmentOneEach.

@Test
public void testHBaseGroupScanAssignmentOneEach() throws Exception {
    NavigableMap<HRegionInfo, ServerName> regionsToScan = Maps.newTreeMap();
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[0], splits[1]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[1], splits[2]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[2], splits[3]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[3], splits[4]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[4], splits[5]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[5], splits[6]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[6], splits[7]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[7], splits[8]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[8], splits[0]), SERVER_A);
    final List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_A).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_A).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_B).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_C).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_D).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_E).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_F).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_G).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_H).setControlPort(1234).build());
    HBaseGroupScan scan = new HBaseGroupScan();
    scan.setRegionsToScan(regionsToScan);
    scan.setHBaseScanSpec(new HBaseScanSpec(TABLE_NAME_STR, splits[0], splits[0], null));
    scan.applyAssignments(endpoints);
    int i = 0;
    // 'A'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'A'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'B'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'C'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'D'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'E'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'F'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'G'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'H'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    testParallelizationWidth(scan, i);
}

Example 44 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class TestHBaseRegionScanAssignments method testHBaseGroupScanAssignmentAllPreferred.

@Test
public void testHBaseGroupScanAssignmentAllPreferred() throws Exception {
    NavigableMap<HRegionInfo, ServerName> regionsToScan = Maps.newTreeMap();
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[0], splits[1]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[1], splits[2]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[2], splits[3]), SERVER_B);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[3], splits[4]), SERVER_B);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[4], splits[5]), SERVER_C);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[5], splits[6]), SERVER_C);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[6], splits[7]), SERVER_D);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[7], splits[0]), SERVER_D);
    final List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    final DrillbitEndpoint DB_A = DrillbitEndpoint.newBuilder().setAddress(HOST_A).setControlPort(1234).build();
    endpoints.add(DB_A);
    final DrillbitEndpoint DB_B = DrillbitEndpoint.newBuilder().setAddress(HOST_B).setControlPort(1234).build();
    endpoints.add(DB_B);
    final DrillbitEndpoint DB_D = DrillbitEndpoint.newBuilder().setAddress(HOST_C).setControlPort(1234).build();
    endpoints.add(DB_D);
    final DrillbitEndpoint DB_X = DrillbitEndpoint.newBuilder().setAddress(HOST_D).setControlPort(1234).build();
    endpoints.add(DB_X);
    HBaseGroupScan scan = new HBaseGroupScan();
    scan.setRegionsToScan(regionsToScan);
    scan.setHBaseScanSpec(new HBaseScanSpec(TABLE_NAME_STR, splits[0], splits[0], null));
    scan.applyAssignments(endpoints);
    int i = 0;
    // 'A'
    assertEquals(2, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'B'
    assertEquals(2, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'C'
    assertEquals(2, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'D'
    assertEquals(2, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    testParallelizationWidth(scan, i);
}

Example 45 with DrillbitEndpoint

use of org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint in project drill by apache.

the class ZKClusterCoordinator method updateEndpoints.

private synchronized void updateEndpoints() {
    try {
        Collection<DrillbitEndpoint> newDrillbitSet = transform(discovery.queryForInstances(serviceName), new Function<ServiceInstance<DrillbitEndpoint>, DrillbitEndpoint>() {

            @Override
            public DrillbitEndpoint apply(ServiceInstance<DrillbitEndpoint> input) {
                return input.getPayload();
            }
        });
        // set of newly dead bits : original bits - new set of active bits.
        Set<DrillbitEndpoint> unregisteredBits = new HashSet<>(endpoints);
        unregisteredBits.removeAll(newDrillbitSet);
        // Set of newly live bits : new set of active bits - original bits.
        Set<DrillbitEndpoint> registeredBits = new HashSet<>(newDrillbitSet);
        registeredBits.removeAll(endpoints);
        endpoints = newDrillbitSet;
        if (logger.isDebugEnabled()) {
            StringBuilder builder = new StringBuilder();
            builder.append("Active drillbit set changed.  Now includes ");
            builder.append(newDrillbitSet.size());
            builder.append(" total bits. New active drillbits:\n");
            builder.append("Address | User Port | Control Port | Data Port | Version |\n");
            for (DrillbitEndpoint bit : newDrillbitSet) {
                builder.append(bit.getAddress()).append(" | ");
                builder.append(bit.getUserPort()).append(" | ");
                builder.append(bit.getControlPort()).append(" | ");
                builder.append(bit.getDataPort()).append(" | ");
                builder.append(bit.getVersion()).append(" |");
                builder.append('\n');
            }
            logger.debug(builder.toString());
        }
        // Notify listeners of newly unregistered Drillbits.
        if (!unregisteredBits.isEmpty()) {
            drillbitUnregistered(unregisteredBits);
        }
        // Notify listeners of newly registered Drillbits.
        if (!registeredBits.isEmpty()) {
            drillbitRegistered(registeredBits);
        }
    } catch (Exception e) {
        logger.error("Failure while update Drillbit service location cache.", e);
    }
}