Examples with HBaseGroupScan org.apache.drill.exec.store.hbase.HBaseGroupScan used on opensource projects

Example 1 with HBaseGroupScan

use of org.apache.drill.exec.store.hbase.HBaseGroupScan in project drill by apache.

the class TestHBaseRegionScanAssignments method testHBaseGroupScanAssignmentSomeAfinedWithOrphans.

@Test
public void testHBaseGroupScanAssignmentSomeAfinedWithOrphans() throws Exception {
    NavigableMap<HRegionInfo, ServerName> regionsToScan = Maps.newTreeMap();
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[0], splits[1]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[1], splits[2]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[2], splits[3]), SERVER_B);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[3], splits[4]), SERVER_B);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[4], splits[5]), SERVER_C);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[5], splits[6]), SERVER_C);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[6], splits[7]), SERVER_D);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[7], splits[8]), SERVER_D);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[8], splits[9]), SERVER_E);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[9], splits[10]), SERVER_E);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[10], splits[11]), SERVER_F);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[11], splits[12]), SERVER_F);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[12], splits[13]), SERVER_G);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[13], splits[14]), SERVER_G);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[14], splits[15]), SERVER_H);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[15], splits[16]), SERVER_H);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[16], splits[17]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[17], splits[0]), SERVER_A);
    final List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_A).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_B).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_C).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_D).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_E).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_F).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_G).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_H).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_I).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_J).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_K).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_L).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_M).setControlPort(1234).build());
    HBaseGroupScan scan = new HBaseGroupScan();
    scan.setRegionsToScan(regionsToScan);
    scan.setHBaseScanSpec(new HBaseScanSpec(TABLE_NAME_STR, splits[0], splits[0], null));
    scan.applyAssignments(endpoints);
    LinkedList<Integer> sizes = Lists.newLinkedList();
    sizes.add(1);
    sizes.add(1);
    sizes.add(1);
    sizes.add(1);
    sizes.add(1);
    sizes.add(1);
    sizes.add(1);
    sizes.add(1);
    sizes.add(2);
    sizes.add(2);
    sizes.add(2);
    sizes.add(2);
    sizes.add(2);
    for (int i = 0; i < endpoints.size(); i++) {
        assertTrue(sizes.remove((Integer) scan.getSpecificScan(i).getRegionScanSpecList().size()));
    }
    assertEquals(0, sizes.size());
    testParallelizationWidth(scan, endpoints.size());
}

Example 2 with HBaseGroupScan

use of org.apache.drill.exec.store.hbase.HBaseGroupScan in project drill by apache.

the class TestHBaseRegionScanAssignments method testHBaseGroupScanAssignmentMix.

@Test
public void testHBaseGroupScanAssignmentMix() throws Exception {
    NavigableMap<HRegionInfo, ServerName> regionsToScan = Maps.newTreeMap();
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[0], splits[1]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[1], splits[2]), SERVER_B);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[2], splits[3]), SERVER_B);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[3], splits[4]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[4], splits[5]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[5], splits[6]), SERVER_D);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[6], splits[7]), SERVER_C);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[7], splits[0]), SERVER_D);
    final List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    final DrillbitEndpoint DB_A = DrillbitEndpoint.newBuilder().setAddress(HOST_A).setControlPort(1234).build();
    endpoints.add(DB_A);
    endpoints.add(DB_A);
    final DrillbitEndpoint DB_B = DrillbitEndpoint.newBuilder().setAddress(HOST_B).setControlPort(1234).build();
    endpoints.add(DB_B);
    final DrillbitEndpoint DB_D = DrillbitEndpoint.newBuilder().setAddress(HOST_D).setControlPort(1234).build();
    endpoints.add(DB_D);
    final DrillbitEndpoint DB_X = DrillbitEndpoint.newBuilder().setAddress(HOST_X).setControlPort(1234).build();
    endpoints.add(DB_X);
    HBaseGroupScan scan = new HBaseGroupScan();
    scan.setRegionsToScan(regionsToScan);
    scan.setHBaseScanSpec(new HBaseScanSpec(TABLE_NAME_STR, splits[0], splits[0], null));
    scan.applyAssignments(endpoints);
    int i = 0;
    // 'A'
    assertEquals(2, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'A'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'B'
    assertEquals(2, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'D'
    assertEquals(2, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'X'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    testParallelizationWidth(scan, i);
}

Example 3 with HBaseGroupScan

use of org.apache.drill.exec.store.hbase.HBaseGroupScan in project drill by apache.

the class TestHBaseRegionScanAssignments method testHBaseGroupScanAssignmentNoAfinity.

@Test
public void testHBaseGroupScanAssignmentNoAfinity() throws Exception {
    NavigableMap<HRegionInfo, ServerName> regionsToScan = Maps.newTreeMap();
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[0], splits[1]), SERVER_X);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[1], splits[2]), SERVER_X);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[2], splits[3]), SERVER_X);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[3], splits[4]), SERVER_X);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[4], splits[5]), SERVER_X);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[5], splits[6]), SERVER_X);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[6], splits[7]), SERVER_X);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[7], splits[0]), SERVER_X);
    final List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_A).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_B).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_C).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_D).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_E).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_F).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_G).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_H).setControlPort(1234).build());
    HBaseGroupScan scan = new HBaseGroupScan();
    scan.setRegionsToScan(regionsToScan);
    scan.setHBaseScanSpec(new HBaseScanSpec(TABLE_NAME_STR, splits[0], splits[0], null));
    scan.applyAssignments(endpoints);
    int i = 0;
    // 'A'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'B'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'C'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'D'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'E'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'F'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'G'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'H'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    testParallelizationWidth(scan, i);
}

Example 4 with HBaseGroupScan

use of org.apache.drill.exec.store.hbase.HBaseGroupScan in project drill by apache.

the class TestHBaseRegionScanAssignments method testHBaseGroupScanAssignmentOneEach.

@Test
public void testHBaseGroupScanAssignmentOneEach() throws Exception {
    NavigableMap<HRegionInfo, ServerName> regionsToScan = Maps.newTreeMap();
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[0], splits[1]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[1], splits[2]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[2], splits[3]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[3], splits[4]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[4], splits[5]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[5], splits[6]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[6], splits[7]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[7], splits[8]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[8], splits[0]), SERVER_A);
    final List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_A).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_A).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_B).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_C).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_D).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_E).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_F).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_G).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_H).setControlPort(1234).build());
    HBaseGroupScan scan = new HBaseGroupScan();
    scan.setRegionsToScan(regionsToScan);
    scan.setHBaseScanSpec(new HBaseScanSpec(TABLE_NAME_STR, splits[0], splits[0], null));
    scan.applyAssignments(endpoints);
    int i = 0;
    // 'A'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'A'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'B'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'C'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'D'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'E'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'F'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'G'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'H'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    testParallelizationWidth(scan, i);
}

Example 5 with HBaseGroupScan

use of org.apache.drill.exec.store.hbase.HBaseGroupScan in project drill by apache.

the class TestHBaseRegionScanAssignments method testHBaseGroupScanAssignmentAllPreferred.

@Test
public void testHBaseGroupScanAssignmentAllPreferred() throws Exception {
    NavigableMap<HRegionInfo, ServerName> regionsToScan = Maps.newTreeMap();
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[0], splits[1]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[1], splits[2]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[2], splits[3]), SERVER_B);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[3], splits[4]), SERVER_B);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[4], splits[5]), SERVER_C);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[5], splits[6]), SERVER_C);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[6], splits[7]), SERVER_D);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[7], splits[0]), SERVER_D);
    final List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    final DrillbitEndpoint DB_A = DrillbitEndpoint.newBuilder().setAddress(HOST_A).setControlPort(1234).build();
    endpoints.add(DB_A);
    final DrillbitEndpoint DB_B = DrillbitEndpoint.newBuilder().setAddress(HOST_B).setControlPort(1234).build();
    endpoints.add(DB_B);
    final DrillbitEndpoint DB_D = DrillbitEndpoint.newBuilder().setAddress(HOST_C).setControlPort(1234).build();
    endpoints.add(DB_D);
    final DrillbitEndpoint DB_X = DrillbitEndpoint.newBuilder().setAddress(HOST_D).setControlPort(1234).build();
    endpoints.add(DB_X);
    HBaseGroupScan scan = new HBaseGroupScan();
    scan.setRegionsToScan(regionsToScan);
    scan.setHBaseScanSpec(new HBaseScanSpec(TABLE_NAME_STR, splits[0], splits[0], null));
    scan.applyAssignments(endpoints);
    int i = 0;
    // 'A'
    assertEquals(2, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'B'
    assertEquals(2, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'C'
    assertEquals(2, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'D'
    assertEquals(2, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    testParallelizationWidth(scan, i);
}