Examples with HBaseScanSpec org.apache.drill.exec.store.hbase.HBaseScanSpec used on opensource projects

Example 1 with HBaseScanSpec

use of org.apache.drill.exec.store.hbase.HBaseScanSpec in project drill by apache.

the class TestHBaseRegionScanAssignments method testHBaseGroupScanAssignmentSomeAfinedWithOrphans.

@Test
public void testHBaseGroupScanAssignmentSomeAfinedWithOrphans() throws Exception {
    NavigableMap<HRegionInfo, ServerName> regionsToScan = Maps.newTreeMap();
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[0], splits[1]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[1], splits[2]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[2], splits[3]), SERVER_B);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[3], splits[4]), SERVER_B);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[4], splits[5]), SERVER_C);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[5], splits[6]), SERVER_C);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[6], splits[7]), SERVER_D);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[7], splits[8]), SERVER_D);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[8], splits[9]), SERVER_E);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[9], splits[10]), SERVER_E);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[10], splits[11]), SERVER_F);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[11], splits[12]), SERVER_F);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[12], splits[13]), SERVER_G);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[13], splits[14]), SERVER_G);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[14], splits[15]), SERVER_H);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[15], splits[16]), SERVER_H);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[16], splits[17]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[17], splits[0]), SERVER_A);
    final List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_A).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_B).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_C).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_D).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_E).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_F).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_G).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_H).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_I).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_J).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_K).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_L).setControlPort(1234).build());
    endpoints.add(DrillbitEndpoint.newBuilder().setAddress(HOST_M).setControlPort(1234).build());
    HBaseGroupScan scan = new HBaseGroupScan();
    scan.setRegionsToScan(regionsToScan);
    scan.setHBaseScanSpec(new HBaseScanSpec(TABLE_NAME_STR, splits[0], splits[0], null));
    scan.applyAssignments(endpoints);
    LinkedList<Integer> sizes = Lists.newLinkedList();
    sizes.add(1);
    sizes.add(1);
    sizes.add(1);
    sizes.add(1);
    sizes.add(1);
    sizes.add(1);
    sizes.add(1);
    sizes.add(1);
    sizes.add(2);
    sizes.add(2);
    sizes.add(2);
    sizes.add(2);
    sizes.add(2);
    for (int i = 0; i < endpoints.size(); i++) {
        assertTrue(sizes.remove((Integer) scan.getSpecificScan(i).getRegionScanSpecList().size()));
    }
    assertEquals(0, sizes.size());
    testParallelizationWidth(scan, endpoints.size());
}

Example 2 with HBaseScanSpec

use of org.apache.drill.exec.store.hbase.HBaseScanSpec in project drill by apache.

the class TestHBaseRegionScanAssignments method testHBaseGroupScanAssignmentMix.

@Test
public void testHBaseGroupScanAssignmentMix() throws Exception {
    NavigableMap<HRegionInfo, ServerName> regionsToScan = Maps.newTreeMap();
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[0], splits[1]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[1], splits[2]), SERVER_B);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[2], splits[3]), SERVER_B);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[3], splits[4]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[4], splits[5]), SERVER_A);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[5], splits[6]), SERVER_D);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[6], splits[7]), SERVER_C);
    regionsToScan.put(new HRegionInfo(TABLE_NAME, splits[7], splits[0]), SERVER_D);
    final List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    final DrillbitEndpoint DB_A = DrillbitEndpoint.newBuilder().setAddress(HOST_A).setControlPort(1234).build();
    endpoints.add(DB_A);
    endpoints.add(DB_A);
    final DrillbitEndpoint DB_B = DrillbitEndpoint.newBuilder().setAddress(HOST_B).setControlPort(1234).build();
    endpoints.add(DB_B);
    final DrillbitEndpoint DB_D = DrillbitEndpoint.newBuilder().setAddress(HOST_D).setControlPort(1234).build();
    endpoints.add(DB_D);
    final DrillbitEndpoint DB_X = DrillbitEndpoint.newBuilder().setAddress(HOST_X).setControlPort(1234).build();
    endpoints.add(DB_X);
    HBaseGroupScan scan = new HBaseGroupScan();
    scan.setRegionsToScan(regionsToScan);
    scan.setHBaseScanSpec(new HBaseScanSpec(TABLE_NAME_STR, splits[0], splits[0], null));
    scan.applyAssignments(endpoints);
    int i = 0;
    // 'A'
    assertEquals(2, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'A'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'B'
    assertEquals(2, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'D'
    assertEquals(2, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    // 'X'
    assertEquals(1, scan.getSpecificScan(i++).getRegionScanSpecList().size());
    testParallelizationWidth(scan, i);
}

Example 3 with HBaseScanSpec

use of org.apache.drill.exec.store.hbase.HBaseScanSpec in project drill by apache.

the class BinaryTableGroupScan method getSubScanSpec.

protected MapRDBSubScanSpec getSubScanSpec(TabletFragmentInfo tfi) {
    HBaseScanSpec spec = hbaseScanSpec;
    MapRDBSubScanSpec subScanSpec = new MapRDBSubScanSpec(spec.getTableName(), regionsToScan.get(tfi), (!isNullOrEmpty(spec.getStartRow()) && tfi.containsRow(spec.getStartRow())) ? spec.getStartRow() : tfi.getStartKey(), (!isNullOrEmpty(spec.getStopRow()) && tfi.containsRow(spec.getStopRow())) ? spec.getStopRow() : tfi.getEndKey(), spec.getSerializedFilter(), null);
    return subScanSpec;
}

Example 4 with HBaseScanSpec

use of org.apache.drill.exec.store.hbase.HBaseScanSpec in project drill by apache.

the class MapRDBPushFilterIntoScan method doPushFilterIntoBinaryGroupScan.

protected void doPushFilterIntoBinaryGroupScan(final RelOptRuleCall call, final FilterPrel filter, final ProjectPrel project, final ScanPrel scan, final BinaryTableGroupScan groupScan, final RexNode condition) {
    if (groupScan.isFilterPushedDown()) {
        /*
       * The rule can get triggered again due to the transformed "scan => filter" sequence
       * created by the earlier execution of this rule when we could not do a complete
       * conversion of Optiq Filter's condition to HBase Filter. In such cases, we rely upon
       * this flag to not do a re-processing of the rule on the already transformed call.
       */
        return;
    }
    final LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition);
    final MapRDBFilterBuilder maprdbFilterBuilder = new MapRDBFilterBuilder(groupScan, conditionExp);
    final HBaseScanSpec newScanSpec = maprdbFilterBuilder.parseTree();
    if (newScanSpec == null) {
        //no filter pushdown ==> No transformation.
        return;
    }
    // Pass tableStats from old groupScan so we do not go and fetch stats (an expensive operation) again from MapR DB client.
    final BinaryTableGroupScan newGroupsScan = new BinaryTableGroupScan(groupScan.getUserName(), groupScan.getStoragePlugin(), groupScan.getFormatPlugin(), newScanSpec, groupScan.getColumns(), groupScan.getTableStats());
    newGroupsScan.setFilterPushedDown(true);
    final ScanPrel newScanPrel = ScanPrel.create(scan, filter.getTraitSet(), newGroupsScan, scan.getRowType());
    // Depending on whether is a project in the middle, assign either scan or copy of project to childRel.
    final RelNode childRel = project == null ? newScanPrel : project.copy(project.getTraitSet(), ImmutableList.of((RelNode) newScanPrel));
    ;
    if (maprdbFilterBuilder.isAllExpressionsConverted()) {
        /*
         * Since we could convert the entire filter condition expression into an HBase filter,
         * we can eliminate the filter operator altogether.
         */
        call.transformTo(childRel);
    } else {
        call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of(childRel)));
    }
}

Example 5 with HBaseScanSpec

use of org.apache.drill.exec.store.hbase.HBaseScanSpec in project drill by apache.

the class MapRDBFilterBuilder method parseTree.

public HBaseScanSpec parseTree() {
    HBaseScanSpec parsedSpec = le.accept(this, null);
    if (parsedSpec != null) {
        parsedSpec = mergeScanSpecs("booleanAnd", this.groupScan.getHBaseScanSpec(), parsedSpec);
        /*
       * If RowFilter is THE filter attached to the scan specification,
       * remove it since its effect is also achieved through startRow and stopRow.
       */
        Filter filter = parsedSpec.getFilter();
        if (filter instanceof RowFilter && ((RowFilter) filter).getOperator() != CompareOp.NOT_EQUAL && ((RowFilter) filter).getComparator() instanceof BinaryComparator) {
            parsedSpec = new HBaseScanSpec(parsedSpec.getTableName(), parsedSpec.getStartRow(), parsedSpec.getStopRow(), null);
        }
    }
    return parsedSpec;
}