Examples with PlanResult org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.PlanResult used on opensource projects

Example 1 with PlanResult

use of org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.PlanResult in project hive by apache.

the class TestHBaseFilterPlanUtil method testPartitionKeyScannerMixedType.

@Test
public void testPartitionKeyScannerMixedType() throws Exception {
    List<FieldSchema> parts = new ArrayList<FieldSchema>();
    parts.add(new FieldSchema("year", "int", null));
    parts.add(new FieldSchema("month", "int", null));
    parts.add(new FieldSchema("state", "string", null));
    // One prefix key and one minor key range
    ExpressionTree exprTree = PartFilterExprUtil.getFilterParser("year = 2015 and state = 'CA'").tree;
    PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
    Assert.assertEquals(planRes.plan.getPlans().size(), 1);
    ScanPlan sp = planRes.plan.getPlans().get(0);
    byte[] startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
    byte[] endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
    RowFilter filter = (RowFilter) sp.getFilter(parts);
    // scan range contains the major key year, rowfilter contains minor key state
    Assert.assertTrue(Bytes.contains(startRowSuffix, Shorts.toByteArray((short) 2015)));
    Assert.assertTrue(Bytes.contains(endRowSuffix, Shorts.toByteArray((short) 2016)));
    PartitionKeyComparator comparator = (PartitionKeyComparator) filter.getComparator();
    Assert.assertEquals(comparator.ranges.size(), 1);
    Assert.assertEquals(comparator.ranges.get(0).keyName, "state");
}

Example 2 with PlanResult

use of org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.PlanResult in project hive by apache.

the class TestHBaseFilterPlanUtil method testPartitionKeyScannerAllString.

@Test
public void testPartitionKeyScannerAllString() throws Exception {
    List<FieldSchema> parts = new ArrayList<FieldSchema>();
    parts.add(new FieldSchema("year", "string", null));
    parts.add(new FieldSchema("month", "string", null));
    parts.add(new FieldSchema("state", "string", null));
    // One prefix key and one minor key range
    ExpressionTree exprTree = PartFilterExprUtil.getFilterParser("year = 2015 and state = 'CA'").tree;
    PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
    Assert.assertEquals(planRes.plan.getPlans().size(), 1);
    ScanPlan sp = planRes.plan.getPlans().get(0);
    byte[] startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
    byte[] endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
    RowFilter filter = (RowFilter) sp.getFilter(parts);
    // scan range contains the major key year, rowfilter contains minor key state
    Assert.assertTrue(Bytes.contains(startRowSuffix, "2015".getBytes()));
    Assert.assertTrue(Bytes.contains(endRowSuffix, "2015".getBytes()));
    Assert.assertFalse(Bytes.contains(startRowSuffix, "CA".getBytes()));
    Assert.assertFalse(Bytes.contains(endRowSuffix, "CA".getBytes()));
    PartitionKeyComparator comparator = (PartitionKeyComparator) filter.getComparator();
    Assert.assertEquals(comparator.ranges.size(), 1);
    Assert.assertEquals(comparator.ranges.get(0).keyName, "state");
    // Two prefix key and one LIKE operator
    exprTree = PartFilterExprUtil.getFilterParser("year = 2015 and month > 10 " + "and month <= 11 and state like 'C%'").tree;
    planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
    Assert.assertEquals(planRes.plan.getPlans().size(), 1);
    sp = planRes.plan.getPlans().get(0);
    startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
    endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
    filter = (RowFilter) sp.getFilter(parts);
    // scan range contains the major key value year/month, rowfilter contains LIKE operator
    Assert.assertTrue(Bytes.contains(startRowSuffix, "2015".getBytes()));
    Assert.assertTrue(Bytes.contains(endRowSuffix, "2015".getBytes()));
    Assert.assertTrue(Bytes.contains(startRowSuffix, "10".getBytes()));
    Assert.assertTrue(Bytes.contains(endRowSuffix, "11".getBytes()));
    comparator = (PartitionKeyComparator) filter.getComparator();
    Assert.assertEquals(comparator.ops.size(), 1);
    Assert.assertEquals(comparator.ops.get(0).keyName, "state");
    // One prefix key, one minor key range and one LIKE operator
    exprTree = PartFilterExprUtil.getFilterParser("year >= 2014 and month > 10 " + "and month <= 11 and state like 'C%'").tree;
    planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
    Assert.assertEquals(planRes.plan.getPlans().size(), 1);
    sp = planRes.plan.getPlans().get(0);
    startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
    endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
    filter = (RowFilter) sp.getFilter(parts);
    // scan range contains the major key value year (low bound), rowfilter contains minor key state
    // and LIKE operator
    Assert.assertTrue(Bytes.contains(startRowSuffix, "2014".getBytes()));
    comparator = (PartitionKeyComparator) filter.getComparator();
    Assert.assertEquals(comparator.ranges.size(), 1);
    Assert.assertEquals(comparator.ranges.get(0).keyName, "month");
    Assert.assertEquals(comparator.ops.size(), 1);
    Assert.assertEquals(comparator.ops.get(0).keyName, "state");
    // Condition contains or
    exprTree = PartFilterExprUtil.getFilterParser("year = 2014 and (month > 10 " + "or month < 3)").tree;
    planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
    sp = planRes.plan.getPlans().get(0);
    startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
    endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
    filter = (RowFilter) sp.getFilter(parts);
    // The first ScanPlan contains year = 2014 and month > 10
    Assert.assertTrue(Bytes.contains(startRowSuffix, "2014".getBytes()));
    Assert.assertTrue(Bytes.contains(endRowSuffix, "2014".getBytes()));
    Assert.assertTrue(Bytes.contains(startRowSuffix, "10".getBytes()));
    sp = planRes.plan.getPlans().get(1);
    startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
    endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
    filter = (RowFilter) sp.getFilter(parts);
    // The first ScanPlan contains year = 2014 and month < 3
    Assert.assertTrue(Bytes.contains(startRowSuffix, "2014".getBytes()));
    Assert.assertTrue(Bytes.contains(endRowSuffix, "2014".getBytes()));
    Assert.assertTrue(Bytes.contains(endRowSuffix, "3".getBytes()));
}

Example 3 with PlanResult

use of org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.PlanResult in project hive by apache.

the class TestHBaseFilterPlanUtil method testTreeNodePlan.

/**
   * Test plan generation from TreeNode
   *
   * @throws MetaException
   */
@Test
public void testTreeNodePlan() throws MetaException {
    final String KEY = "k1";
    final String VAL1 = "10";
    final String VAL2 = "11";
    LeafNode l = new LeafNode();
    l.keyName = KEY;
    l.value = VAL1;
    final ScanMarker DEFAULT_SCANMARKER = null;
    List<FieldSchema> parts = new ArrayList<FieldSchema>();
    parts.add(new FieldSchema("k1", "int", null));
    LeafNode r = new LeafNode();
    r.keyName = KEY;
    r.value = VAL2;
    TreeNode tn = new TreeNode(l, LogicalOperator.AND, r);
    // verify plan for - k1 >= '10' and k1 < '11'
    l.operator = Operator.GREATERTHANOREQUALTO;
    r.operator = Operator.LESSTHAN;
    verifyPlan(tn, parts, KEY, new ScanMarker(VAL1, INCLUSIVE, "int"), new ScanMarker(VAL2, !INCLUSIVE, "int"));
    // verify plan for - k1 >= '10' and k1 > '11'
    l.operator = Operator.GREATERTHANOREQUALTO;
    r.operator = Operator.GREATERTHAN;
    verifyPlan(tn, parts, KEY, new ScanMarker(VAL2, !INCLUSIVE, "int"), DEFAULT_SCANMARKER);
    // verify plan for - k1 >= '10' or k1 > '11'
    tn = new TreeNode(l, LogicalOperator.OR, r);
    ExpressionTree e = new ExpressionTree();
    e.setRootForTest(tn);
    PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
    Assert.assertEquals(2, planRes.plan.getPlans().size());
    Assert.assertEquals(false, planRes.hasUnsupportedCondition);
    // verify plan for - k1 >= '10' and (k1 >= '10' or k1 > '11')
    TreeNode tn2 = new TreeNode(l, LogicalOperator.AND, tn);
    e = new ExpressionTree();
    e.setRootForTest(tn2);
    planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
    Assert.assertEquals(2, planRes.plan.getPlans().size());
    Assert.assertEquals(false, planRes.hasUnsupportedCondition);
    // verify plan for  (k1 >= '10' and (k1 >= '10' or k1 > '11')) or k1 LIKE '2'
    // plan should return true for hasUnsupportedCondition
    LeafNode klike = new LeafNode();
    klike.keyName = KEY;
    klike.value = VAL1;
    klike.operator = Operator.LIKE;
    TreeNode tn3 = new TreeNode(tn2, LogicalOperator.OR, klike);
    e = new ExpressionTree();
    e.setRootForTest(tn3);
    planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
    Assert.assertEquals(3, planRes.plan.getPlans().size());
    Assert.assertEquals(false, planRes.hasUnsupportedCondition);
}

Example 4 with PlanResult

use of org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.PlanResult in project hive by apache.

the class HBaseStore method getPartitionsByExprInternal.

private boolean getPartitionsByExprInternal(String dbName, String tblName, ExpressionTree exprTree, short maxParts, List<Partition> result) throws MetaException, NoSuchObjectException {
    dbName = HiveStringUtils.normalizeIdentifier(dbName);
    tblName = HiveStringUtils.normalizeIdentifier(tblName);
    Table table = getTable(dbName, tblName);
    if (table == null) {
        throw new NoSuchObjectException("Unable to find table " + dbName + "." + tblName);
    }
    // general hbase filter plan from expression tree
    PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, table.getPartitionKeys());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Hbase Filter Plan generated : " + planRes.plan);
    }
    // results from scans need to be merged as there can be overlapping results between
    // the scans. Use a map of list of partition values to partition for this.
    Map<List<String>, Partition> mergedParts = new HashMap<List<String>, Partition>();
    for (ScanPlan splan : planRes.plan.getPlans()) {
        try {
            List<Partition> parts = getHBase().scanPartitions(dbName, tblName, splan.getStartRowSuffix(dbName, tblName, table.getPartitionKeys()), splan.getEndRowSuffix(dbName, tblName, table.getPartitionKeys()), splan.getFilter(table.getPartitionKeys()), -1);
            boolean reachedMax = false;
            for (Partition part : parts) {
                mergedParts.put(part.getValues(), part);
                if (mergedParts.size() == maxParts) {
                    reachedMax = true;
                    break;
                }
            }
            if (reachedMax) {
                break;
            }
        } catch (IOException e) {
            LOG.error("Unable to get partitions", e);
            throw new MetaException("Error scanning partitions" + tableNameForErrorMsg(dbName, tblName) + ": " + e);
        }
    }
    for (Entry<List<String>, Partition> mp : mergedParts.entrySet()) {
        result.add(mp.getValue());
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Matched partitions " + result);
    }
    // being returned
    return !planRes.hasUnsupportedCondition;
}

Example 5 with PlanResult

use of org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.PlanResult in project hive by apache.

the class TestHBaseFilterPlanUtil method verifyPlan.

private void verifyPlan(TreeNode l, List<FieldSchema> parts, String keyName, ScanMarker startMarker, ScanMarker endMarker, boolean hasUnsupportedCondition) throws MetaException {
    ExpressionTree e = null;
    if (l != null) {
        e = new ExpressionTree();
        e.setRootForTest(l);
    }
    PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
    FilterPlan plan = planRes.plan;
    Assert.assertEquals("Has unsupported condition", hasUnsupportedCondition, planRes.hasUnsupportedCondition);
    Assert.assertEquals(1, plan.getPlans().size());
    ScanPlan splan = plan.getPlans().get(0);
    if (startMarker != null) {
        Assert.assertEquals(startMarker, splan.markers.get(keyName).startMarker);
    } else {
        Assert.assertTrue(splan.markers.get(keyName) == null || splan.markers.get(keyName).startMarker == null);
    }
    if (endMarker != null) {
        Assert.assertEquals(endMarker, splan.markers.get(keyName).endMarker);
    } else {
        Assert.assertTrue(splan.markers.get(keyName) == null || splan.markers.get(keyName).endMarker == null);
    }
}