Examples with FieldSchema org.apache.hadoop.hive.metastore.api.FieldSchema used on opensource projects

Example 76 with FieldSchema

use of org.apache.hadoop.hive.metastore.api.FieldSchema in project hive by apache.

the class TestHBaseStore method alterIndex.

@Test
public void alterIndex() throws Exception {
    String tableName = "mytable";
    int startTime = (int) (System.currentTimeMillis() / 1000);
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(new FieldSchema("col1", "int", ""));
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    Map<String, String> params = new HashMap<String, String>();
    params.put("key", "value");
    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17, serde, Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params);
    Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null);
    store.createTable(table);
    String indexName = "myindex";
    Index index = new Index(indexName, null, "default", tableName, startTime, startTime, tableName + "__" + indexName + "__", sd, emptyParameters, false);
    store.addIndex(index);
    startTime += 10;
    index.setLastAccessTime(startTime);
    store.alterIndex("default", tableName, indexName, index);
    Index ind = store.getIndex("default", tableName, indexName);
    Assert.assertEquals(1, ind.getSd().getColsSize());
    Assert.assertEquals("col1", ind.getSd().getCols().get(0).getName());
    Assert.assertEquals("int", ind.getSd().getCols().get(0).getType());
    Assert.assertEquals("", ind.getSd().getCols().get(0).getComment());
    Assert.assertEquals("serde", ind.getSd().getSerdeInfo().getName());
    Assert.assertEquals("seriallib", ind.getSd().getSerdeInfo().getSerializationLib());
    Assert.assertEquals("file:/tmp", ind.getSd().getLocation());
    Assert.assertEquals("input", ind.getSd().getInputFormat());
    Assert.assertEquals("output", ind.getSd().getOutputFormat());
    Assert.assertFalse(ind.getSd().isCompressed());
    Assert.assertEquals(17, ind.getSd().getNumBuckets());
    Assert.assertEquals(1, ind.getSd().getBucketColsSize());
    Assert.assertEquals("bucketcol", ind.getSd().getBucketCols().get(0));
    Assert.assertEquals(1, ind.getSd().getSortColsSize());
    Assert.assertEquals("sortcol", ind.getSd().getSortCols().get(0).getCol());
    Assert.assertEquals(1, ind.getSd().getSortCols().get(0).getOrder());
    Assert.assertEquals(1, ind.getSd().getParametersSize());
    Assert.assertEquals("value", ind.getSd().getParameters().get("key"));
    Assert.assertEquals(indexName, ind.getIndexName());
    Assert.assertNull(ind.getIndexHandlerClass());
    Assert.assertEquals("default", ind.getDbName());
    Assert.assertEquals(tableName, ind.getOrigTableName());
    Assert.assertEquals(0, ind.getParametersSize());
    Assert.assertEquals(startTime, ind.getLastAccessTime());
    Assert.assertEquals(false, ind.isDeferredRebuild());
}

Example 77 with FieldSchema

use of org.apache.hadoop.hive.metastore.api.FieldSchema in project hive by apache.

the class TestHBaseAggregateStatsNDVUniformDist method TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusLong.

@Test
public void TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusLong() throws Exception {
    String dbName = "default";
    String tableName = "snp";
    long now = System.currentTimeMillis();
    List<FieldSchema> cols = new ArrayList<>();
    cols.add(new FieldSchema("col5_long", "long", "nocomment"));
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, Collections.<String, String>emptyMap());
    List<FieldSchema> partCols = new ArrayList<>();
    partCols.add(new FieldSchema("ds", "string", ""));
    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols, Collections.<String, String>emptyMap(), null, null, null);
    store.createTable(table);
    List<List<String>> partVals = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        List<String> partVal = Arrays.asList("" + i);
        partVals.add(partVal);
        StorageDescriptor psd = new StorageDescriptor(sd);
        psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
        Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd, Collections.<String, String>emptyMap());
        store.addPartition(part);
        if (i == 0 || i == 2 || i == 3 || i == 5 || i == 6 || i == 8) {
            ColumnStatistics cs = new ColumnStatistics();
            ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
            desc.setLastAnalyzed(now);
            desc.setPartName("ds=" + partVal);
            cs.setStatsDesc(desc);
            ColumnStatisticsObj obj = new ColumnStatisticsObj();
            obj.setColName("col5_long");
            obj.setColType("long");
            ColumnStatisticsData data = new ColumnStatisticsData();
            LongColumnStatsData dcsd = new LongColumnStatsData();
            dcsd.setHighValue(1000 + i);
            dcsd.setLowValue(-1000 - i);
            dcsd.setNumNulls(i);
            dcsd.setNumDVs(10 * i + 1);
            dcsd.setBitVectors(bitVectors[i / 5]);
            data.setLongStats(dcsd);
            obj.setStatsData(data);
            cs.addToStatsObj(obj);
            store.updatePartitionColumnStatistics(cs, partVal);
        }
    }
    Checker statChecker = new Checker() {

        @Override
        public void checkStats(AggrStats aggrStats) throws Exception {
            Assert.assertEquals(6, aggrStats.getPartsFound());
            Assert.assertEquals(1, aggrStats.getColStatsSize());
            ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
            Assert.assertEquals("col5_long", cso.getColName());
            Assert.assertEquals("long", cso.getColType());
            LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
            Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
            Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
            Assert.assertEquals(40, lcsd.getNumNulls());
            Assert.assertEquals(12, lcsd.getNumDVs());
        }
    };
    List<String> partNames = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        partNames.add("ds=" + i);
    }
    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames, Arrays.asList("col5_long"));
    statChecker.checkStats(aggrStats);
}

Example 78 with FieldSchema

use of org.apache.hadoop.hive.metastore.api.FieldSchema in project hive by apache.

the class TestHBaseFilterPlanUtil method testPartitionKeyScannerMixedType.

@Test
public void testPartitionKeyScannerMixedType() throws Exception {
    List<FieldSchema> parts = new ArrayList<FieldSchema>();
    parts.add(new FieldSchema("year", "int", null));
    parts.add(new FieldSchema("month", "int", null));
    parts.add(new FieldSchema("state", "string", null));
    // One prefix key and one minor key range
    ExpressionTree exprTree = PartFilterExprUtil.getFilterParser("year = 2015 and state = 'CA'").tree;
    PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
    Assert.assertEquals(planRes.plan.getPlans().size(), 1);
    ScanPlan sp = planRes.plan.getPlans().get(0);
    byte[] startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
    byte[] endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
    RowFilter filter = (RowFilter) sp.getFilter(parts);
    // scan range contains the major key year, rowfilter contains minor key state
    Assert.assertTrue(Bytes.contains(startRowSuffix, Shorts.toByteArray((short) 2015)));
    Assert.assertTrue(Bytes.contains(endRowSuffix, Shorts.toByteArray((short) 2016)));
    PartitionKeyComparator comparator = (PartitionKeyComparator) filter.getComparator();
    Assert.assertEquals(comparator.ranges.size(), 1);
    Assert.assertEquals(comparator.ranges.get(0).keyName, "state");
}

Example 79 with FieldSchema

use of org.apache.hadoop.hive.metastore.api.FieldSchema in project hive by apache.

the class TestHBaseFilterPlanUtil method testPartitionKeyScannerAllString.

@Test
public void testPartitionKeyScannerAllString() throws Exception {
    List<FieldSchema> parts = new ArrayList<FieldSchema>();
    parts.add(new FieldSchema("year", "string", null));
    parts.add(new FieldSchema("month", "string", null));
    parts.add(new FieldSchema("state", "string", null));
    // One prefix key and one minor key range
    ExpressionTree exprTree = PartFilterExprUtil.getFilterParser("year = 2015 and state = 'CA'").tree;
    PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
    Assert.assertEquals(planRes.plan.getPlans().size(), 1);
    ScanPlan sp = planRes.plan.getPlans().get(0);
    byte[] startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
    byte[] endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
    RowFilter filter = (RowFilter) sp.getFilter(parts);
    // scan range contains the major key year, rowfilter contains minor key state
    Assert.assertTrue(Bytes.contains(startRowSuffix, "2015".getBytes()));
    Assert.assertTrue(Bytes.contains(endRowSuffix, "2015".getBytes()));
    Assert.assertFalse(Bytes.contains(startRowSuffix, "CA".getBytes()));
    Assert.assertFalse(Bytes.contains(endRowSuffix, "CA".getBytes()));
    PartitionKeyComparator comparator = (PartitionKeyComparator) filter.getComparator();
    Assert.assertEquals(comparator.ranges.size(), 1);
    Assert.assertEquals(comparator.ranges.get(0).keyName, "state");
    // Two prefix key and one LIKE operator
    exprTree = PartFilterExprUtil.getFilterParser("year = 2015 and month > 10 " + "and month <= 11 and state like 'C%'").tree;
    planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
    Assert.assertEquals(planRes.plan.getPlans().size(), 1);
    sp = planRes.plan.getPlans().get(0);
    startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
    endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
    filter = (RowFilter) sp.getFilter(parts);
    // scan range contains the major key value year/month, rowfilter contains LIKE operator
    Assert.assertTrue(Bytes.contains(startRowSuffix, "2015".getBytes()));
    Assert.assertTrue(Bytes.contains(endRowSuffix, "2015".getBytes()));
    Assert.assertTrue(Bytes.contains(startRowSuffix, "10".getBytes()));
    Assert.assertTrue(Bytes.contains(endRowSuffix, "11".getBytes()));
    comparator = (PartitionKeyComparator) filter.getComparator();
    Assert.assertEquals(comparator.ops.size(), 1);
    Assert.assertEquals(comparator.ops.get(0).keyName, "state");
    // One prefix key, one minor key range and one LIKE operator
    exprTree = PartFilterExprUtil.getFilterParser("year >= 2014 and month > 10 " + "and month <= 11 and state like 'C%'").tree;
    planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
    Assert.assertEquals(planRes.plan.getPlans().size(), 1);
    sp = planRes.plan.getPlans().get(0);
    startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
    endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
    filter = (RowFilter) sp.getFilter(parts);
    // scan range contains the major key value year (low bound), rowfilter contains minor key state
    // and LIKE operator
    Assert.assertTrue(Bytes.contains(startRowSuffix, "2014".getBytes()));
    comparator = (PartitionKeyComparator) filter.getComparator();
    Assert.assertEquals(comparator.ranges.size(), 1);
    Assert.assertEquals(comparator.ranges.get(0).keyName, "month");
    Assert.assertEquals(comparator.ops.size(), 1);
    Assert.assertEquals(comparator.ops.get(0).keyName, "state");
    // Condition contains or
    exprTree = PartFilterExprUtil.getFilterParser("year = 2014 and (month > 10 " + "or month < 3)").tree;
    planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
    sp = planRes.plan.getPlans().get(0);
    startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
    endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
    filter = (RowFilter) sp.getFilter(parts);
    // The first ScanPlan contains year = 2014 and month > 10
    Assert.assertTrue(Bytes.contains(startRowSuffix, "2014".getBytes()));
    Assert.assertTrue(Bytes.contains(endRowSuffix, "2014".getBytes()));
    Assert.assertTrue(Bytes.contains(startRowSuffix, "10".getBytes()));
    sp = planRes.plan.getPlans().get(1);
    startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
    endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
    filter = (RowFilter) sp.getFilter(parts);
    // The first ScanPlan contains year = 2014 and month < 3
    Assert.assertTrue(Bytes.contains(startRowSuffix, "2014".getBytes()));
    Assert.assertTrue(Bytes.contains(endRowSuffix, "2014".getBytes()));
    Assert.assertTrue(Bytes.contains(endRowSuffix, "3".getBytes()));
}

Example 80 with FieldSchema

use of org.apache.hadoop.hive.metastore.api.FieldSchema in project hive by apache.

the class TestHBaseFilterPlanUtil method testTreeNodePlan.

/**
   * Test plan generation from TreeNode
   *
   * @throws MetaException
   */
@Test
public void testTreeNodePlan() throws MetaException {
    final String KEY = "k1";
    final String VAL1 = "10";
    final String VAL2 = "11";
    LeafNode l = new LeafNode();
    l.keyName = KEY;
    l.value = VAL1;
    final ScanMarker DEFAULT_SCANMARKER = null;
    List<FieldSchema> parts = new ArrayList<FieldSchema>();
    parts.add(new FieldSchema("k1", "int", null));
    LeafNode r = new LeafNode();
    r.keyName = KEY;
    r.value = VAL2;
    TreeNode tn = new TreeNode(l, LogicalOperator.AND, r);
    // verify plan for - k1 >= '10' and k1 < '11'
    l.operator = Operator.GREATERTHANOREQUALTO;
    r.operator = Operator.LESSTHAN;
    verifyPlan(tn, parts, KEY, new ScanMarker(VAL1, INCLUSIVE, "int"), new ScanMarker(VAL2, !INCLUSIVE, "int"));
    // verify plan for - k1 >= '10' and k1 > '11'
    l.operator = Operator.GREATERTHANOREQUALTO;
    r.operator = Operator.GREATERTHAN;
    verifyPlan(tn, parts, KEY, new ScanMarker(VAL2, !INCLUSIVE, "int"), DEFAULT_SCANMARKER);
    // verify plan for - k1 >= '10' or k1 > '11'
    tn = new TreeNode(l, LogicalOperator.OR, r);
    ExpressionTree e = new ExpressionTree();
    e.setRootForTest(tn);
    PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
    Assert.assertEquals(2, planRes.plan.getPlans().size());
    Assert.assertEquals(false, planRes.hasUnsupportedCondition);
    // verify plan for - k1 >= '10' and (k1 >= '10' or k1 > '11')
    TreeNode tn2 = new TreeNode(l, LogicalOperator.AND, tn);
    e = new ExpressionTree();
    e.setRootForTest(tn2);
    planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
    Assert.assertEquals(2, planRes.plan.getPlans().size());
    Assert.assertEquals(false, planRes.hasUnsupportedCondition);
    // verify plan for  (k1 >= '10' and (k1 >= '10' or k1 > '11')) or k1 LIKE '2'
    // plan should return true for hasUnsupportedCondition
    LeafNode klike = new LeafNode();
    klike.keyName = KEY;
    klike.value = VAL1;
    klike.operator = Operator.LIKE;
    TreeNode tn3 = new TreeNode(tn2, LogicalOperator.OR, klike);
    e = new ExpressionTree();
    e.setRootForTest(tn3);
    planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
    Assert.assertEquals(3, planRes.plan.getPlans().size());
    Assert.assertEquals(false, planRes.hasUnsupportedCondition);
}