use of org.apache.hadoop.hive.metastore.api.ColumnStatisticsData in project hive by apache.
the class TestHBaseStoreBitVector method stringPartitionStatistics.
@Test
public void stringPartitionStatistics() throws Exception {
createMockTableAndPartition(STRING_TYPE, STRING_VAL);
// Add partition stats for: STRING_COL and partition: {PART_KEY, STRING_VAL} to DB
// Because of the way our mock implementation works we actually need to not create the table
// before we set statistics on it.
ColumnStatistics stats = new ColumnStatistics();
// Get a default ColumnStatisticsDesc for partition level stats
ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, STRING_VAL);
stats.setStatsDesc(desc);
// Get one of the pre-created ColumnStatisticsObj
ColumnStatisticsObj obj = stringColStatsObjs.get(0);
StringColumnStatsData stringData = obj.getStatsData().getStringStats();
// Add to DB
stats.addToStatsObj(obj);
List<String> parVals = new ArrayList<String>();
parVals.add(STRING_VAL);
store.updatePartitionColumnStatistics(stats, parVals);
// Get from DB
List<String> partNames = new ArrayList<String>();
partNames.add(desc.getPartName());
List<String> colNames = new ArrayList<String>();
colNames.add(obj.getColName());
List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
// Compare ColumnStatisticsDesc
Assert.assertEquals(1, statsFromDB.size());
Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
// Compare ColumnStatisticsObj
Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
// Compare ColumnStatisticsData
Assert.assertEquals(ColumnStatisticsData._Fields.STRING_STATS, dataFromDB.getSetField());
// Compare StringColumnStatsData
StringColumnStatsData stringDataFromDB = dataFromDB.getStringStats();
Assert.assertEquals(stringData.getMaxColLen(), stringDataFromDB.getMaxColLen());
Assert.assertEquals(stringData.getAvgColLen(), stringDataFromDB.getAvgColLen(), 0.01);
Assert.assertEquals(stringData.getNumNulls(), stringDataFromDB.getNumNulls());
Assert.assertEquals(stringData.getNumDVs(), stringDataFromDB.getNumDVs());
Assert.assertEquals(stringData.getBitVectors(), stringDataFromDB.getBitVectors());
}
use of org.apache.hadoop.hive.metastore.api.ColumnStatisticsData in project hive by apache.
the class TestHBaseStoreBitVector method decimalPartitionStatistics.
@Test
public void decimalPartitionStatistics() throws Exception {
createMockTableAndPartition(DECIMAL_TYPE, DECIMAL_VAL);
// Add partition stats for: DECIMAL_COL and partition: {PART_KEY, DECIMAL_VAL} to DB
// Because of the way our mock implementation works we actually need to not create the table
// before we set statistics on it.
ColumnStatistics stats = new ColumnStatistics();
// Get a default ColumnStatisticsDesc for partition level stats
ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, DECIMAL_VAL);
stats.setStatsDesc(desc);
// Get one of the pre-created ColumnStatisticsObj
ColumnStatisticsObj obj = decimalColStatsObjs.get(0);
DecimalColumnStatsData decimalData = obj.getStatsData().getDecimalStats();
// Add to DB
stats.addToStatsObj(obj);
List<String> parVals = new ArrayList<String>();
parVals.add(DECIMAL_VAL);
store.updatePartitionColumnStatistics(stats, parVals);
// Get from DB
List<String> partNames = new ArrayList<String>();
partNames.add(desc.getPartName());
List<String> colNames = new ArrayList<String>();
colNames.add(obj.getColName());
List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
// Compare ColumnStatisticsDesc
Assert.assertEquals(1, statsFromDB.size());
Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
// Compare ColumnStatisticsObj
Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
// Compare ColumnStatisticsData
Assert.assertEquals(ColumnStatisticsData._Fields.DECIMAL_STATS, dataFromDB.getSetField());
// Compare DecimalColumnStatsData
DecimalColumnStatsData decimalDataFromDB = dataFromDB.getDecimalStats();
Assert.assertEquals(decimalData.getHighValue(), decimalDataFromDB.getHighValue());
Assert.assertEquals(decimalData.getLowValue(), decimalDataFromDB.getLowValue());
Assert.assertEquals(decimalData.getNumNulls(), decimalDataFromDB.getNumNulls());
Assert.assertEquals(decimalData.getNumDVs(), decimalDataFromDB.getNumDVs());
Assert.assertEquals(decimalData.getBitVectors(), decimalDataFromDB.getBitVectors());
}
use of org.apache.hadoop.hive.metastore.api.ColumnStatisticsData in project hive by apache.
the class TestHBaseStoreIntegration method partitionStatistics.
@Test
public void partitionStatistics() throws Exception {
long now = System.currentTimeMillis();
String dbname = "default";
String tableName = "statspart";
String[] partNames = { "ds=today", "ds=yesterday" };
String[] partVals = { "today", "yesterday" };
String boolcol = "boolcol";
String longcol = "longcol";
String doublecol = "doublecol";
String stringcol = "stringcol";
String binarycol = "bincol";
String decimalcol = "deccol";
long trues = 37;
long falses = 12;
long booleanNulls = 2;
long strMaxLen = 1234;
double strAvgLen = 32.3;
long strNulls = 987;
long strDVs = 906;
List<FieldSchema> cols = new ArrayList<FieldSchema>();
cols.add(new FieldSchema(boolcol, "boolean", "nocomment"));
cols.add(new FieldSchema(longcol, "long", "nocomment"));
cols.add(new FieldSchema(doublecol, "double", "nocomment"));
cols.add(new FieldSchema(stringcol, "varchar(32)", "nocomment"));
cols.add(new FieldSchema(binarycol, "binary", "nocomment"));
cols.add(new FieldSchema(decimalcol, "decimal(5, 3)", "nocomment"));
SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters);
List<FieldSchema> partCols = new ArrayList<FieldSchema>();
partCols.add(new FieldSchema("ds", "string", ""));
Table table = new Table(tableName, dbname, "me", (int) now / 1000, (int) now / 1000, 0, sd, partCols, emptyParameters, null, null, null);
store.createTable(table);
for (String partVal : partVals) {
Partition part = new Partition(Arrays.asList(partVal), dbname, tableName, (int) now / 1000, (int) now / 1000, sd, emptyParameters);
store.addPartition(part);
}
for (int i = 0; i < partNames.length; i++) {
ColumnStatistics stats = new ColumnStatistics();
ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
desc.setLastAnalyzed(now);
desc.setDbName(dbname);
desc.setTableName(tableName);
desc.setIsTblLevel(false);
desc.setPartName(partNames[i]);
stats.setStatsDesc(desc);
ColumnStatisticsObj obj = new ColumnStatisticsObj();
obj.setColName(boolcol);
obj.setColType("boolean");
ColumnStatisticsData data = new ColumnStatisticsData();
BooleanColumnStatsData boolData = new BooleanColumnStatsData();
boolData.setNumTrues(trues);
boolData.setNumFalses(falses);
boolData.setNumNulls(booleanNulls);
data.setBooleanStats(boolData);
obj.setStatsData(data);
stats.addToStatsObj(obj);
store.updatePartitionColumnStatistics(stats, Arrays.asList(partVals[i]));
}
List<ColumnStatistics> statsList = store.getPartitionColumnStatistics(dbname, tableName, Arrays.asList(partNames), Arrays.asList(boolcol));
Assert.assertEquals(2, statsList.size());
for (int i = 0; i < partNames.length; i++) {
Assert.assertEquals(1, statsList.get(i).getStatsObjSize());
}
for (int i = 0; i < partNames.length; i++) {
ColumnStatistics stats = new ColumnStatistics();
ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
desc.setLastAnalyzed(now);
desc.setDbName(dbname);
desc.setTableName(tableName);
desc.setIsTblLevel(false);
desc.setPartName(partNames[i]);
stats.setStatsDesc(desc);
ColumnStatisticsObj obj = new ColumnStatisticsObj();
obj.setColName(stringcol);
obj.setColType("string");
ColumnStatisticsData data = new ColumnStatisticsData();
StringColumnStatsData strData = new StringColumnStatsData();
strData.setMaxColLen(strMaxLen);
strData.setAvgColLen(strAvgLen);
strData.setNumNulls(strNulls);
strData.setNumDVs(strDVs);
data.setStringStats(strData);
obj.setStatsData(data);
stats.addToStatsObj(obj);
store.updatePartitionColumnStatistics(stats, Arrays.asList(partVals[i]));
}
// Make sure when we ask for one we only get one
statsList = store.getPartitionColumnStatistics(dbname, tableName, Arrays.asList(partNames), Arrays.asList(boolcol));
Assert.assertEquals(2, statsList.size());
for (int i = 0; i < partNames.length; i++) {
Assert.assertEquals(1, statsList.get(i).getStatsObjSize());
}
statsList = store.getPartitionColumnStatistics(dbname, tableName, Arrays.asList(partNames), Arrays.asList(boolcol, stringcol));
Assert.assertEquals(2, statsList.size());
for (int i = 0; i < partNames.length; i++) {
Assert.assertEquals(2, statsList.get(i).getStatsObjSize());
// Just check one piece of the data, I don't need to check it all again
Assert.assertEquals(booleanNulls, statsList.get(i).getStatsObj().get(0).getStatsData().getBooleanStats().getNumNulls());
Assert.assertEquals(strDVs, statsList.get(i).getStatsObj().get(1).getStatsData().getStringStats().getNumDVs());
}
}
use of org.apache.hadoop.hive.metastore.api.ColumnStatisticsData in project hive by apache.
the class TestHBaseAggrStatsCacheIntegration method alterInvalidation.
@Test
public void alterInvalidation() throws Exception {
try {
String dbName = "default";
String tableName = "ai";
List<String> partVals1 = Arrays.asList("today");
List<String> partVals2 = Arrays.asList("yesterday");
List<String> partVals3 = Arrays.asList("tomorrow");
long now = System.currentTimeMillis();
List<FieldSchema> cols = new ArrayList<>();
cols.add(new FieldSchema("col1", "boolean", "nocomment"));
SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, Collections.<String, String>emptyMap());
List<FieldSchema> partCols = new ArrayList<>();
partCols.add(new FieldSchema("ds", "string", ""));
Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols, Collections.<String, String>emptyMap(), null, null, null);
store.createTable(table);
Partition[] partitions = new Partition[3];
int partnum = 0;
for (List<String> partVals : Arrays.asList(partVals1, partVals2, partVals3)) {
StorageDescriptor psd = new StorageDescriptor(sd);
psd.setLocation("file:/tmp/default/invalidation/ds=" + partVals.get(0));
Partition part = new Partition(partVals, dbName, tableName, (int) now, (int) now, psd, Collections.<String, String>emptyMap());
partitions[partnum++] = part;
store.addPartition(part);
ColumnStatistics cs = new ColumnStatistics();
ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
desc.setLastAnalyzed(now);
desc.setPartName("ds=" + partVals.get(0));
cs.setStatsDesc(desc);
ColumnStatisticsObj obj = new ColumnStatisticsObj();
obj.setColName("col1");
obj.setColType("boolean");
ColumnStatisticsData data = new ColumnStatisticsData();
BooleanColumnStatsData bcsd = new BooleanColumnStatsData();
bcsd.setNumFalses(10);
bcsd.setNumTrues(20);
bcsd.setNumNulls(30);
data.setBooleanStats(bcsd);
obj.setStatsData(data);
cs.addToStatsObj(obj);
store.updatePartitionColumnStatistics(cs, partVals);
}
AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, Arrays.asList("ds=today", "ds=tomorrow"), Arrays.asList("col1"));
aggrStats = store.get_aggr_stats_for(dbName, tableName, Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
// Check that we had to build it from the stats
Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(2, store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(2, store.backdoor().getStatsCache().misses.getCnt());
// wake the invalidator and check again to make sure it isn't too aggressive about
// removing our stuff.
store.backdoor().getStatsCache().wakeInvalidator();
Partition newPart = new Partition(partitions[2]);
newPart.setLastAccessTime((int) System.currentTimeMillis());
store.alterPartition(dbName, tableName, partVals3, newPart);
store.backdoor().getStatsCache().setRunInvalidatorEvery(100);
store.backdoor().getStatsCache().wakeInvalidator();
aggrStats = store.get_aggr_stats_for(dbName, tableName, Arrays.asList("ds=tomorrow", "ds=today"), Arrays.asList("col1"));
// Check that we missed, which means this aggregate was dropped from the cache.
Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(3, store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(3, store.backdoor().getStatsCache().misses.getCnt());
// Check that our other aggregate is still in the cache.
aggrStats = store.get_aggr_stats_for(dbName, tableName, Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
Assert.assertEquals(4, store.backdoor().getStatsCache().totalGets.getCnt());
Assert.assertEquals(3, store.backdoor().getStatsCache().misses.getCnt());
} finally {
store.backdoor().getStatsCache().setRunInvalidatorEvery(5000);
store.backdoor().getStatsCache().setMaxTimeInCache(500000);
store.backdoor().getStatsCache().wakeInvalidator();
}
}
use of org.apache.hadoop.hive.metastore.api.ColumnStatisticsData in project hive by apache.
the class TestHBaseStore method mockLongStats.
private static ColumnStatisticsObj mockLongStats(int i) {
long high = 120938479124L + 100 * i;
long low = -12341243213412124L - 50 * i;
long nulls = 23 + i;
long dVs = 213L + 10 * i;
ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
colStatsObj.setColName(LONG_COL);
colStatsObj.setColType(LONG_TYPE);
ColumnStatisticsData data = new ColumnStatisticsData();
LongColumnStatsData longData = new LongColumnStatsData();
longData.setHighValue(high);
longData.setLowValue(low);
longData.setNumNulls(nulls);
longData.setNumDVs(dVs);
data.setLongStats(longData);
colStatsObj.setStatsData(data);
return colStatsObj;
}
Aggregations