use of org.apache.hadoop.hive.metastore.api.Partition in project hive by apache.
the class HCatClientHMSImpl method getPartitions.
@Override
public List<HCatPartition> getPartitions(String dbName, String tblName) throws HCatException {
List<HCatPartition> hcatPtns = new ArrayList<HCatPartition>();
try {
HCatTable hcatTable = getTable(dbName, tblName);
List<Partition> hivePtns = hmsClient.listPartitions(checkDB(dbName), tblName, (short) -1);
for (Partition ptn : hivePtns) {
hcatPtns.add(new HCatPartition(hcatTable, ptn));
}
} catch (NoSuchObjectException e) {
throw new ObjectNotFoundException("NoSuchObjectException while retrieving partition.", e);
} catch (MetaException e) {
throw new HCatException("MetaException while retrieving partition.", e);
} catch (TException e) {
throw new ConnectionFailureException("TException while retrieving partition.", e);
}
return hcatPtns;
}
use of org.apache.hadoop.hive.metastore.api.Partition in project hive by apache.
the class TestMetastoreAuthorizationProvider method testSimplePrivileges.
public void testSimplePrivileges() throws Exception {
if (!isTestEnabled()) {
System.out.println("Skipping test " + this.getClass().getName());
return;
}
String dbName = getTestDbName();
String tblName = getTestTableName();
String userName = setupUser();
allowCreateDatabase(userName);
CommandProcessorResponse ret = driver.run("create database " + dbName);
assertEquals(0, ret.getResponseCode());
Database db = msc.getDatabase(dbName);
String dbLocn = db.getLocationUri();
validateCreateDb(db, dbName);
disallowCreateInDb(dbName, userName, dbLocn);
disallowCreateDatabase(userName);
driver.run("use " + dbName);
ret = driver.run(String.format("create table %s (a string) partitioned by (b string)", tblName));
assertEquals(1, ret.getResponseCode());
// Even if table location is specified table creation should fail
String tblNameLoc = tblName + "_loc";
String tblLocation = new Path(dbLocn).getParent().toUri() + "/" + tblNameLoc;
driver.run("use " + dbName);
ret = driver.run(String.format("create table %s (a string) partitioned by (b string) location '" + tblLocation + "'", tblNameLoc));
assertEquals(1, ret.getResponseCode());
// failure from not having permissions to create table
ArrayList<FieldSchema> fields = new ArrayList<FieldSchema>(2);
fields.add(new FieldSchema("a", serdeConstants.STRING_TYPE_NAME, ""));
Table ttbl = new Table();
ttbl.setDbName(dbName);
ttbl.setTableName(tblName);
StorageDescriptor sd = new StorageDescriptor();
ttbl.setSd(sd);
sd.setCols(fields);
sd.setParameters(new HashMap<String, String>());
sd.getParameters().put("test_param_1", "Use this for comments etc");
sd.setSerdeInfo(new SerDeInfo());
sd.getSerdeInfo().setName(ttbl.getTableName());
sd.getSerdeInfo().setParameters(new HashMap<String, String>());
sd.getSerdeInfo().getParameters().put(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1");
sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
sd.setInputFormat(HiveInputFormat.class.getName());
sd.setOutputFormat(HiveOutputFormat.class.getName());
ttbl.setPartitionKeys(new ArrayList<FieldSchema>());
MetaException me = null;
try {
msc.createTable(ttbl);
} catch (MetaException e) {
me = e;
}
assertNoPrivileges(me);
allowCreateInDb(dbName, userName, dbLocn);
driver.run("use " + dbName);
ret = driver.run(String.format("create table %s (a string) partitioned by (b string)", tblName));
// now it succeeds.
assertEquals(0, ret.getResponseCode());
Table tbl = msc.getTable(dbName, tblName);
validateCreateTable(tbl, tblName, dbName);
// Table creation should succeed even if location is specified
driver.run("use " + dbName);
ret = driver.run(String.format("create table %s (a string) partitioned by (b string) location '" + tblLocation + "'", tblNameLoc));
assertEquals(0, ret.getResponseCode());
Table tblLoc = msc.getTable(dbName, tblNameLoc);
validateCreateTable(tblLoc, tblNameLoc, dbName);
String fakeUser = "mal";
List<String> fakeGroupNames = new ArrayList<String>();
fakeGroupNames.add("groupygroup");
InjectableDummyAuthenticator.injectUserName(fakeUser);
InjectableDummyAuthenticator.injectGroupNames(fakeGroupNames);
InjectableDummyAuthenticator.injectMode(true);
ret = driver.run(String.format("create table %s (a string) partitioned by (b string)", tblName + "mal"));
assertEquals(1, ret.getResponseCode());
ttbl.setTableName(tblName + "mal");
me = null;
try {
msc.createTable(ttbl);
} catch (MetaException e) {
me = e;
}
assertNoPrivileges(me);
disallowCreateInTbl(tbl.getTableName(), userName, tbl.getSd().getLocation());
ret = driver.run("alter table " + tblName + " add partition (b='2011')");
assertEquals(1, ret.getResponseCode());
List<String> ptnVals = new ArrayList<String>();
ptnVals.add("b=2011");
Partition tpart = new Partition();
tpart.setDbName(dbName);
tpart.setTableName(tblName);
tpart.setValues(ptnVals);
tpart.setParameters(new HashMap<String, String>());
tpart.setSd(tbl.getSd().deepCopy());
tpart.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo().deepCopy());
tpart.getSd().setLocation(tbl.getSd().getLocation() + "/tpart");
me = null;
try {
msc.add_partition(tpart);
} catch (MetaException e) {
me = e;
}
assertNoPrivileges(me);
InjectableDummyAuthenticator.injectMode(false);
allowCreateInTbl(tbl.getTableName(), userName, tbl.getSd().getLocation());
ret = driver.run("alter table " + tblName + " add partition (b='2011')");
assertEquals(0, ret.getResponseCode());
allowDropOnTable(tblName, userName, tbl.getSd().getLocation());
allowDropOnDb(dbName, userName, db.getLocationUri());
ret = driver.run("drop database if exists " + getTestDbName() + " cascade");
assertEquals(0, ret.getResponseCode());
InjectableDummyAuthenticator.injectUserName(userName);
InjectableDummyAuthenticator.injectGroupNames(Arrays.asList(ugi.getGroupNames()));
InjectableDummyAuthenticator.injectMode(true);
allowCreateDatabase(userName);
driver.run("create database " + dbName);
allowCreateInDb(dbName, userName, dbLocn);
tbl.setTableType("EXTERNAL_TABLE");
msc.createTable(tbl);
disallowDropOnTable(tblName, userName, tbl.getSd().getLocation());
ret = driver.run("drop table " + tbl.getTableName());
assertEquals(1, ret.getResponseCode());
}
use of org.apache.hadoop.hive.metastore.api.Partition in project hive by apache.
the class TestHBaseStoreIntegration method partitionStatistics.
@Test
public void partitionStatistics() throws Exception {
long now = System.currentTimeMillis();
String dbname = "default";
String tableName = "statspart";
String[] partNames = { "ds=today", "ds=yesterday" };
String[] partVals = { "today", "yesterday" };
String boolcol = "boolcol";
String longcol = "longcol";
String doublecol = "doublecol";
String stringcol = "stringcol";
String binarycol = "bincol";
String decimalcol = "deccol";
long trues = 37;
long falses = 12;
long booleanNulls = 2;
long strMaxLen = 1234;
double strAvgLen = 32.3;
long strNulls = 987;
long strDVs = 906;
List<FieldSchema> cols = new ArrayList<FieldSchema>();
cols.add(new FieldSchema(boolcol, "boolean", "nocomment"));
cols.add(new FieldSchema(longcol, "long", "nocomment"));
cols.add(new FieldSchema(doublecol, "double", "nocomment"));
cols.add(new FieldSchema(stringcol, "varchar(32)", "nocomment"));
cols.add(new FieldSchema(binarycol, "binary", "nocomment"));
cols.add(new FieldSchema(decimalcol, "decimal(5, 3)", "nocomment"));
SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters);
List<FieldSchema> partCols = new ArrayList<FieldSchema>();
partCols.add(new FieldSchema("ds", "string", ""));
Table table = new Table(tableName, dbname, "me", (int) now / 1000, (int) now / 1000, 0, sd, partCols, emptyParameters, null, null, null);
store.createTable(table);
for (String partVal : partVals) {
Partition part = new Partition(Arrays.asList(partVal), dbname, tableName, (int) now / 1000, (int) now / 1000, sd, emptyParameters);
store.addPartition(part);
}
for (int i = 0; i < partNames.length; i++) {
ColumnStatistics stats = new ColumnStatistics();
ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
desc.setLastAnalyzed(now);
desc.setDbName(dbname);
desc.setTableName(tableName);
desc.setIsTblLevel(false);
desc.setPartName(partNames[i]);
stats.setStatsDesc(desc);
ColumnStatisticsObj obj = new ColumnStatisticsObj();
obj.setColName(boolcol);
obj.setColType("boolean");
ColumnStatisticsData data = new ColumnStatisticsData();
BooleanColumnStatsData boolData = new BooleanColumnStatsData();
boolData.setNumTrues(trues);
boolData.setNumFalses(falses);
boolData.setNumNulls(booleanNulls);
data.setBooleanStats(boolData);
obj.setStatsData(data);
stats.addToStatsObj(obj);
store.updatePartitionColumnStatistics(stats, Arrays.asList(partVals[i]));
}
List<ColumnStatistics> statsList = store.getPartitionColumnStatistics(dbname, tableName, Arrays.asList(partNames), Arrays.asList(boolcol));
Assert.assertEquals(2, statsList.size());
for (int i = 0; i < partNames.length; i++) {
Assert.assertEquals(1, statsList.get(i).getStatsObjSize());
}
for (int i = 0; i < partNames.length; i++) {
ColumnStatistics stats = new ColumnStatistics();
ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
desc.setLastAnalyzed(now);
desc.setDbName(dbname);
desc.setTableName(tableName);
desc.setIsTblLevel(false);
desc.setPartName(partNames[i]);
stats.setStatsDesc(desc);
ColumnStatisticsObj obj = new ColumnStatisticsObj();
obj.setColName(stringcol);
obj.setColType("string");
ColumnStatisticsData data = new ColumnStatisticsData();
StringColumnStatsData strData = new StringColumnStatsData();
strData.setMaxColLen(strMaxLen);
strData.setAvgColLen(strAvgLen);
strData.setNumNulls(strNulls);
strData.setNumDVs(strDVs);
data.setStringStats(strData);
obj.setStatsData(data);
stats.addToStatsObj(obj);
store.updatePartitionColumnStatistics(stats, Arrays.asList(partVals[i]));
}
// Make sure when we ask for one we only get one
statsList = store.getPartitionColumnStatistics(dbname, tableName, Arrays.asList(partNames), Arrays.asList(boolcol));
Assert.assertEquals(2, statsList.size());
for (int i = 0; i < partNames.length; i++) {
Assert.assertEquals(1, statsList.get(i).getStatsObjSize());
}
statsList = store.getPartitionColumnStatistics(dbname, tableName, Arrays.asList(partNames), Arrays.asList(boolcol, stringcol));
Assert.assertEquals(2, statsList.size());
for (int i = 0; i < partNames.length; i++) {
Assert.assertEquals(2, statsList.get(i).getStatsObjSize());
// Just check one piece of the data, I don't need to check it all again
Assert.assertEquals(booleanNulls, statsList.get(i).getStatsObj().get(0).getStatsData().getBooleanStats().getNumNulls());
Assert.assertEquals(strDVs, statsList.get(i).getStatsObj().get(1).getStatsData().getStringStats().getNumDVs());
}
}
use of org.apache.hadoop.hive.metastore.api.Partition in project hive by apache.
the class TestHBaseStoreIntegration method listPartitions.
@Test
public void listPartitions() throws Exception {
String dbName = "default";
String tableName = "listParts";
int startTime = (int) (System.currentTimeMillis() / 1000);
List<FieldSchema> cols = new ArrayList<FieldSchema>();
cols.add(new FieldSchema("col1", "int", "nocomment"));
SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters);
List<FieldSchema> partCols = new ArrayList<FieldSchema>();
partCols.add(new FieldSchema("pc", "string", ""));
partCols.add(new FieldSchema("region", "string", ""));
Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null);
store.createTable(table);
String[][] partVals = new String[][] { { "today", "north america" }, { "tomorrow", "europe" } };
for (String[] pv : partVals) {
List<String> vals = new ArrayList<String>();
for (String v : pv) vals.add(v);
StorageDescriptor psd = new StorageDescriptor(sd);
psd.setLocation("file:/tmp/pc=" + pv[0] + "/region=" + pv[1]);
Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd, emptyParameters);
store.addPartition(part);
}
List<String> names = store.listPartitionNames(dbName, tableName, (short) -1);
Assert.assertEquals(2, names.size());
String[] resultNames = names.toArray(new String[names.size()]);
Arrays.sort(resultNames);
Assert.assertArrayEquals(resultNames, new String[] { "pc=today/region=north america", "pc=tomorrow/region=europe" });
List<Partition> parts = store.getPartitionsByNames(dbName, tableName, names);
Assert.assertArrayEquals(partVals[0], parts.get(0).getValues().toArray(new String[2]));
Assert.assertArrayEquals(partVals[1], parts.get(1).getValues().toArray(new String[2]));
store.dropPartitions(dbName, tableName, names);
List<Partition> afterDropParts = store.getPartitions(dbName, tableName, -1);
Assert.assertEquals(0, afterDropParts.size());
}
use of org.apache.hadoop.hive.metastore.api.Partition in project hive by apache.
the class TestHBaseStoreIntegration method listPartitionsWithPs.
@Test
public void listPartitionsWithPs() throws Exception {
String dbName = "default";
String tableName = "listPartitionsWithPs";
int startTime = (int) (System.currentTimeMillis() / 1000);
List<FieldSchema> cols = new ArrayList<FieldSchema>();
cols.add(new FieldSchema("col1", "int", "nocomment"));
SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters);
List<FieldSchema> partCols = new ArrayList<FieldSchema>();
partCols.add(new FieldSchema("ds", "string", ""));
partCols.add(new FieldSchema("region", "string", ""));
Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null);
store.createTable(table);
String[][] partVals = new String[][] { { "today", "north america" }, { "today", "europe" }, { "tomorrow", "north america" }, { "tomorrow", "europe" } };
for (String[] pv : partVals) {
List<String> vals = new ArrayList<String>();
for (String v : pv) vals.add(v);
StorageDescriptor psd = new StorageDescriptor(sd);
psd.setLocation("file:/tmp/ds=" + pv[0] + "/region=" + pv[1]);
Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd, emptyParameters);
store.addPartition(part);
}
// We only test listPartitionNamesPs since it calls listPartitionsPsWithAuth anyway.
// Test the case where we completely specify the partition
List<String> partitionNames = store.listPartitionNamesPs(dbName, tableName, Arrays.asList(partVals[0]), (short) -1);
Assert.assertEquals(1, partitionNames.size());
Assert.assertEquals("ds=today/region=north america", partitionNames.get(0));
// Leave off the last value of the partition
partitionNames = store.listPartitionNamesPs(dbName, tableName, Arrays.asList(partVals[0][0]), (short) -1);
Assert.assertEquals(2, partitionNames.size());
String[] names = partitionNames.toArray(new String[partitionNames.size()]);
Arrays.sort(names);
Assert.assertArrayEquals(new String[] { "ds=today/region=europe", "ds=today/region=north america" }, names);
// Put a star in the last value of the partition
partitionNames = store.listPartitionNamesPs(dbName, tableName, Arrays.asList("today", "*"), (short) -1);
Assert.assertEquals(2, partitionNames.size());
names = partitionNames.toArray(new String[partitionNames.size()]);
Arrays.sort(names);
Assert.assertArrayEquals(new String[] { "ds=today/region=europe", "ds=today/region=north america" }, names);
// Put a star in the first value of the partition
partitionNames = store.listPartitionNamesPs(dbName, tableName, Arrays.asList("*", "europe"), (short) -1);
Assert.assertEquals(2, partitionNames.size());
names = partitionNames.toArray(new String[partitionNames.size()]);
Arrays.sort(names);
Assert.assertArrayEquals(new String[] { "ds=today/region=europe", "ds=tomorrow/region=europe" }, names);
}
Aggregations