Search in sources :

Example 86 with SerDeInfo

use of org.apache.hadoop.hive.metastore.api.SerDeInfo in project hive by apache.

the class TestAccumuloStorageHandler method testExternalNonExistentTableFails.

@Test(expected = MetaException.class)
public void testExternalNonExistentTableFails() throws Exception {
    MockInstance inst = new MockInstance(test.getMethodName());
    Connector conn = inst.getConnector("root", new PasswordToken(""));
    String tableName = "table";
    // Define the SerDe Parameters
    Map<String, String> params = new HashMap<String, String>();
    params.put(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:cq");
    AccumuloConnectionParameters connectionParams = Mockito.mock(AccumuloConnectionParameters.class);
    AccumuloStorageHandler storageHandler = Mockito.mock(AccumuloStorageHandler.class);
    StorageDescriptor sd = Mockito.mock(StorageDescriptor.class);
    Table table = Mockito.mock(Table.class);
    SerDeInfo serDeInfo = Mockito.mock(SerDeInfo.class);
    // Call the real preCreateTable method
    Mockito.doCallRealMethod().when(storageHandler).preCreateTable(table);
    // Return our known table name
    Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
    // Is an EXTERNAL table
    Mockito.when(storageHandler.isExternalTable(table)).thenReturn(true);
    // Return the mocked StorageDescriptor
    Mockito.when(table.getSd()).thenReturn(sd);
    // No location expected with AccumuloStorageHandler
    Mockito.when(sd.getLocation()).thenReturn(null);
    // Return mocked SerDeInfo
    Mockito.when(sd.getSerdeInfo()).thenReturn(serDeInfo);
    // Custom parameters
    Mockito.when(serDeInfo.getParameters()).thenReturn(params);
    // Return the MockInstance's Connector
    Mockito.when(connectionParams.getConnector()).thenReturn(conn);
    storageHandler.connectionParams = connectionParams;
    storageHandler.preCreateTable(table);
}
Also used : Connector(org.apache.accumulo.core.client.Connector) PasswordToken(org.apache.accumulo.core.client.security.tokens.PasswordToken) Table(org.apache.hadoop.hive.metastore.api.Table) MockInstance(org.apache.accumulo.core.client.mock.MockInstance) HashMap(java.util.HashMap) SerDeInfo(org.apache.hadoop.hive.metastore.api.SerDeInfo) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) Test(org.junit.Test)

Example 87 with SerDeInfo

use of org.apache.hadoop.hive.metastore.api.SerDeInfo in project hive by apache.

the class TestAccumuloStorageHandler method testMissingColumnMappingFails.

@Test(expected = MetaException.class)
public void testMissingColumnMappingFails() throws Exception {
    MockInstance inst = new MockInstance(test.getMethodName());
    Connector conn = inst.getConnector("root", new PasswordToken(""));
    String tableName = "table";
    // Empty parameters are sent, no COLUMN_MAPPING
    Map<String, String> params = new HashMap<String, String>();
    AccumuloConnectionParameters connectionParams = Mockito.mock(AccumuloConnectionParameters.class);
    AccumuloStorageHandler storageHandler = Mockito.mock(AccumuloStorageHandler.class);
    StorageDescriptor sd = Mockito.mock(StorageDescriptor.class);
    Table table = Mockito.mock(Table.class);
    SerDeInfo serDeInfo = Mockito.mock(SerDeInfo.class);
    // Call the real preCreateTable method
    Mockito.doCallRealMethod().when(storageHandler).preCreateTable(table);
    // Return our known table name
    Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
    // Not an EXTERNAL table
    Mockito.when(storageHandler.isExternalTable(table)).thenReturn(false);
    // Return the mocked StorageDescriptor
    Mockito.when(table.getSd()).thenReturn(sd);
    // No location expected with AccumuloStorageHandler
    Mockito.when(sd.getLocation()).thenReturn(null);
    // Return mocked SerDeInfo
    Mockito.when(sd.getSerdeInfo()).thenReturn(serDeInfo);
    // Custom parameters
    Mockito.when(serDeInfo.getParameters()).thenReturn(params);
    // Return the MockInstance's Connector
    Mockito.when(connectionParams.getConnector()).thenReturn(conn);
    storageHandler.connectionParams = connectionParams;
    storageHandler.preCreateTable(table);
}
Also used : Connector(org.apache.accumulo.core.client.Connector) PasswordToken(org.apache.accumulo.core.client.security.tokens.PasswordToken) Table(org.apache.hadoop.hive.metastore.api.Table) MockInstance(org.apache.accumulo.core.client.mock.MockInstance) HashMap(java.util.HashMap) SerDeInfo(org.apache.hadoop.hive.metastore.api.SerDeInfo) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) Test(org.junit.Test)

Example 88 with SerDeInfo

use of org.apache.hadoop.hive.metastore.api.SerDeInfo in project hive by apache.

the class TestAccumuloStorageHandler method testPreCreateTable.

@Test
public void testPreCreateTable() throws Exception {
    MockInstance inst = new MockInstance(test.getMethodName());
    Connector conn = inst.getConnector("root", new PasswordToken(""));
    String tableName = "table";
    // Define the SerDe Parameters
    Map<String, String> params = new HashMap<String, String>();
    params.put(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:cq");
    AccumuloConnectionParameters connectionParams = Mockito.mock(AccumuloConnectionParameters.class);
    AccumuloStorageHandler storageHandler = Mockito.mock(AccumuloStorageHandler.class);
    StorageDescriptor sd = Mockito.mock(StorageDescriptor.class);
    Table table = Mockito.mock(Table.class);
    SerDeInfo serDeInfo = Mockito.mock(SerDeInfo.class);
    // Call the real preCreateTable method
    Mockito.doCallRealMethod().when(storageHandler).preCreateTable(table);
    // Return our known table name
    Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
    // Not an EXTERNAL table
    Mockito.when(storageHandler.isExternalTable(table)).thenReturn(false);
    // Return the mocked StorageDescriptor
    Mockito.when(table.getSd()).thenReturn(sd);
    // No location expected with AccumuloStorageHandler
    Mockito.when(sd.getLocation()).thenReturn(null);
    // Return mocked SerDeInfo
    Mockito.when(sd.getSerdeInfo()).thenReturn(serDeInfo);
    // Custom parameters
    Mockito.when(serDeInfo.getParameters()).thenReturn(params);
    // Return the MockInstance's Connector
    Mockito.when(connectionParams.getConnector()).thenReturn(conn);
    storageHandler.connectionParams = connectionParams;
    storageHandler.preCreateTable(table);
    Assert.assertTrue("Table does not exist when we expect it to", conn.tableOperations().exists(tableName));
}
Also used : Connector(org.apache.accumulo.core.client.Connector) PasswordToken(org.apache.accumulo.core.client.security.tokens.PasswordToken) Table(org.apache.hadoop.hive.metastore.api.Table) MockInstance(org.apache.accumulo.core.client.mock.MockInstance) HashMap(java.util.HashMap) SerDeInfo(org.apache.hadoop.hive.metastore.api.SerDeInfo) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) Test(org.junit.Test)

Example 89 with SerDeInfo

use of org.apache.hadoop.hive.metastore.api.SerDeInfo in project hive by apache.

the class TestAccumuloStorageHandler method testNonNullLocation.

@Test(expected = MetaException.class)
public void testNonNullLocation() throws Exception {
    MockInstance inst = new MockInstance(test.getMethodName());
    Connector conn = inst.getConnector("root", new PasswordToken(""));
    String tableName = "table";
    // Empty parameters are sent, no COLUMN_MAPPING
    Map<String, String> params = new HashMap<String, String>();
    params.put(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:cq");
    AccumuloConnectionParameters connectionParams = Mockito.mock(AccumuloConnectionParameters.class);
    AccumuloStorageHandler storageHandler = Mockito.mock(AccumuloStorageHandler.class);
    StorageDescriptor sd = Mockito.mock(StorageDescriptor.class);
    Table table = Mockito.mock(Table.class);
    SerDeInfo serDeInfo = Mockito.mock(SerDeInfo.class);
    // Call the real preCreateTable method
    Mockito.doCallRealMethod().when(storageHandler).preCreateTable(table);
    // Return our known table name
    Mockito.when(storageHandler.getTableName(table)).thenReturn(tableName);
    // Not an EXTERNAL table
    Mockito.when(storageHandler.isExternalTable(table)).thenReturn(false);
    // Return the mocked StorageDescriptor
    Mockito.when(table.getSd()).thenReturn(sd);
    // No location expected with AccumuloStorageHandler
    Mockito.when(sd.getLocation()).thenReturn("foobar");
    // Return mocked SerDeInfo
    Mockito.when(sd.getSerdeInfo()).thenReturn(serDeInfo);
    // Custom parameters
    Mockito.when(serDeInfo.getParameters()).thenReturn(params);
    // Return the MockInstance's Connector
    Mockito.when(connectionParams.getConnector()).thenReturn(conn);
    storageHandler.connectionParams = connectionParams;
    storageHandler.preCreateTable(table);
}
Also used : Connector(org.apache.accumulo.core.client.Connector) PasswordToken(org.apache.accumulo.core.client.security.tokens.PasswordToken) Table(org.apache.hadoop.hive.metastore.api.Table) MockInstance(org.apache.accumulo.core.client.mock.MockInstance) HashMap(java.util.HashMap) SerDeInfo(org.apache.hadoop.hive.metastore.api.SerDeInfo) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) Test(org.junit.Test)

Example 90 with SerDeInfo

use of org.apache.hadoop.hive.metastore.api.SerDeInfo in project hive by apache.

the class TestHCatMultiOutputFormat method createTable.

private static void createTable(String tableName, String tablePerm) throws Exception {
    Table tbl = new Table();
    tbl.setDbName(DATABASE);
    tbl.setTableName(tableName);
    StorageDescriptor sd = new StorageDescriptor();
    sd.setCols(ColumnHolder.colMapping.get(tableName));
    tbl.setSd(sd);
    sd.setParameters(new HashMap<String, String>());
    sd.setSerdeInfo(new SerDeInfo());
    sd.getSerdeInfo().setName(tbl.getTableName());
    sd.getSerdeInfo().setParameters(new HashMap<String, String>());
    sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName());
    sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName());
    sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1");
    sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName());
    tbl.setPartitionKeys(ColumnHolder.partitionCols);
    hmsc.createTable(tbl);
    Path path = new Path(warehousedir, tableName);
    FileSystem fs = path.getFileSystem(hiveConf);
    fs.setPermission(path, new FsPermission(tablePerm));
}
Also used : Path(org.apache.hadoop.fs.Path) Table(org.apache.hadoop.hive.metastore.api.Table) SerDeInfo(org.apache.hadoop.hive.metastore.api.SerDeInfo) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) FileSystem(org.apache.hadoop.fs.FileSystem) FsPermission(org.apache.hadoop.fs.permission.FsPermission)

Aggregations

SerDeInfo (org.apache.hadoop.hive.metastore.api.SerDeInfo)152 StorageDescriptor (org.apache.hadoop.hive.metastore.api.StorageDescriptor)137 FieldSchema (org.apache.hadoop.hive.metastore.api.FieldSchema)115 Table (org.apache.hadoop.hive.metastore.api.Table)114 ArrayList (java.util.ArrayList)112 Test (org.junit.Test)105 Partition (org.apache.hadoop.hive.metastore.api.Partition)65 HashMap (java.util.HashMap)44 ColumnStatistics (org.apache.hadoop.hive.metastore.api.ColumnStatistics)31 ColumnStatisticsData (org.apache.hadoop.hive.metastore.api.ColumnStatisticsData)31 ColumnStatisticsDesc (org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc)31 ColumnStatisticsObj (org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj)31 AggrStats (org.apache.hadoop.hive.metastore.api.AggrStats)30 List (java.util.List)26 Order (org.apache.hadoop.hive.metastore.api.Order)25 Database (org.apache.hadoop.hive.metastore.api.Database)22 MetaException (org.apache.hadoop.hive.metastore.api.MetaException)14 MetastoreCheckinTest (org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest)13 LongColumnStatsData (org.apache.hadoop.hive.metastore.api.LongColumnStatsData)13 NotificationEvent (org.apache.hadoop.hive.metastore.api.NotificationEvent)12