Examples with Type org.apache.hadoop.hive.metastore.api.Type used on opensource projects

Example 1 with Type

use of org.apache.hadoop.hive.metastore.api.Type in project hive by apache.

the class ObjectStore method getType.

private Type getType(MType mtype) {
    List<FieldSchema> fields = new ArrayList<FieldSchema>();
    if (mtype.getFields() != null) {
        for (MFieldSchema field : mtype.getFields()) {
            fields.add(new FieldSchema(field.getName(), field.getType(), field.getComment()));
        }
    }
    Type ret = new Type();
    ret.setName(mtype.getName());
    ret.setType1(mtype.getType1());
    ret.setType2(mtype.getType2());
    ret.setFields(fields);
    return ret;
}

Example 2 with Type

use of org.apache.hadoop.hive.metastore.api.Type in project hive by apache.

the class TestHiveMetaStore method testSimpleTable.

public void testSimpleTable() throws Exception {
    try {
        String dbName = "simpdb";
        String tblName = "simptbl";
        String tblName2 = "simptbl2";
        String typeName = "Person";
        client.dropTable(dbName, tblName);
        silentDropDatabase(dbName);
        Database db = new Database();
        db.setName(dbName);
        client.createDatabase(db);
        client.dropType(typeName);
        Type typ1 = new Type();
        typ1.setName(typeName);
        typ1.setFields(new ArrayList<FieldSchema>(2));
        typ1.getFields().add(new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, ""));
        typ1.getFields().add(new FieldSchema("income", serdeConstants.INT_TYPE_NAME, ""));
        client.createType(typ1);
        Table tbl = new Table();
        tbl.setDbName(dbName);
        tbl.setTableName(tblName);
        StorageDescriptor sd = new StorageDescriptor();
        tbl.setSd(sd);
        sd.setCols(typ1.getFields());
        sd.setCompressed(false);
        sd.setNumBuckets(1);
        sd.setParameters(new HashMap<String, String>());
        sd.getParameters().put("test_param_1", "Use this for comments etc");
        sd.setBucketCols(new ArrayList<String>(2));
        sd.getBucketCols().add("name");
        sd.setSerdeInfo(new SerDeInfo());
        sd.getSerdeInfo().setName(tbl.getTableName());
        sd.getSerdeInfo().setParameters(new HashMap<String, String>());
        sd.getSerdeInfo().getParameters().put(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1");
        sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
        sd.setInputFormat(HiveInputFormat.class.getName());
        sd.setInputFormat(HiveOutputFormat.class.getName());
        tbl.setPartitionKeys(new ArrayList<FieldSchema>());
        client.createTable(tbl);
        if (isThriftClient) {
            // the createTable() above does not update the location in the 'tbl'
            // object when the client is a thrift client and the code below relies
            // on the location being present in the 'tbl' object - so get the table
            // from the metastore
            tbl = client.getTable(dbName, tblName);
        }
        Table tbl2 = client.getTable(dbName, tblName);
        assertNotNull(tbl2);
        assertEquals(tbl2.getDbName(), dbName);
        assertEquals(tbl2.getTableName(), tblName);
        assertEquals(tbl2.getSd().getCols().size(), typ1.getFields().size());
        assertEquals(tbl2.getSd().isCompressed(), false);
        assertEquals(tbl2.getSd().getNumBuckets(), 1);
        assertEquals(tbl2.getSd().getLocation(), tbl.getSd().getLocation());
        assertNotNull(tbl2.getSd().getSerdeInfo());
        sd.getSerdeInfo().setParameters(new HashMap<String, String>());
        sd.getSerdeInfo().getParameters().put(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1");
        tbl2.setTableName(tblName2);
        tbl2.setParameters(new HashMap<String, String>());
        tbl2.getParameters().put("EXTERNAL", "TRUE");
        tbl2.getSd().setLocation(tbl.getSd().getLocation() + "-2");
        List<FieldSchema> fieldSchemas = client.getFields(dbName, tblName);
        assertNotNull(fieldSchemas);
        assertEquals(fieldSchemas.size(), tbl.getSd().getCols().size());
        for (FieldSchema fs : tbl.getSd().getCols()) {
            assertTrue(fieldSchemas.contains(fs));
        }
        List<FieldSchema> fieldSchemasFull = client.getSchema(dbName, tblName);
        assertNotNull(fieldSchemasFull);
        assertEquals(fieldSchemasFull.size(), tbl.getSd().getCols().size() + tbl.getPartitionKeys().size());
        for (FieldSchema fs : tbl.getSd().getCols()) {
            assertTrue(fieldSchemasFull.contains(fs));
        }
        for (FieldSchema fs : tbl.getPartitionKeys()) {
            assertTrue(fieldSchemasFull.contains(fs));
        }
        client.createTable(tbl2);
        if (isThriftClient) {
            tbl2 = client.getTable(tbl2.getDbName(), tbl2.getTableName());
        }
        Table tbl3 = client.getTable(dbName, tblName2);
        assertNotNull(tbl3);
        assertEquals(tbl3.getDbName(), dbName);
        assertEquals(tbl3.getTableName(), tblName2);
        assertEquals(tbl3.getSd().getCols().size(), typ1.getFields().size());
        assertEquals(tbl3.getSd().isCompressed(), false);
        assertEquals(tbl3.getSd().getNumBuckets(), 1);
        assertEquals(tbl3.getSd().getLocation(), tbl2.getSd().getLocation());
        assertEquals(tbl3.getParameters(), tbl2.getParameters());
        fieldSchemas = client.getFields(dbName, tblName2);
        assertNotNull(fieldSchemas);
        assertEquals(fieldSchemas.size(), tbl2.getSd().getCols().size());
        for (FieldSchema fs : tbl2.getSd().getCols()) {
            assertTrue(fieldSchemas.contains(fs));
        }
        fieldSchemasFull = client.getSchema(dbName, tblName2);
        assertNotNull(fieldSchemasFull);
        assertEquals(fieldSchemasFull.size(), tbl2.getSd().getCols().size() + tbl2.getPartitionKeys().size());
        for (FieldSchema fs : tbl2.getSd().getCols()) {
            assertTrue(fieldSchemasFull.contains(fs));
        }
        for (FieldSchema fs : tbl2.getPartitionKeys()) {
            assertTrue(fieldSchemasFull.contains(fs));
        }
        assertEquals("Use this for comments etc", tbl2.getSd().getParameters().get("test_param_1"));
        assertEquals("name", tbl2.getSd().getBucketCols().get(0));
        assertTrue("Partition key list is not empty", (tbl2.getPartitionKeys() == null) || (tbl2.getPartitionKeys().size() == 0));
        //test get_table_objects_by_name functionality
        ArrayList<String> tableNames = new ArrayList<String>();
        tableNames.add(tblName2);
        tableNames.add(tblName);
        tableNames.add(tblName2);
        List<Table> foundTables = client.getTableObjectsByName(dbName, tableNames);
        assertEquals(2, foundTables.size());
        for (Table t : foundTables) {
            if (t.getTableName().equals(tblName2)) {
                assertEquals(t.getSd().getLocation(), tbl2.getSd().getLocation());
            } else {
                assertEquals(t.getTableName(), tblName);
                assertEquals(t.getSd().getLocation(), tbl.getSd().getLocation());
            }
            assertEquals(t.getSd().getCols().size(), typ1.getFields().size());
            assertEquals(t.getSd().isCompressed(), false);
            assertEquals(foundTables.get(0).getSd().getNumBuckets(), 1);
            assertNotNull(t.getSd().getSerdeInfo());
            assertEquals(t.getDbName(), dbName);
        }
        tableNames.add(1, "table_that_doesnt_exist");
        foundTables = client.getTableObjectsByName(dbName, tableNames);
        assertEquals(foundTables.size(), 2);
        InvalidOperationException ioe = null;
        try {
            foundTables = client.getTableObjectsByName(dbName, null);
        } catch (InvalidOperationException e) {
            ioe = e;
        }
        assertNotNull(ioe);
        assertTrue("Table not found", ioe.getMessage().contains("null tables"));
        UnknownDBException udbe = null;
        try {
            foundTables = client.getTableObjectsByName("db_that_doesnt_exist", tableNames);
        } catch (UnknownDBException e) {
            udbe = e;
        }
        assertNotNull(udbe);
        assertTrue("DB not found", udbe.getMessage().contains("not find database db_that_doesnt_exist"));
        udbe = null;
        try {
            foundTables = client.getTableObjectsByName("", tableNames);
        } catch (UnknownDBException e) {
            udbe = e;
        }
        assertNotNull(udbe);
        assertTrue("DB not found", udbe.getMessage().contains("is null or empty"));
        FileSystem fs = FileSystem.get((new Path(tbl.getSd().getLocation())).toUri(), hiveConf);
        client.dropTable(dbName, tblName);
        assertFalse(fs.exists(new Path(tbl.getSd().getLocation())));
        client.dropTable(dbName, tblName2);
        assertTrue(fs.exists(new Path(tbl2.getSd().getLocation())));
        client.dropType(typeName);
        client.dropDatabase(dbName);
    } catch (Exception e) {
        System.err.println(StringUtils.stringifyException(e));
        System.err.println("testSimpleTable() failed.");
        throw e;
    }
}

Example 3 with Type

use of org.apache.hadoop.hive.metastore.api.Type in project hive by apache.

the class TestHiveMetaTool method setUp.

@Override
protected void setUp() throws Exception {
    super.setUp();
    try {
        HiveConf hiveConf = new HiveConf(HiveMetaTool.class);
        client = new HiveMetaStoreClient(hiveConf);
        // Setup output stream to redirect output to
        os = new ByteArrayOutputStream();
        ps = new PrintStream(os);
        // create a dummy database and a couple of dummy tables
        Database db = new Database();
        db.setName(dbName);
        client.dropTable(dbName, tblName);
        client.dropTable(dbName, badTblName);
        dropDatabase(dbName);
        client.createDatabase(db);
        locationUri = db.getLocationUri();
        String avroUri = "hdfs://nn.example.com/warehouse/hive/ab.avsc";
        String badAvroUri = new String("hdfs:/hive");
        client.dropType(typeName);
        Type typ1 = new Type();
        typ1.setName(typeName);
        typ1.setFields(new ArrayList<FieldSchema>(2));
        typ1.getFields().add(new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, ""));
        typ1.getFields().add(new FieldSchema("income", serdeConstants.INT_TYPE_NAME, ""));
        client.createType(typ1);
        Table tbl = new Table();
        tbl.setDbName(dbName);
        tbl.setTableName(tblName);
        Map<String, String> parameters = new HashMap<>();
        parameters.put(AvroSerdeUtils.SCHEMA_URL, avroUri);
        tbl.setParameters(parameters);
        StorageDescriptor sd = new StorageDescriptor();
        tbl.setSd(sd);
        sd.setCols(typ1.getFields());
        sd.setCompressed(false);
        sd.setNumBuckets(1);
        sd.setParameters(new HashMap<String, String>());
        sd.getParameters().put("test_param_1", "Use this for comments etc");
        sd.setBucketCols(new ArrayList<String>(2));
        sd.getBucketCols().add("name");
        sd.setSerdeInfo(new SerDeInfo());
        sd.getSerdeInfo().setName(tbl.getTableName());
        sd.getSerdeInfo().setParameters(new HashMap<String, String>());
        sd.getSerdeInfo().getParameters().put(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1");
        sd.getParameters().put(AvroSerdeUtils.SCHEMA_URL, avroUri);
        sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.avro.AvroSerDe.class.getName());
        sd.setInputFormat(AvroContainerInputFormat.class.getName());
        sd.setOutputFormat(AvroContainerOutputFormat.class.getName());
        tbl.setPartitionKeys(new ArrayList<FieldSchema>());
        client.createTable(tbl);
        //create a table with bad avro uri
        tbl = new Table();
        tbl.setDbName(dbName);
        tbl.setTableName(badTblName);
        sd = new StorageDescriptor();
        tbl.setSd(sd);
        sd.setCols(typ1.getFields());
        sd.setCompressed(false);
        sd.setNumBuckets(1);
        sd.setParameters(new HashMap<String, String>());
        sd.getParameters().put("test_param_1", "Use this for comments etc");
        sd.setBucketCols(new ArrayList<String>(2));
        sd.getBucketCols().add("name");
        sd.setSerdeInfo(new SerDeInfo());
        sd.getSerdeInfo().setName(tbl.getTableName());
        sd.getSerdeInfo().setParameters(new HashMap<String, String>());
        sd.getSerdeInfo().getParameters().put(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1");
        sd.getParameters().put(AvroSerdeUtils.SCHEMA_URL, badAvroUri);
        sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.avro.AvroSerDe.class.getName());
        sd.setInputFormat(AvroContainerInputFormat.class.getName());
        sd.setOutputFormat(AvroContainerOutputFormat.class.getName());
        tbl.setPartitionKeys(new ArrayList<FieldSchema>());
        client.createTable(tbl);
        client.close();
    } catch (Exception e) {
        System.err.println("Unable to setup the hive metatool test");
        System.err.println(StringUtils.stringifyException(e));
        throw new Exception(e);
    }
}

Example 4 with Type

use of org.apache.hadoop.hive.metastore.api.Type in project hive by apache.

the class TestHiveMetaStore method testComplexTypeApi.

// TODO:pc need to enhance this with complex fields and getType_all function
public void testComplexTypeApi() throws Exception {
    try {
        client.dropType("Person");
        Type typ1 = new Type();
        typ1.setName("Person");
        typ1.setFields(new ArrayList<FieldSchema>(2));
        typ1.getFields().add(new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, ""));
        typ1.getFields().add(new FieldSchema("income", serdeConstants.INT_TYPE_NAME, ""));
        boolean ret = client.createType(typ1);
        assertTrue("Unable to create type", ret);
        Type typ1_2 = client.getType("Person");
        assertNotNull("type Person not found", typ1_2);
        assertEquals(typ1.getName(), typ1_2.getName());
        assertEquals(typ1.getFields().size(), typ1_2.getFields().size());
        assertEquals(typ1.getFields().get(0), typ1_2.getFields().get(0));
        assertEquals(typ1.getFields().get(1), typ1_2.getFields().get(1));
        client.dropType("Family");
        Type fam = new Type();
        fam.setName("Family");
        fam.setFields(new ArrayList<FieldSchema>(2));
        fam.getFields().add(new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, ""));
        fam.getFields().add(new FieldSchema("members", MetaStoreUtils.getListType(typ1.getName()), ""));
        ret = client.createType(fam);
        assertTrue("Unable to create type " + fam.getName(), ret);
        Type fam2 = client.getType("Family");
        assertNotNull("type Person not found", fam2);
        assertEquals(fam.getName(), fam2.getName());
        assertEquals(fam.getFields().size(), fam2.getFields().size());
        assertEquals(fam.getFields().get(0), fam2.getFields().get(0));
        assertEquals(fam.getFields().get(1), fam2.getFields().get(1));
        ret = client.dropType("Family");
        assertTrue("unable to drop type Family", ret);
        ret = client.dropType("Person");
        assertTrue("unable to drop type Person", ret);
        boolean exceptionThrown = false;
        try {
            client.getType("Person");
        } catch (NoSuchObjectException e) {
            exceptionThrown = true;
        }
        assertTrue("Expected NoSuchObjectException", exceptionThrown);
    } catch (Exception e) {
        System.err.println(StringUtils.stringifyException(e));
        System.err.println("testComplexTypeApi() failed.");
        throw e;
    }
}

Example 5 with Type

use of org.apache.hadoop.hive.metastore.api.Type in project hive by apache.

the class ObjectStore method getType.

@Override
public Type getType(String typeName) {
    Type type = null;
    boolean commited = false;
    Query query = null;
    try {
        openTransaction();
        query = pm.newQuery(MType.class, "name == typeName");
        query.declareParameters("java.lang.String typeName");
        query.setUnique(true);
        MType mtype = (MType) query.execute(typeName.trim());
        pm.retrieve(type);
        if (mtype != null) {
            type = getType(mtype);
        }
        commited = commitTransaction();
    } finally {
        if (!commited) {
            rollbackTransaction();
        }
        if (query != null) {
            query.closeAll();
        }
    }
    return type;
}