Examples with Catalog org.apache.hadoop.hive.metastore.api.Catalog used on opensource projects

Example 26 with Catalog

use of org.apache.hadoop.hive.metastore.api.Catalog in project hive by apache.

the class TestAlterPartitions method otherCatalog.

@Test
@ConditionalIgnoreOnSessionHiveMetastoreClient
public void otherCatalog() throws TException {
    String catName = "alter_partition_catalog";
    Catalog cat = new CatalogBuilder().setName(catName).setLocation(MetaStoreTestUtils.getTestWarehouseDir(catName)).build();
    client.createCatalog(cat);
    String dbName = "alter_partition_database_in_other_catalog";
    Database db = new DatabaseBuilder().setName(dbName).setCatalogName(catName).create(client, metaStore.getConf());
    String tableName = "table_in_other_catalog";
    Table table = new TableBuilder().inDb(db).setTableName(tableName).addCol("id", "int").addCol("name", "string").addPartCol("partcol", "string").create(client, metaStore.getConf());
    Partition[] parts = new Partition[5];
    for (int i = 0; i < 5; i++) {
        parts[i] = new PartitionBuilder().inTable(table).addValue("a" + i).setLocation(MetaStoreTestUtils.getTestWarehouseDir("b" + i)).build(metaStore.getConf());
    }
    client.add_partitions(Arrays.asList(parts));
    Partition newPart = client.getPartition(catName, dbName, tableName, Collections.singletonList("a0"));
    newPart.getParameters().put("test_key", "test_value");
    client.alter_partition(catName, dbName, tableName, newPart);
    Partition fetched = client.getPartition(catName, dbName, tableName, Collections.singletonList("a0"));
    Assert.assertEquals(catName, fetched.getCatName());
    Assert.assertEquals("test_value", fetched.getParameters().get("test_key"));
    newPart = client.getPartition(catName, dbName, tableName, Collections.singletonList("a1"));
    newPart.setLastAccessTime(3);
    Partition newPart1 = client.getPartition(catName, dbName, tableName, Collections.singletonList("a2"));
    newPart1.getSd().setLocation(MetaStoreTestUtils.getTestWarehouseDir("somewhere"));
    client.alter_partitions(catName, dbName, tableName, Arrays.asList(newPart, newPart1));
    fetched = client.getPartition(catName, dbName, tableName, Collections.singletonList("a1"));
    Assert.assertEquals(catName, fetched.getCatName());
    Assert.assertEquals(3L, fetched.getLastAccessTime());
    fetched = client.getPartition(catName, dbName, tableName, Collections.singletonList("a2"));
    Assert.assertEquals(catName, fetched.getCatName());
    Assert.assertTrue(fetched.getSd().getLocation().contains("somewhere"));
    newPart = client.getPartition(catName, dbName, tableName, Collections.singletonList("a4"));
    newPart.getParameters().put("test_key", "test_value");
    EnvironmentContext ec = new EnvironmentContext();
    ec.setProperties(Collections.singletonMap("a", "b"));
    client.alter_partition(catName, dbName, tableName, newPart, ec);
    fetched = client.getPartition(catName, dbName, tableName, Collections.singletonList("a4"));
    Assert.assertEquals(catName, fetched.getCatName());
    Assert.assertEquals("test_value", fetched.getParameters().get("test_key"));
    client.dropDatabase(catName, dbName, true, true, true);
    client.dropCatalog(catName);
}

Example 27 with Catalog

use of org.apache.hadoop.hive.metastore.api.Catalog in project hive by apache.

the class TestAddPartitions method addPartitionOtherCatalog.

@Test
@ConditionalIgnoreOnSessionHiveMetastoreClient
public void addPartitionOtherCatalog() throws TException {
    String catName = "add_partition_catalog";
    Catalog cat = new CatalogBuilder().setName(catName).setLocation(MetaStoreTestUtils.getTestWarehouseDir(catName)).build();
    client.createCatalog(cat);
    String dbName = "add_partition_database_in_other_catalog";
    Database db = new DatabaseBuilder().setName(dbName).setCatalogName(catName).create(client, metaStore.getConf());
    String tableName = "table_in_other_catalog";
    Table table = new TableBuilder().inDb(db).setTableName(tableName).addCol("id", "int").addCol("name", "string").addPartCol("partcol", "string").create(client, metaStore.getConf());
    Partition[] parts = new Partition[5];
    for (int i = 0; i < parts.length; i++) {
        parts[i] = new PartitionBuilder().inTable(table).addValue("a" + i).build(metaStore.getConf());
    }
    client.add_partition(parts[0]);
    Assert.assertEquals(2, client.add_partitions(Arrays.asList(parts[1], parts[2])));
    client.add_partitions(Arrays.asList(parts), true, false);
    for (int i = 0; i < parts.length; i++) {
        Partition fetched = client.getPartition(catName, dbName, tableName, Collections.singletonList("a" + i));
        Assert.assertEquals(catName, fetched.getCatName());
        Assert.assertEquals(dbName, fetched.getDbName());
        Assert.assertEquals(tableName, fetched.getTableName());
    }
    client.dropDatabase(catName, dbName, true, true, true);
    client.dropCatalog(catName);
}

Example 28 with Catalog

use of org.apache.hadoop.hive.metastore.api.Catalog in project hive by apache.

the class TestCatalogs method createCatalogWithBadLocation.

@Test(expected = MetaException.class)
// TODO This test passes fine locally but fails on Linux, not sure why
@Ignore
public void createCatalogWithBadLocation() throws TException {
    Catalog cat = new CatalogBuilder().setName("goodluck").setLocation("/nosuch/nosuch").build();
    client.createCatalog(cat);
}

Example 29 with Catalog

use of org.apache.hadoop.hive.metastore.api.Catalog in project hive by apache.

the class TestTenantBasedStorageHierarchy method createTableWithCapabilities.

private Table createTableWithCapabilities(Map<String, Object> props) throws Exception {
    String catalog = (String) props.getOrDefault("CATALOG", MetaStoreUtils.getDefaultCatalog(conf));
    String dbName = (String) props.getOrDefault("DBNAME", "simpdb");
    String tblName = (String) props.getOrDefault("TBLNAME", "test_table");
    TableType type = (TableType) props.getOrDefault("TBLTYPE", TableType.MANAGED_TABLE);
    int buckets = ((Integer) props.getOrDefault("BUCKETS", -1)).intValue();
    String properties = (String) props.getOrDefault("PROPERTIES", "");
    String location = (String) (props.get("LOCATION"));
    boolean dropDb = ((Boolean) props.getOrDefault("DROPDB", Boolean.TRUE)).booleanValue();
    int partitionCount = ((Integer) props.getOrDefault("PARTITIONS", 0)).intValue();
    final String typeName = "Person";
    if (type == TableType.EXTERNAL_TABLE) {
        if (!properties.contains("EXTERNAL=TRUE")) {
            properties.concat(";EXTERNAL=TRUE;");
        }
    }
    Map<String, String> table_params = new HashMap();
    if (properties.length() > 0) {
        String[] propArray = properties.split(";");
        for (String prop : propArray) {
            String[] keyValue = prop.split("=");
            table_params.put(keyValue[0], keyValue[1]);
        }
    }
    Catalog cat = null;
    try {
        cat = client.getCatalog(catalog);
    } catch (NoSuchObjectException e) {
        LOG.debug("Catalog does not exist, creating a new one");
        try {
            if (cat == null) {
                cat = new Catalog();
                cat.setName(catalog.toLowerCase());
                Warehouse wh = new Warehouse(conf);
                cat.setLocationUri(wh.getWhRootExternal().toString() + File.separator + catalog);
                cat.setDescription("Non-hive catalog");
                client.createCatalog(cat);
                LOG.debug("Catalog " + catalog + " created");
            }
        } catch (Exception ce) {
            LOG.warn("Catalog " + catalog + " could not be created");
        }
    } catch (Exception e) {
        LOG.error("Creation of a new catalog failed, aborting test");
        throw e;
    }
    try {
        client.dropTable(dbName, tblName);
    } catch (Exception e) {
        LOG.info("Drop table failed for " + dbName + "." + tblName);
    }
    try {
        if (dropDb)
            silentDropDatabase(dbName);
    } catch (Exception e) {
        LOG.info("Drop database failed for " + dbName);
    }
    if (dropDb)
        new DatabaseBuilder().setName(dbName).setCatalogName(catalog).create(client, conf);
    try {
        client.dropType(typeName);
    } catch (Exception e) {
        LOG.info("Drop type failed for " + typeName);
    }
    Type typ1 = new Type();
    typ1.setName(typeName);
    typ1.setFields(new ArrayList<>(2));
    typ1.getFields().add(new FieldSchema("name", ColumnType.STRING_TYPE_NAME, ""));
    typ1.getFields().add(new FieldSchema("income", ColumnType.INT_TYPE_NAME, ""));
    client.createType(typ1);
    TableBuilder builder = new TableBuilder().setCatName(catalog).setDbName(dbName).setTableName(tblName).setCols(typ1.getFields()).setType(type.name()).setLocation(location).setNumBuckets(buckets).setTableParams(table_params).addBucketCol("name").addStorageDescriptorParam("test_param_1", "Use this for comments etc");
    if (location != null)
        builder.setLocation(location);
    if (buckets > 0)
        builder.setNumBuckets(buckets).addBucketCol("name");
    if (partitionCount > 0) {
        builder.addPartCol("partcol", "string");
    }
    if (type == TableType.MANAGED_TABLE) {
        if (properties.contains("transactional=true") && !properties.contains("transactional_properties=insert_only")) {
            builder.setInputFormat("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
            builder.setOutputFormat("org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat");
            builder.setSerdeLib("org.apache.hadoop.hive.ql.io.orc.OrcSerde");
            builder.addStorageDescriptorParam("inputFormat", "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
            builder.addStorageDescriptorParam("outputFormat", "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat");
        }
    }
    Table tbl = builder.create(client, conf);
    LOG.info("Table " + tbl.getTableName() + " created:type=" + tbl.getTableType());
    if (partitionCount > 0) {
        List<Partition> partitions = new ArrayList<>();
        List<List<String>> partValues = new ArrayList<>();
        for (int i = 1; i <= partitionCount; i++) {
            partValues.add(Lists.newArrayList("" + i));
        }
        for (List<String> vals : partValues) {
            addPartition(client, tbl, vals);
        }
    }
    if (isThriftClient) {
        // the createTable() above does not update the location in the 'tbl'
        // object when the client is a thrift client and the code below relies
        // on the location being present in the 'tbl' object - so get the table
        // from the metastore
        tbl = client.getTable(catalog, dbName, tblName);
        LOG.info("Fetched Table " + tbl.getTableName() + " created:type=" + tbl.getTableType());
    }
    return tbl;
}

Example 30 with Catalog

use of org.apache.hadoop.hive.metastore.api.Catalog in project hive by apache.

the class SharedCache method populateCatalogsInCache.

public void populateCatalogsInCache(Collection<Catalog> catalogs) {
    for (Catalog cat : catalogs) {
        Catalog catCopy = cat.deepCopy();
        // ObjectStore also stores db name in lowercase
        catCopy.setName(catCopy.getName().toLowerCase());
        try {
            cacheLock.writeLock().lock();
            // (which is present because it was added after prewarm started)
            if (catalogsDeletedDuringPrewarm.contains(catCopy.getName())) {
                continue;
            }
            catalogCache.putIfAbsent(catCopy.getName(), catCopy);
            catalogsDeletedDuringPrewarm.clear();
            isCatalogCachePrewarmed = true;
        } finally {
            cacheLock.writeLock().unlock();
        }
    }
}