Examples with Catalog org.apache.hadoop.hive.metastore.api.Catalog used on opensource projects

Example 31 with Catalog

use of org.apache.hadoop.hive.metastore.api.Catalog in project hive by apache.

the class SharedCache method addCatalogToCache.

public void addCatalogToCache(Catalog cat) {
    try {
        cacheLock.writeLock().lock();
        Catalog catCopy = cat.deepCopy();
        // ObjectStore also stores db name in lowercase
        catCopy.setName(catCopy.getName().toLowerCase());
        catalogCache.put(cat.getName(), catCopy);
        isCatalogCacheDirty.set(true);
    } finally {
        cacheLock.writeLock().unlock();
    }
}

Example 32 with Catalog

use of org.apache.hadoop.hive.metastore.api.Catalog in project hive by apache.

the class TestCatalogCaching method createObjectStore.

@Before
public void createObjectStore() throws MetaException, InvalidOperationException {
    conf = MetastoreConf.newMetastoreConf();
    MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST, true);
    MetaStoreTestUtils.setConfForStandloneMode(conf);
    objectStore = new ObjectStore();
    objectStore.setConf(conf);
    // Create three catalogs
    HMSHandler.createDefaultCatalog(objectStore, new Warehouse(conf));
    Catalog cat1 = new CatalogBuilder().setName(CAT1_NAME).setLocation("/tmp/cat1").build();
    objectStore.createCatalog(cat1);
    Catalog cat2 = new CatalogBuilder().setName(CAT2_NAME).setLocation("/tmp/cat2").build();
    objectStore.createCatalog(cat2);
}

Example 33 with Catalog

use of org.apache.hadoop.hive.metastore.api.Catalog in project hive by apache.

the class TestAppendPartitions method otherCatalog.

@Test
@ConditionalIgnoreOnSessionHiveMetastoreClient
public void otherCatalog() throws TException {
    String catName = "append_partition_catalog";
    Catalog cat = new CatalogBuilder().setName(catName).setLocation(MetaStoreTestUtils.getTestWarehouseDir(catName)).build();
    client.createCatalog(cat);
    String dbName = "append_partition_database_in_other_catalog";
    Database db = new DatabaseBuilder().setName(dbName).setCatalogName(catName).create(client, metaStore.getConf());
    String tableName = "table_in_other_catalog";
    new TableBuilder().inDb(db).setTableName(tableName).addCol("id", "int").addCol("name", "string").addPartCol("partcol", "string").create(client, metaStore.getConf());
    Partition created = client.appendPartition(catName, dbName, tableName, Collections.singletonList("a1"));
    Assert.assertEquals(1, created.getValuesSize());
    Assert.assertEquals("a1", created.getValues().get(0));
    Partition fetched = client.getPartition(catName, dbName, tableName, Collections.singletonList("a1"));
    created.setWriteId(fetched.getWriteId());
    Assert.assertEquals(created, fetched);
    created = client.appendPartition(catName, dbName, tableName, "partcol=a2");
    Assert.assertEquals(1, created.getValuesSize());
    Assert.assertEquals("a2", created.getValues().get(0));
    fetched = client.getPartition(catName, dbName, tableName, Collections.singletonList("a2"));
    created.setWriteId(fetched.getWriteId());
    Assert.assertEquals(created, fetched);
}

Example 34 with Catalog

use of org.apache.hadoop.hive.metastore.api.Catalog in project hive by apache.

the class TestCheckConstraint method setUp.

@Before
public void setUp() throws Exception {
    // Get new client
    client = metaStore.getClient();
    // Clean up the database
    client.dropDatabase(OTHER_DATABASE, true, true, true);
    // Drop every table in the default database
    for (String tableName : client.getAllTables(DEFAULT_DATABASE_NAME)) {
        client.dropTable(DEFAULT_DATABASE_NAME, tableName, true, true, true);
    }
    client.dropDatabase(OTHER_CATALOG, DATABASE_IN_OTHER_CATALOG, true, true, true);
    try {
        client.dropCatalog(OTHER_CATALOG);
    } catch (NoSuchObjectException e) {
    // NOP
    }
    // Clean up trash
    metaStore.cleanWarehouseDirs();
    new DatabaseBuilder().setName(OTHER_DATABASE).create(client, metaStore.getConf());
    Catalog cat = new CatalogBuilder().setName(OTHER_CATALOG).setLocation(MetaStoreTestUtils.getTestWarehouseDir(OTHER_CATALOG)).build();
    client.createCatalog(cat);
    // For this one don't specify a location to make sure it gets put in the catalog directory
    inOtherCatalog = new DatabaseBuilder().setName(DATABASE_IN_OTHER_CATALOG).setCatalogName(OTHER_CATALOG).create(client, metaStore.getConf());
    testTables[0] = new TableBuilder().setTableName("test_table_1").addCol("col1", "int").addCol("col2", "varchar(32)").create(client, metaStore.getConf());
    testTables[1] = new TableBuilder().setDbName(OTHER_DATABASE).setTableName("test_table_2").addCol("col1", "int").addCol("col2", "varchar(32)").create(client, metaStore.getConf());
    testTables[2] = new TableBuilder().inDb(inOtherCatalog).setTableName("test_table_3").addCol("col1", "int").addCol("col2", "varchar(32)").create(client, metaStore.getConf());
    // Reload tables from the MetaStore
    for (int i = 0; i < testTables.length; i++) {
        testTables[i] = client.getTable(testTables[i].getCatName(), testTables[i].getDbName(), testTables[i].getTableName());
    }
}

Example 35 with Catalog

use of org.apache.hadoop.hive.metastore.api.Catalog in project hive by apache.

the class TestTablesCreateDropAlterTruncate method tablesInOtherCatalogs.

@Test
public void tablesInOtherCatalogs() throws TException, URISyntaxException {
    String catName = "create_etc_tables_in_other_catalogs";
    Catalog cat = new CatalogBuilder().setName(catName).setLocation(MetaStoreTestUtils.getTestWarehouseDir(catName)).build();
    client.createCatalog(cat);
    String dbName = "db_in_other_catalog";
    // For this one don't specify a location to make sure it gets put in the catalog directory
    Database db = new DatabaseBuilder().setName(dbName).setCatalogName(catName).create(client, metaStore.getConf());
    Table table = new TableBuilder().inDb(db).setTableName("mvSource").addCol("col1_1", ColumnType.STRING_TYPE_NAME).addCol("col2_2", ColumnType.INT_TYPE_NAME).build(metaStore.getConf());
    client.createTable(table);
    SourceTable sourceTable = createSourceTable(table);
    String[] tableNames = new String[4];
    for (int i = 0; i < tableNames.length; i++) {
        tableNames[i] = "table_in_other_catalog_" + i;
        TableBuilder builder = new TableBuilder().inDb(db).setTableName(tableNames[i]).addCol("col1_" + i, ColumnType.STRING_TYPE_NAME).addCol("col2_" + i, ColumnType.INT_TYPE_NAME);
        // Make one have a non-standard location
        if (i == 0) {
            builder.setLocation(MetaStoreTestUtils.getTestWarehouseDir(tableNames[i]));
        }
        // Make one partitioned
        if (i == 2) {
            builder.addPartCol("pcol1", ColumnType.STRING_TYPE_NAME);
        }
        // Make one a materialized view
        if (i == 3) {
            builder.setType(TableType.MATERIALIZED_VIEW.name()).setRewriteEnabled(true).addMaterializedViewReferencedTable(sourceTable);
        }
        client.createTable(builder.build(metaStore.getConf()));
    }
    // Add partitions for the partitioned table
    String[] partVals = new String[3];
    Table partitionedTable = client.getTable(catName, dbName, tableNames[2]);
    for (int i = 0; i < partVals.length; i++) {
        partVals[i] = "part" + i;
        new PartitionBuilder().inTable(partitionedTable).addValue(partVals[i]).addToTable(client, metaStore.getConf());
    }
    // Get tables, make sure the locations are correct
    for (int i = 0; i < tableNames.length; i++) {
        Table t = client.getTable(catName, dbName, tableNames[i]);
        Assert.assertEquals(catName, t.getCatName());
        String expectedLocation = (i < 1) ? new File(MetaStoreTestUtils.getTestWarehouseDir(tableNames[i])).toURI().toString() : new File(cat.getLocationUri() + File.separatorChar + dbName + ".db", tableNames[i]).toURI().toString();
        Assert.assertEquals(expectedLocation, t.getSd().getLocation() + "/");
        File dir = new File(new URI(t.getSd().getLocation()).getPath());
        Assert.assertTrue(dir.exists() && dir.isDirectory());
    }
    // Make sure getting table in the wrong catalog does not work
    try {
        Table t = client.getTable(DEFAULT_DATABASE_NAME, tableNames[0]);
        Assert.fail();
    } catch (NoSuchObjectException e) {
    // NOP
    }
    // test getAllTables
    Set<String> fetchedNames = new HashSet<>(client.getAllTables(catName, dbName));
    Assert.assertEquals(tableNames.length + 1, fetchedNames.size());
    for (String tableName : tableNames) {
        Assert.assertTrue(fetchedNames.contains(tableName));
    }
    fetchedNames = new HashSet<>(client.getAllTables(DEFAULT_DATABASE_NAME));
    for (String tableName : tableNames) {
        Assert.assertFalse(fetchedNames.contains(tableName));
    }
    // test getMaterializedViewsForRewriting
    List<String> materializedViews = client.getMaterializedViewsForRewriting(catName, dbName);
    Assert.assertEquals(1, materializedViews.size());
    Assert.assertEquals(tableNames[3], materializedViews.get(0));
    fetchedNames = new HashSet<>(client.getMaterializedViewsForRewriting(DEFAULT_DATABASE_NAME));
    Assert.assertFalse(fetchedNames.contains(tableNames[3]));
    // test getTableObjectsByName
    List<Table> fetchedTables = client.getTableObjectsByName(catName, dbName, Arrays.asList(tableNames[0], tableNames[1]));
    Assert.assertEquals(2, fetchedTables.size());
    Collections.sort(fetchedTables);
    Assert.assertEquals(tableNames[0], fetchedTables.get(0).getTableName());
    Assert.assertEquals(tableNames[1], fetchedTables.get(1).getTableName());
    fetchedTables = client.getTableObjectsByName(DEFAULT_DATABASE_NAME, Arrays.asList(tableNames[0], tableNames[1]));
    Assert.assertEquals(0, fetchedTables.size());
    // Test altering the table
    Table t = client.getTable(catName, dbName, tableNames[0]).deepCopy();
    t.getParameters().put("test", "test");
    client.alter_table(catName, dbName, tableNames[0], t);
    t = client.getTable(catName, dbName, tableNames[0]).deepCopy();
    Assert.assertEquals("test", t.getParameters().get("test"));
    // Alter a table in the wrong catalog
    try {
        client.alter_table(DEFAULT_DATABASE_NAME, tableNames[0], t);
        Assert.fail();
    } catch (InvalidOperationException e) {
    // NOP
    }
    // Update the metadata for the materialized view
    CreationMetadata cm = client.getTable(catName, dbName, tableNames[3]).getCreationMetadata();
    Table table1 = new TableBuilder().inDb(db).setTableName("mvSource2").addCol("col1_1", ColumnType.STRING_TYPE_NAME).addCol("col2_2", ColumnType.INT_TYPE_NAME).build(metaStore.getConf());
    client.createTable(table1);
    sourceTable = createSourceTable(table1);
    cm.addToTablesUsed(TableName.getDbTable(sourceTable.getTable().getDbName(), sourceTable.getTable().getTableName()));
    cm.addToSourceTables(sourceTable);
    cm.unsetMaterializationTime();
    client.updateCreationMetadata(catName, dbName, tableNames[3], cm);
    List<String> partNames = new ArrayList<>();
    for (String partVal : partVals) {
        partNames.add("pcol1=" + partVal);
    }
    // Truncate a table
    client.truncateTable(catName, dbName, tableNames[0], partNames);
    // Truncate a table in the wrong catalog
    try {
        client.truncateTable(DEFAULT_DATABASE_NAME, tableNames[0], partNames);
        Assert.fail();
    } catch (NoSuchObjectException | TApplicationException e) {
    // NOP
    }
    // Drop a table from the wrong catalog
    try {
        client.dropTable(DEFAULT_DATABASE_NAME, tableNames[0], true, false);
        Assert.fail();
    } catch (NoSuchObjectException | TApplicationException e) {
    // NOP
    }
    // Should ignore the failure
    client.dropTable(DEFAULT_DATABASE_NAME, tableNames[0], false, true);
    // Have to do this in reverse order so that we drop the materialized view first.
    for (int i = tableNames.length - 1; i >= 0; i--) {
        t = client.getTable(catName, dbName, tableNames[i]);
        File tableDir = new File(new URI(t.getSd().getLocation()).getPath());
        Assert.assertTrue(tableDir.exists() && tableDir.isDirectory());
        if (tableNames[i].equalsIgnoreCase(tableNames[0])) {
            client.dropTable(catName, dbName, tableNames[i], false, false);
            Assert.assertTrue(tableDir.exists() && tableDir.isDirectory());
        } else {
            client.dropTable(catName, dbName, tableNames[i]);
            Assert.assertFalse(tableDir.exists());
        }
    }
    client.dropTable(table.getCatName(), table.getDbName(), table.getTableName());
    client.dropTable(table1.getCatName(), table1.getDbName(), table1.getTableName());
    Assert.assertEquals(0, client.getAllTables(catName, dbName).size());
}