Examples with HiveMetastoreClientPool org.apache.gobblin.hive.HiveMetastoreClientPool used on opensource projects

Example 6 with HiveMetastoreClientPool

use of org.apache.gobblin.hive.HiveMetastoreClientPool in project incubator-gobblin by apache.

the class HiveDatasetFinderTest method testDatasetConfig.

@Test
public void testDatasetConfig() throws Exception {
    List<HiveDatasetFinder.DbAndTable> dbAndTables = Lists.newArrayList();
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table1"));
    HiveMetastoreClientPool pool = getTestPool(dbAndTables);
    Properties properties = new Properties();
    properties.put(HiveDatasetFinder.HIVE_DATASET_PREFIX + "." + WhitelistBlacklist.WHITELIST, "");
    properties.put("hive.dataset.test.conf1", "conf1-val1");
    properties.put("hive.dataset.test.conf2", "conf2-val2");
    HiveDatasetFinder finder = new TestHiveDatasetFinder(FileSystem.getLocal(new Configuration()), properties, pool);
    List<HiveDataset> datasets = Lists.newArrayList(finder.getDatasetsIterator());
    Assert.assertEquals(datasets.size(), 1);
    HiveDataset hiveDataset = datasets.get(0);
    Assert.assertEquals(hiveDataset.getDatasetConfig().getString("hive.dataset.test.conf1"), "conf1-val1");
    Assert.assertEquals(hiveDataset.getDatasetConfig().getString("hive.dataset.test.conf2"), "conf2-val2");
    // Test scoped configs with prefix
    properties.put(HiveDatasetFinder.HIVE_DATASET_CONFIG_PREFIX_KEY, "hive.dataset.test");
    finder = new TestHiveDatasetFinder(FileSystem.getLocal(new Configuration()), properties, pool);
    datasets = Lists.newArrayList(finder.getDatasetsIterator());
    Assert.assertEquals(datasets.size(), 1);
    hiveDataset = datasets.get(0);
    Assert.assertEquals(hiveDataset.getDatasetConfig().getString("conf1"), "conf1-val1");
    Assert.assertEquals(hiveDataset.getDatasetConfig().getString("conf2"), "conf2-val2");
}

Example 7 with HiveMetastoreClientPool

use of org.apache.gobblin.hive.HiveMetastoreClientPool in project incubator-gobblin by apache.

the class HiveDatasetFinderTest method testException.

@Test
public void testException() throws Exception {
    List<HiveDatasetFinder.DbAndTable> dbAndTables = Lists.newArrayList();
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table1"));
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", TestHiveDatasetFinder.THROW_EXCEPTION));
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table3"));
    HiveMetastoreClientPool pool = getTestPool(dbAndTables);
    Properties properties = new Properties();
    properties.put(HiveDatasetFinder.HIVE_DATASET_PREFIX + "." + WhitelistBlacklist.WHITELIST, "");
    HiveDatasetFinder finder = new TestHiveDatasetFinder(FileSystem.getLocal(new Configuration()), properties, pool);
    List<HiveDataset> datasets = Lists.newArrayList(finder.getDatasetsIterator());
    Assert.assertEquals(datasets.size(), 2);
}

Example 8 with HiveMetastoreClientPool

use of org.apache.gobblin.hive.HiveMetastoreClientPool in project incubator-gobblin by apache.

the class HiveDatasetFinderTest method getTestPool.

private HiveMetastoreClientPool getTestPool(List<HiveDatasetFinder.DbAndTable> dbAndTables) throws Exception {
    SetMultimap<String, String> entities = HashMultimap.create();
    for (HiveDatasetFinder.DbAndTable dbAndTable : dbAndTables) {
        entities.put(dbAndTable.getDb(), dbAndTable.getTable());
    }
    HiveMetastoreClientPool pool = Mockito.mock(HiveMetastoreClientPool.class);
    IMetaStoreClient client = Mockito.mock(IMetaStoreClient.class);
    Mockito.when(client.getAllDatabases()).thenReturn(Lists.newArrayList(entities.keySet()));
    for (String db : entities.keySet()) {
        Mockito.doReturn(Lists.newArrayList(entities.get(db))).when(client).getAllTables(db);
    }
    for (HiveDatasetFinder.DbAndTable dbAndTable : dbAndTables) {
        Table table = new Table();
        table.setDbName(dbAndTable.getDb());
        table.setTableName(dbAndTable.getTable());
        StorageDescriptor sd = new StorageDescriptor();
        sd.setLocation("/tmp/test");
        table.setSd(sd);
        Mockito.doReturn(table).when(client).getTable(dbAndTable.getDb(), dbAndTable.getTable());
    }
    @SuppressWarnings("unchecked") AutoReturnableObject<IMetaStoreClient> aro = Mockito.mock(AutoReturnableObject.class);
    Mockito.when(aro.get()).thenReturn(client);
    Mockito.when(pool.getHiveRegProps()).thenReturn(null);
    Mockito.when(pool.getClient()).thenReturn(aro);
    return pool;
}

Example 9 with HiveMetastoreClientPool

use of org.apache.gobblin.hive.HiveMetastoreClientPool in project incubator-gobblin by apache.

the class HiveDatasetFinderTest method testDatasetFinder.

@Test
public void testDatasetFinder() throws Exception {
    List<HiveDatasetFinder.DbAndTable> dbAndTables = Lists.newArrayList();
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table1"));
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table2"));
    dbAndTables.add(new HiveDatasetFinder.DbAndTable("db1", "table3"));
    HiveMetastoreClientPool pool = getTestPool(dbAndTables);
    Properties properties = new Properties();
    properties.put(HiveDatasetFinder.HIVE_DATASET_PREFIX + "." + WhitelistBlacklist.WHITELIST, "");
    HiveDatasetFinder finder = new TestHiveDatasetFinder(FileSystem.getLocal(new Configuration()), properties, pool);
    List<HiveDataset> datasets = Lists.newArrayList(finder.getDatasetsIterator());
    Assert.assertEquals(datasets.size(), 3);
}

Example 10 with HiveMetastoreClientPool

use of org.apache.gobblin.hive.HiveMetastoreClientPool in project incubator-gobblin by apache.

the class HiveConverterUtils method getDestinationPartitionLocation.

/**
 * Returns the partition data location of a given table and partition
 * @param table Hive table
 * @param state workunit state
 * @param partitionName partition name
 * @return partition data location
 * @throws DataConversionException
 */
public static Optional<Path> getDestinationPartitionLocation(Optional<Table> table, WorkUnitState state, String partitionName) throws DataConversionException {
    Optional<org.apache.hadoop.hive.metastore.api.Partition> partitionOptional;
    if (!table.isPresent()) {
        return Optional.absent();
    }
    try {
        HiveMetastoreClientPool pool = HiveMetastoreClientPool.get(state.getJobState().getProperties(), Optional.fromNullable(state.getJobState().getProp(HiveDatasetFinder.HIVE_METASTORE_URI_KEY)));
        try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) {
            partitionOptional = Optional.of(client.get().getPartition(table.get().getDbName(), table.get().getTableName(), partitionName));
        } catch (NoSuchObjectException e) {
            return Optional.absent();
        }
        if (partitionOptional.isPresent()) {
            org.apache.hadoop.hive.ql.metadata.Table qlTable = new org.apache.hadoop.hive.ql.metadata.Table(table.get());
            Partition qlPartition = new Partition(qlTable, partitionOptional.get());
            return Optional.of(qlPartition.getDataLocation());
        }
    } catch (IOException | TException | HiveException e) {
        throw new DataConversionException("Could not fetch destination table metadata", e);
    }
    return Optional.absent();
}