Examples with HiveMetaStoreClient org.apache.hadoop.hive.metastore.HiveMetaStoreClient used on opensource projects

Example 81 with HiveMetaStoreClient

use of org.apache.hadoop.hive.metastore.HiveMetaStoreClient in project hive by apache.

the class TestStatsUpdaterThread method testSimpleUpdateWithThreads.

@Test(timeout = 80000)
public void testSimpleUpdateWithThreads() throws Exception {
    StatsUpdaterThread su = createUpdater();
    su.startWorkers();
    IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf);
    executeQuery("create table simple_stats (i int, s string)");
    executeQuery("insert into simple_stats (i, s) values (1, 'test')");
    verifyAndUnsetColStats("simple_stats", Lists.newArrayList("i"), msClient);
    assertTrue(su.runOneIteration());
    su.waitForQueuedCommands();
    verifyStatsUpToDate("simple_stats", Lists.newArrayList("i"), msClient, true);
    msClient.close();
}

Example 82 with HiveMetaStoreClient

use of org.apache.hadoop.hive.metastore.HiveMetaStoreClient in project hive by apache.

the class TestStatsUpdaterThread method testNoStatsUpdateForFailoverDb.

private void testNoStatsUpdateForFailoverDb(String tblNamePrefix, String txnProperty) throws Exception {
    // Set high worker count so we get a longer queue.
    hiveConf.setInt(MetastoreConf.ConfVars.STATS_AUTO_UPDATE_WORKER_COUNT.getVarname(), 4);
    String tblWOStats = tblNamePrefix + "_repl_failover_nostats";
    String ptnTblWOStats = tblNamePrefix + "_ptn_repl_failover_nostats";
    String newTable = "new_table";
    String dbName = ss.getCurrentDatabase();
    StatsUpdaterThread su = createUpdater();
    IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf);
    hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false);
    hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false);
    executeQuery("create table " + tblWOStats + "(i int, s string) " + txnProperty);
    executeQuery("insert into " + tblWOStats + "(i, s) values (1, 'test')");
    verifyStatsUpToDate(tblWOStats, Lists.newArrayList("i"), msClient, false);
    executeQuery("create table " + ptnTblWOStats + "(s string) partitioned by (i int) " + txnProperty);
    executeQuery("insert into " + ptnTblWOStats + "(i, s) values (1, 'test')");
    executeQuery("insert into " + ptnTblWOStats + "(i, s) values (2, 'test2')");
    executeQuery("insert into " + ptnTblWOStats + "(i, s) values (3, 'test3')");
    verifyPartStatsUpToDate(3, 1, msClient, ptnTblWOStats, false);
    assertTrue(su.runOneIteration());
    Assert.assertEquals(2, su.getQueueLength());
    executeQuery("alter database " + dbName + " set dbproperties('" + ReplConst.REPL_FAILOVER_ENDPOINT + "'='" + MetaStoreUtils.FailoverEndpoint.SOURCE + "')");
    // StatsUpdaterThread would not run analyze commands for the tables which were inserted before
    // failover property was enabled for that database
    drainWorkQueue(su, 2);
    verifyStatsUpToDate(tblWOStats, Lists.newArrayList("i"), msClient, false);
    verifyPartStatsUpToDate(3, 1, msClient, ptnTblWOStats, false);
    Assert.assertEquals(0, su.getQueueLength());
    executeQuery("create table " + newTable + "(i int, s string) " + txnProperty);
    executeQuery("insert into " + newTable + "(i, s) values (4, 'test4')");
    assertFalse(su.runOneIteration());
    Assert.assertEquals(0, su.getQueueLength());
    verifyStatsUpToDate(tblWOStats, Lists.newArrayList("i"), msClient, false);
    verifyPartStatsUpToDate(3, 1, msClient, ptnTblWOStats, false);
    executeQuery("alter database " + dbName + " set dbproperties('" + ReplConst.REPL_FAILOVER_ENDPOINT + "'='')");
    assertTrue(su.runOneIteration());
    Assert.assertEquals(3, su.getQueueLength());
    drainWorkQueue(su, 3);
    verifyStatsUpToDate(newTable, Lists.newArrayList("i"), msClient, true);
    verifyStatsUpToDate(tblWOStats, Lists.newArrayList("i"), msClient, true);
    verifyPartStatsUpToDate(3, 1, msClient, ptnTblWOStats, true);
    executeQuery("drop table " + tblWOStats);
    executeQuery("drop table " + ptnTblWOStats);
    executeQuery("drop table " + newTable);
    msClient.close();
}

Example 83 with HiveMetaStoreClient

use of org.apache.hadoop.hive.metastore.HiveMetaStoreClient in project hive by apache.

the class TestStatsUpdaterThread method testAllPartitions.

@Test(timeout = 80000)
public void testAllPartitions() throws Exception {
    final int PART_COUNT = 3;
    StatsUpdaterThread su = createUpdater();
    IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf);
    hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false);
    hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false);
    executeQuery("create table simple_stats (s string) partitioned by (i int)");
    for (int i = 0; i < PART_COUNT; ++i) {
        executeQuery("insert into simple_stats partition(i='" + i + "') values ('test')");
    }
    verifyPartStatsUpToDate(PART_COUNT, 0, msClient, "simple_stats", false);
    assertTrue(su.runOneIteration());
    // All the partitions need to be updated; a single command can be used.
    drainWorkQueue(su, 1);
    verifyPartStatsUpToDate(PART_COUNT, 0, msClient, "simple_stats", true);
    assertFalse(su.runOneIteration());
    // Nothing else is updated after the first update.
    drainWorkQueue(su, 0);
    msClient.close();
}

Example 84 with HiveMetaStoreClient

use of org.apache.hadoop.hive.metastore.HiveMetaStoreClient in project hive by apache.

the class TestStatsUpdaterThread method testQueueingWithThreads.

@Ignore("HIVE-25363")
@Test(timeout = 160000)
public void testQueueingWithThreads() throws Exception {
    final int PART_COUNT = 12;
    hiveConf.setInt(MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX.getVarname(), 5);
    hiveConf.setInt(MetastoreConf.ConfVars.STATS_AUTO_UPDATE_WORKER_COUNT.getVarname(), 2);
    StatsUpdaterThread su = createUpdater();
    su.startWorkers();
    IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf);
    hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false);
    hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false);
    executeQuery("create table simple_stats (s string) partitioned by (i int)");
    for (int i = 0; i < PART_COUNT; ++i) {
        executeQuery("insert into simple_stats partition(i='" + i + "') values ('test')");
    }
    verifyPartStatsUpToDate(PART_COUNT, 0, msClient, "simple_stats", false);
    // Set one of the partitions to be skipped, so that a command is created for every other one.
    setPartitionSkipProperty(msClient, "simple_stats", "i=0", "true");
    assertTrue(su.runOneIteration());
    su.waitForQueuedCommands();
    verifyStatsUpToDate("simple_stats", "i=0", Lists.newArrayList("s"), msClient, false);
    verifyPartStatsUpToDate(PART_COUNT, 1, msClient, "simple_stats", true);
    assertFalse(su.runOneIteration());
    // Nothing else is updated after the first update.
    drainWorkQueue(su, 0);
    msClient.close();
}

Example 85 with HiveMetaStoreClient

use of org.apache.hadoop.hive.metastore.HiveMetaStoreClient in project hive by apache.

the class TestStatsUpdaterThread method testPartitionSubset.

@Test(timeout = 80000)
public void testPartitionSubset() throws Exception {
    final int NONSTAT_PART_COUNT = 3;
    StatsUpdaterThread su = createUpdater();
    IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf);
    hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false);
    hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false);
    executeQuery("create table simple_stats (s string) partitioned by (i int)");
    for (int i = 0; i < NONSTAT_PART_COUNT; ++i) {
        executeQuery("insert into simple_stats partition(i='" + i + "') values ('test')");
    }
    hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, true);
    hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, true);
    executeQuery("insert into simple_stats partition(i='" + NONSTAT_PART_COUNT + "') values ('test')");
    verifyPartStatsUpToDate(NONSTAT_PART_COUNT, 0, msClient, "simple_stats", false);
    verifyStatsUpToDate("simple_stats", "i=" + NONSTAT_PART_COUNT, Lists.newArrayList("s"), msClient, true);
    final int EXCLUDED_PART = 1;
    setPartitionSkipProperty(msClient, "simple_stats", "i=" + EXCLUDED_PART, "true");
    assertTrue(su.runOneIteration());
    // 1 is excluded via property, 3 already has stats, so we only expect two updates.
    drainWorkQueue(su, NONSTAT_PART_COUNT - 1);
    for (int i = 0; i < NONSTAT_PART_COUNT; ++i) {
        verifyStatsUpToDate("simple_stats", "i=" + i, Lists.newArrayList("s"), msClient, i != EXCLUDED_PART);
    }
    verifyStatsUpToDate("simple_stats", "i=" + EXCLUDED_PART, Lists.newArrayList("s"), msClient, false);
    msClient.close();
}