Examples with HiveMetaStoreChecker org.apache.hadoop.hive.metastore.HiveMetaStoreChecker used on opensource projects

Example 1 with HiveMetaStoreChecker

use of org.apache.hadoop.hive.metastore.HiveMetaStoreChecker in project hive by apache.

the class TestHiveMetaStoreChecker method testSkipInvalidOrderForPartitionKeysOnFS.

/**
 * In skip mode msck should ignore invalid partitions instead of throwing exception.
 * @throws Exception ex
 */
@Test
public void testSkipInvalidOrderForPartitionKeysOnFS() throws Exception {
    hive.getConf().set(MetastoreConf.ConfVars.MSCK_PATH_VALIDATION.getVarname(), "skip");
    checker = new HiveMetaStoreChecker(msc, hive.getConf());
    Table testTable = createPartitionedTestTable(dbName, tableName, 2, 0);
    // add 10 partitions on the filesystem
    createInvalidPartitionDirsOnFS(testTable, 2);
    // add 10 partitions on the filesystem
    createPartitionsDirectoriesOnFS(testTable, 2);
    CheckResult result = checker.checkMetastore(catName, dbName, tableName, null, null);
    assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
    assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
    assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
    // only 2 valid partitions should be added
    assertEquals(2, result.getPartitionsNotInMs().size());
}

Example 2 with HiveMetaStoreChecker

use of org.apache.hadoop.hive.metastore.HiveMetaStoreChecker in project hive by apache.

the class TestHiveMetaStoreChecker method setUp.

@Before
public void setUp() throws Exception {
    hive = Hive.get();
    hive.getConf().set(MetastoreConf.ConfVars.FS_HANDLER_THREADS_COUNT.getVarname(), "15");
    hive.getConf().set(MetastoreConf.ConfVars.MSCK_PATH_VALIDATION.getVarname(), "throw");
    msc = new HiveMetaStoreClient(hive.getConf());
    checker = new HiveMetaStoreChecker(msc, hive.getConf());
    hive.getConf().setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
    HiveConf.setBoolVar(hive.getConf(), HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
    SessionState ss = SessionState.start(hive.getConf());
    ss.initTxnMgr(hive.getConf());
    partCols = new ArrayList<>();
    partCols.add(new FieldSchema(partDateName, serdeConstants.STRING_TYPE_NAME, ""));
    partCols.add(new FieldSchema(partCityName, serdeConstants.STRING_TYPE_NAME, ""));
    parts = new ArrayList<>();
    Map<String, String> part1 = new HashMap<>();
    part1.put(partDateName, "2008-01-01");
    part1.put(partCityName, "london");
    parts.add(part1);
    Map<String, String> part2 = new HashMap<>();
    part2.put(partDateName, "2008-01-02");
    part2.put(partCityName, "stockholm");
    parts.add(part2);
    // cleanup just in case something is left over from previous run
    dropDbTable();
}

Example 3 with HiveMetaStoreChecker

use of org.apache.hadoop.hive.metastore.HiveMetaStoreChecker in project hive by apache.

the class TestHiveMetaStoreChecker method testSkipInvalidPartitionKeyName.

/*
   * skip mode should not throw exception when a invalid partition directory
   * is found. It should just ignore it
   */
@Test
public void testSkipInvalidPartitionKeyName() throws HiveException, AlreadyExistsException, IOException, MetastoreException {
    hive.getConf().set(MetastoreConf.ConfVars.MSCK_PATH_VALIDATION.getVarname(), "skip");
    checker = new HiveMetaStoreChecker(msc, hive.getConf());
    Table table = createTestTable(false);
    List<Partition> partitions = hive.getPartitions(table);
    assertEquals(2, partitions.size());
    // add a fake partition dir on fs
    fs = partitions.get(0).getDataLocation().getFileSystem(hive.getConf());
    addFolderToPath(fs, table.getDataLocation().toString(), "fakedate=2009-01-01/fakecity=sanjose");
    createPartitionsDirectoriesOnFS(table, 2);
    CheckResult result = checker.checkMetastore(catName, dbName, tableName, null, null);
    assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
    assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
    assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
    // only 2 valid partitions should be added
    assertEquals(2, result.getPartitionsNotInMs().size());
}