Examples with CheckResult org.apache.hadoop.hive.metastore.CheckResult used on opensource projects

Example 6 with CheckResult

use of org.apache.hadoop.hive.metastore.CheckResult in project hive by apache.

the class TestHiveMetaStoreChecker method testDeeplyNestedPartitionedTables.

/**
 * Tests the case when the number of partition keys are more than the threadpool size.
 *
 * @throws Exception ex
 */
@Test
public void testDeeplyNestedPartitionedTables() throws Exception {
    hive.getConf().set(MetastoreConf.ConfVars.FS_HANDLER_THREADS_COUNT.getVarname(), "2");
    int poolSize = 2;
    // create a deeply nested table which has more partition keys than the pool size
    Table testTable = createPartitionedTestTable(dbName, tableName, poolSize + 2, 0);
    // add 10 partitions on the filesystem
    createPartitionsDirectoriesOnFS(testTable, 10);
    CheckResult result = checker.checkMetastore(catName, dbName, tableName, null, null);
    assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
    assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
    assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
    assertEquals(10, result.getPartitionsNotInMs().size());
}

Example 7 with CheckResult

use of org.apache.hadoop.hive.metastore.CheckResult in project hive by apache.

the class TestHiveMetaStoreChecker method testAdditionalPartitionDirs.

/*
   * Tests the case when tblPath/p1=a/p2=b/p3=c/file for a table with partition (p1, p2)
   * does not throw HiveException
   */
@Test
public void testAdditionalPartitionDirs() throws HiveException, AlreadyExistsException, IOException, MetastoreException {
    Table table = createTestTable(false);
    List<Partition> partitions = hive.getPartitions(table);
    assertEquals(2, partitions.size());
    // add a fake partition dir on fs
    fs = partitions.get(0).getDataLocation().getFileSystem(hive.getConf());
    addFolderToPath(fs, table.getDataLocation().toString(), partDateName + "=2017-01-01/" + partCityName + "=paloalto/fakePartCol=fakepartValue");
    CheckResult result = checker.checkMetastore(catName, dbName, tableName, null, null);
    assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
    assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
    assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
    // fakePart path partition is added since the defined partition keys are valid
    assertEquals(1, result.getPartitionsNotInMs().size());
}

Example 8 with CheckResult

use of org.apache.hadoop.hive.metastore.CheckResult in project hive by apache.

the class TestHiveMetaStoreChecker method testSingleThreadedCheckMetastore.

/**
 * Tests single threaded implementation of checkMetastore.
 * @throws Exception ex
 */
@Test
public void testSingleThreadedCheckMetastore() throws Exception {
    // set num of threads to 0 so that single-threaded checkMetastore is called
    hive.getConf().set(MetastoreConf.ConfVars.FS_HANDLER_THREADS_COUNT.getVarname(), "0");
    Table testTable = createPartitionedTestTable(dbName, tableName, 2, 0);
    // add 10 partitions on the filesystem
    createPartitionsDirectoriesOnFS(testTable, 10);
    CheckResult result = checker.checkMetastore(catName, dbName, tableName, null, null);
    assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
    assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
    assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
    assertEquals(10, result.getPartitionsNotInMs().size());
}

Example 9 with CheckResult

use of org.apache.hadoop.hive.metastore.CheckResult in project hive by apache.

the class TestHiveMetaStoreChecker method testAddPartitionCompactedDeltas.

/*
   * Tests the case when we have normal delta_dirs in the partition folder
   * does not throw HiveException
   */
@Test
public void testAddPartitionCompactedDeltas() throws Exception {
    Table table = createTestTable(true);
    List<Partition> partitions = hive.getPartitions(table);
    assertEquals(2, partitions.size());
    // add a partition dir on fs
    fs = partitions.get(0).getDataLocation().getFileSystem(hive.getConf());
    Path newPart = addFolderToPath(fs, table.getDataLocation().toString(), partDateName + "=2017-01-01/" + partCityName + "=paloalto");
    // Add a few deltas
    addFolderToPath(fs, newPart.toString(), "delta_0000001_0000001_0000");
    addFolderToPath(fs, newPart.toString(), "delta_0000010_0000015_v0000067");
    addFolderToPath(fs, newPart.toString(), "delta_0000101_0000120_v0000087");
    CheckResult result = checker.checkMetastore(catName, dbName, tableName, null, null);
    assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
    assertEquals(1, result.getPartitionsNotInMs().size());
    // Found the highest writeId
    assertEquals(120, result.getPartitionsNotInMs().iterator().next().getMaxWriteId());
    assertEquals(87, result.getPartitionsNotInMs().iterator().next().getMaxTxnId());
}

Example 10 with CheckResult

use of org.apache.hadoop.hive.metastore.CheckResult in project hive by apache.

the class TestHiveMetaStoreChecker method testAddPartitionNormalDeltas.

/*
   * Tests the case when we have normal delta_dirs in the partition folder
   * does not throw HiveException
   */
@Test
public void testAddPartitionNormalDeltas() throws Exception {
    Table table = createTestTable(true);
    List<Partition> partitions = hive.getPartitions(table);
    assertEquals(2, partitions.size());
    // add a partition dir on fs
    fs = partitions.get(0).getDataLocation().getFileSystem(hive.getConf());
    Path newPart = addFolderToPath(fs, table.getDataLocation().toString(), partDateName + "=2017-01-01/" + partCityName + "=paloalto");
    // Add a few deltas
    addFolderToPath(fs, newPart.toString(), "delta_0000001_0000001_0000");
    addFolderToPath(fs, newPart.toString(), "delta_0000010_0000010_0000");
    addFolderToPath(fs, newPart.toString(), "delta_0000101_0000101_0000");
    CheckResult result = checker.checkMetastore(catName, dbName, tableName, null, null);
    assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
    assertEquals(1, result.getPartitionsNotInMs().size());
    // Found the highest writeId
    assertEquals(101, result.getPartitionsNotInMs().iterator().next().getMaxWriteId());
    assertEquals(0, result.getPartitionsNotInMs().iterator().next().getMaxTxnId());
}