Examples with CheckResult org.apache.hadoop.hive.metastore.CheckResult used on opensource projects

Example 11 with CheckResult

use of org.apache.hadoop.hive.metastore.CheckResult in project hive by apache.

the class TestHiveMetaStoreChecker method testNoNPartitionedTable.

@Test
public void testNoNPartitionedTable() throws Exception {
    Table table = createNonPartitionedTable();
    // add a partition dir on fs
    fs = table.getDataLocation().getFileSystem(hive.getConf());
    Path tablePath = table.getDataLocation();
    // Add a few deltas
    addFolderToPath(fs, tablePath.toString(), "delta_0000001_0000001_0000");
    addFolderToPath(fs, tablePath.toString(), "delta_0000002_0000002_0000");
    addFolderToPath(fs, tablePath.toString(), "delta_0000003_0000003_0000");
    addFolderToPath(fs, tablePath.toString(), "base_0000003_v0000200");
    CheckResult result = checker.checkMetastore(catName, dbName, tableName, null, null);
    assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
    assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotInMs());
    // Found the highest writeId
    assertEquals(3, result.getMaxWriteId());
    assertEquals(200, result.getMaxTxnId());
}

Example 12 with CheckResult

use of org.apache.hadoop.hive.metastore.CheckResult in project hive by apache.

the class TestHiveMetaStoreChecker method testAddPartitionCompactedBase.

@Test
public void testAddPartitionCompactedBase() throws Exception {
    Table table = createTestTable(true);
    List<Partition> partitions = hive.getPartitions(table);
    assertEquals(2, partitions.size());
    // add a partition dir on fs
    fs = partitions.get(0).getDataLocation().getFileSystem(hive.getConf());
    Path newPart = addFolderToPath(fs, table.getDataLocation().toString(), partDateName + "=2017-01-01/" + partCityName + "=paloalto");
    // Add a few deltas
    addFolderToPath(fs, newPart.toString(), "delta_0000001_0000001_0000");
    addFolderToPath(fs, newPart.toString(), "delta_0000002_0000002_0000");
    addFolderToPath(fs, newPart.toString(), "delta_0000003_0000003_0000");
    addFolderToPath(fs, newPart.toString(), "base_0000003_v0000200");
    CheckResult result = checker.checkMetastore(catName, dbName, tableName, null, null);
    assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
    assertEquals(1, result.getPartitionsNotInMs().size());
    // Found the highest writeId
    assertEquals(3, result.getPartitionsNotInMs().iterator().next().getMaxWriteId());
    assertEquals(200, result.getPartitionsNotInMs().iterator().next().getMaxTxnId());
}

Example 13 with CheckResult

use of org.apache.hadoop.hive.metastore.CheckResult in project hive by apache.

the class TestHiveMetaStoreChecker method testPartitionsCheck.

@Test
public void testPartitionsCheck() throws HiveException, IOException, TException, MetastoreException {
    Table table = createTestTable(false);
    CheckResult result = checker.checkMetastore(catName, dbName, tableName, null, null);
    // all is well
    assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
    assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
    assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
    assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotInMs());
    List<Partition> partitions = hive.getPartitions(table);
    assertEquals(2, partitions.size());
    Partition partToRemove = partitions.get(0);
    // As this partition (partdate=2008-01-01/partcity=london) is the only
    // partition under (partdate=2008-01-01)
    // we also need to delete partdate=2008-01-01 to make it consistent.
    Path partToRemovePath = partToRemove.getDataLocation().getParent();
    fs = partToRemovePath.getFileSystem(hive.getConf());
    fs.delete(partToRemovePath, true);
    result = checker.checkMetastore(catName, dbName, tableName, null, null);
    // missing one partition on fs
    assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
    assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
    assertEquals(1, result.getPartitionsNotOnFs().size());
    assertEquals(partToRemove.getName(), result.getPartitionsNotOnFs().iterator().next().getPartitionName());
    assertEquals(partToRemove.getTable().getTableName(), result.getPartitionsNotOnFs().iterator().next().getTableName());
    assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotInMs());
    // old test is moved to msck_repair_2.q
    // cleanup
    hive.dropTable(dbName, tableName, true, true);
    hive.createTable(table);
    result = checker.checkMetastore(catName, dbName, null, null, null);
    assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
    assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
    assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
    // --0e
    assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotInMs());
    System.err.println("Test completed - partition check");
}

Example 14 with CheckResult

use of org.apache.hadoop.hive.metastore.CheckResult in project hive by apache.

the class TestHiveMetaStoreChecker method testSkipInvalidPartitionKeyName.

/*
   * skip mode should not throw exception when a invalid partition directory
   * is found. It should just ignore it
   */
@Test
public void testSkipInvalidPartitionKeyName() throws HiveException, AlreadyExistsException, IOException, MetastoreException {
    hive.getConf().set(MetastoreConf.ConfVars.MSCK_PATH_VALIDATION.getVarname(), "skip");
    checker = new HiveMetaStoreChecker(msc, hive.getConf());
    Table table = createTestTable(false);
    List<Partition> partitions = hive.getPartitions(table);
    assertEquals(2, partitions.size());
    // add a fake partition dir on fs
    fs = partitions.get(0).getDataLocation().getFileSystem(hive.getConf());
    addFolderToPath(fs, table.getDataLocation().toString(), "fakedate=2009-01-01/fakecity=sanjose");
    createPartitionsDirectoriesOnFS(table, 2);
    CheckResult result = checker.checkMetastore(catName, dbName, tableName, null, null);
    assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
    assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
    assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
    // only 2 valid partitions should be added
    assertEquals(2, result.getPartitionsNotInMs().size());
}