Examples with TableMetadata org.apache.iceberg.TableMetadata used on opensource projects

Example 6 with TableMetadata

use of org.apache.iceberg.TableMetadata in project hive by apache.

the class TestHiveCommits method testThriftExceptionConcurrentCommit.

/**
 * Pretends we threw an exception while persisting, the commit succeeded, the lock expired,
 * and a second committer placed a commit on top of ours before the first committer was able to check
 * if their commit succeeded or not
 *
 * Timeline:
 *   Client 1 commits which throws an exception but suceeded
 *   Client 1's lock expires while waiting to do the recheck for commit success
 *   Client 2 acquires a lock, commits successfully on top of client 1's commit and release lock
 *   Client 1 check's to see if their commit was successful
 *
 * This tests to make sure a disconnected client 1 doesn't think their commit failed just because it isn't the
 * current one during the recheck phase.
 */
@Test
public void testThriftExceptionConcurrentCommit() throws TException, InterruptedException, UnknownHostException {
    Table table = catalog.loadTable(TABLE_IDENTIFIER);
    HiveTableOperations ops = (HiveTableOperations) ((HasTableOperations) table).operations();
    TableMetadata metadataV1 = ops.current();
    table.updateSchema().addColumn("n", Types.IntegerType.get()).commit();
    ops.refresh();
    TableMetadata metadataV2 = ops.current();
    Assert.assertEquals(2, ops.current().schema().columns().size());
    HiveTableOperations spyOps = spy(ops);
    AtomicLong lockId = new AtomicLong();
    doAnswer(i -> {
        lockId.set(ops.acquireLock());
        return lockId.get();
    }).when(spyOps).acquireLock();
    concurrentCommitAndThrowException(ops, spyOps, table, lockId);
    /*
    This commit and our concurrent commit should succeed even though this commit throws an exception
    after the persist operation succeeds
     */
    spyOps.commit(metadataV2, metadataV1);
    ops.refresh();
    Assert.assertNotEquals("Current metadata should have changed", metadataV2, ops.current());
    Assert.assertTrue("Current metadata file should still exist", metadataFileExists(ops.current()));
    Assert.assertEquals("The column addition from the concurrent commit should have been successful", 2, ops.current().schema().columns().size());
}

Example 7 with TableMetadata

use of org.apache.iceberg.TableMetadata in project hive by apache.

the class TestHiveCommits method testSuppressUnlockExceptions.

@Test
public void testSuppressUnlockExceptions() throws TException, InterruptedException {
    Table table = catalog.loadTable(TABLE_IDENTIFIER);
    HiveTableOperations ops = (HiveTableOperations) ((HasTableOperations) table).operations();
    TableMetadata metadataV1 = ops.current();
    table.updateSchema().addColumn("n", Types.IntegerType.get()).commit();
    ops.refresh();
    TableMetadata metadataV2 = ops.current();
    Assert.assertEquals(2, ops.current().schema().columns().size());
    HiveTableOperations spyOps = spy(ops);
    ArgumentCaptor<Long> lockId = ArgumentCaptor.forClass(Long.class);
    doThrow(new RuntimeException()).when(spyOps).doUnlock(lockId.capture());
    try {
        spyOps.commit(metadataV2, metadataV1);
    } finally {
        ops.doUnlock(lockId.getValue());
    }
    ops.refresh();
    // the commit must succeed
    Assert.assertEquals(1, ops.current().schema().columns().size());
}

Example 8 with TableMetadata

use of org.apache.iceberg.TableMetadata in project hive by apache.

the class TestHiveCommits method testThriftExceptionFailureOnCommit.

/**
 * Pretends we throw an error while persisting that actually fails to commit serverside
 */
@Test
public void testThriftExceptionFailureOnCommit() throws TException, InterruptedException {
    Table table = catalog.loadTable(TABLE_IDENTIFIER);
    HiveTableOperations ops = (HiveTableOperations) ((HasTableOperations) table).operations();
    TableMetadata metadataV1 = ops.current();
    table.updateSchema().addColumn("n", Types.IntegerType.get()).commit();
    ops.refresh();
    TableMetadata metadataV2 = ops.current();
    Assert.assertEquals(2, ops.current().schema().columns().size());
    HiveTableOperations spyOps = spy(ops);
    failCommitAndThrowException(spyOps);
    AssertHelpers.assertThrows("We should rethrow generic runtime errors if the " + "commit actually doesn't succeed", CommitStateUnknownException.class, "Cannot determine whether the commit was successful or not, the underlying data files may " + "or may not be needed. Manual intervention via the Remove Orphan Files Action can remove these files " + "when a connection to the Catalog can be re-established if the commit was actually unsuccessful.", () -> spyOps.commit(metadataV2, metadataV1));
    ops.refresh();
    Assert.assertEquals("Current metadata should not have changed", metadataV2, ops.current());
    Assert.assertTrue("Current metadata should still exist", metadataFileExists(metadataV2));
    Assert.assertEquals("New non-current metadata file should be added", 3, metadataFileCount(ops.current()));
}

Example 9 with TableMetadata

use of org.apache.iceberg.TableMetadata in project hive by apache.

the class TestHiveCommits method testThriftExceptionsUnknownSuccessCommit.

/**
 * Pretends we throw an exception while persisting and don't know what happened, can't check to find out,
 * but in reality the commit succeeded
 */
@Test
public void testThriftExceptionsUnknownSuccessCommit() throws TException, InterruptedException {
    Table table = catalog.loadTable(TABLE_IDENTIFIER);
    HiveTableOperations ops = (HiveTableOperations) ((HasTableOperations) table).operations();
    TableMetadata metadataV1 = ops.current();
    table.updateSchema().addColumn("n", Types.IntegerType.get()).commit();
    ops.refresh();
    TableMetadata metadataV2 = ops.current();
    Assert.assertEquals(2, ops.current().schema().columns().size());
    HiveTableOperations spyOps = spy(ops);
    commitAndThrowException(ops, spyOps);
    breakFallbackCatalogCommitCheck(spyOps);
    AssertHelpers.assertThrows("Should throw CommitStateUnknownException since the catalog check was blocked", CommitStateUnknownException.class, "Datacenter on fire", () -> spyOps.commit(metadataV2, metadataV1));
    ops.refresh();
    Assert.assertFalse("Current metadata should have changed", ops.current().equals(metadataV2));
    Assert.assertTrue("Current metadata file should still exist", metadataFileExists(ops.current()));
}

Example 10 with TableMetadata

use of org.apache.iceberg.TableMetadata in project presto by prestodb.

the class HiveTableOperations method metadataFileLocation.

@Override
public String metadataFileLocation(String filename) {
    TableMetadata metadata = current();
    String location;
    if (metadata != null) {
        String writeLocation = metadata.properties().get(WRITE_METADATA_LOCATION);
        if (writeLocation != null) {
            return format("%s/%s", writeLocation, filename);
        }
        location = metadata.location();
    } else {
        location = this.location.orElseThrow(() -> new IllegalStateException("Location not set"));
    }
    return format("%s/%s/%s", location, METADATA_FOLDER_NAME, filename);
}