Examples with TableMetadata org.apache.iceberg.TableMetadata used on opensource projects

Example 1 with TableMetadata

use of org.apache.iceberg.TableMetadata in project presto by prestodb.

the class HiveTableOperations method commit.

@Override
public void commit(@Nullable TableMetadata base, TableMetadata metadata) {
    requireNonNull(metadata, "metadata is null");
    // if the metadata is already out of date, reject it
    if (!Objects.equals(base, current())) {
        throw new CommitFailedException("Cannot commit: stale table metadata for %s", getSchemaTableName());
    }
    // if the metadata is not changed, return early
    if (Objects.equals(base, metadata)) {
        return;
    }
    String newMetadataLocation = writeNewMetadata(metadata, version + 1);
    Table table;
    // getting a process-level lock per table to avoid concurrent commit attempts to the same table from the same
    // JVM process, which would result in unnecessary and costly HMS lock acquisition requests
    Optional<Long> lockId = Optional.empty();
    ReentrantLock tableLevelMutex = commitLockCache.getUnchecked(database + "." + tableName);
    tableLevelMutex.lock();
    try {
        try {
            lockId = Optional.of(metastore.lock(metastoreContext, database, tableName));
            if (base == null) {
                String tableComment = metadata.properties().get(TABLE_COMMENT);
                Map<String, String> parameters = new HashMap<>();
                parameters.put("EXTERNAL", "TRUE");
                parameters.put(TABLE_TYPE_PROP, ICEBERG_TABLE_TYPE_VALUE);
                parameters.put(METADATA_LOCATION, newMetadataLocation);
                if (tableComment != null) {
                    parameters.put(TABLE_COMMENT, tableComment);
                }
                Table.Builder builder = Table.builder().setDatabaseName(database).setTableName(tableName).setOwner(owner.orElseThrow(() -> new IllegalStateException("Owner not set"))).setTableType(PrestoTableType.EXTERNAL_TABLE).setDataColumns(toHiveColumns(metadata.schema().columns())).withStorage(storage -> storage.setLocation(metadata.location())).withStorage(storage -> storage.setStorageFormat(STORAGE_FORMAT)).setParameters(parameters);
                table = builder.build();
            } else {
                Table currentTable = getTable();
                checkState(currentMetadataLocation != null, "No current metadata location for existing table");
                String metadataLocation = currentTable.getParameters().get(METADATA_LOCATION);
                if (!currentMetadataLocation.equals(metadataLocation)) {
                    throw new CommitFailedException("Metadata location [%s] is not same as table metadata location [%s] for %s", currentMetadataLocation, metadataLocation, getSchemaTableName());
                }
                table = Table.builder(currentTable).setDataColumns(toHiveColumns(metadata.schema().columns())).withStorage(storage -> storage.setLocation(metadata.location())).setParameter(METADATA_LOCATION, newMetadataLocation).setParameter(PREVIOUS_METADATA_LOCATION, currentMetadataLocation).build();
            }
        } catch (RuntimeException e) {
            try {
                io().deleteFile(newMetadataLocation);
            } catch (RuntimeException exception) {
                e.addSuppressed(exception);
            }
            throw e;
        }
        PrestoPrincipal owner = new PrestoPrincipal(USER, table.getOwner());
        PrincipalPrivileges privileges = new PrincipalPrivileges(ImmutableMultimap.<String, HivePrivilegeInfo>builder().put(table.getOwner(), new HivePrivilegeInfo(SELECT, true, owner, owner)).put(table.getOwner(), new HivePrivilegeInfo(INSERT, true, owner, owner)).put(table.getOwner(), new HivePrivilegeInfo(UPDATE, true, owner, owner)).put(table.getOwner(), new HivePrivilegeInfo(DELETE, true, owner, owner)).build(), ImmutableMultimap.of());
        if (base == null) {
            metastore.createTable(metastoreContext, table, privileges);
        } else {
            metastore.replaceTable(metastoreContext, database, tableName, table, privileges);
        }
    } finally {
        shouldRefresh = true;
        try {
            lockId.ifPresent(id -> metastore.unlock(metastoreContext, id));
        } catch (Exception e) {
            log.error(e, "Failed to unlock: %s", lockId.orElse(null));
        } finally {
            tableLevelMutex.unlock();
        }
    }
}

Example 2 with TableMetadata

use of org.apache.iceberg.TableMetadata in project hive by apache.

the class TestHiveCommits method testThriftExceptionUnknownFailedCommit.

/**
 * Pretends we throw an exception while persisting and don't know what happened, can't check to find out,
 * but in reality the commit failed
 */
@Test
public void testThriftExceptionUnknownFailedCommit() throws TException, InterruptedException {
    Table table = catalog.loadTable(TABLE_IDENTIFIER);
    HiveTableOperations ops = (HiveTableOperations) ((HasTableOperations) table).operations();
    TableMetadata metadataV1 = ops.current();
    table.updateSchema().addColumn("n", Types.IntegerType.get()).commit();
    ops.refresh();
    TableMetadata metadataV2 = ops.current();
    Assert.assertEquals(2, ops.current().schema().columns().size());
    HiveTableOperations spyOps = spy(ops);
    failCommitAndThrowException(spyOps);
    breakFallbackCatalogCommitCheck(spyOps);
    AssertHelpers.assertThrows("Should throw CommitStateUnknownException since the catalog check was blocked", CommitStateUnknownException.class, "Datacenter on fire", () -> spyOps.commit(metadataV2, metadataV1));
    ops.refresh();
    Assert.assertEquals("Current metadata should not have changed", metadataV2, ops.current());
    Assert.assertTrue("Current metadata file should still exist", metadataFileExists(ops.current()));
    Assert.assertEquals("Client could not determine outcome so new metadata file should also exist", 3, metadataFileCount(ops.current()));
}

Example 3 with TableMetadata

use of org.apache.iceberg.TableMetadata in project hive by apache.

the class TestInputFormatReaderDeletes method createTable.

@Override
protected Table createTable(String name, Schema schema, PartitionSpec spec) throws IOException {
    Table table;
    File location = temp.newFolder(inputFormat, fileFormat.name());
    Assert.assertTrue(location.delete());
    helper = new TestHelper(conf, tables, location.toString(), schema, spec, fileFormat, temp);
    table = helper.createTable();
    TableOperations ops = ((BaseTable) table).operations();
    TableMetadata meta = ops.current();
    ops.commit(meta, meta.upgradeToFormatVersion(2));
    return table;
}

Example 4 with TableMetadata

use of org.apache.iceberg.TableMetadata in project hive by apache.

the class HiveCatalog method dropTable.

@Override
public boolean dropTable(TableIdentifier identifier, boolean purge) {
    if (!isValidIdentifier(identifier)) {
        return false;
    }
    String database = identifier.namespace().level(0);
    TableOperations ops = newTableOps(identifier);
    TableMetadata lastMetadata;
    if (purge && ops.current() != null) {
        lastMetadata = ops.current();
    } else {
        lastMetadata = null;
    }
    try {
        clients.run(client -> {
            client.dropTable(database, identifier.name(), false, /* do not delete data */
            false);
            return null;
        });
        if (purge && lastMetadata != null) {
            CatalogUtil.dropTableData(ops.io(), lastMetadata);
        }
        LOG.info("Dropped table: {}", identifier);
        return true;
    } catch (NoSuchTableException | NoSuchObjectException e) {
        LOG.info("Skipping drop, table does not exist: {}", identifier, e);
        return false;
    } catch (TException e) {
        throw new RuntimeException("Failed to drop " + identifier, e);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new RuntimeException("Interrupted in call to dropTable", e);
    }
}

Example 5 with TableMetadata

use of org.apache.iceberg.TableMetadata in project hive by apache.

the class TestHiveCommits method testThriftExceptionSuccessOnCommit.

/**
 * Pretends we throw an error while persisting that actually does commit serverside
 */
@Test
public void testThriftExceptionSuccessOnCommit() throws TException, InterruptedException {
    Table table = catalog.loadTable(TABLE_IDENTIFIER);
    HiveTableOperations ops = (HiveTableOperations) ((HasTableOperations) table).operations();
    TableMetadata metadataV1 = ops.current();
    table.updateSchema().addColumn("n", Types.IntegerType.get()).commit();
    ops.refresh();
    TableMetadata metadataV2 = ops.current();
    Assert.assertEquals(2, ops.current().schema().columns().size());
    HiveTableOperations spyOps = spy(ops);
    // Simulate a communication error after a successful commit
    commitAndThrowException(ops, spyOps);
    // Shouldn't throw because the commit actually succeeds even though persistTable throws an exception
    spyOps.commit(metadataV2, metadataV1);
    ops.refresh();
    Assert.assertNotEquals("Current metadata should have changed", metadataV2, ops.current());
    Assert.assertTrue("Current metadata file should still exist", metadataFileExists(ops.current()));
    Assert.assertEquals("Commit should have been successful and new metadata file should be made", 3, metadataFileCount(ops.current()));
}