Examples with ImmutableMap org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap used on opensource projects

Example 1 with ImmutableMap

use of org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap in project hive by apache.

the class HiveTableOperations method doCommit.

@SuppressWarnings("checkstyle:CyclomaticComplexity")
@Override
protected void doCommit(TableMetadata base, TableMetadata metadata) {
    String newMetadataLocation = writeNewMetadata(metadata, currentVersion() + 1);
    boolean hiveEngineEnabled = hiveEngineEnabled(metadata, conf);
    boolean keepHiveStats = conf.getBoolean(KEEP_HIVE_STATS, false);
    CommitStatus commitStatus = CommitStatus.FAILURE;
    boolean updateHiveTable = false;
    Optional<Long> lockId = Optional.empty();
    // getting a process-level lock per table to avoid concurrent commit attempts to the same table from the same
    // JVM process, which would result in unnecessary and costly HMS lock acquisition requests
    ReentrantLock tableLevelMutex = commitLockCache.get(fullName, t -> new ReentrantLock(true));
    tableLevelMutex.lock();
    try {
        lockId = Optional.of(acquireLock());
        // TODO add lock heart beating for cases where default lock timeout is too low.
        Table tbl = loadHmsTable();
        if (tbl != null) {
            // If we try to create the table but the metadata location is already set, then we had a concurrent commit
            if (base == null && tbl.getParameters().get(BaseMetastoreTableOperations.METADATA_LOCATION_PROP) != null) {
                throw new AlreadyExistsException("Table already exists: %s.%s", database, tableName);
            }
            updateHiveTable = true;
            LOG.debug("Committing existing table: {}", fullName);
        } else {
            tbl = newHmsTable();
            LOG.debug("Committing new table: {}", fullName);
        }
        // set to pickup any schema changes
        tbl.setSd(storageDescriptor(metadata, hiveEngineEnabled));
        String metadataLocation = tbl.getParameters().get(METADATA_LOCATION_PROP);
        String baseMetadataLocation = base != null ? base.metadataFileLocation() : null;
        if (!Objects.equals(baseMetadataLocation, metadataLocation)) {
            throw new CommitFailedException("Base metadata location '%s' is not same as the current table metadata location '%s' for %s.%s", baseMetadataLocation, metadataLocation, database, tableName);
        }
        // get Iceberg props that have been removed
        Set<String> removedProps = Collections.emptySet();
        if (base != null) {
            removedProps = base.properties().keySet().stream().filter(key -> !metadata.properties().containsKey(key)).collect(Collectors.toSet());
        }
        Map<String, String> summary = Optional.ofNullable(metadata.currentSnapshot()).map(Snapshot::summary).orElseGet(ImmutableMap::of);
        setHmsTableParameters(newMetadataLocation, tbl, metadata.properties(), removedProps, hiveEngineEnabled, summary);
        if (!keepHiveStats) {
            StatsSetupConst.setBasicStatsState(tbl.getParameters(), StatsSetupConst.FALSE);
            StatsSetupConst.clearColumnStatsState(tbl.getParameters());
        }
        try {
            persistTable(tbl, updateHiveTable);
            commitStatus = CommitStatus.SUCCESS;
        } catch (Throwable persistFailure) {
            LOG.error("Cannot tell if commit to {}.{} succeeded, attempting to reconnect and check.", database, tableName, persistFailure);
            commitStatus = checkCommitStatus(newMetadataLocation, metadata);
            switch(commitStatus) {
                case SUCCESS:
                    break;
                case FAILURE:
                    throw persistFailure;
                case UNKNOWN:
                    throw new CommitStateUnknownException(persistFailure);
            }
        }
    } catch (org.apache.hadoop.hive.metastore.api.AlreadyExistsException e) {
        throw new AlreadyExistsException("Table already exists: %s.%s", database, tableName);
    } catch (TException | UnknownHostException e) {
        if (e.getMessage() != null && e.getMessage().contains("Table/View 'HIVE_LOCKS' does not exist")) {
            throw new RuntimeException("Failed to acquire locks from metastore because 'HIVE_LOCKS' doesn't " + "exist, this probably happened when using embedded metastore or doesn't create a " + "transactional meta table. To fix this, use an alternative metastore", e);
        }
        throw new RuntimeException(String.format("Metastore operation failed for %s.%s", database, tableName), e);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new RuntimeException("Interrupted during commit", e);
    } finally {
        cleanupMetadataAndUnlock(commitStatus, newMetadataLocation, lockId, tableLevelMutex);
    }
}

Example 2 with ImmutableMap

use of org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap in project hive by apache.

the class TestHiveCatalog method testCreateNamespace.

@Test
public void testCreateNamespace() throws TException {
    Namespace namespace1 = Namespace.of("noLocation");
    catalog.createNamespace(namespace1, meta);
    Database database1 = metastoreClient.getDatabase(namespace1.toString());
    Assert.assertTrue(database1.getParameters().get("owner").equals("apache"));
    Assert.assertTrue(database1.getParameters().get("group").equals("iceberg"));
    Assert.assertEquals("There no same location for db and namespace", database1.getLocationUri(), defaultUri(namespace1));
    AssertHelpers.assertThrows("Should fail to create when namespace already exist " + namespace1, AlreadyExistsException.class, "Namespace '" + namespace1 + "' already exists!", () -> {
        catalog.createNamespace(namespace1);
    });
    ImmutableMap newMeta = ImmutableMap.<String, String>builder().putAll(meta).put("location", hiveLocalDir).build();
    Namespace namespace2 = Namespace.of("haveLocation");
    catalog.createNamespace(namespace2, newMeta);
    Database database2 = metastoreClient.getDatabase(namespace2.toString());
    Assert.assertEquals("There no same location for db and namespace", database2.getLocationUri(), hiveLocalDir);
}