Examples with CommitStateUnknownException org.apache.iceberg.exceptions.CommitStateUnknownException used on opensource projects

Example 1 with CommitStateUnknownException

use of org.apache.iceberg.exceptions.CommitStateUnknownException in project hive by apache.

the class HiveTableOperations method doCommit.

@SuppressWarnings("checkstyle:CyclomaticComplexity")
@Override
protected void doCommit(TableMetadata base, TableMetadata metadata) {
    String newMetadataLocation = writeNewMetadata(metadata, currentVersion() + 1);
    boolean hiveEngineEnabled = hiveEngineEnabled(metadata, conf);
    boolean keepHiveStats = conf.getBoolean(KEEP_HIVE_STATS, false);
    CommitStatus commitStatus = CommitStatus.FAILURE;
    boolean updateHiveTable = false;
    Optional<Long> lockId = Optional.empty();
    // getting a process-level lock per table to avoid concurrent commit attempts to the same table from the same
    // JVM process, which would result in unnecessary and costly HMS lock acquisition requests
    ReentrantLock tableLevelMutex = commitLockCache.get(fullName, t -> new ReentrantLock(true));
    tableLevelMutex.lock();
    try {
        lockId = Optional.of(acquireLock());
        // TODO add lock heart beating for cases where default lock timeout is too low.
        Table tbl = loadHmsTable();
        if (tbl != null) {
            // If we try to create the table but the metadata location is already set, then we had a concurrent commit
            if (base == null && tbl.getParameters().get(BaseMetastoreTableOperations.METADATA_LOCATION_PROP) != null) {
                throw new AlreadyExistsException("Table already exists: %s.%s", database, tableName);
            }
            updateHiveTable = true;
            LOG.debug("Committing existing table: {}", fullName);
        } else {
            tbl = newHmsTable();
            LOG.debug("Committing new table: {}", fullName);
        }
        // set to pickup any schema changes
        tbl.setSd(storageDescriptor(metadata, hiveEngineEnabled));
        String metadataLocation = tbl.getParameters().get(METADATA_LOCATION_PROP);
        String baseMetadataLocation = base != null ? base.metadataFileLocation() : null;
        if (!Objects.equals(baseMetadataLocation, metadataLocation)) {
            throw new CommitFailedException("Base metadata location '%s' is not same as the current table metadata location '%s' for %s.%s", baseMetadataLocation, metadataLocation, database, tableName);
        }
        // get Iceberg props that have been removed
        Set<String> removedProps = Collections.emptySet();
        if (base != null) {
            removedProps = base.properties().keySet().stream().filter(key -> !metadata.properties().containsKey(key)).collect(Collectors.toSet());
        }
        Map<String, String> summary = Optional.ofNullable(metadata.currentSnapshot()).map(Snapshot::summary).orElseGet(ImmutableMap::of);
        setHmsTableParameters(newMetadataLocation, tbl, metadata.properties(), removedProps, hiveEngineEnabled, summary);
        if (!keepHiveStats) {
            StatsSetupConst.setBasicStatsState(tbl.getParameters(), StatsSetupConst.FALSE);
            StatsSetupConst.clearColumnStatsState(tbl.getParameters());
        }
        try {
            persistTable(tbl, updateHiveTable);
            commitStatus = CommitStatus.SUCCESS;
        } catch (Throwable persistFailure) {
            LOG.error("Cannot tell if commit to {}.{} succeeded, attempting to reconnect and check.", database, tableName, persistFailure);
            commitStatus = checkCommitStatus(newMetadataLocation, metadata);
            switch(commitStatus) {
                case SUCCESS:
                    break;
                case FAILURE:
                    throw persistFailure;
                case UNKNOWN:
                    throw new CommitStateUnknownException(persistFailure);
            }
        }
    } catch (org.apache.hadoop.hive.metastore.api.AlreadyExistsException e) {
        throw new AlreadyExistsException("Table already exists: %s.%s", database, tableName);
    } catch (TException | UnknownHostException e) {
        if (e.getMessage() != null && e.getMessage().contains("Table/View 'HIVE_LOCKS' does not exist")) {
            throw new RuntimeException("Failed to acquire locks from metastore because 'HIVE_LOCKS' doesn't " + "exist, this probably happened when using embedded metastore or doesn't create a " + "transactional meta table. To fix this, use an alternative metastore", e);
        }
        throw new RuntimeException(String.format("Metastore operation failed for %s.%s", database, tableName), e);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new RuntimeException("Interrupted during commit", e);
    } finally {
        cleanupMetadataAndUnlock(commitStatus, newMetadataLocation, lockId, tableLevelMutex);
    }
}