Examples with HoodieInstant org.apache.hudi.common.table.timeline.HoodieInstant used on opensource projects

Example 56 with HoodieInstant

use of org.apache.hudi.common.table.timeline.HoodieInstant in project hudi by apache.

the class ITTestCompactionCommand method prepareScheduleCompaction.

private String prepareScheduleCompaction() {
    // generate requested compaction
    CommandResult cr = getShell().executeCommand(String.format("compaction schedule --hoodieConfigs hoodie.compact.inline.max.delta.commits=1 --sparkMaster %s", "local"));
    assertTrue(cr.isSuccess());
    // get compaction instance
    HoodieActiveTimeline timeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
    Option<String> instance = timeline.filterPendingCompactionTimeline().firstInstant().map(HoodieInstant::getTimestamp);
    assertTrue(instance.isPresent(), "Must have pending compaction.");
    return instance.get();
}

Example 57 with HoodieInstant

use of org.apache.hudi.common.table.timeline.HoodieInstant in project hudi by apache.

the class AsyncCompactService method startService.

/**
 * Start Compaction Service.
 */
@Override
protected Pair<CompletableFuture, ExecutorService> startService() {
    ExecutorService executor = Executors.newFixedThreadPool(maxConcurrentCompaction, r -> {
        Thread t = new Thread(r, "async_compact_thread");
        t.setDaemon(isRunInDaemonMode());
        return t;
    });
    return Pair.of(CompletableFuture.allOf(IntStream.range(0, maxConcurrentCompaction).mapToObj(i -> CompletableFuture.supplyAsync(() -> {
        try {
            // Set Compactor Pool Name for allowing users to prioritize compaction
            LOG.info("Setting pool name for compaction to " + COMPACT_POOL_NAME);
            context.setProperty(EngineProperty.COMPACTION_POOL_NAME, COMPACT_POOL_NAME);
            while (!isShutdownRequested()) {
                final HoodieInstant instant = fetchNextAsyncServiceInstant();
                if (null != instant) {
                    LOG.info("Starting Compaction for instant " + instant);
                    compactor.compact(instant);
                    LOG.info("Finished Compaction for instant " + instant);
                }
            }
            LOG.info("Compactor shutting down properly!!");
        } catch (InterruptedException ie) {
            hasError = true;
            LOG.warn("Compactor executor thread got interrupted exception. Stopping", ie);
        } catch (IOException e) {
            hasError = true;
            LOG.error("Compactor executor failed due to IOException", e);
            throw new HoodieIOException(e.getMessage(), e);
        } catch (Exception e) {
            hasError = true;
            LOG.error("Compactor executor failed", e);
            throw e;
        }
        return true;
    }, executor)).toArray(CompletableFuture[]::new)), executor);
}

Example 58 with HoodieInstant

use of org.apache.hudi.common.table.timeline.HoodieInstant in project hudi by apache.

the class HoodieAsyncService method fetchNextAsyncServiceInstant.

/**
 * Fetch next pending compaction/clustering instant if available.
 *
 * @return {@link HoodieInstant} corresponding to the next pending compaction/clustering.
 * @throws InterruptedException
 */
HoodieInstant fetchNextAsyncServiceInstant() throws InterruptedException {
    LOG.info(String.format("Waiting for next instant up to %d seconds", POLLING_SECONDS));
    HoodieInstant instant = pendingInstants.poll(POLLING_SECONDS, TimeUnit.SECONDS);
    if (instant != null) {
        try {
            queueLock.lock();
            // Signal waiting thread
            consumed.signal();
        } finally {
            queueLock.unlock();
        }
    }
    return instant;
}

Example 59 with HoodieInstant

use of org.apache.hudi.common.table.timeline.HoodieInstant in project hudi by apache.

the class TransactionUtils method resolveWriteConflictIfAny.

/**
 * Resolve any write conflicts when committing data.
 *
 * @param table
 * @param currentTxnOwnerInstant
 * @param thisCommitMetadata
 * @param config
 * @param lastCompletedTxnOwnerInstant
 * @return
 * @throws HoodieWriteConflictException
 */
public static Option<HoodieCommitMetadata> resolveWriteConflictIfAny(final HoodieTable table, final Option<HoodieInstant> currentTxnOwnerInstant, final Option<HoodieCommitMetadata> thisCommitMetadata, final HoodieWriteConfig config, Option<HoodieInstant> lastCompletedTxnOwnerInstant, boolean reloadActiveTimeline) throws HoodieWriteConflictException {
    if (config.getWriteConcurrencyMode().supportsOptimisticConcurrencyControl()) {
        ConflictResolutionStrategy resolutionStrategy = config.getWriteConflictResolutionStrategy();
        Stream<HoodieInstant> instantStream = resolutionStrategy.getCandidateInstants(reloadActiveTimeline ? table.getMetaClient().reloadActiveTimeline() : table.getActiveTimeline(), currentTxnOwnerInstant.get(), lastCompletedTxnOwnerInstant);
        final ConcurrentOperation thisOperation = new ConcurrentOperation(currentTxnOwnerInstant.get(), thisCommitMetadata.orElse(new HoodieCommitMetadata()));
        instantStream.forEach(instant -> {
            try {
                ConcurrentOperation otherOperation = new ConcurrentOperation(instant, table.getMetaClient());
                if (resolutionStrategy.hasConflict(thisOperation, otherOperation)) {
                    LOG.info("Conflict encountered between current instant = " + thisOperation + " and instant = " + otherOperation + ", attempting to resolve it...");
                    resolutionStrategy.resolveConflict(table, thisOperation, otherOperation);
                }
            } catch (IOException io) {
                throw new HoodieWriteConflictException("Unable to resolve conflict, if present", io);
            }
        });
        LOG.info("Successfully resolved conflicts, if any");
        return thisOperation.getCommitMetadataOption();
    }
    return thisCommitMetadata;
}

Example 60 with HoodieInstant

use of org.apache.hudi.common.table.timeline.HoodieInstant in project hudi by apache.

the class IncrementalTimelineSyncFileSystemView method removeFileSlicesForPartition.

private void removeFileSlicesForPartition(HoodieTimeline timeline, HoodieInstant instant, String partition, List<String> paths) {
    if (isPartitionAvailableInStore(partition)) {
        LOG.info("Removing file slices for partition (" + partition + ") for instant (" + instant + ")");
        FileStatus[] statuses = paths.stream().map(p -> {
            FileStatus status = new FileStatus();
            status.setPath(new Path(p));
            return status;
        }).toArray(FileStatus[]::new);
        List<HoodieFileGroup> fileGroups = buildFileGroups(statuses, timeline.filterCompletedAndCompactionInstants(), false);
        applyDeltaFileSlicesToPartitionView(partition, fileGroups, DeltaApplyMode.REMOVE);
    } else {
        LOG.warn("Skipping partition (" + partition + ") when syncing instant (" + instant + ") as it is not loaded");
    }
}