Search in sources :

Example 11 with HoodieTimer

use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.

the class BaseRestoreActionExecutor method execute.

@Override
public HoodieRestoreMetadata execute() {
    HoodieTimer restoreTimer = new HoodieTimer();
    restoreTimer.startTimer();
    Option<HoodieInstant> restoreInstant = table.getRestoreTimeline().filterInflightsAndRequested().filter(instant -> instant.getTimestamp().equals(instantTime)).firstInstant();
    if (!restoreInstant.isPresent()) {
        throw new HoodieRollbackException("No pending restore instants found to execute restore");
    }
    try {
        List<HoodieInstant> instantsToRollback = getInstantsToRollback(restoreInstant.get());
        ValidationUtils.checkArgument(restoreInstant.get().getState().equals(HoodieInstant.State.REQUESTED) || restoreInstant.get().getState().equals(HoodieInstant.State.INFLIGHT));
        Map<String, List<HoodieRollbackMetadata>> instantToMetadata = new HashMap<>();
        if (restoreInstant.get().isRequested()) {
            table.getActiveTimeline().transitionRestoreRequestedToInflight(restoreInstant.get());
        }
        instantsToRollback.forEach(instant -> {
            instantToMetadata.put(instant.getTimestamp(), Collections.singletonList(rollbackInstant(instant)));
            LOG.info("Deleted instant " + instant);
        });
        return finishRestore(instantToMetadata, instantsToRollback, restoreTimer.endTimer());
    } catch (IOException io) {
        throw new HoodieRestoreException("unable to Restore instant " + restoreInstant.get(), io);
    }
}
Also used : HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) HoodieTable(org.apache.hudi.table.HoodieTable) BaseActionExecutor(org.apache.hudi.table.action.BaseActionExecutor) HoodieRestorePlan(org.apache.hudi.avro.model.HoodieRestorePlan) HoodieRestoreException(org.apache.hudi.exception.HoodieRestoreException) HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) Option(org.apache.hudi.common.util.Option) HashMap(java.util.HashMap) HoodieEngineContext(org.apache.hudi.common.engine.HoodieEngineContext) TransactionManager(org.apache.hudi.client.transaction.TransactionManager) HoodieTimer(org.apache.hudi.common.util.HoodieTimer) ArrayList(java.util.ArrayList) Logger(org.apache.log4j.Logger) Map(java.util.Map) HoodieRollbackMetadata(org.apache.hudi.avro.model.HoodieRollbackMetadata) HoodieActiveTimeline(org.apache.hudi.common.table.timeline.HoodieActiveTimeline) HoodieTimeline(org.apache.hudi.common.table.timeline.HoodieTimeline) ValidationUtils(org.apache.hudi.common.util.ValidationUtils) HoodieRollbackException(org.apache.hudi.exception.HoodieRollbackException) HoodieWriteConfig(org.apache.hudi.config.HoodieWriteConfig) TimelineMetadataUtils(org.apache.hudi.common.table.timeline.TimelineMetadataUtils) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) HoodieInstantInfo(org.apache.hudi.avro.model.HoodieInstantInfo) HoodieRecordPayload(org.apache.hudi.common.model.HoodieRecordPayload) List(java.util.List) HoodieRestoreMetadata(org.apache.hudi.avro.model.HoodieRestoreMetadata) LogManager(org.apache.log4j.LogManager) Collections(java.util.Collections) HoodieRollbackException(org.apache.hudi.exception.HoodieRollbackException) HashMap(java.util.HashMap) HoodieTimer(org.apache.hudi.common.util.HoodieTimer) ArrayList(java.util.ArrayList) List(java.util.List) IOException(java.io.IOException) HoodieRestoreException(org.apache.hudi.exception.HoodieRestoreException)

Example 12 with HoodieTimer

use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.

the class MergeOnReadRollbackActionExecutor method executeRollback.

@Override
protected List<HoodieRollbackStat> executeRollback(HoodieRollbackPlan hoodieRollbackPlan) {
    HoodieTimer rollbackTimer = new HoodieTimer();
    rollbackTimer.startTimer();
    LOG.info("Rolling back instant " + instantToRollback);
    HoodieInstant resolvedInstant = instantToRollback;
    // Atomically un-publish all non-inflight commits
    if (instantToRollback.isCompleted()) {
        LOG.info("Un-publishing instant " + instantToRollback + ", deleteInstants=" + deleteInstants);
        resolvedInstant = table.getActiveTimeline().revertToInflight(instantToRollback);
        // reload meta-client to reflect latest timeline status
        table.getMetaClient().reloadActiveTimeline();
    }
    List<HoodieRollbackStat> allRollbackStats = new ArrayList<>();
    // deleting the timeline file
    if (!resolvedInstant.isRequested()) {
        LOG.info("Unpublished " + resolvedInstant);
        allRollbackStats = executeRollback(instantToRollback, hoodieRollbackPlan);
    }
    dropBootstrapIndexIfNeeded(resolvedInstant);
    // Delete Inflight instants if enabled
    deleteInflightAndRequestedInstant(deleteInstants, table.getActiveTimeline(), resolvedInstant);
    LOG.info("Time(in ms) taken to finish rollback " + rollbackTimer.endTimer());
    return allRollbackStats;
}
Also used : HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) HoodieRollbackStat(org.apache.hudi.common.HoodieRollbackStat) ArrayList(java.util.ArrayList) HoodieTimer(org.apache.hudi.common.util.HoodieTimer)

Example 13 with HoodieTimer

use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.

the class DirectWriteMarkers method create.

private Option<Path> create(Path markerPath, boolean checkIfExists) {
    HoodieTimer timer = new HoodieTimer().startTimer();
    Path dirPath = markerPath.getParent();
    try {
        if (!fs.exists(dirPath)) {
            // create a new partition as needed.
            fs.mkdirs(dirPath);
        }
    } catch (IOException e) {
        throw new HoodieIOException("Failed to make dir " + dirPath, e);
    }
    try {
        if (checkIfExists && fs.exists(markerPath)) {
            LOG.warn("Marker Path=" + markerPath + " already exists, cancel creation");
            return Option.empty();
        }
        LOG.info("Creating Marker Path=" + markerPath);
        fs.create(markerPath, false).close();
    } catch (IOException e) {
        throw new HoodieException("Failed to create marker file " + markerPath, e);
    }
    LOG.info("[direct] Created marker file " + markerPath.toString() + " in " + timer.endTimer() + " ms");
    return Option.of(markerPath);
}
Also used : Path(org.apache.hadoop.fs.Path) HoodieIOException(org.apache.hudi.exception.HoodieIOException) HoodieTimer(org.apache.hudi.common.util.HoodieTimer) HoodieException(org.apache.hudi.exception.HoodieException) IOException(java.io.IOException) HoodieIOException(org.apache.hudi.exception.HoodieIOException)

Example 14 with HoodieTimer

use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.

the class SparkDeletePartitionCommitActionExecutor method execute.

@Override
public HoodieWriteMetadata<HoodieData<WriteStatus>> execute() {
    HoodieTimer timer = new HoodieTimer().startTimer();
    context.setJobStatus(this.getClass().getSimpleName(), "Gather all file ids from all deleting partitions.");
    Map<String, List<String>> partitionToReplaceFileIds = HoodieJavaPairRDD.getJavaPairRDD(context.parallelize(partitions).distinct().mapToPair(partitionPath -> Pair.of(partitionPath, getAllExistingFileIds(partitionPath)))).collectAsMap();
    HoodieWriteMetadata<HoodieData<WriteStatus>> result = new HoodieWriteMetadata<>();
    result.setPartitionToReplaceFileIds(partitionToReplaceFileIds);
    result.setIndexUpdateDuration(Duration.ofMillis(timer.endTimer()));
    result.setWriteStatuses(context.emptyHoodieData());
    this.saveWorkloadProfileMetadataToInflight(new WorkloadProfile(Pair.of(new HashMap<>(), new WorkloadStat())), instantTime);
    this.commitOnAutoCommit(result);
    return result;
}
Also used : HoodieData(org.apache.hudi.common.data.HoodieData) WorkloadProfile(org.apache.hudi.table.WorkloadProfile) WorkloadStat(org.apache.hudi.table.WorkloadStat) HoodieTimer(org.apache.hudi.common.util.HoodieTimer) List(java.util.List) HoodieWriteMetadata(org.apache.hudi.table.action.HoodieWriteMetadata)

Example 15 with HoodieTimer

use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.

the class TimelineServerBasedWriteMarkers method create.

@Override
protected Option<Path> create(String partitionPath, String dataFileName, IOType type, boolean checkIfExists) {
    HoodieTimer timer = new HoodieTimer().startTimer();
    String markerFileName = getMarkerFileName(dataFileName, type);
    Map<String, String> paramsMap = new HashMap<>();
    paramsMap.put(MARKER_DIR_PATH_PARAM, markerDirPath.toString());
    if (StringUtils.isNullOrEmpty(partitionPath)) {
        paramsMap.put(MARKER_NAME_PARAM, markerFileName);
    } else {
        paramsMap.put(MARKER_NAME_PARAM, partitionPath + "/" + markerFileName);
    }
    boolean success;
    try {
        success = executeRequestToTimelineServer(CREATE_MARKER_URL, paramsMap, new TypeReference<Boolean>() {
        }, RequestMethod.POST);
    } catch (IOException e) {
        throw new HoodieRemoteException("Failed to create marker file " + partitionPath + "/" + markerFileName, e);
    }
    LOG.info("[timeline-server-based] Created marker file " + partitionPath + "/" + markerFileName + " in " + timer.endTimer() + " ms");
    if (success) {
        return Option.of(new Path(FSUtils.getPartitionPath(markerDirPath, partitionPath), markerFileName));
    } else {
        return Option.empty();
    }
}
Also used : Path(org.apache.hadoop.fs.Path) HashMap(java.util.HashMap) HoodieTimer(org.apache.hudi.common.util.HoodieTimer) HoodieRemoteException(org.apache.hudi.exception.HoodieRemoteException) TypeReference(com.fasterxml.jackson.core.type.TypeReference) IOException(java.io.IOException)

Aggregations

HoodieTimer (org.apache.hudi.common.util.HoodieTimer)35 ArrayList (java.util.ArrayList)16 Path (org.apache.hadoop.fs.Path)15 IOException (java.io.IOException)14 HashMap (java.util.HashMap)12 Option (org.apache.hudi.common.util.Option)12 HoodieInstant (org.apache.hudi.common.table.timeline.HoodieInstant)11 HoodieRecord (org.apache.hudi.common.model.HoodieRecord)10 Map (java.util.Map)9 Pair (org.apache.hudi.common.util.collection.Pair)9 List (java.util.List)8 FileStatus (org.apache.hadoop.fs.FileStatus)8 HoodieIOException (org.apache.hudi.exception.HoodieIOException)7 LogManager (org.apache.log4j.LogManager)7 Logger (org.apache.log4j.Logger)7 Collectors (java.util.stream.Collectors)6 HoodieSparkEngineContext (org.apache.hudi.client.common.HoodieSparkEngineContext)6 HoodieTableMetaClient (org.apache.hudi.common.table.HoodieTableMetaClient)6 HoodieTimeline (org.apache.hudi.common.table.timeline.HoodieTimeline)6 HoodieWriteConfig (org.apache.hudi.config.HoodieWriteConfig)6