use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.
the class BaseRestoreActionExecutor method execute.
@Override
public HoodieRestoreMetadata execute() {
HoodieTimer restoreTimer = new HoodieTimer();
restoreTimer.startTimer();
Option<HoodieInstant> restoreInstant = table.getRestoreTimeline().filterInflightsAndRequested().filter(instant -> instant.getTimestamp().equals(instantTime)).firstInstant();
if (!restoreInstant.isPresent()) {
throw new HoodieRollbackException("No pending restore instants found to execute restore");
}
try {
List<HoodieInstant> instantsToRollback = getInstantsToRollback(restoreInstant.get());
ValidationUtils.checkArgument(restoreInstant.get().getState().equals(HoodieInstant.State.REQUESTED) || restoreInstant.get().getState().equals(HoodieInstant.State.INFLIGHT));
Map<String, List<HoodieRollbackMetadata>> instantToMetadata = new HashMap<>();
if (restoreInstant.get().isRequested()) {
table.getActiveTimeline().transitionRestoreRequestedToInflight(restoreInstant.get());
}
instantsToRollback.forEach(instant -> {
instantToMetadata.put(instant.getTimestamp(), Collections.singletonList(rollbackInstant(instant)));
LOG.info("Deleted instant " + instant);
});
return finishRestore(instantToMetadata, instantsToRollback, restoreTimer.endTimer());
} catch (IOException io) {
throw new HoodieRestoreException("unable to Restore instant " + restoreInstant.get(), io);
}
}
use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.
the class MergeOnReadRollbackActionExecutor method executeRollback.
@Override
protected List<HoodieRollbackStat> executeRollback(HoodieRollbackPlan hoodieRollbackPlan) {
HoodieTimer rollbackTimer = new HoodieTimer();
rollbackTimer.startTimer();
LOG.info("Rolling back instant " + instantToRollback);
HoodieInstant resolvedInstant = instantToRollback;
// Atomically un-publish all non-inflight commits
if (instantToRollback.isCompleted()) {
LOG.info("Un-publishing instant " + instantToRollback + ", deleteInstants=" + deleteInstants);
resolvedInstant = table.getActiveTimeline().revertToInflight(instantToRollback);
// reload meta-client to reflect latest timeline status
table.getMetaClient().reloadActiveTimeline();
}
List<HoodieRollbackStat> allRollbackStats = new ArrayList<>();
// deleting the timeline file
if (!resolvedInstant.isRequested()) {
LOG.info("Unpublished " + resolvedInstant);
allRollbackStats = executeRollback(instantToRollback, hoodieRollbackPlan);
}
dropBootstrapIndexIfNeeded(resolvedInstant);
// Delete Inflight instants if enabled
deleteInflightAndRequestedInstant(deleteInstants, table.getActiveTimeline(), resolvedInstant);
LOG.info("Time(in ms) taken to finish rollback " + rollbackTimer.endTimer());
return allRollbackStats;
}
use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.
the class DirectWriteMarkers method create.
private Option<Path> create(Path markerPath, boolean checkIfExists) {
HoodieTimer timer = new HoodieTimer().startTimer();
Path dirPath = markerPath.getParent();
try {
if (!fs.exists(dirPath)) {
// create a new partition as needed.
fs.mkdirs(dirPath);
}
} catch (IOException e) {
throw new HoodieIOException("Failed to make dir " + dirPath, e);
}
try {
if (checkIfExists && fs.exists(markerPath)) {
LOG.warn("Marker Path=" + markerPath + " already exists, cancel creation");
return Option.empty();
}
LOG.info("Creating Marker Path=" + markerPath);
fs.create(markerPath, false).close();
} catch (IOException e) {
throw new HoodieException("Failed to create marker file " + markerPath, e);
}
LOG.info("[direct] Created marker file " + markerPath.toString() + " in " + timer.endTimer() + " ms");
return Option.of(markerPath);
}
use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.
the class SparkDeletePartitionCommitActionExecutor method execute.
@Override
public HoodieWriteMetadata<HoodieData<WriteStatus>> execute() {
HoodieTimer timer = new HoodieTimer().startTimer();
context.setJobStatus(this.getClass().getSimpleName(), "Gather all file ids from all deleting partitions.");
Map<String, List<String>> partitionToReplaceFileIds = HoodieJavaPairRDD.getJavaPairRDD(context.parallelize(partitions).distinct().mapToPair(partitionPath -> Pair.of(partitionPath, getAllExistingFileIds(partitionPath)))).collectAsMap();
HoodieWriteMetadata<HoodieData<WriteStatus>> result = new HoodieWriteMetadata<>();
result.setPartitionToReplaceFileIds(partitionToReplaceFileIds);
result.setIndexUpdateDuration(Duration.ofMillis(timer.endTimer()));
result.setWriteStatuses(context.emptyHoodieData());
this.saveWorkloadProfileMetadataToInflight(new WorkloadProfile(Pair.of(new HashMap<>(), new WorkloadStat())), instantTime);
this.commitOnAutoCommit(result);
return result;
}
use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.
the class TimelineServerBasedWriteMarkers method create.
@Override
protected Option<Path> create(String partitionPath, String dataFileName, IOType type, boolean checkIfExists) {
HoodieTimer timer = new HoodieTimer().startTimer();
String markerFileName = getMarkerFileName(dataFileName, type);
Map<String, String> paramsMap = new HashMap<>();
paramsMap.put(MARKER_DIR_PATH_PARAM, markerDirPath.toString());
if (StringUtils.isNullOrEmpty(partitionPath)) {
paramsMap.put(MARKER_NAME_PARAM, markerFileName);
} else {
paramsMap.put(MARKER_NAME_PARAM, partitionPath + "/" + markerFileName);
}
boolean success;
try {
success = executeRequestToTimelineServer(CREATE_MARKER_URL, paramsMap, new TypeReference<Boolean>() {
}, RequestMethod.POST);
} catch (IOException e) {
throw new HoodieRemoteException("Failed to create marker file " + partitionPath + "/" + markerFileName, e);
}
LOG.info("[timeline-server-based] Created marker file " + partitionPath + "/" + markerFileName + " in " + timer.endTimer() + " ms");
if (success) {
return Option.of(new Path(FSUtils.getPartitionPath(markerDirPath, partitionPath), markerFileName));
} else {
return Option.empty();
}
}
Aggregations