Examples with HoodieTimer org.apache.hudi.common.util.HoodieTimer used on opensource projects

Example 6 with HoodieTimer

use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.

the class MetadataCommand method create.

@CliCommand(value = "metadata create", help = "Create the Metadata Table if it does not exist")
public String create(@CliOption(key = "sparkMaster", unspecifiedDefaultValue = SparkUtil.DEFAULT_SPARK_MASTER, help = "Spark master") final String master) throws IOException {
    HoodieCLI.getTableMetaClient();
    Path metadataPath = new Path(getMetadataTableBasePath(HoodieCLI.basePath));
    try {
        FileStatus[] statuses = HoodieCLI.fs.listStatus(metadataPath);
        if (statuses.length > 0) {
            throw new RuntimeException("Metadata directory (" + metadataPath.toString() + ") not empty.");
        }
    } catch (FileNotFoundException e) {
        // Metadata directory does not exist yet
        HoodieCLI.fs.mkdirs(metadataPath);
    }
    HoodieTimer timer = new HoodieTimer().startTimer();
    HoodieWriteConfig writeConfig = getWriteConfig();
    initJavaSparkContext(Option.of(master));
    SparkHoodieBackedTableMetadataWriter.create(HoodieCLI.conf, writeConfig, new HoodieSparkEngineContext(jsc));
    return String.format("Created Metadata Table in %s (duration=%.2f secs)", metadataPath, timer.endTimer() / 1000.0);
}

Example 7 with HoodieTimer

use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.

the class MetadataCommand method listPartitions.

@CliCommand(value = "metadata list-partitions", help = "List all partitions from metadata")
public String listPartitions(@CliOption(key = "sparkMaster", unspecifiedDefaultValue = SparkUtil.DEFAULT_SPARK_MASTER, help = "Spark master") final String master) throws IOException {
    HoodieCLI.getTableMetaClient();
    initJavaSparkContext(Option.of(master));
    HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().enable(true).build();
    HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(new HoodieSparkEngineContext(jsc), config, HoodieCLI.basePath, "/tmp");
    if (!metadata.enabled()) {
        return "[ERROR] Metadata Table not enabled/initialized\n\n";
    }
    HoodieTimer timer = new HoodieTimer().startTimer();
    List<String> partitions = metadata.getAllPartitionPaths();
    LOG.debug("Took " + timer.endTimer() + " ms");
    final List<Comparable[]> rows = new ArrayList<>();
    partitions.stream().sorted(Comparator.reverseOrder()).forEach(p -> {
        Comparable[] row = new Comparable[1];
        row[0] = p;
        rows.add(row);
    });
    TableHeader header = new TableHeader().addTableHeaderField("partition");
    return HoodiePrintHelper.print(header, new HashMap<>(), "", false, Integer.MAX_VALUE, false, rows);
}

Example 8 with HoodieTimer

use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.

the class BaseTableMetadata method fetchAllPartitionPaths.

/**
 * Returns a list of all partitions.
 */
protected List<String> fetchAllPartitionPaths() {
    HoodieTimer timer = new HoodieTimer().startTimer();
    Option<HoodieRecord<HoodieMetadataPayload>> hoodieRecord = getRecordByKey(RECORDKEY_PARTITION_LIST, MetadataPartitionType.FILES.getPartitionPath());
    metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_PARTITIONS_STR, timer.endTimer()));
    List<String> partitions = Collections.emptyList();
    if (hoodieRecord.isPresent()) {
        mayBeHandleSpuriousDeletes(hoodieRecord, "\"all partitions\"");
        partitions = hoodieRecord.get().getData().getFilenames();
        // Partition-less tables have a single empty partition
        if (partitions.contains(NON_PARTITIONED_NAME)) {
            partitions.remove(NON_PARTITIONED_NAME);
            partitions.add("");
        }
    }
    LOG.info("Listed partitions from metadata: #partitions=" + partitions.size());
    return partitions;
}

Example 9 with HoodieTimer

use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.

the class BaseTableMetadata method fetchAllFilesInPartition.

/**
 * Return all the files from the partition.
 *
 * @param partitionPath The absolute path of the partition
 */
FileStatus[] fetchAllFilesInPartition(Path partitionPath) throws IOException {
    String partitionName = FSUtils.getRelativePartitionPath(new Path(dataBasePath), partitionPath);
    if (partitionName.isEmpty()) {
        partitionName = NON_PARTITIONED_NAME;
    }
    HoodieTimer timer = new HoodieTimer().startTimer();
    Option<HoodieRecord<HoodieMetadataPayload>> hoodieRecord = getRecordByKey(partitionName, MetadataPartitionType.FILES.getPartitionPath());
    metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_FILES_STR, timer.endTimer()));
    FileStatus[] statuses = {};
    if (hoodieRecord.isPresent()) {
        mayBeHandleSpuriousDeletes(hoodieRecord, partitionName);
        statuses = hoodieRecord.get().getData().getFileStatuses(hadoopConf.get(), partitionPath);
    }
    LOG.info("Listed file in partition from metadata: partition=" + partitionName + ", #files=" + statuses.length);
    return statuses;
}

Example 10 with HoodieTimer

use of org.apache.hudi.common.util.HoodieTimer in project hudi by apache.

the class CleanActionExecutor method runClean.

private HoodieCleanMetadata runClean(HoodieTable<T, I, K, O> table, HoodieInstant cleanInstant, HoodieCleanerPlan cleanerPlan) {
    ValidationUtils.checkArgument(cleanInstant.getState().equals(HoodieInstant.State.REQUESTED) || cleanInstant.getState().equals(HoodieInstant.State.INFLIGHT));
    try {
        final HoodieInstant inflightInstant;
        final HoodieTimer timer = new HoodieTimer();
        timer.startTimer();
        if (cleanInstant.isRequested()) {
            inflightInstant = table.getActiveTimeline().transitionCleanRequestedToInflight(cleanInstant, TimelineMetadataUtils.serializeCleanerPlan(cleanerPlan));
        } else {
            inflightInstant = cleanInstant;
        }
        List<HoodieCleanStat> cleanStats = clean(context, cleanerPlan);
        if (cleanStats.isEmpty()) {
            return HoodieCleanMetadata.newBuilder().build();
        }
        table.getMetaClient().reloadActiveTimeline();
        HoodieCleanMetadata metadata = CleanerUtils.convertCleanMetadata(inflightInstant.getTimestamp(), Option.of(timer.endTimer()), cleanStats);
        if (!skipLocking) {
            this.txnManager.beginTransaction(Option.empty(), Option.empty());
        }
        writeTableMetadata(metadata, inflightInstant.getTimestamp());
        table.getActiveTimeline().transitionCleanInflightToComplete(inflightInstant, TimelineMetadataUtils.serializeCleanMetadata(metadata));
        LOG.info("Marked clean started on " + inflightInstant.getTimestamp() + " as complete");
        return metadata;
    } catch (IOException e) {
        throw new HoodieIOException("Failed to clean up after commit", e);
    } finally {
        if (!skipLocking) {
            this.txnManager.endTransaction(Option.empty());
        }
    }
}