Examples with DatePartitionType org.apache.gobblin.util.DatePartitionType used on opensource projects

Example 1 with DatePartitionType

use of org.apache.gobblin.util.DatePartitionType in project incubator-gobblin by apache.

the class DatePartitionedNestedRetriever method initDatePartitionFromGranularity.

private void initDatePartitionFromGranularity(State state) {
    String granularityProp = state.getProp(PartitionedFileSourceBase.DATE_PARTITIONED_SOURCE_PARTITION_GRANULARITY);
    DatePartitionType partitionType = null;
    if (granularityProp == null) {
        partitionType = PartitionedFileSourceBase.DEFAULT_DATE_PARTITIONED_SOURCE_PARTITION_GRANULARITY;
    } else {
        Optional<DatePartitionType> partitionTypeOpt = Enums.getIfPresent(DatePartitionType.class, granularityProp.toUpperCase());
        Preconditions.checkState(partitionTypeOpt.isPresent(), "Invalid source partition granularity: " + granularityProp);
        partitionType = partitionTypeOpt.get();
    }
    this.partitionPatternFormatter = DateTimeFormat.forPattern(partitionType.getDateTimePattern());
    this.incrementalUnit = partitionType.getDateTimeFieldType().getDurationType();
}

Example 2 with DatePartitionType

use of org.apache.gobblin.util.DatePartitionType in project incubator-gobblin by apache.

the class PartitionAwareFileRetrieverUtils method getLeadTimeDurationFromConfig.

/**
 * Retrieve the lead time duration from the LEAD_TIME and LEAD_TIME granularity config settings.
 */
public static Duration getLeadTimeDurationFromConfig(State state) {
    String leadTimeProp = state.getProp(DATE_PARTITIONED_SOURCE_PARTITION_LEAD_TIME);
    if (leadTimeProp == null || leadTimeProp.length() == 0) {
        return DEFAULT_PARTITIONED_SOURCE_PARTITION_LEAD_TIME;
    }
    int leadTime = Integer.parseInt(leadTimeProp);
    DatePartitionType leadTimeGranularity = DEFAULT_DATE_PARTITIONED_SOURCE_PARTITION_LEAD_TIME_GRANULARITY;
    String leadTimeGranularityProp = state.getProp(DATE_PARTITIONED_SOURCE_PARTITION_LEAD_TIME_GRANULARITY);
    if (leadTimeGranularityProp != null) {
        leadTimeGranularity = DatePartitionType.valueOf(leadTimeGranularityProp);
    }
    return new Duration(leadTime * leadTimeGranularity.getUnitMilliseconds());
}