Examples with ResourceEstimator org.apache.gobblin.util.request_allocation.ResourceEstimator used on opensource projects

Example 1 with ResourceEstimator

use of org.apache.gobblin.util.request_allocation.ResourceEstimator in project incubator-gobblin by apache.

the class CompactionSource method initRequestAllocator.

private void initRequestAllocator(State state) {
    try {
        ResourceEstimator estimator = GobblinConstructorUtils.<ResourceEstimator>invokeLongestConstructor(new ClassAliasResolver(ResourceEstimator.class).resolveClass(state.getProp(ConfigurationKeys.COMPACTION_ESTIMATOR, SimpleDatasetRequest.SimpleDatasetCountEstimator.class.getName())));
        RequestAllocatorConfig.Builder<SimpleDatasetRequest> configBuilder = RequestAllocatorConfig.builder(estimator).allowParallelization(1).withLimitedScopeConfig(ConfigBuilder.create().loadProps(state.getProperties(), ConfigurationKeys.COMPACTION_PRIORITIZATION_PREFIX).build());
        if (!state.contains(ConfigurationKeys.COMPACTION_PRIORITIZER_ALIAS)) {
            allocator = new GreedyAllocator<>(configBuilder.build());
            return;
        }
        Comparator<SimpleDatasetRequest> prioritizer = GobblinConstructorUtils.<Comparator>invokeLongestConstructor(new ClassAliasResolver(Comparator.class).resolveClass(state.getProp(ConfigurationKeys.COMPACTION_PRIORITIZER_ALIAS)), state);
        configBuilder.withPrioritizer(prioritizer);
        if (prioritizer instanceof HierarchicalPrioritizer) {
            allocator = new HierarchicalAllocator.Factory().createRequestAllocator(configBuilder.build());
        } else {
            allocator = RequestAllocatorUtils.inferFromConfig(configBuilder.build());
        }
    } catch (RuntimeException e) {
        throw e;
    } catch (Exception e) {
        throw new RuntimeException("Cannot initialize allocator", e);
    }
}