Examples with TaskConfig org.apache.flink.runtime.operators.util.TaskConfig used on opensource projects

Example 56 with TaskConfig

use of org.apache.flink.runtime.operators.util.TaskConfig in project flink by apache.

the class IterationHeadTask method initOutputs.

@Override
protected void initOutputs() throws Exception {
    // initialize the regular outputs first (the ones into the step function).
    super.initOutputs();
    // at this time, the outputs to the step function are created
    // add the outputs for the final solution
    List<RecordWriter<?>> finalOutputWriters = new ArrayList<RecordWriter<?>>();
    final TaskConfig finalOutConfig = this.config.getIterationHeadFinalOutputConfig();
    final ClassLoader userCodeClassLoader = getUserCodeClassLoader();
    this.finalOutputCollector = BatchTask.getOutputCollector(this, finalOutConfig, userCodeClassLoader, finalOutputWriters, config.getNumOutputs(), finalOutConfig.getNumOutputs());
    // sanity check the setup
    final int writersIntoStepFunction = this.eventualOutputs.size();
    final int writersIntoFinalResult = finalOutputWriters.size();
    final int syncGateIndex = this.config.getIterationHeadIndexOfSyncOutput();
    if (writersIntoStepFunction + writersIntoFinalResult != syncGateIndex) {
        throw new Exception("Error: Inconsistent head task setup - wrong mapping of output gates.");
    }
    // now, we can instantiate the sync gate
    this.toSync = new RecordWriterBuilder<>().build(getEnvironment().getWriter(syncGateIndex));
    this.toSyncPartitionId = getEnvironment().getWriter(syncGateIndex).getPartitionId();
}

Example 57 with TaskConfig

use of org.apache.flink.runtime.operators.util.TaskConfig in project flink by apache.

the class AbstractIterativeTask method initialize.

// --------------------------------------------------------------------------------------------
// Main life cycle methods that implement the iterative behavior
// --------------------------------------------------------------------------------------------
@Override
protected void initialize() throws Exception {
    super.initialize();
    // check if the driver is resettable
    if (this.driver instanceof ResettableDriver) {
        final ResettableDriver<?, ?> resDriver = (ResettableDriver<?, ?>) this.driver;
        // make sure that the according inputs are not reset
        for (int i = 0; i < resDriver.getNumberOfInputs(); i++) {
            if (resDriver.isInputResettable(i)) {
                excludeFromReset(i);
            }
        }
    }
    TaskConfig config = getLastTasksConfig();
    isWorksetIteration = config.getIsWorksetIteration();
    isWorksetUpdate = config.getIsWorksetUpdate();
    isSolutionSetUpdate = config.getIsSolutionSetUpdate();
    if (isWorksetUpdate) {
        worksetBackChannel = BlockingBackChannelBroker.instance().getAndRemove(brokerKey());
        if (isWorksetIteration) {
            worksetAggregator = getIterationAggregators().getAggregator(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME);
            if (worksetAggregator == null) {
                throw new RuntimeException("Missing workset elements count aggregator.");
            }
        }
    }
}

Example 58 with TaskConfig

use of org.apache.flink.runtime.operators.util.TaskConfig in project flink by apache.

the class IterationSynchronizationSinkTask method invoke.

// --------------------------------------------------------------------------------------------
@Override
public void invoke() throws Exception {
    this.headEventReader = new MutableRecordReader<>(getEnvironment().getInputGate(0), getEnvironment().getTaskManagerInfo().getTmpDirectories());
    TaskConfig taskConfig = new TaskConfig(getTaskConfiguration());
    // store all aggregators
    this.aggregators = new HashMap<>();
    for (AggregatorWithName<?> aggWithName : taskConfig.getIterationAggregators(getUserCodeClassLoader())) {
        aggregators.put(aggWithName.getName(), aggWithName.getAggregator());
    }
    // store the aggregator convergence criterion
    if (taskConfig.usesConvergenceCriterion()) {
        convergenceCriterion = taskConfig.getConvergenceCriterion(getUserCodeClassLoader());
        convergenceAggregatorName = taskConfig.getConvergenceCriterionAggregatorName();
        Preconditions.checkNotNull(convergenceAggregatorName);
    }
    // store the default aggregator convergence criterion
    if (taskConfig.usesImplicitConvergenceCriterion()) {
        implicitConvergenceCriterion = taskConfig.getImplicitConvergenceCriterion(getUserCodeClassLoader());
        implicitConvergenceAggregatorName = taskConfig.getImplicitConvergenceCriterionAggregatorName();
        Preconditions.checkNotNull(implicitConvergenceAggregatorName);
    }
    maxNumberOfIterations = taskConfig.getNumberOfIterations();
    // set up the event handler
    int numEventsTillEndOfSuperstep = taskConfig.getNumberOfEventsUntilInterruptInIterativeGate(0);
    eventHandler = new SyncEventHandler(numEventsTillEndOfSuperstep, aggregators, getEnvironment().getUserCodeClassLoader().asClassLoader());
    headEventReader.registerTaskEventListener(eventHandler, WorkerDoneEvent.class);
    IntValue dummy = new IntValue();
    while (!terminationRequested()) {
        if (log.isInfoEnabled()) {
            log.info(formatLogString("starting iteration [" + currentIteration + "]"));
        }
        // this call listens for events until the end-of-superstep is reached
        readHeadEventChannel(dummy);
        if (log.isInfoEnabled()) {
            log.info(formatLogString("finishing iteration [" + currentIteration + "]"));
        }
        if (checkForConvergence()) {
            if (log.isInfoEnabled()) {
                log.info(formatLogString("signaling that all workers are to terminate in iteration [" + currentIteration + "]"));
            }
            requestTermination();
            sendToAllWorkers(new TerminationEvent());
        } else {
            if (log.isInfoEnabled()) {
                log.info(formatLogString("signaling that all workers are done in iteration [" + currentIteration + "]"));
            }
            AllWorkersDoneEvent allWorkersDoneEvent = new AllWorkersDoneEvent(aggregators);
            sendToAllWorkers(allWorkersDoneEvent);
            // reset all aggregators
            for (Aggregator<?> agg : aggregators.values()) {
                agg.reset();
            }
            currentIteration++;
        }
    }
}

Example 59 with TaskConfig

use of org.apache.flink.runtime.operators.util.TaskConfig in project flink by apache.

the class ChainedAllReduceDriverTest method testMapTask.

@Test
public void testMapTask() throws Exception {
    final int keyCnt = 100;
    final int valCnt = 20;
    final double memoryFraction = 1.0;
    // environment
    initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
    mockEnv.getExecutionConfig().enableObjectReuse();
    addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
    addOutput(this.outList);
    // chained reduce config
    {
        final TaskConfig reduceConfig = new TaskConfig(new Configuration());
        // input
        reduceConfig.addInputToGroup(0);
        reduceConfig.setInputSerializer(serFact, 0);
        // output
        reduceConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
        reduceConfig.setOutputSerializer(serFact);
        // driver
        reduceConfig.setDriverStrategy(DriverStrategy.ALL_REDUCE);
        reduceConfig.setDriverComparator(compFact, 0);
        reduceConfig.setDriverComparator(compFact, 1);
        reduceConfig.setRelativeMemoryDriver(memoryFraction);
        // udf
        reduceConfig.setStubWrapper(new UserCodeClassWrapper<>(MockReduceStub.class));
        getTaskConfig().addChainedTask(ChainedAllReduceDriver.class, reduceConfig, "reduce");
    }
    // chained map+reduce
    {
        registerTask(FlatMapDriver.class, MockMapStub.class);
        BatchTask<FlatMapFunction<Record, Record>, Record> testTask = new BatchTask<>(mockEnv);
        testTask.invoke();
    }
    int sumTotal = valCnt * keyCnt * (keyCnt - 1) / 2;
    Assert.assertEquals(1, this.outList.size());
    Assert.assertEquals(sumTotal, this.outList.get(0).getField(0, IntValue.class).getValue());
}

Example 60 with TaskConfig

use of org.apache.flink.runtime.operators.util.TaskConfig in project flink by apache.

the class ChainTaskTest method testBatchTaskOutputInCloseMethod.

@Test
public void testBatchTaskOutputInCloseMethod() {
    final int numChainedTasks = 10;
    final int keyCnt = 100;
    final int valCnt = 10;
    try {
        initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
        addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
        addOutput(outList);
        registerTask(FlatMapDriver.class, MockMapStub.class);
        for (int i = 0; i < numChainedTasks; i++) {
            final TaskConfig taskConfig = new TaskConfig(new Configuration());
            taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
            taskConfig.setOutputSerializer(serFact);
            taskConfig.setStubWrapper(new UserCodeClassWrapper<>(MockDuplicateLastValueMapFunction.class));
            getTaskConfig().addChainedTask(ChainedFlatMapDriver.class, taskConfig, "chained-" + i);
        }
        final BatchTask<FlatMapFunction<Record, Record>, Record> testTask = new BatchTask<>(mockEnv);
        testTask.invoke();
        Assert.assertEquals(keyCnt * valCnt + numChainedTasks, outList.size());
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}