Examples with TaskConfig org.apache.flink.runtime.operators.util.TaskConfig used on opensource projects

Example 16 with TaskConfig

use of org.apache.flink.runtime.operators.util.TaskConfig in project flink by apache.

the class CoGroupRawDriver method prepare.

@Override
public void prepare() throws Exception {
    final TaskConfig config = this.taskContext.getTaskConfig();
    if (config.getDriverStrategy() != DriverStrategy.CO_GROUP_RAW) {
        throw new Exception("Unrecognized driver strategy for CoGoup Python driver: " + config.getDriverStrategy().name());
    }
    final MutableObjectIterator<IT1> in1 = this.taskContext.getInput(0);
    final MutableObjectIterator<IT2> in2 = this.taskContext.getInput(1);
    IT1 reuse1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer().createInstance();
    IT2 reuse2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer().createInstance();
    this.coGroupIterator1 = new SimpleIterable<IT1>(reuse1, in1);
    this.coGroupIterator2 = new SimpleIterable<IT2>(reuse2, in2);
    if (LOG.isDebugEnabled()) {
        LOG.debug(this.taskContext.formatLogString("CoGroup task iterator ready."));
    }
}

Example 17 with TaskConfig

use of org.apache.flink.runtime.operators.util.TaskConfig in project flink by apache.

the class CoGroupWithSolutionSetSecondDriver method initialize.

// --------------------------------------------------------------------------------------------
@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
    final TypeComparator<IT2> solutionSetComparator;
    // grab a handle to the hash table from the iteration broker
    if (taskContext instanceof AbstractIterativeTask) {
        AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
        String identifier = iterativeTaskContext.brokerKey();
        Object table = SolutionSetBroker.instance().get(identifier);
        if (table instanceof CompactingHashTable) {
            this.hashTable = (CompactingHashTable<IT2>) table;
            solutionSetSerializer = this.hashTable.getBuildSideSerializer();
            solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
        } else if (table instanceof JoinHashMap) {
            this.objectMap = (JoinHashMap<IT2>) table;
            solutionSetSerializer = this.objectMap.getBuildSerializer();
            solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
        } else {
            throw new RuntimeException("Unrecognized solution set index: " + table);
        }
    } else {
        throw new Exception("The task context of this driver is no iterative task context.");
    }
    TaskConfig config = taskContext.getTaskConfig();
    ClassLoader classLoader = taskContext.getUserCodeClassLoader();
    TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
    this.probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
    this.probeSideComparator = probeSideComparatorFactory.createComparator();
    ExecutionConfig executionConfig = taskContext.getExecutionConfig();
    objectReuseEnabled = executionConfig.isObjectReuseEnabled();
    if (objectReuseEnabled) {
        solutionSideRecord = solutionSetSerializer.createInstance();
    }
    ;
    TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
    pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}

Example 18 with TaskConfig

use of org.apache.flink.runtime.operators.util.TaskConfig in project flink by apache.

the class CrossDriver method prepare.

@Override
public void prepare() throws Exception {
    final TaskConfig config = this.taskContext.getTaskConfig();
    final DriverStrategy ls = config.getDriverStrategy();
    switch(ls) {
        case NESTEDLOOP_BLOCKED_OUTER_FIRST:
            this.blocked = true;
            this.firstIsOuter = true;
            break;
        case NESTEDLOOP_BLOCKED_OUTER_SECOND:
            this.blocked = true;
            this.firstIsOuter = false;
            break;
        case NESTEDLOOP_STREAMED_OUTER_FIRST:
            this.blocked = false;
            this.firstIsOuter = true;
            break;
        case NESTEDLOOP_STREAMED_OUTER_SECOND:
            this.blocked = false;
            this.firstIsOuter = false;
            break;
        default:
            throw new RuntimeException("Invalid local strategy for CROSS: " + ls);
    }
    this.memManager = this.taskContext.getMemoryManager();
    final int numPages = this.memManager.computeNumberOfPages(config.getRelativeMemoryDriver());
    if (numPages < 2) {
        throw new RuntimeException("The Cross task was initialized with too little memory. " + "Cross requires at least 2 memory pages.");
    }
    // divide memory between spilling and blocking side
    if (ls == DriverStrategy.NESTEDLOOP_STREAMED_OUTER_FIRST || ls == DriverStrategy.NESTEDLOOP_STREAMED_OUTER_SECOND) {
        this.memPagesForSpillingSide = numPages;
        this.memPagesForBlockSide = 0;
    } else {
        if (numPages > 32) {
            this.memPagesForSpillingSide = 2;
        } else {
            this.memPagesForSpillingSide = 1;
        }
        this.memPagesForBlockSide = numPages - this.memPagesForSpillingSide;
    }
    ExecutionConfig executionConfig = taskContext.getExecutionConfig();
    this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();
    if (LOG.isDebugEnabled()) {
        LOG.debug("CrossDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
    }
}

Example 19 with TaskConfig

use of org.apache.flink.runtime.operators.util.TaskConfig in project flink by apache.

the class DataSinkTask method initOutputFormat.

/**
	 * Initializes the OutputFormat implementation and configuration.
	 * 
	 * @throws RuntimeException
	 *         Throws if instance of OutputFormat implementation can not be
	 *         obtained.
	 */
private void initOutputFormat() {
    ClassLoader userCodeClassLoader = getUserCodeClassLoader();
    // obtain task configuration (including stub parameters)
    Configuration taskConf = getTaskConfiguration();
    this.config = new TaskConfig(taskConf);
    try {
        this.format = config.<OutputFormat<IT>>getStubWrapper(userCodeClassLoader).getUserCodeObject(OutputFormat.class, userCodeClassLoader);
        // check if the class is a subclass, if the check is required
        if (!OutputFormat.class.isAssignableFrom(this.format.getClass())) {
            throw new RuntimeException("The class '" + this.format.getClass().getName() + "' is not a subclass of '" + OutputFormat.class.getName() + "' as is required.");
        }
    } catch (ClassCastException ccex) {
        throw new RuntimeException("The stub class is not a proper subclass of " + OutputFormat.class.getName(), ccex);
    }
    Thread thread = Thread.currentThread();
    ClassLoader original = thread.getContextClassLoader();
    // configure the stub. catch exceptions here extra, to report them as originating from the user code 
    try {
        thread.setContextClassLoader(userCodeClassLoader);
        this.format.configure(this.config.getStubParameters());
    } catch (Throwable t) {
        throw new RuntimeException("The user defined 'configure()' method in the Output Format caused an error: " + t.getMessage(), t);
    } finally {
        thread.setContextClassLoader(original);
    }
}

Example 20 with TaskConfig

use of org.apache.flink.runtime.operators.util.TaskConfig in project flink by apache.

the class AbstractOuterJoinDriver method prepare.

@Override
public void prepare() throws Exception {
    final TaskConfig config = this.taskContext.getTaskConfig();
    // obtain task manager's memory manager and I/O manager
    final MemoryManager memoryManager = this.taskContext.getMemoryManager();
    final IOManager ioManager = this.taskContext.getIOManager();
    // set up memory and I/O parameters
    final double driverMemFraction = config.getRelativeMemoryDriver();
    final DriverStrategy ls = config.getDriverStrategy();
    final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
    final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
    // get serializers and comparators
    final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
    final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
    final TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
    final TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
    final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());
    if (pairComparatorFactory == null) {
        throw new Exception("Missing pair comparator factory for outer join driver");
    }
    ExecutionConfig executionConfig = taskContext.getExecutionConfig();
    boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();
    if (LOG.isDebugEnabled()) {
        LOG.debug("Outer Join Driver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
    }
    // create and return outer join iterator according to provided local strategy.
    if (objectReuseEnabled) {
        this.outerJoinIterator = getReusingOuterJoinIterator(ls, in1, in2, serializer1, comparator1, serializer2, comparator2, pairComparatorFactory, memoryManager, ioManager, driverMemFraction);
    } else {
        this.outerJoinIterator = getNonReusingOuterJoinIterator(ls, in1, in2, serializer1, comparator1, serializer2, comparator2, pairComparatorFactory, memoryManager, ioManager, driverMemFraction);
    }
    this.outerJoinIterator.open();
    if (LOG.isDebugEnabled()) {
        LOG.debug(this.taskContext.formatLogString("outer join task iterator ready."));
    }
}