Examples with TypeSerializerFactory org.apache.flink.api.common.typeutils.TypeSerializerFactory used on opensource projects

Example 1 with TypeSerializerFactory

use of org.apache.flink.api.common.typeutils.TypeSerializerFactory in project flink by apache.

the class BroadcastVariableMaterialization method materializeVariable.

// --------------------------------------------------------------------------------------------
public void materializeVariable(MutableReader<?> reader, TypeSerializerFactory<?> serializerFactory, BatchTask<?, ?> referenceHolder) throws MaterializationExpiredException, IOException {
    Preconditions.checkNotNull(reader);
    Preconditions.checkNotNull(serializerFactory);
    Preconditions.checkNotNull(referenceHolder);
    final boolean materializer;
    // that way, other tasks can de-register (in case of failure) while materialization is happening
    synchronized (references) {
        if (disposed) {
            throw new MaterializationExpiredException();
        }
        // sanity check
        if (!references.add(referenceHolder)) {
            throw new IllegalStateException(String.format("The task %s already holds a reference to the broadcast variable %s.", referenceHolder.getEnvironment().getTaskInfo().getTaskNameWithSubtasks(), key.toString()));
        }
        materializer = references.size() == 1;
    }
    try {
        @SuppressWarnings("unchecked") final MutableReader<DeserializationDelegate<T>> typedReader = (MutableReader<DeserializationDelegate<T>>) reader;
        @SuppressWarnings("unchecked") final TypeSerializer<T> serializer = ((TypeSerializerFactory<T>) serializerFactory).getSerializer();
        final ReaderIterator<T> readerIterator = new ReaderIterator<T>(typedReader, serializer);
        if (materializer) {
            // first one, so we need to materialize;
            if (LOG.isDebugEnabled()) {
                LOG.debug("Getting Broadcast Variable (" + key + ") - First access, materializing.");
            }
            ArrayList<T> data = new ArrayList<T>();
            T element;
            while ((element = readerIterator.next()) != null) {
                data.add(element);
            }
            synchronized (materializationMonitor) {
                this.data = data;
                this.materialized = true;
                materializationMonitor.notifyAll();
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug("Materialization of Broadcast Variable (" + key + ") finished.");
            }
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Getting Broadcast Variable (" + key + ") - shared access.");
            }
            T element = serializer.createInstance();
            while ((element = readerIterator.next(element)) != null) ;
            synchronized (materializationMonitor) {
                while (!this.materialized && !disposed) {
                    materializationMonitor.wait();
                }
            }
        }
    } catch (Throwable t) {
        // in case of an exception, we need to clean up big time
        decrementReferenceIfHeld(referenceHolder);
        if (t instanceof IOException) {
            throw (IOException) t;
        } else {
            throw new IOException("Materialization of the broadcast variable failed.", t);
        }
    }
}

Example 2 with TypeSerializerFactory

use of org.apache.flink.api.common.typeutils.TypeSerializerFactory in project flink by apache.

the class IterationHeadTask method run.

@Override
public void run() throws Exception {
    final String brokerKey = brokerKey();
    final int workerIndex = getEnvironment().getTaskInfo().getIndexOfThisSubtask();
    final boolean objectSolutionSet = config.isSolutionSetUnmanaged();
    // if workset iteration
    CompactingHashTable<X> solutionSet = null;
    // if workset iteration with unmanaged solution set
    JoinHashMap<X> solutionSetObjectMap = null;
    boolean waitForSolutionSetUpdate = config.getWaitForSolutionSetUpdate();
    boolean isWorksetIteration = config.getIsWorksetIteration();
    try {
        /* used for receiving the current iteration result from iteration tail */
        SuperstepKickoffLatch nextStepKickoff = new SuperstepKickoffLatch();
        SuperstepKickoffLatchBroker.instance().handIn(brokerKey, nextStepKickoff);
        BlockingBackChannel backChannel = initBackChannel();
        SuperstepBarrier barrier = initSuperstepBarrier();
        SolutionSetUpdateBarrier solutionSetUpdateBarrier = null;
        feedbackDataInput = config.getIterationHeadPartialSolutionOrWorksetInputIndex();
        feedbackTypeSerializer = this.getInputSerializer(feedbackDataInput);
        excludeFromReset(feedbackDataInput);
        int initialSolutionSetInput;
        if (isWorksetIteration) {
            initialSolutionSetInput = config.getIterationHeadSolutionSetInputIndex();
            solutionTypeSerializer = config.getSolutionSetSerializer(getUserCodeClassLoader());
            // setup the index for the solution set
            @SuppressWarnings("unchecked") MutableObjectIterator<X> solutionSetInput = (MutableObjectIterator<X>) createInputIterator(inputReaders[initialSolutionSetInput], solutionTypeSerializer);
            // read the initial solution set
            if (objectSolutionSet) {
                solutionSetObjectMap = initJoinHashMap();
                readInitialSolutionSet(solutionSetObjectMap, solutionSetInput);
                SolutionSetBroker.instance().handIn(brokerKey, solutionSetObjectMap);
            } else {
                solutionSet = initCompactingHashTable();
                readInitialSolutionSet(solutionSet, solutionSetInput);
                SolutionSetBroker.instance().handIn(brokerKey, solutionSet);
            }
            if (waitForSolutionSetUpdate) {
                solutionSetUpdateBarrier = new SolutionSetUpdateBarrier();
                SolutionSetUpdateBarrierBroker.instance().handIn(brokerKey, solutionSetUpdateBarrier);
            }
        } else {
            // bulk iteration case
            @SuppressWarnings("unchecked") TypeSerializerFactory<X> solSer = (TypeSerializerFactory<X>) feedbackTypeSerializer;
            solutionTypeSerializer = solSer;
            // = termination Criterion tail
            if (waitForSolutionSetUpdate) {
                solutionSetUpdateBarrier = new SolutionSetUpdateBarrier();
                SolutionSetUpdateBarrierBroker.instance().handIn(brokerKey, solutionSetUpdateBarrier);
            }
        }
        // instantiate all aggregators and register them at the iteration global registry
        RuntimeAggregatorRegistry aggregatorRegistry = new RuntimeAggregatorRegistry(config.getIterationAggregators(getUserCodeClassLoader()));
        IterationAggregatorBroker.instance().handIn(brokerKey, aggregatorRegistry);
        DataInputView superstepResult = null;
        while (this.running && !terminationRequested()) {
            if (log.isInfoEnabled()) {
                log.info(formatLogString("starting iteration [" + currentIteration() + "]"));
            }
            barrier.setup();
            if (waitForSolutionSetUpdate) {
                solutionSetUpdateBarrier.setup();
            }
            if (!inFirstIteration()) {
                feedBackSuperstepResult(superstepResult);
            }
            super.run();
            // signal to connected tasks that we are done with the superstep
            sendEndOfSuperstepToAllIterationOutputs();
            if (waitForSolutionSetUpdate) {
                solutionSetUpdateBarrier.waitForSolutionSetUpdate();
            }
            // blocking call to wait for the result
            superstepResult = backChannel.getReadEndAfterSuperstepEnded();
            if (log.isInfoEnabled()) {
                log.info(formatLogString("finishing iteration [" + currentIteration() + "]"));
            }
            sendEventToSync(new WorkerDoneEvent(workerIndex, aggregatorRegistry.getAllAggregators()));
            if (log.isInfoEnabled()) {
                log.info(formatLogString("waiting for other workers in iteration [" + currentIteration() + "]"));
            }
            barrier.waitForOtherWorkers();
            if (barrier.terminationSignaled()) {
                if (log.isInfoEnabled()) {
                    log.info(formatLogString("head received termination request in iteration [" + currentIteration() + "]"));
                }
                requestTermination();
                nextStepKickoff.signalTermination();
            } else {
                incrementIterationCounter();
                String[] globalAggregateNames = barrier.getAggregatorNames();
                Value[] globalAggregates = barrier.getAggregates();
                aggregatorRegistry.updateGlobalAggregatesAndReset(globalAggregateNames, globalAggregates);
                nextStepKickoff.triggerNextSuperstep();
            }
        }
        if (log.isInfoEnabled()) {
            log.info(formatLogString("streaming out final result after [" + currentIteration() + "] iterations"));
        }
        if (isWorksetIteration) {
            if (objectSolutionSet) {
                streamSolutionSetToFinalOutput(solutionSetObjectMap);
            } else {
                streamSolutionSetToFinalOutput(solutionSet);
            }
        } else {
            streamOutFinalOutputBulk(new InputViewIterator<X>(superstepResult, this.solutionTypeSerializer.getSerializer()));
        }
        this.finalOutputCollector.close();
    } finally {
        // make sure we unregister everything from the broker:
        // - backchannel
        // - aggregator registry
        // - solution set index
        IterationAggregatorBroker.instance().remove(brokerKey);
        BlockingBackChannelBroker.instance().remove(brokerKey);
        SuperstepKickoffLatchBroker.instance().remove(brokerKey);
        SolutionSetBroker.instance().remove(brokerKey);
        SolutionSetUpdateBarrierBroker.instance().remove(brokerKey);
        if (solutionSet != null) {
            solutionSet.close();
        }
    }
}

Example 3 with TypeSerializerFactory

use of org.apache.flink.api.common.typeutils.TypeSerializerFactory in project flink by apache.

the class TaskConfig method getTypeSerializerFactory.

private <T> TypeSerializerFactory<T> getTypeSerializerFactory(String classNameKey, String parametersPrefix, ClassLoader cl) {
    // check the class name
    final String className = this.config.getString(classNameKey, null);
    if (className == null) {
        return null;
    }
    // instantiate the class
    @SuppressWarnings("unchecked") final Class<TypeSerializerFactory<T>> superClass = (Class<TypeSerializerFactory<T>>) (Class<?>) TypeSerializerFactory.class;
    final TypeSerializerFactory<T> factory;
    try {
        Class<? extends TypeSerializerFactory<T>> clazz = Class.forName(className, true, cl).asSubclass(superClass);
        factory = InstantiationUtil.instantiate(clazz, superClass);
    } catch (ClassNotFoundException cnfex) {
        throw new RuntimeException("The class '" + className + "', noted in the configuration as " + "serializer factory, could not be found. It is not part of the user code's class loader resources.");
    } catch (ClassCastException ccex) {
        throw new CorruptConfigurationException("The class noted in the configuration as the serializer factory " + "is no subclass of TypeSerializerFactory.");
    }
    // parameterize the comparator factory
    final Configuration parameters = new DelegatingConfiguration(this.config, parametersPrefix);
    try {
        factory.readParametersFromConfig(parameters, cl);
    } catch (ClassNotFoundException cnfex) {
        throw new RuntimeException("The type serializer factory could not load its parameters from the " + "configuration due to missing classes.", cnfex);
    }
    return factory;
}