Examples with Function org.apache.flink.api.common.functions.Function used on opensource projects

Example 1 with Function

use of org.apache.flink.api.common.functions.Function in project flink by apache.

the class TypeExtractor method createTypeInfoFromInputs.

private <IN1, IN2> TypeInformation<?> createTypeInfoFromInputs(TypeVariable<?> returnTypeVar, List<Type> returnTypeHierarchy, TypeInformation<IN1> in1TypeInfo, TypeInformation<IN2> in2TypeInfo) {
    Type matReturnTypeVar = materializeTypeVariable(returnTypeHierarchy, returnTypeVar);
    // variable could be resolved
    if (!(matReturnTypeVar instanceof TypeVariable)) {
        return createTypeInfoWithTypeHierarchy(returnTypeHierarchy, matReturnTypeVar, in1TypeInfo, in2TypeInfo);
    } else {
        returnTypeVar = (TypeVariable<?>) matReturnTypeVar;
    }
    // no input information exists
    if (in1TypeInfo == null && in2TypeInfo == null) {
        return null;
    }
    // create a new type hierarchy for the input
    List<Type> inputTypeHierarchy = new ArrayList<>();
    // copy the function part of the type hierarchy
    for (Type t : returnTypeHierarchy) {
        Class<?> clazz = typeToClass(t);
        if (isClassType(t) && Function.class.isAssignableFrom(clazz) && clazz != Function.class) {
            inputTypeHierarchy.add(t);
        } else {
            break;
        }
    }
    if (inputTypeHierarchy.size() == 0) {
        return null;
    }
    ParameterizedType baseClass = (ParameterizedType) inputTypeHierarchy.get(inputTypeHierarchy.size() - 1);
    TypeInformation<?> info = null;
    if (in1TypeInfo != null) {
        // find the deepest type variable that describes the type of input 1
        Type in1Type = baseClass.getActualTypeArguments()[0];
        info = createTypeInfoFromInput(returnTypeVar, new ArrayList<>(inputTypeHierarchy), in1Type, in1TypeInfo);
    }
    if (info == null && in2TypeInfo != null) {
        // find the deepest type variable that describes the type of input 2
        Type in2Type = baseClass.getActualTypeArguments()[1];
        info = createTypeInfoFromInput(returnTypeVar, new ArrayList<>(inputTypeHierarchy), in2Type, in2TypeInfo);
    }
    return info;
}

Example 2 with Function

use of org.apache.flink.api.common.functions.Function in project flink by apache.

the class TypeExtractor method createTypeInfoFromInputs.

private <IN1, IN2> TypeInformation<?> createTypeInfoFromInputs(TypeVariable<?> returnTypeVar, ArrayList<Type> returnTypeHierarchy, TypeInformation<IN1> in1TypeInfo, TypeInformation<IN2> in2TypeInfo) {
    Type matReturnTypeVar = materializeTypeVariable(returnTypeHierarchy, returnTypeVar);
    // variable could be resolved
    if (!(matReturnTypeVar instanceof TypeVariable)) {
        return createTypeInfoWithTypeHierarchy(returnTypeHierarchy, matReturnTypeVar, in1TypeInfo, in2TypeInfo);
    } else {
        returnTypeVar = (TypeVariable<?>) matReturnTypeVar;
    }
    // no input information exists
    if (in1TypeInfo == null && in2TypeInfo == null) {
        return null;
    }
    // create a new type hierarchy for the input
    ArrayList<Type> inputTypeHierarchy = new ArrayList<Type>();
    // copy the function part of the type hierarchy
    for (Type t : returnTypeHierarchy) {
        if (isClassType(t) && Function.class.isAssignableFrom(typeToClass(t)) && typeToClass(t) != Function.class) {
            inputTypeHierarchy.add(t);
        } else {
            break;
        }
    }
    ParameterizedType baseClass = (ParameterizedType) inputTypeHierarchy.get(inputTypeHierarchy.size() - 1);
    TypeInformation<?> info = null;
    if (in1TypeInfo != null) {
        // find the deepest type variable that describes the type of input 1
        Type in1Type = baseClass.getActualTypeArguments()[0];
        info = createTypeInfoFromInput(returnTypeVar, new ArrayList<Type>(inputTypeHierarchy), in1Type, in1TypeInfo);
    }
    if (info == null && in2TypeInfo != null) {
        // find the deepest type variable that describes the type of input 2
        Type in2Type = baseClass.getActualTypeArguments()[1];
        info = createTypeInfoFromInput(returnTypeVar, new ArrayList<Type>(inputTypeHierarchy), in2Type, in2TypeInfo);
    }
    if (info != null) {
        return info;
    }
    return null;
}

Example 3 with Function

use of org.apache.flink.api.common.functions.Function in project flink by apache.

the class StreamingJobGraphGenerator method configureCheckpointing.

private void configureCheckpointing() {
    CheckpointConfig cfg = streamGraph.getCheckpointConfig();
    long interval = cfg.getCheckpointInterval();
    if (interval < MINIMAL_CHECKPOINT_TIME) {
        // interval of max value means disable periodic checkpoint
        interval = Long.MAX_VALUE;
    }
    // --- configure options ---
    CheckpointRetentionPolicy retentionAfterTermination;
    if (cfg.isExternalizedCheckpointsEnabled()) {
        CheckpointConfig.ExternalizedCheckpointCleanup cleanup = cfg.getExternalizedCheckpointCleanup();
        // Sanity check
        if (cleanup == null) {
            throw new IllegalStateException("Externalized checkpoints enabled, but no cleanup mode configured.");
        }
        retentionAfterTermination = cleanup.deleteOnCancellation() ? CheckpointRetentionPolicy.RETAIN_ON_FAILURE : CheckpointRetentionPolicy.RETAIN_ON_CANCELLATION;
    } else {
        retentionAfterTermination = CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION;
    }
    // --- configure the master-side checkpoint hooks ---
    final ArrayList<MasterTriggerRestoreHook.Factory> hooks = new ArrayList<>();
    for (StreamNode node : streamGraph.getStreamNodes()) {
        if (node.getOperatorFactory() instanceof UdfStreamOperatorFactory) {
            Function f = ((UdfStreamOperatorFactory) node.getOperatorFactory()).getUserFunction();
            if (f instanceof WithMasterCheckpointHook) {
                hooks.add(new FunctionMasterCheckpointHookFactory((WithMasterCheckpointHook<?>) f));
            }
        }
    }
    // because the hooks can have user-defined code, they need to be stored as
    // eagerly serialized values
    final SerializedValue<MasterTriggerRestoreHook.Factory[]> serializedHooks;
    if (hooks.isEmpty()) {
        serializedHooks = null;
    } else {
        try {
            MasterTriggerRestoreHook.Factory[] asArray = hooks.toArray(new MasterTriggerRestoreHook.Factory[hooks.size()]);
            serializedHooks = new SerializedValue<>(asArray);
        } catch (IOException e) {
            throw new FlinkRuntimeException("Trigger/restore hook is not serializable", e);
        }
    }
    // because the state backend can have user-defined code, it needs to be stored as
    // eagerly serialized value
    final SerializedValue<StateBackend> serializedStateBackend;
    if (streamGraph.getStateBackend() == null) {
        serializedStateBackend = null;
    } else {
        try {
            serializedStateBackend = new SerializedValue<StateBackend>(streamGraph.getStateBackend());
        } catch (IOException e) {
            throw new FlinkRuntimeException("State backend is not serializable", e);
        }
    }
    // because the checkpoint storage can have user-defined code, it needs to be stored as
    // eagerly serialized value
    final SerializedValue<CheckpointStorage> serializedCheckpointStorage;
    if (streamGraph.getCheckpointStorage() == null) {
        serializedCheckpointStorage = null;
    } else {
        try {
            serializedCheckpointStorage = new SerializedValue<>(streamGraph.getCheckpointStorage());
        } catch (IOException e) {
            throw new FlinkRuntimeException("Checkpoint storage is not serializable", e);
        }
    }
    // --- done, put it all together ---
    JobCheckpointingSettings settings = new JobCheckpointingSettings(CheckpointCoordinatorConfiguration.builder().setCheckpointInterval(interval).setCheckpointTimeout(cfg.getCheckpointTimeout()).setMinPauseBetweenCheckpoints(cfg.getMinPauseBetweenCheckpoints()).setMaxConcurrentCheckpoints(cfg.getMaxConcurrentCheckpoints()).setCheckpointRetentionPolicy(retentionAfterTermination).setExactlyOnce(getCheckpointingMode(cfg) == CheckpointingMode.EXACTLY_ONCE).setTolerableCheckpointFailureNumber(cfg.getTolerableCheckpointFailureNumber()).setUnalignedCheckpointsEnabled(cfg.isUnalignedCheckpointsEnabled()).setCheckpointIdOfIgnoredInFlightData(cfg.getCheckpointIdOfIgnoredInFlightData()).setAlignedCheckpointTimeout(cfg.getAlignedCheckpointTimeout().toMillis()).setEnableCheckpointsAfterTasksFinish(streamGraph.isEnableCheckpointsAfterTasksFinish()).build(), serializedStateBackend, streamGraph.isChangelogStateBackendEnabled(), serializedCheckpointStorage, serializedHooks);
    jobGraph.setSnapshotSettings(settings);
}

Example 4 with Function

use of org.apache.flink.api.common.functions.Function in project flink by apache.

the class DataStreamTest method testWindowOperatorDescription.

/**
 * Tests that verifies window operator has different name and description.
 */
@Test
public void testWindowOperatorDescription() {
    // global window
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Long> dataStream1 = env.generateSequence(0, 0).windowAll(GlobalWindows.create()).trigger(PurgingTrigger.of(CountTrigger.of(10))).reduce(new ReduceFunction<Long>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Long reduce(Long value1, Long value2) throws Exception {
            return null;
        }
    });
    // name is simplified
    assertEquals("GlobalWindows", dataStream1.getTransformation().getName());
    // description contains detail of function:
    // TriggerWindow(GlobalWindows(), ReducingStateDescriptor{name=window-contents,
    // defaultValue=null,
    // serializer=org.apache.flink.api.common.typeutils.base.LongSerializer@6af9fcb2},
    // PurgingTrigger(CountTrigger(10)), AllWindowedStream.reduce(AllWindowedStream.java:229))
    assertTrue(dataStream1.getTransformation().getDescription().contains("PurgingTrigger"));
    // keyed window
    DataStream<Long> dataStream2 = env.generateSequence(0, 0).keyBy(value -> value).window(TumblingEventTimeWindows.of(Time.milliseconds(1000))).trigger(PurgingTrigger.of(CountTrigger.of(10))).reduce(new ReduceFunction<Long>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Long reduce(Long value1, Long value2) throws Exception {
            return null;
        }
    });
    // name is simplified
    assertEquals("TumblingEventTimeWindows", dataStream2.getTransformation().getName());
    // description contains detail of function:
    // Window(TumblingEventTimeWindows(1000), PurgingTrigger, ReduceFunction$36,
    // PassThroughWindowFunction)
    assertTrue(dataStream2.getTransformation().getDescription().contains("PurgingTrigger"));
}