Search in sources :

Example 16 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class RemoteExecutionTest method testExecutionWithMultipleStages.

@Test
public void testExecutionWithMultipleStages() throws Exception {
    launchSdkHarness(PipelineOptionsFactory.create());
    Pipeline p = Pipeline.create();
    Function<String, PCollection<String>> pCollectionGenerator = suffix -> p.apply("impulse" + suffix, Impulse.create()).apply("create" + suffix, ParDo.of(new DoFn<byte[], String>() {

        @ProcessElement
        public void process(ProcessContext c) {
            try {
                c.output(CoderUtils.decodeFromByteArray(StringUtf8Coder.of(), c.element()));
            } catch (CoderException e) {
                throw new RuntimeException(e);
            }
        }
    })).setCoder(StringUtf8Coder.of()).apply(ParDo.of(new DoFn<String, String>() {

        @ProcessElement
        public void processElement(ProcessContext c) {
            c.output("stream" + suffix + c.element());
        }
    }));
    PCollection<String> input1 = pCollectionGenerator.apply("1");
    PCollection<String> input2 = pCollectionGenerator.apply("2");
    PCollection<String> outputMerged = PCollectionList.of(input1).and(input2).apply(Flatten.pCollections());
    outputMerged.apply("createKV", ParDo.of(new DoFn<String, KV<String, String>>() {

        @ProcessElement
        public void process(ProcessContext c) {
            c.output(KV.of(c.element(), ""));
        }
    })).setCoder(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of())).apply("gbk", GroupByKey.create());
    RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(p);
    FusedPipeline fused = GreedyPipelineFuser.fuse(pipelineProto);
    Set<ExecutableStage> stages = fused.getFusedStages();
    assertThat(stages.size(), equalTo(2));
    List<WindowedValue<?>> outputValues = Collections.synchronizedList(new ArrayList<>());
    for (ExecutableStage stage : stages) {
        ExecutableProcessBundleDescriptor descriptor = ProcessBundleDescriptors.fromExecutableStage(stage.toString(), stage, dataServer.getApiServiceDescriptor(), stateServer.getApiServiceDescriptor());
        BundleProcessor processor = controlClient.getProcessor(descriptor.getProcessBundleDescriptor(), descriptor.getRemoteInputDestinations(), stateDelegator);
        Map<String, Coder> remoteOutputCoders = descriptor.getRemoteOutputCoders();
        Map<String, RemoteOutputReceiver<?>> outputReceivers = new HashMap<>();
        for (Entry<String, Coder> remoteOutputCoder : remoteOutputCoders.entrySet()) {
            outputReceivers.putIfAbsent(remoteOutputCoder.getKey(), RemoteOutputReceiver.of((Coder<WindowedValue<?>>) remoteOutputCoder.getValue(), outputValues::add));
        }
        try (RemoteBundle bundle = processor.newBundle(outputReceivers, StateRequestHandler.unsupported(), BundleProgressHandler.ignored())) {
            Iterables.getOnlyElement(bundle.getInputReceivers().values()).accept(valueInGlobalWindow(CoderUtils.encodeToByteArray(StringUtf8Coder.of(), "X")));
        }
    }
    assertThat(outputValues, containsInAnyOrder(valueInGlobalWindow(KV.of("stream1X", "")), valueInGlobalWindow(KV.of("stream2X", ""))));
}
Also used : Arrays(java.util.Arrays) CoderUtils(org.apache.beam.sdk.util.CoderUtils) TimerSpecs(org.apache.beam.sdk.state.TimerSpecs) Matchers.not(org.hamcrest.Matchers.not) WindowedValue.valueInGlobalWindow(org.apache.beam.sdk.util.WindowedValue.valueInGlobalWindow) ImmutableMap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap) Metrics(org.apache.beam.sdk.metrics.Metrics) Future(java.util.concurrent.Future) GrpcDataService(org.apache.beam.runners.fnexecution.data.GrpcDataService) Map(java.util.Map) SimpleMonitoringInfoBuilder(org.apache.beam.runners.core.metrics.SimpleMonitoringInfoBuilder) GlobalWindow(org.apache.beam.sdk.transforms.windowing.GlobalWindow) ExecutableProcessBundleDescriptor(org.apache.beam.runners.fnexecution.control.ProcessBundleDescriptors.ExecutableProcessBundleDescriptor) Iterators(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterators) BagUserStateHandlerFactory(org.apache.beam.runners.fnexecution.state.StateRequestHandlers.BagUserStateHandlerFactory) KvCoder(org.apache.beam.sdk.coders.KvCoder) PTransformTranslation(org.apache.beam.runners.core.construction.PTransformTranslation) Matchers.allOf(org.hamcrest.Matchers.allOf) FnDataReceiver(org.apache.beam.sdk.fn.data.FnDataReceiver) Set(java.util.Set) BeamFnApi(org.apache.beam.model.fnexecution.v1.BeamFnApi) ExecutableStage(org.apache.beam.runners.core.construction.graph.ExecutableStage) Executors(java.util.concurrent.Executors) GrpcLoggingService(org.apache.beam.runners.fnexecution.logging.GrpcLoggingService) Serializable(java.io.Serializable) ManagedChannelFactory(org.apache.beam.sdk.fn.channel.ManagedChannelFactory) MultimapSideInputHandler(org.apache.beam.runners.fnexecution.state.StateRequestHandlers.MultimapSideInputHandler) CountDownLatch(java.util.concurrent.CountDownLatch) CoderException(org.apache.beam.sdk.coders.CoderException) CompletionStage(java.util.concurrent.CompletionStage) ProtoOverrides(org.apache.beam.runners.core.construction.graph.ProtoOverrides) Assert.assertFalse(org.junit.Assert.assertFalse) KV(org.apache.beam.sdk.values.KV) ExperimentalOptions(org.apache.beam.sdk.options.ExperimentalOptions) Duration(org.joda.time.Duration) RunWith(org.junit.runner.RunWith) Impulse(org.apache.beam.sdk.transforms.Impulse) View(org.apache.beam.sdk.transforms.View) Optional(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Optional) GrpcStateService(org.apache.beam.runners.fnexecution.state.GrpcStateService) ArrayList(java.util.ArrayList) TimerSpec(org.apache.beam.sdk.state.TimerSpec) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Pipeline(org.apache.beam.sdk.Pipeline) StateRequestHandler(org.apache.beam.runners.fnexecution.state.StateRequestHandler) RestrictionTracker(org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker) InProcessServerFactory(org.apache.beam.sdk.fn.server.InProcessServerFactory) RunnerApi(org.apache.beam.model.pipeline.v1.RunnerApi) DoFn(org.apache.beam.sdk.transforms.DoFn) Assert.assertTrue(org.junit.Assert.assertTrue) StateRequestHandlers(org.apache.beam.runners.fnexecution.state.StateRequestHandlers) Test(org.junit.Test) SingleOutput(org.apache.beam.sdk.transforms.ParDo.SingleOutput) ExecutionException(java.util.concurrent.ExecutionException) Preconditions.checkState(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState) PCollectionView(org.apache.beam.sdk.values.PCollectionView) BoundedWindow(org.apache.beam.sdk.transforms.windowing.BoundedWindow) Matcher(org.hamcrest.Matcher) TimeDomain(org.apache.beam.sdk.state.TimeDomain) Assert.assertEquals(org.junit.Assert.assertEquals) IsEmptyIterable(org.hamcrest.collection.IsEmptyIterable) StateSpec(org.apache.beam.sdk.state.StateSpec) IsIterableContainingInOrder(org.hamcrest.collection.IsIterableContainingInOrder) ScheduledFuture(java.util.concurrent.ScheduledFuture) WindowedValue(org.apache.beam.sdk.util.WindowedValue) ChannelSplit(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleSplitResponse.ChannelSplit) Urns(org.apache.beam.runners.core.metrics.MonitoringInfoConstants.Urns) GreedyPipelineFuser(org.apache.beam.runners.core.construction.graph.GreedyPipelineFuser) ExperimentalOptions.addExperiment(org.apache.beam.sdk.options.ExperimentalOptions.addExperiment) PCollectionList(org.apache.beam.sdk.values.PCollectionList) GrpcContextHeaderAccessorProvider(org.apache.beam.sdk.fn.server.GrpcContextHeaderAccessorProvider) ResetDateTimeProvider(org.apache.beam.sdk.testing.ResetDateTimeProvider) Iterables(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables) After(org.junit.After) Assert.fail(org.junit.Assert.fail) ProcessBundleSplitResponse(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleSplitResponse) ThreadFactory(java.util.concurrent.ThreadFactory) Flatten(org.apache.beam.sdk.transforms.Flatten) FusedPipeline(org.apache.beam.runners.core.construction.graph.FusedPipeline) IterableSideInputHandler(org.apache.beam.runners.fnexecution.state.StateRequestHandlers.IterableSideInputHandler) PaneInfo(org.apache.beam.sdk.transforms.windowing.PaneInfo) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) OutboundObserverFactory(org.apache.beam.sdk.fn.stream.OutboundObserverFactory) UUID(java.util.UUID) ThreadFactoryBuilder(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder) List(java.util.List) ParDo(org.apache.beam.sdk.transforms.ParDo) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) Timer(org.apache.beam.sdk.state.Timer) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Entry(java.util.Map.Entry) ProcessBundleResponse(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleResponse) FnHarness(org.apache.beam.fn.harness.FnHarness) Slf4jLogWriter(org.apache.beam.runners.fnexecution.logging.Slf4jLogWriter) DistributionData(org.apache.beam.runners.core.metrics.DistributionData) BundleProcessor(org.apache.beam.runners.fnexecution.control.SdkHarnessClient.BundleProcessor) DateTimeUtils(org.joda.time.DateTimeUtils) SideInputHandlerFactory(org.apache.beam.runners.fnexecution.state.StateRequestHandlers.SideInputHandlerFactory) Coder(org.apache.beam.sdk.coders.Coder) HashMap(java.util.HashMap) ExecutionStateSampler(org.apache.beam.runners.core.metrics.ExecutionStateSampler) PipelineTranslation(org.apache.beam.runners.core.construction.PipelineTranslation) PipelineOptionsFactory(org.apache.beam.sdk.options.PipelineOptionsFactory) Function(java.util.function.Function) ConcurrentMap(java.util.concurrent.ConcurrentMap) StringUtf8Coder(org.apache.beam.sdk.coders.StringUtf8Coder) MonitoringInfoConstants(org.apache.beam.runners.core.metrics.MonitoringInfoConstants) ByteString(org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString) PTransformNode(org.apache.beam.runners.core.construction.graph.PipelineNode.PTransformNode) TypeUrns(org.apache.beam.runners.core.metrics.MonitoringInfoConstants.TypeUrns) PipelineOptions(org.apache.beam.sdk.options.PipelineOptions) ExecutorService(java.util.concurrent.ExecutorService) ProcessBundleProgressResponse(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleProgressResponse) MonitoringInfo(org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo) GrpcFnServer(org.apache.beam.sdk.fn.server.GrpcFnServer) GroupByKey(org.apache.beam.sdk.transforms.GroupByKey) WithKeys(org.apache.beam.sdk.transforms.WithKeys) Iterator(java.util.Iterator) BigEndianLongCoder(org.apache.beam.sdk.coders.BigEndianLongCoder) Matchers(org.hamcrest.Matchers) JUnit4(org.junit.runners.JUnit4) PCollection(org.apache.beam.sdk.values.PCollection) TimeUnit(java.util.concurrent.TimeUnit) SplittableParDoExpander(org.apache.beam.runners.core.construction.graph.SplittableParDoExpander) BagState(org.apache.beam.sdk.state.BagState) StateSpecs(org.apache.beam.sdk.state.StateSpecs) Rule(org.junit.Rule) MonitoringInfoMatchers(org.apache.beam.runners.core.metrics.MonitoringInfoMatchers) Caches(org.apache.beam.fn.harness.Caches) SplitResult(org.apache.beam.sdk.transforms.splittabledofn.SplitResult) Collections(java.util.Collections) BagUserStateHandler(org.apache.beam.runners.fnexecution.state.StateRequestHandlers.BagUserStateHandler) ReadableState(org.apache.beam.sdk.state.ReadableState) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) ByteString(org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString) RunnerApi(org.apache.beam.model.pipeline.v1.RunnerApi) WindowedValue(org.apache.beam.sdk.util.WindowedValue) BundleProcessor(org.apache.beam.runners.fnexecution.control.SdkHarnessClient.BundleProcessor) ExecutableStage(org.apache.beam.runners.core.construction.graph.ExecutableStage) ExecutableProcessBundleDescriptor(org.apache.beam.runners.fnexecution.control.ProcessBundleDescriptors.ExecutableProcessBundleDescriptor) KvCoder(org.apache.beam.sdk.coders.KvCoder) Coder(org.apache.beam.sdk.coders.Coder) StringUtf8Coder(org.apache.beam.sdk.coders.StringUtf8Coder) BigEndianLongCoder(org.apache.beam.sdk.coders.BigEndianLongCoder) FusedPipeline(org.apache.beam.runners.core.construction.graph.FusedPipeline) Pipeline(org.apache.beam.sdk.Pipeline) FusedPipeline(org.apache.beam.runners.core.construction.graph.FusedPipeline) PCollection(org.apache.beam.sdk.values.PCollection) DoFn(org.apache.beam.sdk.transforms.DoFn) CoderException(org.apache.beam.sdk.coders.CoderException) Test(org.junit.Test)

Example 17 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class ConfigGeneratorTest method testDuplicateStateIdConfig.

@Test
public void testDuplicateStateIdConfig() {
    SamzaPipelineOptions options = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
    options.setJobName("TestStoreConfig");
    options.setRunner(SamzaRunner.class);
    Pipeline pipeline = Pipeline.create(options);
    pipeline.apply(Create.empty(TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.strings()))).apply(ParDo.of(new DoFn<KV<String, String>, KV<String, String>>() {

        private static final String testState = "testState";

        @StateId(testState)
        private final StateSpec<ValueState<Integer>> state = StateSpecs.value();

        @ProcessElement
        public void processElement(ProcessContext context, @StateId(testState) ValueState<Integer> state) {
            context.output(context.element());
        }
    })).apply(ParDo.of(new DoFn<KV<String, String>, Void>() {

        private static final String testState = "testState";

        @StateId(testState)
        private final StateSpec<ValueState<Integer>> state = StateSpecs.value();

        @ProcessElement
        public void processElement(ProcessContext context, @StateId(testState) ValueState<Integer> state) {
        }
    }));
    final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline);
    final ConfigBuilder configBuilder = new ConfigBuilder(options);
    assertThrows(IllegalStateException.class, () -> SamzaPipelineTranslator.createConfig(pipeline, options, idMap, configBuilder));
}
Also used : KV(org.apache.beam.sdk.values.KV) PValue(org.apache.beam.sdk.values.PValue) Pipeline(org.apache.beam.sdk.Pipeline) StateSpec(org.apache.beam.sdk.state.StateSpec) ValueState(org.apache.beam.sdk.state.ValueState) DoFn(org.apache.beam.sdk.transforms.DoFn) SamzaPipelineOptions(org.apache.beam.runners.samza.SamzaPipelineOptions) Test(org.junit.Test)

Example 18 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class DoFnSignatures method analyzeSplitRestrictionMethod.

@VisibleForTesting
static DoFnSignature.SplitRestrictionMethod analyzeSplitRestrictionMethod(ErrorReporter errors, TypeDescriptor<? extends DoFn<?, ?>> fnT, Method m, TypeDescriptor<?> inputT, TypeDescriptor<?> outputT, TypeDescriptor<?> restrictionT, FnAnalysisContext fnContext) {
    // Method is of the form:
    // @SplitRestriction
    // void splitRestriction(... parameters ...);
    errors.checkArgument(void.class.equals(m.getReturnType()), "Must return void");
    Type[] params = m.getGenericParameterTypes();
    MethodAnalysisContext methodContext = MethodAnalysisContext.create();
    TypeDescriptor<? extends BoundedWindow> windowT = getWindowType(fnT, m);
    for (int i = 0; i < params.length; ++i) {
        Parameter extraParam = analyzeExtraParameter(errors, fnContext, methodContext, ParameterDescription.of(m, i, fnT.resolveType(params[i]), Arrays.asList(m.getParameterAnnotations()[i])), inputT, restrictionT);
        if (extraParam instanceof SchemaElementParameter) {
            errors.throwIllegalArgument("Schema @%s are not supported for @%s method. Found %s, did you mean to use %s?", format(DoFn.Element.class), format(DoFn.SplitRestriction.class), format(((SchemaElementParameter) extraParam).elementT()), format(inputT));
        } else if (extraParam instanceof RestrictionParameter) {
            errors.checkArgument(restrictionT.equals(((RestrictionParameter) extraParam).restrictionT()), "Uses restriction type %s, but @%s method uses restriction type %s", format(((RestrictionParameter) extraParam).restrictionT()), format(DoFn.GetInitialRestriction.class), format(restrictionT));
        }
        methodContext.addParameter(extraParam);
    }
    for (Parameter parameter : methodContext.getExtraParameters()) {
        checkParameterOneOf(errors, parameter, ALLOWED_SPLIT_RESTRICTION_PARAMETERS);
    }
    return DoFnSignature.SplitRestrictionMethod.create(m, windowT, methodContext.getExtraParameters());
}
Also used : RestrictionParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionParameter) Type(java.lang.reflect.Type) ParameterizedType(java.lang.reflect.ParameterizedType) DoFn(org.apache.beam.sdk.transforms.DoFn) Parameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter) WatermarkEstimatorParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WatermarkEstimatorParameter) BundleFinalizerParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.BundleFinalizerParameter) PipelineOptionsParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.PipelineOptionsParameter) WindowParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WindowParameter) TimerFamilyParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.TimerFamilyParameter) TypeParameter(org.apache.beam.sdk.values.TypeParameter) RestrictionTrackerParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionTrackerParameter) TimerParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.TimerParameter) WatermarkEstimatorStateParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WatermarkEstimatorStateParameter) SchemaElementParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.SchemaElementParameter) RestrictionParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionParameter) StateParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.StateParameter) SchemaElementParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.SchemaElementParameter) VisibleForTesting(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting)

Example 19 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class DoFnSignatures method analyzeProcessElementMethod.

@VisibleForTesting
static DoFnSignature.ProcessElementMethod analyzeProcessElementMethod(ErrorReporter errors, TypeDescriptor<? extends DoFn<?, ?>> fnClass, Method m, TypeDescriptor<?> inputT, TypeDescriptor<?> outputT, FnAnalysisContext fnContext) {
    errors.checkArgument(void.class.equals(m.getReturnType()) || DoFn.ProcessContinuation.class.equals(m.getReturnType()), "Must return void or %s", format(DoFn.ProcessContinuation.class));
    MethodAnalysisContext methodContext = MethodAnalysisContext.create();
    boolean requiresStableInput = m.isAnnotationPresent(DoFn.RequiresStableInput.class);
    boolean requiresTimeSortedInput = m.isAnnotationPresent(DoFn.RequiresTimeSortedInput.class);
    TypeDescriptor<? extends BoundedWindow> windowT = getWindowType(fnClass, m);
    Type[] params = m.getGenericParameterTypes();
    for (int i = 0; i < params.length; ++i) {
        Parameter extraParam = analyzeExtraParameter(errors.forMethod(DoFn.ProcessElement.class, m), fnContext, methodContext, ParameterDescription.of(m, i, fnClass.resolveType(params[i]), Arrays.asList(m.getParameterAnnotations()[i])), inputT, outputT);
        methodContext.addParameter(extraParam);
    }
    int schemaElementIndex = 0;
    for (int i = 0; i < methodContext.getExtraParameters().size(); ++i) {
        Parameter parameter = methodContext.getExtraParameters().get(i);
        if (parameter instanceof SchemaElementParameter) {
            SchemaElementParameter schemaParameter = (SchemaElementParameter) parameter;
            schemaParameter = schemaParameter.toBuilder().setIndex(schemaElementIndex).build();
            methodContext.setParameter(i, schemaParameter);
            ++schemaElementIndex;
        }
    }
    TypeDescriptor<?> trackerT = methodContext.findParameter(RestrictionTrackerParameter.class).map(p -> p.trackerT()).orElse(null);
    TypeDescriptor<?> watermarkEstimatorT = methodContext.findParameter(WatermarkEstimatorParameter.class).map(p -> p.estimatorT()).orElse(null);
    // The allowed parameters depend on whether this DoFn is splittable
    if (trackerT != null) {
        for (Parameter parameter : methodContext.getExtraParameters()) {
            checkParameterOneOf(errors, parameter, ALLOWED_SPLITTABLE_PROCESS_ELEMENT_PARAMETERS);
        }
    } else {
        for (Parameter parameter : methodContext.getExtraParameters()) {
            checkParameterOneOf(errors, parameter, ALLOWED_NON_SPLITTABLE_PROCESS_ELEMENT_PARAMETERS);
        }
    }
    return DoFnSignature.ProcessElementMethod.create(m, methodContext.getExtraParameters(), requiresStableInput, requiresTimeSortedInput, trackerT, watermarkEstimatorT, windowT, DoFn.ProcessContinuation.class.equals(m.getReturnType()));
}
Also used : MethodWithExtraParameters(org.apache.beam.sdk.transforms.reflect.DoFnSignature.MethodWithExtraParameters) StateSpec(org.apache.beam.sdk.state.StateSpec) Arrays(java.util.Arrays) StateDeclaration(org.apache.beam.sdk.transforms.reflect.DoFnSignature.StateDeclaration) Parameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter) ValueState(org.apache.beam.sdk.state.ValueState) ImmutableMap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap) TruncateResult(org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker.TruncateResult) WatermarkEstimatorParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WatermarkEstimatorParameter) HasDefaultWatermarkEstimator(org.apache.beam.sdk.transforms.splittabledofn.HasDefaultWatermarkEstimator) Map(java.util.Map) OutputReceiver(org.apache.beam.sdk.transforms.DoFn.OutputReceiver) Predicates(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Predicates) BundleFinalizerParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.BundleFinalizerParameter) Method(java.lang.reflect.Method) Internal(org.apache.beam.sdk.annotations.Internal) MultiOutputReceiver(org.apache.beam.sdk.transforms.DoFn.MultiOutputReceiver) GetInitialRestrictionMethod(org.apache.beam.sdk.transforms.reflect.DoFnSignature.GetInitialRestrictionMethod) TruncateRestriction(org.apache.beam.sdk.transforms.DoFn.TruncateRestriction) OrderedListState(org.apache.beam.sdk.state.OrderedListState) PaneInfo(org.apache.beam.sdk.transforms.windowing.PaneInfo) Collection(java.util.Collection) SideInput(org.apache.beam.sdk.transforms.DoFn.SideInput) Collectors(java.util.stream.Collectors) List(java.util.List) WatermarkHoldState(org.apache.beam.sdk.state.WatermarkHoldState) PipelineOptionsParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.PipelineOptionsParameter) Type(java.lang.reflect.Type) Timer(org.apache.beam.sdk.state.Timer) VisibleForTesting(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting) Modifier(java.lang.reflect.Modifier) ReflectHelpers(org.apache.beam.sdk.util.common.ReflectHelpers) TypeDescriptors(org.apache.beam.sdk.values.TypeDescriptors) AutoValue(com.google.auto.value.AutoValue) Annotation(java.lang.annotation.Annotation) Optional(java.util.Optional) HasDefaultTracker(org.apache.beam.sdk.transforms.splittabledofn.HasDefaultTracker) ImmutableList(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList) FieldAccessDeclaration(org.apache.beam.sdk.transforms.reflect.DoFnSignature.FieldAccessDeclaration) KV(org.apache.beam.sdk.values.KV) TypeDescriptor(org.apache.beam.sdk.values.TypeDescriptor) TimerMap(org.apache.beam.sdk.state.TimerMap) SetState(org.apache.beam.sdk.state.SetState) WindowParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WindowParameter) TimerFamilyParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.TimerFamilyParameter) Coder(org.apache.beam.sdk.coders.Coder) WatermarkEstimator(org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimator) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) TypeParameter(org.apache.beam.sdk.values.TypeParameter) LinkedHashMap(java.util.LinkedHashMap) MapState(org.apache.beam.sdk.state.MapState) TimerSpec(org.apache.beam.sdk.state.TimerSpec) RestrictionTrackerParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionTrackerParameter) TimerParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.TimerParameter) WatermarkEstimatorStateParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WatermarkEstimatorStateParameter) SchemaElementParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.SchemaElementParameter) FieldAccessDescriptor(org.apache.beam.sdk.schemas.FieldAccessDescriptor) FormatMethod(com.google.errorprone.annotations.FormatMethod) FormatString(com.google.errorprone.annotations.FormatString) Maps(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps) RestrictionTracker(org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker) Row(org.apache.beam.sdk.values.Row) PipelineOptions(org.apache.beam.sdk.options.PipelineOptions) Nullable(org.checkerframework.checker.nullness.qual.Nullable) DoFn(org.apache.beam.sdk.transforms.DoFn) GetInitialWatermarkEstimatorStateMethod(org.apache.beam.sdk.transforms.reflect.DoFnSignature.GetInitialWatermarkEstimatorStateMethod) State(org.apache.beam.sdk.state.State) TimerFamilyDeclaration(org.apache.beam.sdk.transforms.reflect.DoFnSignature.TimerFamilyDeclaration) Field(java.lang.reflect.Field) ManualWatermarkEstimator(org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator) PCollection(org.apache.beam.sdk.values.PCollection) StateId(org.apache.beam.sdk.transforms.DoFn.StateId) RestrictionParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionParameter) BagState(org.apache.beam.sdk.state.BagState) StateParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.StateParameter) TimerDeclaration(org.apache.beam.sdk.transforms.reflect.DoFnSignature.TimerDeclaration) ParameterizedType(java.lang.reflect.ParameterizedType) Preconditions.checkState(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState) TimerId(org.apache.beam.sdk.transforms.DoFn.TimerId) BoundedWindow(org.apache.beam.sdk.transforms.windowing.BoundedWindow) Instant(org.joda.time.Instant) Collections(java.util.Collections) TimeDomain(org.apache.beam.sdk.state.TimeDomain) ReadableState(org.apache.beam.sdk.state.ReadableState) Type(java.lang.reflect.Type) ParameterizedType(java.lang.reflect.ParameterizedType) DoFn(org.apache.beam.sdk.transforms.DoFn) Parameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter) WatermarkEstimatorParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WatermarkEstimatorParameter) BundleFinalizerParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.BundleFinalizerParameter) PipelineOptionsParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.PipelineOptionsParameter) WindowParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WindowParameter) TimerFamilyParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.TimerFamilyParameter) TypeParameter(org.apache.beam.sdk.values.TypeParameter) RestrictionTrackerParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionTrackerParameter) TimerParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.TimerParameter) WatermarkEstimatorStateParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WatermarkEstimatorStateParameter) SchemaElementParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.SchemaElementParameter) RestrictionParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionParameter) StateParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.StateParameter) SchemaElementParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.SchemaElementParameter) VisibleForTesting(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting)

Example 20 with DoFn

use of org.apache.beam.sdk.transforms.DoFn in project beam by apache.

the class DoFnSignatures method analyzeOnTimerFamilyMethod.

@VisibleForTesting
static DoFnSignature.OnTimerFamilyMethod analyzeOnTimerFamilyMethod(ErrorReporter errors, TypeDescriptor<? extends DoFn<?, ?>> fnClass, Method m, String timerFamilyId, TypeDescriptor<?> inputT, TypeDescriptor<?> outputT, FnAnalysisContext fnContext) {
    errors.checkArgument(void.class.equals(m.getReturnType()), "Must return void");
    Type[] params = m.getGenericParameterTypes();
    MethodAnalysisContext methodContext = MethodAnalysisContext.create();
    boolean requiresStableInput = m.isAnnotationPresent(DoFn.RequiresStableInput.class);
    @Nullable TypeDescriptor<? extends BoundedWindow> windowT = getWindowType(fnClass, m);
    List<DoFnSignature.Parameter> extraParameters = new ArrayList<>();
    ErrorReporter onTimerErrors = errors.forMethod(DoFn.OnTimerFamily.class, m);
    for (int i = 0; i < params.length; ++i) {
        Parameter parameter = analyzeExtraParameter(onTimerErrors, fnContext, methodContext, ParameterDescription.of(m, i, fnClass.resolveType(params[i]), Arrays.asList(m.getParameterAnnotations()[i])), inputT, outputT);
        checkParameterOneOf(errors, parameter, ALLOWED_ON_TIMER_FAMILY_PARAMETERS);
        extraParameters.add(parameter);
    }
    return DoFnSignature.OnTimerFamilyMethod.create(m, timerFamilyId, requiresStableInput, windowT, extraParameters);
}
Also used : Type(java.lang.reflect.Type) ParameterizedType(java.lang.reflect.ParameterizedType) DoFn(org.apache.beam.sdk.transforms.DoFn) ArrayList(java.util.ArrayList) Parameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter) WatermarkEstimatorParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WatermarkEstimatorParameter) BundleFinalizerParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.BundleFinalizerParameter) PipelineOptionsParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.PipelineOptionsParameter) WindowParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WindowParameter) TimerFamilyParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.TimerFamilyParameter) TypeParameter(org.apache.beam.sdk.values.TypeParameter) RestrictionTrackerParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionTrackerParameter) TimerParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.TimerParameter) WatermarkEstimatorStateParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WatermarkEstimatorStateParameter) SchemaElementParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.SchemaElementParameter) RestrictionParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionParameter) StateParameter(org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.StateParameter) Nullable(org.checkerframework.checker.nullness.qual.Nullable) VisibleForTesting(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting)

Aggregations

DoFn (org.apache.beam.sdk.transforms.DoFn)154 Test (org.junit.Test)98 Pipeline (org.apache.beam.sdk.Pipeline)60 KV (org.apache.beam.sdk.values.KV)45 TupleTag (org.apache.beam.sdk.values.TupleTag)28 StateSpec (org.apache.beam.sdk.state.StateSpec)26 Instant (org.joda.time.Instant)26 ArrayList (java.util.ArrayList)23 TestPipeline (org.apache.beam.sdk.testing.TestPipeline)23 BoundedWindow (org.apache.beam.sdk.transforms.windowing.BoundedWindow)22 PCollection (org.apache.beam.sdk.values.PCollection)21 TimerSpec (org.apache.beam.sdk.state.TimerSpec)19 WindowedValue (org.apache.beam.sdk.util.WindowedValue)18 PCollectionView (org.apache.beam.sdk.values.PCollectionView)18 HashMap (java.util.HashMap)17 Coder (org.apache.beam.sdk.coders.Coder)17 List (java.util.List)16 Map (java.util.Map)14 ValueState (org.apache.beam.sdk.state.ValueState)14 RunnerApi (org.apache.beam.model.pipeline.v1.RunnerApi)13