Examples with ParDoPayload org.apache.beam.sdk.common.runner.v1.RunnerApi.ParDoPayload used on opensource projects

Example 11 with ParDoPayload

use of org.apache.beam.sdk.common.runner.v1.RunnerApi.ParDoPayload in project beam by apache.

the class JavaClassLookupTransformProviderTest method testClassLookupExpansionRequestConstruction.

void testClassLookupExpansionRequestConstruction(ExternalTransforms.JavaClassLookupPayload payload, Map<String, Object> fieldsToVerify) {
    Pipeline p = Pipeline.create();
    RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(p);
    ExpansionApi.ExpansionRequest request = ExpansionApi.ExpansionRequest.newBuilder().setComponents(pipelineProto.getComponents()).setTransform(RunnerApi.PTransform.newBuilder().setUniqueName(TEST_NAME).setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(getUrn(ExpansionMethods.Enum.JAVA_CLASS_LOOKUP)).setPayload(payload.toByteString()))).setNamespace(TEST_NAMESPACE).build();
    ExpansionApi.ExpansionResponse response = expansionService.expand(request);
    RunnerApi.PTransform expandedTransform = response.getTransform();
    assertEquals(TEST_NAMESPACE + TEST_NAME, expandedTransform.getUniqueName());
    assertThat(expandedTransform.getInputsCount(), Matchers.is(0));
    assertThat(expandedTransform.getOutputsCount(), Matchers.is(1));
    assertEquals(2, expandedTransform.getSubtransformsCount());
    assertEquals(2, expandedTransform.getSubtransformsCount());
    assertThat(expandedTransform.getSubtransforms(0), anyOf(containsString("MyCreateTransform"), containsString("MyParDoTransform")));
    assertThat(expandedTransform.getSubtransforms(1), anyOf(containsString("MyCreateTransform"), containsString("MyParDoTransform")));
    org.apache.beam.model.pipeline.v1.RunnerApi.PTransform userParDoTransform = null;
    for (String transformId : response.getComponents().getTransformsMap().keySet()) {
        if (transformId.contains("ParMultiDo-Dummy-")) {
            userParDoTransform = response.getComponents().getTransformsMap().get(transformId);
        }
    }
    assertNotNull(userParDoTransform);
    ParDoPayload parDoPayload = null;
    try {
        parDoPayload = ParDoPayload.parseFrom(userParDoTransform.getSpec().getPayload());
    } catch (InvalidProtocolBufferException e) {
        throw new RuntimeException(e);
    }
    assertNotNull(parDoPayload);
    DummyDoFn doFn = (DummyDoFn) ParDoTranslation.doFnWithExecutionInformationFromProto(parDoPayload.getDoFn()).getDoFn();
    System.out.println("DoFn" + doFn);
    List<String> verifiedFields = new ArrayList<>();
    if (fieldsToVerify.keySet().contains("strField1")) {
        assertEquals(doFn.strField1, fieldsToVerify.get("strField1"));
        verifiedFields.add("strField1");
    }
    if (fieldsToVerify.keySet().contains("strField2")) {
        assertEquals(doFn.strField2, fieldsToVerify.get("strField2"));
        verifiedFields.add("strField2");
    }
    if (fieldsToVerify.keySet().contains("intField1")) {
        assertEquals(doFn.intField1, fieldsToVerify.get("intField1"));
        verifiedFields.add("intField1");
    }
    if (fieldsToVerify.keySet().contains("doubleWrapperField")) {
        assertEquals(doFn.doubleWrapperField, fieldsToVerify.get("doubleWrapperField"));
        verifiedFields.add("doubleWrapperField");
    }
    if (fieldsToVerify.containsKey("complexTypeStrField")) {
        assertEquals(doFn.complexTypeField.complexTypeStrField, fieldsToVerify.get("complexTypeStrField"));
        verifiedFields.add("complexTypeStrField");
    }
    if (fieldsToVerify.containsKey("complexTypeIntField")) {
        assertEquals(doFn.complexTypeField.complexTypeIntField, fieldsToVerify.get("complexTypeIntField"));
        verifiedFields.add("complexTypeIntField");
    }
    if (fieldsToVerify.keySet().contains("strArrayField")) {
        assertArrayEquals(doFn.strArrayField, (String[]) fieldsToVerify.get("strArrayField"));
        verifiedFields.add("strArrayField");
    }
    if (fieldsToVerify.keySet().contains("strListField")) {
        assertEquals(doFn.strListField, (List) fieldsToVerify.get("strListField"));
        verifiedFields.add("strListField");
    }
    if (fieldsToVerify.keySet().contains("complexTypeArrayField")) {
        assertArrayEquals(doFn.complexTypeArrayField, (DummyComplexType[]) fieldsToVerify.get("complexTypeArrayField"));
        verifiedFields.add("complexTypeArrayField");
    }
    if (fieldsToVerify.keySet().contains("complexTypeListField")) {
        assertEquals(doFn.complexTypeListField, (List) fieldsToVerify.get("complexTypeListField"));
        verifiedFields.add("complexTypeListField");
    }
    List<String> unverifiedFields = new ArrayList<>(fieldsToVerify.keySet());
    unverifiedFields.removeAll(verifiedFields);
    if (!unverifiedFields.isEmpty()) {
        throw new RuntimeException("Failed to verify some fields: " + unverifiedFields);
    }
}

Example 12 with ParDoPayload

use of org.apache.beam.sdk.common.runner.v1.RunnerApi.ParDoPayload in project beam by apache.

the class ProcessBundleHandlerTest method setupProcessBundleHandlerForSimpleRecordingDoFn.

private ProcessBundleHandler setupProcessBundleHandlerForSimpleRecordingDoFn(List<String> dataOutput, List<Timers> timerOutput, boolean enableOutputEmbedding) throws Exception {
    DoFnWithExecutionInformation doFnWithExecutionInformation = DoFnWithExecutionInformation.of(new SimpleDoFn(), SimpleDoFn.MAIN_OUTPUT_TAG, Collections.emptyMap(), DoFnSchemaInformation.create());
    RunnerApi.FunctionSpec functionSpec = RunnerApi.FunctionSpec.newBuilder().setUrn(ParDoTranslation.CUSTOM_JAVA_DO_FN_URN).setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(doFnWithExecutionInformation))).build();
    RunnerApi.ParDoPayload parDoPayload = ParDoPayload.newBuilder().setDoFn(functionSpec).putTimerFamilySpecs("tfs-" + SimpleDoFn.TIMER_FAMILY_ID, TimerFamilySpec.newBuilder().setTimeDomain(RunnerApi.TimeDomain.Enum.EVENT_TIME).setTimerFamilyCoderId("timer-coder").build()).build();
    BeamFnApi.ProcessBundleDescriptor processBundleDescriptor = ProcessBundleDescriptor.newBuilder().putTransforms("2L", PTransform.newBuilder().setSpec(FunctionSpec.newBuilder().setUrn(DATA_INPUT_URN).build()).putOutputs("2L-output", "2L-output-pc").build()).putTransforms("3L", PTransform.newBuilder().setSpec(FunctionSpec.newBuilder().setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN).setPayload(parDoPayload.toByteString())).putInputs("3L-input", "2L-output-pc").build()).putPcollections("2L-output-pc", PCollection.newBuilder().setWindowingStrategyId("window-strategy").setCoderId("2L-output-coder").setIsBounded(IsBounded.Enum.BOUNDED).build()).putWindowingStrategies("window-strategy", WindowingStrategy.newBuilder().setWindowCoderId("window-strategy-coder").setWindowFn(FunctionSpec.newBuilder().setUrn("beam:window_fn:global_windows:v1")).setOutputTime(OutputTime.Enum.END_OF_WINDOW).setAccumulationMode(AccumulationMode.Enum.ACCUMULATING).setTrigger(Trigger.newBuilder().setAlways(Always.getDefaultInstance())).setClosingBehavior(ClosingBehavior.Enum.EMIT_ALWAYS).setOnTimeBehavior(OnTimeBehavior.Enum.FIRE_ALWAYS).build()).setTimerApiServiceDescriptor(ApiServiceDescriptor.newBuilder().setUrl("url").build()).putCoders("string_coder", CoderTranslation.toProto(StringUtf8Coder.of()).getCoder()).putCoders("2L-output-coder", Coder.newBuilder().setSpec(FunctionSpec.newBuilder().setUrn(ModelCoders.KV_CODER_URN).build()).addComponentCoderIds("string_coder").addComponentCoderIds("string_coder").build()).putCoders("window-strategy-coder", Coder.newBuilder().setSpec(FunctionSpec.newBuilder().setUrn(ModelCoders.GLOBAL_WINDOW_CODER_URN).build()).build()).putCoders("timer-coder", Coder.newBuilder().setSpec(FunctionSpec.newBuilder().setUrn(ModelCoders.TIMER_CODER_URN)).addComponentCoderIds("string_coder").addComponentCoderIds("window-strategy-coder").build()).build();
    Map<String, BeamFnApi.ProcessBundleDescriptor> fnApiRegistry = ImmutableMap.of("1L", processBundleDescriptor);
    Map<String, PTransformRunnerFactory> urnToPTransformRunnerFactoryMap = Maps.newHashMap(REGISTERED_RUNNER_FACTORIES);
    urnToPTransformRunnerFactoryMap.put(DATA_INPUT_URN, (PTransformRunnerFactory<Object>) (context) -> {
        context.addIncomingDataEndpoint(ApiServiceDescriptor.getDefaultInstance(), KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()), (input) -> {
            dataOutput.add(input.getValue());
        });
        return null;
    });
    Mockito.doAnswer((invocation) -> new BeamFnDataOutboundAggregator(PipelineOptionsFactory.create(), invocation.getArgument(1), new StreamObserver<Elements>() {

        @Override
        public void onNext(Elements elements) {
            for (Timers timer : elements.getTimersList()) {
                timerOutput.addAll(elements.getTimersList());
            }
        }

        @Override
        public void onError(Throwable throwable) {
        }

        @Override
        public void onCompleted() {
        }
    }, invocation.getArgument(2))).when(beamFnDataClient).createOutboundAggregator(any(), any(), anyBoolean());
    return new ProcessBundleHandler(PipelineOptionsFactory.create(), enableOutputEmbedding ? Collections.singleton(BeamUrns.getUrn(StandardRunnerProtocols.Enum.CONTROL_RESPONSE_ELEMENTS_EMBEDDING)) : Collections.emptySet(), fnApiRegistry::get, beamFnDataClient, null, /* beamFnStateClient */
    null, /* finalizeBundleHandler */
    new ShortIdMap(), urnToPTransformRunnerFactoryMap, Caches.noop(), new BundleProcessorCache());
}