Examples with PipelineOptions org.apache.beam.sdk.options.PipelineOptions used on opensource projects

Example 36 with PipelineOptions

use of org.apache.beam.sdk.options.PipelineOptions in project beam by apache.

the class SparkRuntimeContextTest method testSerializingPipelineOptionsWithCustomUserType.

@Test
public void testSerializingPipelineOptionsWithCustomUserType() throws Exception {
    PipelineOptions options = PipelineOptionsFactory.fromArgs("--jacksonIncompatible=\"testValue\"").as(JacksonIncompatibleOptions.class);
    options.setRunner(CrashingRunner.class);
    Pipeline p = Pipeline.create(options);
    SparkRuntimeContext context = new SparkRuntimeContext(p, options);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (ObjectOutputStream outputStream = new ObjectOutputStream(baos)) {
        outputStream.writeObject(context);
    }
    try (ObjectInputStream inputStream = new ObjectInputStream(new ByteArrayInputStream(baos.toByteArray()))) {
        SparkRuntimeContext copy = (SparkRuntimeContext) inputStream.readObject();
        assertEquals("testValue", copy.getPipelineOptions().as(JacksonIncompatibleOptions.class).getJacksonIncompatible().value);
    }
}

Example 37 with PipelineOptions

use of org.apache.beam.sdk.options.PipelineOptions in project beam by apache.

the class FnHarness method main.

public static void main(String[] args) throws Exception {
    System.out.format("SDK Fn Harness started%n");
    System.out.format("Logging location %s%n", System.getenv(LOGGING_API_SERVICE_DESCRIPTOR));
    System.out.format("Control location %s%n", System.getenv(CONTROL_API_SERVICE_DESCRIPTOR));
    System.out.format("Pipeline options %s%n", System.getenv(PIPELINE_OPTIONS));
    ObjectMapper objectMapper = new ObjectMapper().registerModules(ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
    PipelineOptions options = objectMapper.readValue(System.getenv(PIPELINE_OPTIONS), PipelineOptions.class);
    BeamFnApi.ApiServiceDescriptor loggingApiServiceDescriptor = getApiServiceDescriptor(LOGGING_API_SERVICE_DESCRIPTOR);
    BeamFnApi.ApiServiceDescriptor controlApiServiceDescriptor = getApiServiceDescriptor(CONTROL_API_SERVICE_DESCRIPTOR);
    main(options, loggingApiServiceDescriptor, controlApiServiceDescriptor);
}

Example 38 with PipelineOptions

use of org.apache.beam.sdk.options.PipelineOptions in project beam by apache.

the class FnHarnessTest method testLaunchFnHarnessAndTeardownCleanly.

@Test
public void testLaunchFnHarnessAndTeardownCleanly() throws Exception {
    PipelineOptions options = PipelineOptionsFactory.create();
    List<BeamFnApi.LogEntry> logEntries = new ArrayList<>();
    List<BeamFnApi.InstructionResponse> instructionResponses = new ArrayList<>();
    BeamFnLoggingGrpc.BeamFnLoggingImplBase loggingService = new BeamFnLoggingGrpc.BeamFnLoggingImplBase() {

        @Override
        public StreamObserver<BeamFnApi.LogEntry.List> logging(StreamObserver<LogControl> responseObserver) {
            return TestStreams.withOnNext((BeamFnApi.LogEntry.List entries) -> logEntries.addAll(entries.getLogEntriesList())).withOnCompleted(() -> responseObserver.onCompleted()).build();
        }
    };
    BeamFnControlGrpc.BeamFnControlImplBase controlService = new BeamFnControlGrpc.BeamFnControlImplBase() {

        @Override
        public StreamObserver<InstructionResponse> control(StreamObserver<InstructionRequest> responseObserver) {
            CountDownLatch waitForResponses = new CountDownLatch(1);
            options.as(GcsOptions.class).getExecutorService().submit(new Runnable() {

                @Override
                public void run() {
                    responseObserver.onNext(INSTRUCTION_REQUEST);
                    Uninterruptibles.awaitUninterruptibly(waitForResponses);
                    responseObserver.onCompleted();
                }
            });
            return TestStreams.withOnNext(new Consumer<BeamFnApi.InstructionResponse>() {

                @Override
                public void accept(InstructionResponse t) {
                    instructionResponses.add(t);
                    waitForResponses.countDown();
                }
            }).withOnCompleted(waitForResponses::countDown).build();
        }
    };
    Server loggingServer = ServerBuilder.forPort(0).addService(loggingService).build();
    loggingServer.start();
    try {
        Server controlServer = ServerBuilder.forPort(0).addService(controlService).build();
        controlServer.start();
        try {
            BeamFnApi.ApiServiceDescriptor loggingDescriptor = BeamFnApi.ApiServiceDescriptor.newBuilder().setId("1L").setUrl("localhost:" + loggingServer.getPort()).build();
            BeamFnApi.ApiServiceDescriptor controlDescriptor = BeamFnApi.ApiServiceDescriptor.newBuilder().setId("2L").setUrl("localhost:" + controlServer.getPort()).build();
            FnHarness.main(options, loggingDescriptor, controlDescriptor);
            assertThat(instructionResponses, contains(INSTRUCTION_RESPONSE));
        } finally {
            controlServer.shutdownNow();
        }
    } finally {
        loggingServer.shutdownNow();
    }
}

Example 39 with PipelineOptions

use of org.apache.beam.sdk.options.PipelineOptions in project beam by apache.

the class XmlSinkTest method testCreateWriter.

/**
   * An XmlWriteOperation correctly creates an XmlWriter.
   */
@Test
public void testCreateWriter() throws Exception {
    PipelineOptions options = PipelineOptionsFactory.create();
    XmlWriteOperation<Bird> writeOp = XmlIO.<Bird>write().withRecordClass(Bird.class).withRootElement(testRootElement).to(testFilePrefix).createSink().createWriteOperation();
    XmlWriter<Bird> writer = writeOp.createWriter();
    Path outputPath = new File(testFilePrefix).toPath();
    Path tempPath = new File(writer.getWriteOperation().getTemporaryDirectory().toString()).toPath();
    assertThat(tempPath.getParent(), equalTo(outputPath.getParent()));
    assertThat(tempPath.getFileName().toString(), containsString("temp-beam-"));
    assertNotNull(writer.marshaller);
}

Example 40 with PipelineOptions

use of org.apache.beam.sdk.options.PipelineOptions in project beam by apache.

the class XmlSinkTest method testXmlWriter.

/**
   * An XmlWriter correctly writes objects as Xml elements with an enclosing root element.
   */
@Test
public void testXmlWriter() throws Exception {
    PipelineOptions options = PipelineOptionsFactory.create();
    XmlWriteOperation<Bird> writeOp = XmlIO.<Bird>write().to(testFilePrefix).withRecordClass(Bird.class).withRootElement("birds").createSink().createWriteOperation();
    XmlWriter<Bird> writer = writeOp.createWriter();
    List<Bird> bundle = Lists.newArrayList(new Bird("bemused", "robin"), new Bird("evasive", "goose"));
    List<String> lines = Arrays.asList("<birds>", "<bird>", "<species>robin</species>", "<adjective>bemused</adjective>", "</bird>", "<bird>", "<species>goose</species>", "<adjective>evasive</adjective>", "</bird>", "</birds>");
    runTestWrite(writer, bundle, lines, StandardCharsets.UTF_8.name());
}