Examples with PipelineResult org.apache.beam.sdk.PipelineResult used on opensource projects

Example 6 with PipelineResult

use of org.apache.beam.sdk.PipelineResult in project beam by apache.

the class MetricsTest method testAllAttemptedMetrics.

@Category({ ValidatesRunner.class, UsesAttemptedMetrics.class, UsesCounterMetrics.class, UsesDistributionMetrics.class, UsesGaugeMetrics.class })
@Test
public void testAllAttemptedMetrics() {
    PipelineResult result = runPipelineWithMetrics();
    MetricQueryResults metrics = queryTestMetrics(result);
    // TODO: BEAM-1169: Metrics shouldn't verify the physical values tightly.
    assertAllMetrics(metrics, false);
}

Example 7 with PipelineResult

use of org.apache.beam.sdk.PipelineResult in project beam by apache.

the class MetricsTest method testAttemptedCounterMetrics.

@Category({ ValidatesRunner.class, UsesAttemptedMetrics.class, UsesCounterMetrics.class })
@Test
public void testAttemptedCounterMetrics() {
    PipelineResult result = runPipelineWithMetrics();
    MetricQueryResults metrics = queryTestMetrics(result);
    assertCounterMetrics(metrics, false);
}

Example 8 with PipelineResult

use of org.apache.beam.sdk.PipelineResult in project beam by apache.

the class MetricsTest method runPipelineWithMetrics.

private PipelineResult runPipelineWithMetrics() {
    final Counter count = Metrics.counter(MetricsTest.class, "count");
    final TupleTag<Integer> output1 = new TupleTag<Integer>() {
    };
    final TupleTag<Integer> output2 = new TupleTag<Integer>() {
    };
    pipeline.apply(Create.of(5, 8, 13)).apply("MyStep1", ParDo.of(new DoFn<Integer, Integer>() {

        Distribution bundleDist = Metrics.distribution(MetricsTest.class, "bundle");

        @StartBundle
        public void startBundle() {
            bundleDist.update(10L);
        }

        @SuppressWarnings("unused")
        @ProcessElement
        public void processElement(ProcessContext c) {
            Distribution values = Metrics.distribution(MetricsTest.class, "input");
            count.inc();
            values.update(c.element());
            c.output(c.element());
            c.output(c.element());
        }

        @DoFn.FinishBundle
        public void finishBundle() {
            bundleDist.update(40L);
        }
    })).apply("MyStep2", ParDo.of(new DoFn<Integer, Integer>() {

        @SuppressWarnings("unused")
        @ProcessElement
        public void processElement(ProcessContext c) {
            Distribution values = Metrics.distribution(MetricsTest.class, "input");
            Gauge gauge = Metrics.gauge(MetricsTest.class, "my-gauge");
            Integer element = c.element();
            count.inc();
            values.update(element);
            gauge.set(12L);
            c.output(element);
            c.output(output2, element);
        }
    }).withOutputTags(output1, TupleTagList.of(output2)));
    PipelineResult result = pipeline.run();
    result.waitUntilFinish();
    return result;
}

Example 9 with PipelineResult

use of org.apache.beam.sdk.PipelineResult in project beam by apache.

the class MetricsTest method testAttemptedGaugeMetrics.

@Category({ ValidatesRunner.class, UsesAttemptedMetrics.class, UsesGaugeMetrics.class })
@Test
public void testAttemptedGaugeMetrics() {
    PipelineResult result = runPipelineWithMetrics();
    MetricQueryResults metrics = queryTestMetrics(result);
    assertGaugeMetrics(metrics, false);
}

Example 10 with PipelineResult

use of org.apache.beam.sdk.PipelineResult in project components by Talend.

the class JDBCBeamRuntimeTest method testPipeline.

@Test
public void testPipeline() throws Exception {
    JDBCDatastoreProperties jdbcDatastoreProperties;
    jdbcDatastoreProperties = new JDBCDatastoreProperties("datastore");
    jdbcDatastoreProperties.init();
    jdbcDatastoreProperties.dbTypes.setValue("DERBY");
    jdbcDatastoreProperties.jdbcUrl.setValue(JDBC_URL);
    JDBCDatasetProperties inputDatasetProperties = new JDBCDatasetProperties("inputDataset");
    inputDatasetProperties.init();
    inputDatasetProperties.setDatastoreProperties(jdbcDatastoreProperties);
    inputDatasetProperties.sql.setValue("select * from " + TABLE_IN);
    // Schema inputSchema =
    // SchemaBuilder.record("DYNAMIC").fields().name("ID").type(SchemaBuilder.builder().stringType())
    // .noDefault().name("NAME").type(SchemaBuilder.builder().stringType()).noDefault().endRecord();
    // inputDatasetProperties.main.schema.setValue(inputSchema);
    JDBCInputProperties inputProperties = new JDBCInputProperties("input");
    inputProperties.init();
    inputProperties.setDatasetProperties(inputDatasetProperties);
    JDBCDatasetProperties outputDatasetProperties = new JDBCDatasetProperties("outputDataset");
    outputDatasetProperties.init();
    outputDatasetProperties.setDatastoreProperties(jdbcDatastoreProperties);
    outputDatasetProperties.sourceType.setValue(JDBCDatasetProperties.SourceType.TABLE_NAME);
    outputDatasetProperties.tableName.setValue(TABLE_OUT);
    Schema outputSchema = SchemaBuilder.record("DYNAMIC").fields().name("ID").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "ID").type(SchemaBuilder.builder().intType()).noDefault().name("NAME").prop(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, "NAME").type(SchemaBuilder.builder().stringType()).noDefault().endRecord();
    outputDatasetProperties.main.schema.setValue(outputSchema);
    JDBCOutputProperties outputProperties = new JDBCOutputProperties("output");
    outputProperties.init();
    outputProperties.setDatasetProperties(outputDatasetProperties);
    outputProperties.dataAction.setValue(JDBCOutputProperties.DataAction.INSERT);
    JDBCInputPTransformRuntime inputRuntime = new JDBCInputPTransformRuntime();
    inputRuntime.initialize(null, inputProperties);
    JDBCOutputPTransformRuntime outputRuntime = new JDBCOutputPTransformRuntime();
    outputRuntime.initialize(null, outputProperties);
    pipeline.apply(inputRuntime).apply(outputRuntime);
    PipelineResult pipelineResult = pipeline.run();
    Map<Integer, String> results = new HashMap<>();
    try (Connection connection = dataSource.getConnection()) {
        try (Statement statement = connection.createStatement()) {
            try (ResultSet resultSet = statement.executeQuery("select * from " + TABLE_OUT)) {
                while (resultSet.next()) {
                    results.put(resultSet.getInt(1), resultSet.getString(2));
                }
            }
        }
    }
    assertEquals(assertRows, results);
}