Examples with State org.apache.beam.sdk.PipelineResult.State used on opensource projects

Example 11 with State

use of org.apache.beam.sdk.PipelineResult.State in project beam by apache.

the class TpcdsRun method call.

@Override
public TpcdsRunResult call() {
    TpcdsRunResult tpcdsRunResult;
    try {
        PipelineResult pipelineResult = pipeline.run();
        long startTimeStamp = System.currentTimeMillis();
        State state = pipelineResult.waitUntilFinish();
        long endTimeStamp = System.currentTimeMillis();
        // Make sure to set the job status to be successful only when pipelineResult's final state is
        // DONE.
        boolean isSuccessful = state == State.DONE;
        tpcdsRunResult = new TpcdsRunResult(isSuccessful, startTimeStamp, endTimeStamp, pipeline.getOptions(), pipelineResult);
    } catch (Exception e) {
        // If the pipeline execution failed, return a result with failed status but don't interrupt
        // other threads.
        e.printStackTrace();
        tpcdsRunResult = new TpcdsRunResult(false, 0, 0, pipeline.getOptions(), null);
    }
    return tpcdsRunResult;
}

Example 12 with State

use of org.apache.beam.sdk.PipelineResult.State in project beam by apache.

the class DataStoreReadWriteIT method testReadAllSupportedTypes.

@Test
public void testReadAllSupportedTypes() {
    BeamSqlEnv sqlEnv = BeamSqlEnv.inMemory(new DataStoreV1TableProvider());
    String projectId = options.getProject();
    final Schema expectedSchema = Schema.builder().addNullableField("__key__", VARBINARY).addNullableField("boolean", BOOLEAN).addNullableField("datetime", DATETIME).addNullableField("floatingnumber", DOUBLE).addNullableField("integer", INT64).addNullableField("primitivearray", FieldType.array(STRING)).addNullableField("string", STRING).addNullableField("text", STRING).build();
    String createTableStatement = "CREATE EXTERNAL TABLE TEST( \n" + "   `__key__` VARBINARY, \n" + "   `boolean` BOOLEAN, \n" + "   `datetime` TIMESTAMP, \n" + // + "   `embeddedentity` ROW(`property1` VARCHAR, `property2` BIGINT), \n"
    "   `floatingnumber` DOUBLE, \n" + "   `integer` BIGINT, \n" + "   `primitivearray` ARRAY<VARCHAR>, \n" + "   `string` VARCHAR, \n" + "   `text` VARCHAR" + ") \n" + "TYPE 'datastoreV1' \n" + "LOCATION '" + projectId + "/" + KIND_ALL_TYPES + "'";
    sqlEnv.executeDdl(createTableStatement);
    String selectTableStatement = "SELECT * FROM TEST";
    PCollection<Row> output = BeamSqlRelUtils.toPCollection(readPipeline, sqlEnv.parseQuery(selectTableStatement));
    assertThat(output.getSchema(), equalTo(expectedSchema));
    PipelineResult.State state = readPipeline.run().waitUntilFinish(Duration.standardMinutes(5));
    assertThat(state, equalTo(State.DONE));
}

Example 13 with State

use of org.apache.beam.sdk.PipelineResult.State in project beam by apache.

the class DataStoreReadWriteIT method testDataStoreV1SqlWriteRead.

@Test
public void testDataStoreV1SqlWriteRead() {
    BeamSqlEnv sqlEnv = BeamSqlEnv.inMemory(new DataStoreV1TableProvider());
    String projectId = options.getProject();
    String createTableStatement = "CREATE EXTERNAL TABLE TEST( \n" + "   `__key__` VARBINARY, \n" + "   `content` VARCHAR \n" + ") \n" + "TYPE 'datastoreV1' \n" + "LOCATION '" + projectId + "/" + KIND + "'";
    sqlEnv.executeDdl(createTableStatement);
    Key ancestor = makeKey(KIND, UUID.randomUUID().toString()).build();
    Key itemKey = makeKey(ancestor, KIND, UUID.randomUUID().toString()).build();
    String insertStatement = "INSERT INTO TEST VALUES ( \n" + keyToSqlByteString(itemKey) + ", \n" + "'2000' \n" + ")";
    BeamSqlRelUtils.toPCollection(writePipeline, sqlEnv.parseQuery(insertStatement));
    writePipeline.run().waitUntilFinish();
    String selectTableStatement = "SELECT * FROM TEST";
    PCollection<Row> output = BeamSqlRelUtils.toPCollection(readPipeline, sqlEnv.parseQuery(selectTableStatement));
    assertThat(output.getSchema(), equalTo(SOURCE_SCHEMA));
    PipelineResult.State state = readPipeline.run().waitUntilFinish(Duration.standardMinutes(5));
    assertThat(state, equalTo(State.DONE));
}

Example 14 with State

use of org.apache.beam.sdk.PipelineResult.State in project beam by apache.

the class AvroTableProviderTest method testWriteAndReadTable.

@Test
public void testWriteAndReadTable() {
    File destinationFile = new File(tempFolder.getRoot(), "person-info.avro");
    String query = "INSERT INTO PersonInfo VALUES ('Alan', 22, 'England'), ('John', 42, 'USA')";
    String ddl = String.format("CREATE EXTERNAL TABLE PersonInfo %s TYPE avro LOCATION '%s'", FIELD_NAMES, destinationFile.getAbsolutePath());
    writePipeline.apply(SqlTransform.query(query).withDdlString(ddl));
    writePipeline.run().waitUntilFinish();
    String readQuery = "SELECT age, country FROM PersonInfo WHERE age > 25";
    PCollection<Row> rows = readPipeline.apply(SqlTransform.query(readQuery).withDdlString(ddl));
    PAssert.that(rows).containsInAnyOrder(Row.withSchema(OUTPUT_ROW_SCHEMA).addValues(42L, "USA").build());
    PipelineResult.State state = readPipeline.run().waitUntilFinish();
    assertEquals(State.DONE, state);
}

Example 15 with State

use of org.apache.beam.sdk.PipelineResult.State in project beam by apache.

the class BigQueryReadWriteIT method testSQLRead_withExport.

@Test
public void testSQLRead_withExport() throws IOException {
    bigQueryTestingTypes.insertRows(SOURCE_SCHEMA_TWO, row(SOURCE_SCHEMA_TWO, 9223372036854775807L, (byte) 127, (short) 32767, 2147483647, (float) 1.0, 1.0, true, parseTimestampWithUTCTimeZone("2018-05-28 20:17:40.123"), "varchar", "char", Arrays.asList("123", "456")));
    BeamSqlEnv sqlEnv = BeamSqlEnv.inMemory(new BigQueryTableProvider());
    String createTableStatement = "CREATE EXTERNAL TABLE TEST( \n" + "   c_bigint BIGINT, \n" + "   c_tinyint TINYINT, \n" + "   c_smallint SMALLINT, \n" + "   c_integer INTEGER, \n" + "   c_float FLOAT, \n" + "   c_double DOUBLE, \n" + "   c_boolean BOOLEAN, \n" + "   c_timestamp TIMESTAMP, \n" + "   c_varchar VARCHAR, \n " + "   c_char CHAR, \n" + "   c_arr ARRAY<VARCHAR> \n" + ") \n" + "TYPE 'bigquery' \n" + "LOCATION '" + bigQueryTestingTypes.tableSpec() + "'" + "TBLPROPERTIES " + "'{ " + METHOD_PROPERTY + ": \"" + Method.EXPORT.toString() + "\" }'";
    sqlEnv.executeDdl(createTableStatement);
    String selectTableStatement = "SELECT * FROM TEST";
    PCollection<Row> output = BeamSqlRelUtils.toPCollection(readPipeline, sqlEnv.parseQuery(selectTableStatement));
    PAssert.that(output).containsInAnyOrder(row(SOURCE_SCHEMA_TWO, 9223372036854775807L, (byte) 127, (short) 32767, 2147483647, (float) 1.0, 1.0, true, parseTimestampWithUTCTimeZone("2018-05-28 20:17:40.123"), "varchar", "char", Arrays.asList("123", "456")));
    PipelineResult.State state = readPipeline.run().waitUntilFinish(Duration.standardMinutes(5));
    assertThat(state, equalTo(State.DONE));
}