Examples with DataflowPipelineJob org.apache.beam.runners.dataflow.DataflowPipelineJob used on opensource projects

Example 1 with DataflowPipelineJob

use of org.apache.beam.runners.dataflow.DataflowPipelineJob in project java-docs-samples by GoogleCloudPlatform.

the class WorkloadGeneratorTest method testPipeline.

@Test
public void testPipeline() throws IOException, InterruptedException {
    String workloadJobName = "bigtable-workload-generator-test-" + new Date().getTime();
    final int WORKLOAD_DURATION = 10;
    final int WAIT_DURATION = WORKLOAD_DURATION * 60 * 1000;
    int rate = 1000;
    BigtableWorkloadOptions options = PipelineOptionsFactory.create().as(BigtableWorkloadOptions.class);
    options.setBigtableInstanceId(instanceId);
    options.setBigtableTableId(TABLE_ID);
    options.setWorkloadRate(rate);
    options.setRegion(REGION_ID);
    options.setWorkloadDurationMinutes(WORKLOAD_DURATION);
    options.setRunner(DataflowRunner.class);
    options.setJobName(workloadJobName);
    final PipelineResult pipelineResult = WorkloadGenerator.generateWorkload(options);
    MetricServiceClient metricServiceClient = MetricServiceClient.create();
    ProjectName name = ProjectName.of(projectId);
    // Wait X minutes and then get metrics for the X minute period.
    Thread.sleep(WAIT_DURATION);
    long startMillis = System.currentTimeMillis() - WAIT_DURATION;
    TimeInterval interval = TimeInterval.newBuilder().setStartTime(Timestamps.fromMillis(startMillis)).setEndTime(Timestamps.fromMillis(System.currentTimeMillis())).build();
    ListTimeSeriesRequest request = ListTimeSeriesRequest.newBuilder().setName(name.toString()).setFilter("metric.type=\"bigtable.googleapis.com/server/request_count\"").setInterval(interval).build();
    ListTimeSeriesPagedResponse response = metricServiceClient.listTimeSeries(request);
    long startRequestCount = 0;
    long endRequestCount = 0;
    for (TimeSeries ts : response.iterateAll()) {
        startRequestCount = ts.getPoints(0).getValue().getInt64Value();
        endRequestCount = ts.getPoints(ts.getPointsCount() - 1).getValue().getInt64Value();
    }
    assertThat(endRequestCount - startRequestCount > rate);
    // Stop the running job.
    String jobId = ((DataflowPipelineJob) pipelineResult).getJobId();
    DataflowClient client = DataflowClient.create(options);
    Job job = client.getJob(jobId);
    assertThat(job.getCurrentState().equals("JOB_STATE_CANCELLED"));
}

Example 2 with DataflowPipelineJob

use of org.apache.beam.runners.dataflow.DataflowPipelineJob in project java-docs-samples by GoogleCloudPlatform.

the class WorkloadGenerator method cancelJob.

private static void cancelJob(BigtableWorkloadOptions options, DataflowPipelineJob pipelineResult) throws IOException {
    String jobId = pipelineResult.getJobId();
    DataflowClient client = DataflowClient.create(options);
    Job job = client.getJob(jobId);
    job.setRequestedState("JOB_STATE_CANCELLED");
    client.updateJob(jobId, job);
}