Examples with JobMetrics com.google.api.services.dataflow.model.JobMetrics used on opensource projects

Example 11 with JobMetrics

use of com.google.api.services.dataflow.model.JobMetrics in project beam by apache.

the class DataflowMetricsTest method testMultipleCounterUpdatesStreaming.

@Test
public void testMultipleCounterUpdatesStreaming() throws IOException {
    JobMetrics jobMetrics = new JobMetrics();
    DataflowClient dataflowClient = mock(DataflowClient.class);
    when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics);
    DataflowPipelineJob job = mock(DataflowPipelineJob.class);
    DataflowPipelineOptions options = mock(DataflowPipelineOptions.class);
    when(options.isStreaming()).thenReturn(true);
    when(job.getDataflowOptions()).thenReturn(options);
    when(job.getState()).thenReturn(State.RUNNING);
    job.jobId = JOB_ID;
    AppliedPTransform<?, ?, ?> myStep2 = mock(AppliedPTransform.class);
    when(myStep2.getFullName()).thenReturn("myStepName");
    job.transformStepNames = HashBiMap.create();
    job.transformStepNames.put(myStep2, "s2");
    AppliedPTransform<?, ?, ?> myStep3 = mock(AppliedPTransform.class);
    when(myStep3.getFullName()).thenReturn("myStepName3");
    job.transformStepNames.put(myStep3, "s3");
    AppliedPTransform<?, ?, ?> myStep4 = mock(AppliedPTransform.class);
    when(myStep4.getFullName()).thenReturn("myStepName4");
    job.transformStepNames.put(myStep4, "s4");
    // The parser relies on the fact that one tentative and one committed metric update exist in
    // the job metrics results.
    jobMetrics.setMetrics(ImmutableList.of(makeCounterMetricUpdate("counterName", "counterNamespace", "s2", 1233L, false), makeCounterMetricUpdate("counterName", "counterNamespace", "s2", 1234L, true), makeCounterMetricUpdate("otherCounter", "otherNamespace", "s3", 12L, false), makeCounterMetricUpdate("otherCounter", "otherNamespace", "s3", 12L, true), makeCounterMetricUpdate("counterName", "otherNamespace", "s4", 1200L, false), makeCounterMetricUpdate("counterName", "otherNamespace", "s4", 1233L, true)));
    DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient);
    MetricQueryResults result = dataflowMetrics.allMetrics();
    try {
        result.getCounters().iterator().next().getCommitted();
        fail("Expected UnsupportedOperationException");
    } catch (UnsupportedOperationException expected) {
        assertThat(expected.getMessage(), containsString("This runner does not currently support committed" + " metrics results. Please use 'attempted' instead."));
    }
    assertThat(result.getCounters(), containsInAnyOrder(attemptedMetricsResult("counterNamespace", "counterName", "myStepName", 1233L), attemptedMetricsResult("otherNamespace", "otherCounter", "myStepName3", 12L), attemptedMetricsResult("otherNamespace", "counterName", "myStepName4", 1200L)));
}

Example 12 with JobMetrics

use of com.google.api.services.dataflow.model.JobMetrics in project beam by apache.

the class TestDataflowRunner method checkForPAssertSuccess.

/**
 * Check that PAssert expectations were met.
 *
 * <p>If the pipeline is not in a failed/cancelled state and no PAsserts were used within the
 * pipeline, then this method will state that all PAsserts succeeded.
 *
 * @return Optional.of(false) if we are certain a PAssert failed. Optional.of(true) if we are
 *     certain all PAsserts passed. Optional.absent() if the evidence is inconclusive, including
 *     when the pipeline may have failed for other reasons.
 */
@VisibleForTesting
Optional<Boolean> checkForPAssertSuccess(DataflowPipelineJob job) {
    JobMetrics metrics = getJobMetrics(job);
    if (metrics == null || metrics.getMetrics() == null) {
        LOG.warn("Metrics not present for Dataflow job {}.", job.getJobId());
        return Optional.absent();
    }
    int successes = 0;
    int failures = 0;
    for (MetricUpdate metric : metrics.getMetrics()) {
        if (metric.getName() == null || metric.getName().getContext() == null || !metric.getName().getContext().containsKey(TENTATIVE_COUNTER)) {
            // Don't double count using the non-tentative version of the metric.
            continue;
        }
        if (PAssert.SUCCESS_COUNTER.equals(metric.getName().getName())) {
            successes += ((BigDecimal) metric.getScalar()).intValue();
        } else if (PAssert.FAILURE_COUNTER.equals(metric.getName().getName())) {
            failures += ((BigDecimal) metric.getScalar()).intValue();
        }
    }
    if (failures > 0) {
        LOG.info("Failure result for Dataflow job {}. Found {} success, {} failures out of " + "{} expected assertions.", job.getJobId(), successes, failures, expectedNumberOfAssertions);
        return Optional.of(false);
    } else if (successes >= expectedNumberOfAssertions) {
        LOG.info("Success result for Dataflow job {}." + " Found {} success, {} failures out of {} expected assertions.", job.getJobId(), successes, failures, expectedNumberOfAssertions);
        return Optional.of(true);
    }
    // If the job failed, this is a definite failure. We only cancel jobs when they fail.
    State state = job.getState();
    if (state == State.FAILED || state == State.CANCELLED) {
        LOG.info("Dataflow job {} terminated in failure state {} without reporting a failed assertion", job.getJobId(), state);
        return Optional.absent();
    }
    LOG.info("Inconclusive results for Dataflow job {}." + " Found {} success, {} failures out of {} expected assertions.", job.getJobId(), successes, failures, expectedNumberOfAssertions);
    return Optional.absent();
}

Example 13 with JobMetrics

use of com.google.api.services.dataflow.model.JobMetrics in project beam by apache.

the class DataflowMetrics method queryMetrics.

@Override
public MetricQueryResults queryMetrics(MetricsFilter filter) {
    List<MetricUpdate> metricUpdates;
    ImmutableList<MetricResult<Long>> counters = ImmutableList.of();
    ImmutableList<MetricResult<DistributionResult>> distributions = ImmutableList.of();
    ImmutableList<MetricResult<GaugeResult>> gauges = ImmutableList.of();
    JobMetrics jobMetrics;
    try {
        jobMetrics = getJobMetrics();
    } catch (IOException e) {
        LOG.warn("Unable to query job metrics.\n");
        return MetricQueryResults.create(counters, distributions, gauges);
    }
    metricUpdates = firstNonNull(jobMetrics.getMetrics(), Collections.emptyList());
    return populateMetricQueryResults(metricUpdates, filter);
}