Examples with MetricsContainerImpl org.apache.beam.runners.core.metrics.MetricsContainerImpl used on opensource projects

Example 6 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class BigtableServiceImplTest method verifyMetricWasSet.

private void verifyMetricWasSet(String method, String status, long count) {
    // Verify the metric as reported.
    HashMap<String, String> labels = new HashMap<>();
    labels.put(MonitoringInfoConstants.Labels.PTRANSFORM, "");
    labels.put(MonitoringInfoConstants.Labels.SERVICE, "BigTable");
    labels.put(MonitoringInfoConstants.Labels.METHOD, method);
    labels.put(MonitoringInfoConstants.Labels.RESOURCE, GcpResourceIdentifiers.bigtableResource(PROJECT_ID, INSTANCE_ID, TABLE_ID));
    labels.put(MonitoringInfoConstants.Labels.BIGTABLE_PROJECT_ID, PROJECT_ID);
    labels.put(MonitoringInfoConstants.Labels.INSTANCE_ID, INSTANCE_ID);
    labels.put(MonitoringInfoConstants.Labels.TABLE_ID, GcpResourceIdentifiers.bigtableTableID(PROJECT_ID, INSTANCE_ID, TABLE_ID));
    labels.put(MonitoringInfoConstants.Labels.STATUS, status);
    MonitoringInfoMetricName name = MonitoringInfoMetricName.named(MonitoringInfoConstants.Urns.API_REQUEST_COUNT, labels);
    MetricsContainerImpl container = (MetricsContainerImpl) MetricsEnvironment.getProcessWideContainer();
    assertEquals(count, (long) container.getCounter(name).getCumulative());
}

Example 7 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class BigQueryServicesImplTest method testRetryAttemptCounter.

@Test
public void testRetryAttemptCounter() {
    BigQueryServicesImpl.StorageClientImpl.RetryAttemptCounter counter = new BigQueryServicesImpl.StorageClientImpl.RetryAttemptCounter();
    RetryInfo retryInfo = RetryInfo.newBuilder().setRetryDelay(com.google.protobuf.Duration.newBuilder().setSeconds(123).setNanos(456000000).build()).build();
    Metadata metadata = new Metadata();
    metadata.put(Metadata.Key.of("google.rpc.retryinfo-bin", new Metadata.BinaryMarshaller<RetryInfo>() {

        @Override
        public byte[] toBytes(RetryInfo value) {
            return value.toByteArray();
        }

        @Override
        public RetryInfo parseBytes(byte[] serialized) {
            try {
                Parser<RetryInfo> parser = RetryInfo.newBuilder().build().getParserForType();
                return parser.parseFrom(serialized);
            } catch (Exception e) {
                return null;
            }
        }
    }), retryInfo);
    MetricName metricName = MetricName.named("org.apache.beam.sdk.io.gcp.bigquery.BigQueryServicesImpl$StorageClientImpl", "throttling-msecs");
    MetricsContainerImpl container = (MetricsContainerImpl) MetricsEnvironment.getCurrentContainer();
    // Nulls don't bump the counter.
    counter.onRetryAttempt(null, null);
    assertEquals(0, (long) container.getCounter(metricName).getCumulative());
    // Resource exhausted with empty metadata doesn't bump the counter.
    counter.onRetryAttempt(Status.RESOURCE_EXHAUSTED.withDescription("You have consumed some quota"), new Metadata());
    assertEquals(0, (long) container.getCounter(metricName).getCumulative());
    // Resource exhausted with retry info bumps the counter.
    counter.onRetryAttempt(Status.RESOURCE_EXHAUSTED.withDescription("Stop for a while"), metadata);
    assertEquals(123456, (long) container.getCounter(metricName).getCumulative());
    // Other errors with retry info doesn't bump the counter.
    counter.onRetryAttempt(Status.UNAVAILABLE.withDescription("Server is gone"), metadata);
    assertEquals(123456, (long) container.getCounter(metricName).getCumulative());
}

Example 8 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class BigQueryServicesImplTest method setUp.

@Before
public void setUp() {
    MockitoAnnotations.initMocks(this);
    // Set up the MockHttpRequest for future inspection.
    request = new MockLowLevelHttpRequest() {

        int index = 0;

        @Override
        public LowLevelHttpResponse execute() throws IOException {
            Verify.verify(index < responses.length, "The number of HttpRequest invocation exceeded the number of prepared mock requests. Index: %d", index);
            return responses[index++];
        }
    };
    // A mock transport that lets us mock the API responses.
    MockHttpTransport transport = new MockHttpTransport.Builder().setLowLevelHttpRequest(request).build();
    // A sample BigQuery API client that uses default JsonFactory and RetryHttpInitializer.
    bigquery = new Bigquery.Builder(transport, Transport.getJsonFactory(), new RetryHttpRequestInitializer()).build();
    // Setup the ProcessWideContainer for testing metrics are set.
    MetricsContainerImpl container = new MetricsContainerImpl(null);
    MetricsEnvironment.setProcessWideContainer(container);
    MetricsEnvironment.setCurrentContainer(container);
}

Example 9 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class GcsUtilTest method verifyMetricWasSet.

private void verifyMetricWasSet(String projectId, String bucketId, String method, String status, long count) {
    // Verify the metric as reported.
    HashMap<String, String> labels = new HashMap<>();
    labels.put(MonitoringInfoConstants.Labels.PTRANSFORM, "");
    labels.put(MonitoringInfoConstants.Labels.SERVICE, "Storage");
    labels.put(MonitoringInfoConstants.Labels.METHOD, method);
    labels.put(MonitoringInfoConstants.Labels.GCS_PROJECT_ID, projectId);
    labels.put(MonitoringInfoConstants.Labels.GCS_BUCKET, bucketId);
    labels.put(MonitoringInfoConstants.Labels.RESOURCE, GcpResourceIdentifiers.cloudStorageBucket(bucketId));
    labels.put(MonitoringInfoConstants.Labels.STATUS, status);
    MonitoringInfoMetricName name = MonitoringInfoMetricName.named(MonitoringInfoConstants.Urns.API_REQUEST_COUNT, labels);
    MetricsContainerImpl container = (MetricsContainerImpl) MetricsEnvironment.getProcessWideContainer();
    assertEquals(count, (long) container.getCounter(name).getCumulative());
}

Example 10 with MetricsContainerImpl

use of org.apache.beam.runners.core.metrics.MetricsContainerImpl in project beam by apache.

the class ReduceFnRunnerTest method testMergingLateWatermarkHolds.

@Test
public void testMergingLateWatermarkHolds() throws Exception {
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    Duration gapDuration = Duration.millis(10);
    Duration allowedLateness = Duration.standardMinutes(100);
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(WindowingStrategy.of(Sessions.withGapDuration(gapDuration)).withMode(AccumulationMode.DISCARDING_FIRED_PANES).withTrigger(Repeatedly.forever(AfterWatermark.pastEndOfWindow().withLateFirings(AfterPane.elementCountAtLeast(10)))).withAllowedLateness(allowedLateness));
    tester.setAutoAdvanceOutputWatermark(false);
    // Input watermark -> null
    assertEquals(null, tester.getWatermarkHold());
    assertEquals(null, tester.getOutputWatermark());
    tester.advanceInputWatermark(new Instant(20));
    // Add two late elements that cause a window to merge.
    injectElements(tester, Arrays.asList(3));
    assertThat(tester.getWatermarkHold(), nullValue());
    injectElements(tester, Arrays.asList(4));
    Instant endOfWindow = new Instant(4).plus(gapDuration);
    // We expect a GC hold to be one less than the end of window plus the allowed lateness.
    Instant expectedGcHold = endOfWindow.plus(allowedLateness).minus(Duration.millis(1));
    assertEquals(expectedGcHold, tester.getWatermarkHold());
    tester.advanceInputWatermark(new Instant(1000));
    assertEquals(expectedGcHold, tester.getWatermarkHold());
}