Examples with MetricContext org.apache.gobblin.metrics.MetricContext used on opensource projects

Example 6 with MetricContext

use of org.apache.gobblin.metrics.MetricContext in project incubator-gobblin by apache.

the class KafkaAvroEventReporterWithSchemaRegistryTest method test.

@Test
public void test() throws Exception {
    MetricContext context = MetricContext.builder("context").build();
    MockKafkaPusher pusher = new MockKafkaPusher();
    KafkaAvroSchemaRegistry registry = Mockito.mock(KafkaAvroSchemaRegistry.class);
    Mockito.when(registry.register(Mockito.any(Schema.class))).thenAnswer(new Answer<String>() {

        @Override
        public String answer(InvocationOnMock invocation) throws Throwable {
            return register((Schema) invocation.getArguments()[0]);
        }
    });
    Mockito.when(registry.register(Mockito.any(Schema.class), Mockito.anyString())).thenAnswer(new Answer<String>() {

        @Override
        public String answer(InvocationOnMock invocation) throws Throwable {
            return register((Schema) invocation.getArguments()[0]);
        }
    });
    KafkaEventReporter kafkaReporter = KafkaAvroEventReporter.forContext(context).withKafkaPusher(pusher).withSchemaRegistry(registry).build("localhost:0000", "topic");
    GobblinTrackingEvent event = new GobblinTrackingEvent(0l, "namespace", "name", Maps.<String, String>newHashMap());
    context.submitEvent(event);
    try {
        Thread.sleep(100);
    } catch (InterruptedException ex) {
        Thread.currentThread().interrupt();
    }
    kafkaReporter.report();
    try {
        Thread.sleep(100);
    } catch (InterruptedException ex) {
        Thread.currentThread().interrupt();
    }
    byte[] nextMessage = pusher.messageIterator().next();
    DataInputStream is = new DataInputStream(new ByteArrayInputStream(nextMessage));
    Assert.assertEquals(is.readByte(), KafkaAvroSchemaRegistry.MAGIC_BYTE);
    byte[] readId = new byte[20];
    Assert.assertEquals(is.read(readId), 20);
    String readStringId = Hex.encodeHexString(readId);
    Assert.assertTrue(this.schemas.containsKey(readStringId));
    Schema schema = this.schemas.get(readStringId);
    Assert.assertFalse(schema.toString().contains("avro.java.string"));
    is.close();
}

Example 7 with MetricContext

use of org.apache.gobblin.metrics.MetricContext in project incubator-gobblin by apache.

the class LimiterServerResource method get.

/**
 * Request permits from the limiter server. The returned {@link PermitAllocation} specifies the number of permits
 * that the client can use.
 */
@Override
@RestMethod.Get
public void get(ComplexResourceKey<PermitRequest, EmptyRecord> key, @CallbackParam final Callback<PermitAllocation> callback) {
    try (Closeable context = this.requestTimer == null ? NoopCloseable.INSTANCE : this.requestTimer.time()) {
        PermitRequest request = key.getKey();
        String resourceId = request.getResource();
        MetricContext resourceContext = (MetricContext) broker.getSharedResource(new MetricContextFactory(), new SubTaggedMetricContextKey(resourceId, ImmutableMap.of(RESOURCE_ID_TAG, resourceId)));
        Meter permitsRequestedMeter = resourceContext.meter(PERMITS_REQUESTED_METER_NAME);
        Meter permitsGrantedMeter = resourceContext.meter(PERMITS_GRANTED_METER_NAME);
        Timer limiterTimer = resourceContext.timer(LIMITER_TIMER_NAME);
        permitsRequestedMeter.mark(request.getPermits());
        if (this.leaderFinderOpt.isPresent() && !this.leaderFinderOpt.get().isLeader()) {
            URI leaderUri = this.leaderFinderOpt.get().getLeaderMetadata().getUri();
            RestLiServiceException exception = new RestLiServiceException(HttpStatus.S_301_MOVED_PERMANENTLY, String.format("New leader <a href=\"%s\">%s</a>", leaderUri, leaderUri));
            exception.setErrorDetails(new DataMap(ImmutableMap.of(LOCATION_301, leaderUri.toString())));
            throw exception;
        } else {
            ThrottlingPolicy policy = (ThrottlingPolicy) this.broker.getSharedResource(new ThrottlingPolicyFactory(), new SharedLimiterKey(request.getResource()));
            PermitAllocation allocation;
            try (Closeable thisContext = limiterTimer.time()) {
                allocation = policy.computePermitAllocation(request);
            }
            permitsGrantedMeter.mark(allocation.getPermits());
            callback.onSuccess(allocation);
        }
    } catch (NotConfiguredException nce) {
        throw new RestLiServiceException(HttpStatus.S_422_UNPROCESSABLE_ENTITY, "No configuration for the requested resource.");
    } catch (IOException ioe) {
        // Failed to close timer context. This should never happen
        throw new RuntimeException(ioe);
    }
}

Example 8 with MetricContext

use of org.apache.gobblin.metrics.MetricContext in project incubator-gobblin by apache.

the class PoliciesResource method get.

@Override
public Policy get(String resourceId) {
    try {
        ThrottlingPolicy throttlingPolicy = (ThrottlingPolicy) this.broker.getSharedResource(new ThrottlingPolicyFactory(), new SharedLimiterKey(resourceId));
        Policy restliPolicy = new Policy();
        restliPolicy.setPolicyName(throttlingPolicy.getClass().getSimpleName());
        restliPolicy.setResource(resourceId);
        restliPolicy.setParameters(new StringMap(throttlingPolicy.getParameters()));
        restliPolicy.setPolicyDetails(throttlingPolicy.getDescription());
        MetricContext resourceContext = (MetricContext) broker.getSharedResource(new MetricContextFactory(), new SubTaggedMetricContextKey(resourceId, ImmutableMap.of(RESOURCE_ID_TAG, resourceId)));
        StringMap metrics = new StringMap();
        for (Map.Entry<String, Meter> meter : resourceContext.getMeters().entrySet()) {
            metrics.put(meter.getKey(), Double.toString(meter.getValue().getOneMinuteRate()));
        }
        restliPolicy.setMetrics(metrics);
        return restliPolicy;
    } catch (NotConfiguredException nce) {
        throw new RestLiServiceException(HttpStatus.S_404_NOT_FOUND, "Policy not found for resource " + resourceId);
    }
}

Example 9 with MetricContext

use of org.apache.gobblin.metrics.MetricContext in project incubator-gobblin by apache.

the class CopySourceTest method testSubmitUnfulfilledRequestEvents.

@Test
public void testSubmitUnfulfilledRequestEvents() throws IOException, NoSuchMethodException, InvocationTargetException, IllegalAccessException {
    SourceState state = new SourceState();
    state.setProp(ConfigurationKeys.SOURCE_FILEBASED_FS_URI, "file:///");
    state.setProp(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, "file:///");
    state.setProp(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/target/dir");
    state.setProp(DatasetUtils.DATASET_PROFILE_CLASS_KEY, TestCopyablePartitionableDatasedFinder.class.getCanonicalName());
    state.setProp(CopySource.MAX_CONCURRENT_LISTING_SERVICES, 2);
    state.setProp(CopyConfiguration.MAX_COPY_PREFIX + ".size", "50");
    state.setProp(CopyConfiguration.MAX_COPY_PREFIX + ".copyEntities", 2);
    state.setProp(CopyConfiguration.STORE_REJECTED_REQUESTS_KEY, RequestAllocatorConfig.StoreRejectedRequestsConfig.ALL.name().toLowerCase());
    state.setProp(ConfigurationKeys.METRICS_CUSTOM_BUILDERS, "org.apache.gobblin.metrics.ConsoleEventReporterFactory");
    CopySource source = new CopySource();
    final FileSystem sourceFs = HadoopUtils.getSourceFileSystem(state);
    final FileSystem targetFs = HadoopUtils.getWriterFileSystem(state, 1, 0);
    int maxThreads = state.getPropAsInt(CopySource.MAX_CONCURRENT_LISTING_SERVICES, CopySource.DEFAULT_MAX_CONCURRENT_LISTING_SERVICES);
    final CopyConfiguration copyConfiguration = CopyConfiguration.builder(targetFs, state.getProperties()).build();
    MetricContext metricContext = Instrumented.getMetricContext(state, CopySource.class);
    EventSubmitter eventSubmitter = new EventSubmitter.Builder(metricContext, CopyConfiguration.COPY_PREFIX).build();
    DatasetsFinder<CopyableDatasetBase> datasetFinder = DatasetUtils.instantiateDatasetFinder(state.getProperties(), sourceFs, CopySource.DEFAULT_DATASET_PROFILE_CLASS_KEY, eventSubmitter, state);
    IterableDatasetFinder<CopyableDatasetBase> iterableDatasetFinder = datasetFinder instanceof IterableDatasetFinder ? (IterableDatasetFinder<CopyableDatasetBase>) datasetFinder : new IterableDatasetFinderImpl<>(datasetFinder);
    Iterator<CopyableDatasetRequestor> requestorIteratorWithNulls = Iterators.transform(iterableDatasetFinder.getDatasetsIterator(), new CopyableDatasetRequestor.Factory(targetFs, copyConfiguration, log));
    Iterator<CopyableDatasetRequestor> requestorIterator = Iterators.filter(requestorIteratorWithNulls, Predicates.<CopyableDatasetRequestor>notNull());
    Method m = CopySource.class.getDeclaredMethod("createRequestAllocator", CopyConfiguration.class, int.class);
    m.setAccessible(true);
    PriorityIterableBasedRequestAllocator<FileSet<CopyEntity>> allocator = (PriorityIterableBasedRequestAllocator<FileSet<CopyEntity>>) m.invoke(source, copyConfiguration, maxThreads);
    Iterator<FileSet<CopyEntity>> prioritizedFileSets = allocator.allocateRequests(requestorIterator, copyConfiguration.getMaxToCopy());
    List<FileSet<CopyEntity>> fileSetList = allocator.getRequestsExceedingAvailableResourcePool();
    Assert.assertEquals(fileSetList.size(), 2);
    FileSet<CopyEntity> fileSet = fileSetList.get(0);
    Assert.assertEquals(fileSet.getDataset().getUrn(), "/test");
    Assert.assertEquals(fileSet.getTotalEntities(), 5);
    Assert.assertEquals(fileSet.getTotalSizeInBytes(), 50);
    fileSet = fileSetList.get(1);
    Assert.assertEquals(fileSet.getDataset().getUrn(), "/test");
    Assert.assertEquals(fileSet.getTotalEntities(), 5);
    Assert.assertEquals(fileSet.getTotalSizeInBytes(), 50);
}

Example 10 with MetricContext

use of org.apache.gobblin.metrics.MetricContext in project incubator-gobblin by apache.

the class GraphiteReporterTest method testWithoutTags.

@Test
public void testWithoutTags() throws IOException {
    try (MetricContext metricContext = MetricContext.builder(this.getClass().getCanonicalName() + ".testGraphiteReporter").build();
        GraphiteReporter graphiteReporter = GraphiteReporter.Factory.newBuilder().withGraphitePusher(graphitePusher).withMetricContextName(CONTEXT_NAME).build(new Properties())) {
        ContextAwareGauge<Long> contextAwareGauge = metricContext.newContextAwareGauge("com.linkedin.example.gauge", new Gauge<Long>() {

            @Override
            public Long getValue() {
                return 1000l;
            }
        });
        metricContext.register(MetricRegistry.name(METRIC_PREFIX, GAUGE), contextAwareGauge);
        Counter counter = metricContext.counter(MetricRegistry.name(METRIC_PREFIX, COUNTER));
        Meter meter = metricContext.meter(MetricRegistry.name(METRIC_PREFIX, METER));
        Histogram histogram = metricContext.histogram(MetricRegistry.name(METRIC_PREFIX, HISTOGRAM));
        Timer timer = metricContext.timer(MetricRegistry.name(METRIC_PREFIX, TIMER));
        counter.inc(3l);
        meter.mark(1l);
        meter.mark(2l);
        meter.mark(3l);
        histogram.update(1);
        histogram.update(1);
        histogram.update(2);
        timer.update(1, TimeUnit.SECONDS);
        timer.update(2, TimeUnit.SECONDS);
        timer.update(3, TimeUnit.SECONDS);
        graphiteReporter.report(metricContext.getGauges(), metricContext.getCounters(), metricContext.getHistograms(), metricContext.getMeters(), metricContext.getTimers(), metricContext.getTagMap());
        Assert.assertEquals(getMetricValue(COUNTER, Measurements.COUNT), Long.toString(3l));
        Assert.assertEquals(getMetricValue(GAUGE, null), Long.toString(1000l));
        Assert.assertTrue(getMetricTimestamp(GAUGE, null) <= System.currentTimeMillis() / 1000l);
        Assert.assertEquals(getMetricValue(HISTOGRAM, Measurements.PERCENTILE_75TH), Double.toString(2d));
        Assert.assertEquals(getMetricValue(HISTOGRAM, Measurements.PERCENTILE_98TH), Double.toString(2d));
        Assert.assertEquals(getMetricValue(HISTOGRAM, Measurements.PERCENTILE_99TH), Double.toString(2d));
        Assert.assertEquals(getMetricValue(HISTOGRAM, Measurements.PERCENTILE_999TH), Double.toString(2d));
        Assert.assertEquals(getMetricValue(HISTOGRAM, Measurements.COUNT), Long.toString(3l));
        Assert.assertEquals(getMetricValue(HISTOGRAM, Measurements.MIN), Long.toString(1l));
        Assert.assertEquals(getMetricValue(HISTOGRAM, Measurements.MAX), Long.toString(2l));
        Assert.assertEquals(getMetricValue(HISTOGRAM, Measurements.MEDIAN), Double.toString(1d));
        Assert.assertTrue(Double.valueOf(getMetricValue(HISTOGRAM, Measurements.MEAN)) > 1d);
        Assert.assertTrue(Double.valueOf(getMetricValue(HISTOGRAM, Measurements.STDDEV)) < 0.5d);
        Assert.assertEquals(getMetricValue(METER, Measurements.RATE_1MIN), Double.toString(0d));
        Assert.assertEquals(getMetricValue(METER, Measurements.RATE_5MIN), Double.toString(0d));
        Assert.assertEquals(getMetricValue(METER, Measurements.COUNT), Long.toString(6l));
        Assert.assertTrue(Double.valueOf(getMetricValue(METER, Measurements.MEAN_RATE)) > 0d);
        Assert.assertEquals(getMetricValue(TIMER, Measurements.RATE_1MIN), Double.toString(0d));
        Assert.assertEquals(getMetricValue(TIMER, Measurements.RATE_5MIN), Double.toString(0d));
        Assert.assertEquals(getMetricValue(TIMER, Measurements.PERCENTILE_75TH), Double.toString(3000d));
        Assert.assertEquals(getMetricValue(TIMER, Measurements.PERCENTILE_98TH), Double.toString(3000d));
        Assert.assertEquals(getMetricValue(TIMER, Measurements.PERCENTILE_99TH), Double.toString(3000d));
        Assert.assertEquals(getMetricValue(TIMER, Measurements.PERCENTILE_999TH), Double.toString(3000d));
        Assert.assertEquals(getMetricValue(TIMER, Measurements.COUNT), Long.toString(3l));
        Assert.assertEquals(getMetricValue(TIMER, Measurements.MIN), Double.toString(1000d));
        Assert.assertEquals(getMetricValue(TIMER, Measurements.MAX), Double.toString(3000d));
        Assert.assertEquals(getMetricValue(TIMER, Measurements.MEAN), Double.toString(2000d));
        Assert.assertEquals(getMetricValue(TIMER, Measurements.MEDIAN), Double.toString(2000d));
        Assert.assertTrue(Double.valueOf(getMetricValue(TIMER, Measurements.MEAN_RATE)) > 0d);
        Assert.assertTrue(Double.valueOf(getMetricValue(TIMER, Measurements.STDDEV)) > 0d);
    }
}