Examples with MetricsRegistryMap org.apache.samza.metrics.MetricsRegistryMap used on opensource projects

Example 26 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class MetricsSnapshotReporterFactory method getMetricsReporter.

@Override
public MetricsReporter getMetricsReporter(String reporterName, String containerName, Config config) {
    LOG.info("Creating new metrics snapshot reporter.");
    MetricsRegistryMap registry = new MetricsRegistryMap();
    SystemStream systemStream = getSystemStream(reporterName, config);
    SystemProducer producer = getProducer(reporterName, config, registry);
    Duration reportingInterval = Duration.ofSeconds(getReportingInterval(reporterName, config));
    String jobName = getJobName(config);
    String jobId = getJobId(config);
    Serde<MetricsSnapshot> serde = getSerde(reporterName, config);
    Optional<Pattern> blacklist = getBlacklist(reporterName, config);
    MetricsSnapshotReporter reporter = new MetricsSnapshotReporter(producer, systemStream, reportingInterval, jobName, jobId, containerName, Util.getTaskClassVersion(config), Util.getSamzaVersion(), Util.getLocalHost().getHostName(), serde, blacklist, SystemClock.instance());
    reporter.register(this.getClass().getSimpleName(), registry);
    return reporter;
}

Example 27 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class TestJobCoordinatorMetadataManager method setup.

@Before
public void setup() {
    Map<TaskName, TaskModel> tasksForContainer1 = ImmutableMap.of(new TaskName("t1"), new TaskModel(new TaskName("t1"), ImmutableSet.of(), new Partition(0)), new TaskName("t2"), new TaskModel(new TaskName("t2"), ImmutableSet.of(), new Partition(1)));
    Map<TaskName, TaskModel> tasksForContainer2 = ImmutableMap.of(new TaskName("t3"), new TaskModel(new TaskName("t3"), ImmutableSet.of(), new Partition(2)), new TaskName("t4"), new TaskModel(new TaskName("t4"), ImmutableSet.of(), new Partition(3)), new TaskName("t5"), new TaskModel(new TaskName("t5"), ImmutableSet.of(), new Partition(4)));
    ContainerModel containerModel1 = new ContainerModel("0", tasksForContainer1);
    ContainerModel containerModel2 = new ContainerModel("1", tasksForContainer2);
    containerModelMap = ImmutableMap.of("0", containerModel1, "1", containerModel2);
    CoordinatorStreamStoreTestUtil mockCoordinatorStreamStore = new CoordinatorStreamStoreTestUtil(COORDINATOR_STORE_CONFIG);
    metadataStore = spy(new NamespaceAwareCoordinatorStreamStore(mockCoordinatorStreamStore.getCoordinatorStreamStore(), SetJobCoordinatorMetadataMessage.TYPE));
    jobCoordinatorMetadataManager = spy(new JobCoordinatorMetadataManager(metadataStore, ClusterType.YARN, new MetricsRegistryMap()));
}

Example 28 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class TestJobCoordinatorMetadataManager method testReadJobCoordinatorMetadataFailed.

@Test
public void testReadJobCoordinatorMetadataFailed() {
    JobCoordinatorMetadata jobCoordinatorMetadata = new JobCoordinatorMetadata(NEW_EPOCH_ID, NEW_CONFIG_ID, NEW_JOB_MODEL_ID);
    Serde<String> mockSerde = spy(new CoordinatorStreamValueSerde(SetJobCoordinatorMetadataMessage.TYPE));
    doThrow(new RuntimeException("Failed to read coordinator stream")).when(mockSerde).fromBytes(any());
    jobCoordinatorMetadataManager = spy(new JobCoordinatorMetadataManager(metadataStore, ClusterType.YARN, new MetricsRegistryMap(), mockSerde));
    jobCoordinatorMetadataManager.writeJobCoordinatorMetadata(jobCoordinatorMetadata);
    JobCoordinatorMetadata actualMetadata = jobCoordinatorMetadataManager.readJobCoordinatorMetadata();
    assertNull("Read failed should return null", actualMetadata);
    assertEquals("Metadata read failed count should be 1", 1, jobCoordinatorMetadataManager.getMetrics().getMetadataReadFailedCount().getValue().intValue());
}

Example 29 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class TestWindowOperator method setup.

@Before
public void setup() {
    Map<String, String> configMap = new HashMap<>();
    configMap.put("job.default.system", "kafka");
    configMap.put("job.name", "jobName");
    configMap.put("job.id", "jobId");
    this.config = new MapConfig(configMap);
    this.context = new MockContext();
    when(this.context.getJobContext().getConfig()).thenReturn(this.config);
    Serde storeKeySerde = new TimeSeriesKeySerde(new IntegerSerde());
    Serde storeValSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde());
    SystemStreamPartition ssp = new SystemStreamPartition("kafka", "integers", new Partition(0));
    TaskModel taskModel = mock(TaskModel.class);
    when(taskModel.getSystemStreamPartitions()).thenReturn(ImmutableSet.of(ssp));
    when(taskModel.getTaskName()).thenReturn(new TaskName("task 1"));
    when(this.context.getTaskContext().getTaskModel()).thenReturn(taskModel);
    when(((TaskContextImpl) this.context.getTaskContext()).getSspsExcludingSideInputs()).thenReturn(ImmutableSet.of(ssp));
    when(this.context.getTaskContext().getTaskMetricsRegistry()).thenReturn(new MetricsRegistryMap());
    when(this.context.getContainerContext().getContainerMetricsRegistry()).thenReturn(new MetricsRegistryMap());
    when(this.context.getTaskContext().getStore("jobName-jobId-window-w1")).thenReturn(new TestInMemoryStore<>(storeKeySerde, storeValSerde));
}

Example 30 with MetricsRegistryMap

use of org.apache.samza.metrics.MetricsRegistryMap in project samza by apache.

the class TestJoinOperator method createStreamOperatorTask.

private StreamOperatorTask createStreamOperatorTask(Clock clock, StreamApplicationDescriptorImpl graphSpec) throws Exception {
    Map<String, String> mapConfig = new HashMap<>();
    mapConfig.put("job.name", "jobName");
    mapConfig.put("job.id", "jobId");
    StreamTestUtils.addStreamConfigs(mapConfig, "inStream", "insystem", "instream");
    StreamTestUtils.addStreamConfigs(mapConfig, "inStream2", "insystem", "instream2");
    Context context = new MockContext(new MapConfig(mapConfig));
    TaskModel taskModel = mock(TaskModel.class);
    when(taskModel.getSystemStreamPartitions()).thenReturn(ImmutableSet.of(new SystemStreamPartition("insystem", "instream", new Partition(0)), new SystemStreamPartition("insystem", "instream2", new Partition(0))));
    when(context.getTaskContext().getTaskModel()).thenReturn(taskModel);
    when(context.getTaskContext().getTaskMetricsRegistry()).thenReturn(new MetricsRegistryMap());
    when(context.getContainerContext().getContainerMetricsRegistry()).thenReturn(new MetricsRegistryMap());
    // need to return different stores for left and right side
    IntegerSerde integerSerde = new IntegerSerde();
    TimestampedValueSerde timestampedValueSerde = new TimestampedValueSerde(new KVSerde(integerSerde, integerSerde));
    when(context.getTaskContext().getStore(eq("jobName-jobId-join-j1-L"))).thenReturn(new TestInMemoryStore(integerSerde, timestampedValueSerde));
    when(context.getTaskContext().getStore(eq("jobName-jobId-join-j1-R"))).thenReturn(new TestInMemoryStore(integerSerde, timestampedValueSerde));
    StreamOperatorTask sot = new StreamOperatorTask(graphSpec.getOperatorSpecGraph(), clock);
    sot.init(context);
    return sot;
}