Examples with Tag org.apache.gobblin.metrics.Tag used on opensource projects

Example 6 with Tag

use of org.apache.gobblin.metrics.Tag in project incubator-gobblin by apache.

the class JobMetricsTest method testCustomTags.

@Test
public void testCustomTags() {
    Properties testProperties = new Properties();
    Tag<String> expectedPropertyTag = new Tag<>("key1", "value1");
    GobblinMetrics.addCustomTagToProperties(testProperties, expectedPropertyTag);
    State testState = new State(testProperties);
    List<Tag<?>> tags = GobblinMetrics.getCustomTagsFromState(testState);
    Assert.assertEquals(Iterables.getFirst(tags, null), expectedPropertyTag);
    Tag<String> expectedStateTag = new Tag<>("key2", "value2");
    GobblinMetrics.addCustomTagToState(testState, expectedStateTag);
    tags = GobblinMetrics.getCustomTagsFromState(testState);
    Assert.assertTrue(tags.containsAll(ImmutableList.of(expectedPropertyTag, expectedStateTag)));
}

Example 7 with Tag

use of org.apache.gobblin.metrics.Tag in project incubator-gobblin by apache.

the class JobMetricsTest method testJobMetricsGet.

@Test
public void testJobMetricsGet() {
    String jobName = "testJob";
    String jobId = "job_123";
    JobState jobState = new JobState(jobName, jobId);
    JobMetrics jobMetrics = JobMetrics.get(jobState);
    Assert.assertNotNull(jobMetrics.getMetricContext());
    List<Tag<?>> tags = jobMetrics.getMetricContext().getTags();
    Map<String, ?> tagMap = jobMetrics.getMetricContext().getTagMap();
    String contextId = tagMap.get(MetricContext.METRIC_CONTEXT_ID_TAG_NAME).toString();
    String contextName = tagMap.get(MetricContext.METRIC_CONTEXT_NAME_TAG_NAME).toString();
    Assert.assertEquals(tagMap.size(), 4);
    Assert.assertEquals(tagMap.get(JobEvent.METADATA_JOB_ID), jobId);
    Assert.assertEquals(tagMap.get(JobEvent.METADATA_JOB_NAME), jobName);
    Assert.assertEquals(tagMap.get(MetricContext.METRIC_CONTEXT_ID_TAG_NAME), contextId);
    Assert.assertEquals(tagMap.get(MetricContext.METRIC_CONTEXT_NAME_TAG_NAME), contextName);
    // should get the original jobMetrics, can check by the id
    JobMetrics jobMetrics1 = JobMetrics.get(jobName + "_", jobId);
    Assert.assertNotNull(jobMetrics1.getMetricContext());
    tagMap = jobMetrics1.getMetricContext().getTagMap();
    Assert.assertEquals(tags.size(), 4);
    Assert.assertEquals(tagMap.get(MetricContext.METRIC_CONTEXT_ID_TAG_NAME), contextId);
    Assert.assertEquals(tagMap.get(MetricContext.METRIC_CONTEXT_NAME_TAG_NAME), contextName);
    // remove original jobMetrics, should create a new one
    GobblinMetricsRegistry.getInstance().remove(jobMetrics.getId());
    JobMetrics jobMetrics2 = JobMetrics.get(jobName + "_", jobId);
    Assert.assertNotNull(jobMetrics2.getMetricContext());
    tagMap = jobMetrics2.getMetricContext().getTagMap();
    Assert.assertEquals(tags.size(), 4);
    Assert.assertNotEquals(tagMap.get(MetricContext.METRIC_CONTEXT_ID_TAG_NAME), contextId);
    Assert.assertNotEquals(tagMap.get(MetricContext.METRIC_CONTEXT_NAME_TAG_NAME), contextName);
}

Example 8 with Tag

use of org.apache.gobblin.metrics.Tag in project incubator-gobblin by apache.

the class YarnService method buildGobblinMetrics.

private GobblinMetrics buildGobblinMetrics() {
    // Create tags list
    ImmutableList.Builder<Tag<?>> tags = new ImmutableList.Builder<>();
    tags.add(new Tag<>(GobblinClusterMetricTagNames.APPLICATION_ID, this.applicationId));
    tags.add(new Tag<>(GobblinClusterMetricTagNames.APPLICATION_NAME, this.applicationName));
    // Intialize Gobblin metrics and start reporters
    GobblinMetrics gobblinMetrics = GobblinMetrics.get(this.applicationId, null, tags.build());
    gobblinMetrics.startMetricReporting(ConfigUtils.configToProperties(config));
    return gobblinMetrics;
}

Example 9 with Tag

use of org.apache.gobblin.metrics.Tag in project incubator-gobblin by apache.

the class GobblinHelixJobLauncherTest method testLaunchMultipleJobs.

public void testLaunchMultipleJobs() throws Exception {
    final ConcurrentHashMap<String, Boolean> runningMap = new ConcurrentHashMap<>();
    // Job launcher(1) to test parallel job running
    final Properties properties1 = generateJobProperties(this.baseConfig, "2", "_1504201348471");
    final GobblinHelixJobLauncher gobblinHelixJobLauncher1 = this.closer.register(new GobblinHelixJobLauncher(properties1, this.helixManager, this.appWorkDir, ImmutableList.<Tag<?>>of(), runningMap));
    // Job launcher(2) to test parallel job running
    final Properties properties2 = generateJobProperties(this.baseConfig, "2", "_1504201348472");
    final GobblinHelixJobLauncher gobblinHelixJobLauncher2 = this.closer.register(new GobblinHelixJobLauncher(properties2, this.helixManager, this.appWorkDir, ImmutableList.<Tag<?>>of(), runningMap));
    CountDownLatch stg1 = new CountDownLatch(1);
    CountDownLatch stg2 = new CountDownLatch(1);
    CountDownLatch stg3 = new CountDownLatch(1);
    SuspendJobListener testListener = new SuspendJobListener(stg1, stg2);
    (new Thread(() -> {
        try {
            gobblinHelixJobLauncher1.launchJob(testListener);
            stg3.countDown();
        } catch (JobException e) {
        }
    })).start();
    // Wait for the first job to start
    stg1.await();
    // When first job is in the middle of running, launch the second job (which should do NOOP because previous job is still running)
    gobblinHelixJobLauncher2.launchJob(testListener);
    stg2.countDown();
    // Wait for the first job to finish
    stg3.await();
    Assert.assertEquals(testListener.getCompletes().get() == 1, true);
}

Example 10 with Tag

use of org.apache.gobblin.metrics.Tag in project incubator-gobblin by apache.

the class GobblinHelixJobLauncherTest method testLaunchJob.

public void testLaunchJob() throws Exception {
    final ConcurrentHashMap<String, Boolean> runningMap = new ConcurrentHashMap<>();
    // Normal job launcher
    final Properties properties = generateJobProperties(this.baseConfig, "1", "_1504201348470");
    final GobblinHelixJobLauncher gobblinHelixJobLauncher = this.closer.register(new GobblinHelixJobLauncher(properties, this.helixManager, this.appWorkDir, ImmutableList.<Tag<?>>of(), runningMap));
    gobblinHelixJobLauncher.launchJob(null);
    final File jobOutputFile = getJobOutputFile(properties);
    Assert.assertTrue(jobOutputFile.exists());
    Schema schema = new Schema.Parser().parse(TestHelper.SOURCE_SCHEMA);
    TestHelper.assertGenericRecords(jobOutputFile, schema);
    List<JobState.DatasetState> datasetStates = this.datasetStateStore.getAll(properties.getProperty(ConfigurationKeys.JOB_NAME_KEY), FsDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + FsDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX);
    Assert.assertEquals(datasetStates.size(), 1);
    JobState.DatasetState datasetState = datasetStates.get(0);
    Assert.assertEquals(datasetState.getCompletedTasks(), 1);
    Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED);
    Assert.assertEquals(datasetState.getTaskStates().size(), 1);
    Assert.assertEquals(datasetState.getTaskStates().get(0).getWorkingState(), WorkUnitState.WorkingState.COMMITTED);
}