Examples with Progress org.apache.hadoop.hive.ql.parse.repl.metric.event.Progress used on opensource projects

Example 11 with Progress

use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Progress in project hive by apache.

the class ReplicationMetricCollector method reportEnd.

public void reportEnd(Status status) throws SemanticException {
    if (isEnabled) {
        LOG.info("End {}", status);
        Progress progress = replicationMetric.getProgress();
        progress.setStatus(status);
        replicationMetric.setProgress(progress);
        metricCollector.addMetric(replicationMetric);
    }
}

Example 12 with Progress

use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Progress in project hive by apache.

the class ReplicationMetricCollector method reportFailoverStart.

public void reportFailoverStart(String stageName, Map<String, Long> metricMap, FailoverMetaData failoverMd) throws SemanticException {
    if (isEnabled) {
        LOG.info("Failover Stage Started {}, {}, {}", stageName, metricMap.size(), metricMap);
        Progress progress = replicationMetric.getProgress();
        progress.setStatus(Status.FAILOVER_IN_PROGRESS);
        Stage stage = new Stage(stageName, Status.IN_PROGRESS, getCurrentTimeInMillis());
        for (Map.Entry<String, Long> metric : metricMap.entrySet()) {
            stage.addMetric(new Metric(metric.getKey(), metric.getValue()));
        }
        progress.addStage(stage);
        replicationMetric.setProgress(progress);
        Metadata metadata = replicationMetric.getMetadata();
        metadata.setFailoverMetadataLoc(failoverMd.getFilePath());
        metadata.setFailoverEventId(failoverMd.getFailoverEventId());
        replicationMetric.setMetadata(metadata);
        metricCollector.addMetric(replicationMetric);
    }
}

Example 13 with Progress

use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Progress in project hive by apache.

the class TestReplicationMetricCollector method testSuccessIncrLoadMetrics.

@Test
public void testSuccessIncrLoadMetrics() throws Exception {
    ReplicationMetricCollector incrLoadMetricCollector = new IncrementalLoadMetricCollector("db", "dummyDir", 1, conf);
    Map<String, Long> metricMap = new HashMap<>();
    metricMap.put(ReplUtils.MetricName.TABLES.name(), (long) 10);
    metricMap.put(ReplUtils.MetricName.FUNCTIONS.name(), (long) 1);
    incrLoadMetricCollector.reportStageStart("dump", metricMap);
    incrLoadMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.TABLES.name(), 1);
    List<ReplicationMetric> actualMetrics = MetricCollector.getInstance().getMetrics();
    Assert.assertEquals(1, actualMetrics.size());
    incrLoadMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.TABLES.name(), 2);
    incrLoadMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.FUNCTIONS.name(), 1);
    actualMetrics = MetricCollector.getInstance().getMetrics();
    Assert.assertEquals(1, actualMetrics.size());
    incrLoadMetricCollector.reportStageEnd("dump", Status.SUCCESS, 10, new SnapshotUtils.ReplSnapshotCount(), new ReplStatsTracker(0));
    incrLoadMetricCollector.reportEnd(Status.SUCCESS);
    actualMetrics = MetricCollector.getInstance().getMetrics();
    Assert.assertEquals(1, actualMetrics.size());
    Metadata expectedMetadata = new Metadata("db", Metadata.ReplicationType.INCREMENTAL, "dummyDir");
    expectedMetadata.setLastReplId(10);
    Progress expectedProgress = new Progress();
    expectedProgress.setStatus(Status.SUCCESS);
    Stage dumpStage = new Stage("dump", Status.SUCCESS, 0);
    dumpStage.setEndTime(0);
    Metric expectedTableMetric = new Metric(ReplUtils.MetricName.TABLES.name(), 10);
    expectedTableMetric.setCurrentCount(3);
    Metric expectedFuncMetric = new Metric(ReplUtils.MetricName.FUNCTIONS.name(), 1);
    expectedFuncMetric.setCurrentCount(1);
    dumpStage.addMetric(expectedTableMetric);
    dumpStage.addMetric(expectedFuncMetric);
    expectedProgress.addStage(dumpStage);
    ReplicationMetric expectedMetric = new ReplicationMetric(1, "repl", 1, expectedMetadata);
    expectedMetric.setProgress(expectedProgress);
    checkSuccess(actualMetrics.get(0), expectedMetric, "dump", Arrays.asList(ReplUtils.MetricName.TABLES.name(), ReplUtils.MetricName.FUNCTIONS.name()));
}

Example 14 with Progress

use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Progress in project hive by apache.

the class TestReplicationMetricCollector method testSuccessBootstrapDumpMetrics.

@Test
public void testSuccessBootstrapDumpMetrics() throws Exception {
    ReplicationMetricCollector bootstrapDumpMetricCollector = new BootstrapDumpMetricCollector("db", "dummyDir", conf);
    Map<String, Long> metricMap = new HashMap<>();
    metricMap.put(ReplUtils.MetricName.TABLES.name(), (long) 10);
    metricMap.put(ReplUtils.MetricName.FUNCTIONS.name(), (long) 1);
    bootstrapDumpMetricCollector.reportStageStart("dump", metricMap);
    bootstrapDumpMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.TABLES.name(), 1);
    List<ReplicationMetric> actualMetrics = MetricCollector.getInstance().getMetrics();
    Assert.assertEquals(1, actualMetrics.size());
    bootstrapDumpMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.TABLES.name(), 2);
    bootstrapDumpMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.FUNCTIONS.name(), 1);
    actualMetrics = MetricCollector.getInstance().getMetrics();
    Assert.assertEquals(1, actualMetrics.size());
    bootstrapDumpMetricCollector.reportStageEnd("dump", Status.SUCCESS, 10, new SnapshotUtils.ReplSnapshotCount(), new ReplStatsTracker(0));
    bootstrapDumpMetricCollector.reportEnd(Status.SUCCESS);
    actualMetrics = MetricCollector.getInstance().getMetrics();
    Assert.assertEquals(1, actualMetrics.size());
    Metadata expectedMetadata = new Metadata("db", Metadata.ReplicationType.BOOTSTRAP, "dummyDir");
    expectedMetadata.setLastReplId(10);
    Progress expectedProgress = new Progress();
    expectedProgress.setStatus(Status.SUCCESS);
    Stage dumpStage = new Stage("dump", Status.SUCCESS, 0);
    dumpStage.setEndTime(0);
    Metric expectedTableMetric = new Metric(ReplUtils.MetricName.TABLES.name(), 10);
    expectedTableMetric.setCurrentCount(3);
    Metric expectedFuncMetric = new Metric(ReplUtils.MetricName.FUNCTIONS.name(), 1);
    expectedFuncMetric.setCurrentCount(1);
    dumpStage.addMetric(expectedTableMetric);
    dumpStage.addMetric(expectedFuncMetric);
    expectedProgress.addStage(dumpStage);
    ReplicationMetric expectedMetric = new ReplicationMetric(1, "repl", 0, expectedMetadata);
    expectedMetric.setProgress(expectedProgress);
    checkSuccess(actualMetrics.get(0), expectedMetric, "dump", Arrays.asList(ReplUtils.MetricName.TABLES.name(), ReplUtils.MetricName.FUNCTIONS.name()));
}

Example 15 with Progress

use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Progress in project hive by apache.

the class TestReplicationMetricUpdateOnFailure method performRecoverableChecks.

void performRecoverableChecks(String stageName) {
    List<ReplicationMetric> metricList = MetricCollector.getInstance().getMetrics();
    Assert.assertEquals(1, metricList.size());
    ReplicationMetric updatedMetric = metricList.get(0);
    Progress updatedProgress = updatedMetric.getProgress();
    Assert.assertEquals(Status.FAILED, updatedProgress.getStatus());
    Assert.assertEquals(1, updatedProgress.getStages().size());
    Assert.assertEquals(Status.FAILED, updatedProgress.getStageByName(stageName).getStatus());
    Assert.assertNotEquals(0, updatedProgress.getStageByName(stageName).getEndTime());
}