Search in sources :

Example 11 with Stage

use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage in project hive by apache.

the class TestReplicationScenariosUsingSnapshots method assertIncrementalMetricsValues.

private void assertIncrementalMetricsValues(Metadata.ReplicationType replicationType, MetricCollector collector, int numCreated, int numDeleted) {
    Iterator<ReplicationMetric> itr;
    itr = collector.getMetrics().iterator();
    while (itr.hasNext()) {
        ReplicationMetric elem = itr.next();
        assertEquals(replicationType, elem.getMetadata().getReplicationType());
        List<Stage> stages = elem.getProgress().getStages();
        for (Stage stage : stages) {
            SnapshotUtils.ReplSnapshotCount count = stage.getReplSnapshotCount();
            assertEquals(numCreated, count.getNumCreated());
            assertEquals(numDeleted, count.getNumDeleted());
        }
    }
}
Also used : SnapshotUtils(org.apache.hadoop.hive.ql.exec.repl.util.SnapshotUtils) Stage(org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage) ReplicationMetric(org.apache.hadoop.hive.ql.parse.repl.metric.event.ReplicationMetric)

Example 12 with Stage

use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage in project hive by apache.

the class TestReplicationScenariosUsingSnapshots method testSnapshotMetrics.

@Test
public void testSnapshotMetrics() throws Throwable {
    conf.set(Constants.SCHEDULED_QUERY_SCHEDULENAME, "metrics_test");
    List<String> withClause = ReplicationTestUtils.includeExternalTableClause(true);
    MetricCollector collector = MetricCollector.getInstance();
    Path externalDatabaseLocation = new Path("/" + testName.getMethodName() + "/externalDatabase/");
    DistributedFileSystem fs = primary.miniDFSCluster.getFileSystem();
    fs.mkdirs(externalDatabaseLocation, new FsPermission("777"));
    Path externalTableLocation1 = new Path("/" + testName.getMethodName() + "/t1/");
    fs = primary.miniDFSCluster.getFileSystem();
    fs.mkdirs(externalTableLocation1, new FsPermission("777"));
    withClause.add("'hive.repl.external.warehouse.single.copy.task.paths'='" + externalTableLocation1.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString() + "'");
    WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName).run("create external table emp1 (id int)").run("insert into emp1 values(1),(2)").run("create external table exttab (place string) row format delimited fields terminated by ','" + " location '" + externalTableLocation1.toString() + "'").run("insert into exttab values('lucknow')").dump(primaryDbName, withClause);
    // The boootstrap stage, 2 directories for which snapshot is enabled, the database directory and the one table
    // as part of the config. This would be initial copy stage, so only 1 snapshot per directory and none to be deleted.
    assertIncrementalMetricsValues(BOOTSTRAP, collector, 2, 0);
    Iterator<ReplicationMetric> itr = collector.getMetrics().iterator();
    while (itr.hasNext()) {
        ReplicationMetric elem = itr.next();
        assertEquals(BOOTSTRAP, elem.getMetadata().getReplicationType());
        List<Stage> stages = elem.getProgress().getStages();
        for (Stage stage : stages) {
            SnapshotUtils.ReplSnapshotCount counts = stage.getReplSnapshotCount();
            assertEquals(2, counts.getNumCreated());
            assertEquals(0, counts.getNumDeleted());
        }
    }
    // Load and check if the data and table are there.
    replica.load(replicatedDbName, primaryDbName, withClause).run("use " + replicatedDbName).run("show tables like 'emp1'").verifyResults(new String[] { "emp1" }).run("select id from emp1").verifyResults(new String[] { "1", "2" }).run("show tables like 'exttab'").verifyResults(new String[] { "exttab" }).run("select place from exttab").verifyResults(new String[] { "lucknow" }).verifyReplTargetProperty(replicatedDbName);
    // Add some data and try incremental dump.
    tuple = primary.run("use " + primaryDbName).run("insert into emp1 values(3),(4)").run("insert into exttab values('agra')").dump(primaryDbName, withClause);
    // This is from the diff stage, 2 Directories where snapshots were enabled, 1 old snapshots got deleted and 1
    // got created, so 2 created and 2 deleted.
    assertIncrementalMetricsValues(INCREMENTAL, collector, 2, 2);
    // Do a load
    replica.load(replicatedDbName, primaryDbName, withClause);
    // Remove the with clause, hence the external table specified as part of the config.
    tuple = primary.run("use " + primaryDbName).run("insert into exttab values('lucknow')").dump(primaryDbName, null);
    // Only one directory, i.e the database directory is going through snapshot based replication, so only 1 created
    // for it and 1 old deleted for it, 2 deleted for the table removed from the snapshot based replication scope.
    assertIncrementalMetricsValues(INCREMENTAL, collector, 1, 3);
}
Also used : Path(org.apache.hadoop.fs.Path) ReplExternalTables.externalTableDataPath(org.apache.hadoop.hive.ql.exec.repl.ReplExternalTables.externalTableDataPath) DistributedFileSystem(org.apache.hadoop.hdfs.DistributedFileSystem) ReplicationMetric(org.apache.hadoop.hive.ql.parse.repl.metric.event.ReplicationMetric) SnapshotUtils(org.apache.hadoop.hive.ql.exec.repl.util.SnapshotUtils) MetricCollector(org.apache.hadoop.hive.ql.parse.repl.metric.MetricCollector) Stage(org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage) FsPermission(org.apache.hadoop.fs.permission.FsPermission) Test(org.junit.Test)

Example 13 with Stage

use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage in project hive by apache.

the class TestScheduledReplicationScenarios method generateExpectedMetric.

private ReplicationMetric generateExpectedMetric(String policy, long dumpExecId, String dbName, Metadata.ReplicationType replicationType, String staging, long lastReplId, Status status, List<Stage> stages) {
    Metadata metadata = new Metadata(dbName, replicationType, staging);
    metadata.setLastReplId(lastReplId);
    ReplicationMetric replicationMetric = new ReplicationMetric(0, policy, dumpExecId, metadata);
    Progress progress = new Progress();
    progress.setStatus(status);
    for (Stage stage : stages) {
        progress.addStage(stage);
    }
    replicationMetric.setProgress(progress);
    return replicationMetric;
}
Also used : Progress(org.apache.hadoop.hive.ql.parse.repl.metric.event.Progress) Metadata(org.apache.hadoop.hive.ql.parse.repl.metric.event.Metadata) Stage(org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage) ReplicationMetric(org.apache.hadoop.hive.ql.parse.repl.metric.event.ReplicationMetric)

Example 14 with Stage

use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage in project hive by apache.

the class ReplicationMetricCollector method reportFailoverStart.

public void reportFailoverStart(String stageName, Map<String, Long> metricMap, FailoverMetaData failoverMd) throws SemanticException {
    if (isEnabled) {
        LOG.info("Failover Stage Started {}, {}, {}", stageName, metricMap.size(), metricMap);
        Progress progress = replicationMetric.getProgress();
        progress.setStatus(Status.FAILOVER_IN_PROGRESS);
        Stage stage = new Stage(stageName, Status.IN_PROGRESS, getCurrentTimeInMillis());
        for (Map.Entry<String, Long> metric : metricMap.entrySet()) {
            stage.addMetric(new Metric(metric.getKey(), metric.getValue()));
        }
        progress.addStage(stage);
        replicationMetric.setProgress(progress);
        Metadata metadata = replicationMetric.getMetadata();
        metadata.setFailoverMetadataLoc(failoverMd.getFilePath());
        metadata.setFailoverEventId(failoverMd.getFailoverEventId());
        replicationMetric.setMetadata(metadata);
        metricCollector.addMetric(replicationMetric);
    }
}
Also used : Progress(org.apache.hadoop.hive.ql.parse.repl.metric.event.Progress) Metadata(org.apache.hadoop.hive.ql.parse.repl.metric.event.Metadata) Stage(org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage) Metric(org.apache.hadoop.hive.ql.parse.repl.metric.event.Metric) ReplicationMetric(org.apache.hadoop.hive.ql.parse.repl.metric.event.ReplicationMetric) Map(java.util.Map)

Example 15 with Stage

use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage in project hive by apache.

the class TestReplicationMetricCollector method testSuccessIncrLoadMetrics.

@Test
public void testSuccessIncrLoadMetrics() throws Exception {
    ReplicationMetricCollector incrLoadMetricCollector = new IncrementalLoadMetricCollector("db", "dummyDir", 1, conf);
    Map<String, Long> metricMap = new HashMap<>();
    metricMap.put(ReplUtils.MetricName.TABLES.name(), (long) 10);
    metricMap.put(ReplUtils.MetricName.FUNCTIONS.name(), (long) 1);
    incrLoadMetricCollector.reportStageStart("dump", metricMap);
    incrLoadMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.TABLES.name(), 1);
    List<ReplicationMetric> actualMetrics = MetricCollector.getInstance().getMetrics();
    Assert.assertEquals(1, actualMetrics.size());
    incrLoadMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.TABLES.name(), 2);
    incrLoadMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.FUNCTIONS.name(), 1);
    actualMetrics = MetricCollector.getInstance().getMetrics();
    Assert.assertEquals(1, actualMetrics.size());
    incrLoadMetricCollector.reportStageEnd("dump", Status.SUCCESS, 10, new SnapshotUtils.ReplSnapshotCount(), new ReplStatsTracker(0));
    incrLoadMetricCollector.reportEnd(Status.SUCCESS);
    actualMetrics = MetricCollector.getInstance().getMetrics();
    Assert.assertEquals(1, actualMetrics.size());
    Metadata expectedMetadata = new Metadata("db", Metadata.ReplicationType.INCREMENTAL, "dummyDir");
    expectedMetadata.setLastReplId(10);
    Progress expectedProgress = new Progress();
    expectedProgress.setStatus(Status.SUCCESS);
    Stage dumpStage = new Stage("dump", Status.SUCCESS, 0);
    dumpStage.setEndTime(0);
    Metric expectedTableMetric = new Metric(ReplUtils.MetricName.TABLES.name(), 10);
    expectedTableMetric.setCurrentCount(3);
    Metric expectedFuncMetric = new Metric(ReplUtils.MetricName.FUNCTIONS.name(), 1);
    expectedFuncMetric.setCurrentCount(1);
    dumpStage.addMetric(expectedTableMetric);
    dumpStage.addMetric(expectedFuncMetric);
    expectedProgress.addStage(dumpStage);
    ReplicationMetric expectedMetric = new ReplicationMetric(1, "repl", 1, expectedMetadata);
    expectedMetric.setProgress(expectedProgress);
    checkSuccess(actualMetrics.get(0), expectedMetric, "dump", Arrays.asList(ReplUtils.MetricName.TABLES.name(), ReplUtils.MetricName.FUNCTIONS.name()));
}
Also used : ReplStatsTracker(org.apache.hadoop.hive.ql.exec.repl.ReplStatsTracker) Progress(org.apache.hadoop.hive.ql.parse.repl.metric.event.Progress) HashMap(java.util.HashMap) Metadata(org.apache.hadoop.hive.ql.parse.repl.metric.event.Metadata) ReplicationMetric(org.apache.hadoop.hive.ql.parse.repl.metric.event.ReplicationMetric) SnapshotUtils(org.apache.hadoop.hive.ql.exec.repl.util.SnapshotUtils) Stage(org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage) Metric(org.apache.hadoop.hive.ql.parse.repl.metric.event.Metric) ReplicationMetric(org.apache.hadoop.hive.ql.parse.repl.metric.event.ReplicationMetric) IncrementalLoadMetricCollector(org.apache.hadoop.hive.ql.parse.repl.load.metric.IncrementalLoadMetricCollector) Test(org.junit.Test)

Aggregations

Stage (org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage)18 ReplicationMetric (org.apache.hadoop.hive.ql.parse.repl.metric.event.ReplicationMetric)16 Metric (org.apache.hadoop.hive.ql.parse.repl.metric.event.Metric)12 Progress (org.apache.hadoop.hive.ql.parse.repl.metric.event.Progress)12 Metadata (org.apache.hadoop.hive.ql.parse.repl.metric.event.Metadata)9 SnapshotUtils (org.apache.hadoop.hive.ql.exec.repl.util.SnapshotUtils)8 Test (org.junit.Test)8 HashMap (java.util.HashMap)6 ReplStatsTracker (org.apache.hadoop.hive.ql.exec.repl.ReplStatsTracker)6 IncrementalDumpMetricCollector (org.apache.hadoop.hive.ql.parse.repl.dump.metric.IncrementalDumpMetricCollector)3 ArrayList (java.util.ArrayList)2 Map (java.util.Map)2 BootstrapDumpMetricCollector (org.apache.hadoop.hive.ql.parse.repl.dump.metric.BootstrapDumpMetricCollector)2 BootstrapLoadMetricCollector (org.apache.hadoop.hive.ql.parse.repl.load.metric.BootstrapLoadMetricCollector)2 IncrementalLoadMetricCollector (org.apache.hadoop.hive.ql.parse.repl.load.metric.IncrementalLoadMetricCollector)2 MetricCollector (org.apache.hadoop.hive.ql.parse.repl.metric.MetricCollector)2 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 Path (org.apache.hadoop.fs.Path)1 FsPermission (org.apache.hadoop.fs.permission.FsPermission)1 DistributedFileSystem (org.apache.hadoop.hdfs.DistributedFileSystem)1