Examples with Stage org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage used on opensource projects

Example 11 with Stage

use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage in project hive by apache.

the class TestReplicationScenariosUsingSnapshots method assertIncrementalMetricsValues.

private void assertIncrementalMetricsValues(Metadata.ReplicationType replicationType, MetricCollector collector, int numCreated, int numDeleted) {
    Iterator<ReplicationMetric> itr;
    itr = collector.getMetrics().iterator();
    while (itr.hasNext()) {
        ReplicationMetric elem = itr.next();
        assertEquals(replicationType, elem.getMetadata().getReplicationType());
        List<Stage> stages = elem.getProgress().getStages();
        for (Stage stage : stages) {
            SnapshotUtils.ReplSnapshotCount count = stage.getReplSnapshotCount();
            assertEquals(numCreated, count.getNumCreated());
            assertEquals(numDeleted, count.getNumDeleted());
        }
    }
}

Example 12 with Stage

use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage in project hive by apache.

the class TestReplicationScenariosUsingSnapshots method testSnapshotMetrics.

@Test
public void testSnapshotMetrics() throws Throwable {
    conf.set(Constants.SCHEDULED_QUERY_SCHEDULENAME, "metrics_test");
    List<String> withClause = ReplicationTestUtils.includeExternalTableClause(true);
    MetricCollector collector = MetricCollector.getInstance();
    Path externalDatabaseLocation = new Path("/" + testName.getMethodName() + "/externalDatabase/");
    DistributedFileSystem fs = primary.miniDFSCluster.getFileSystem();
    fs.mkdirs(externalDatabaseLocation, new FsPermission("777"));
    Path externalTableLocation1 = new Path("/" + testName.getMethodName() + "/t1/");
    fs = primary.miniDFSCluster.getFileSystem();
    fs.mkdirs(externalTableLocation1, new FsPermission("777"));
    withClause.add("'hive.repl.external.warehouse.single.copy.task.paths'='" + externalTableLocation1.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString() + "'");
    WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName).run("create external table emp1 (id int)").run("insert into emp1 values(1),(2)").run("create external table exttab (place string) row format delimited fields terminated by ','" + " location '" + externalTableLocation1.toString() + "'").run("insert into exttab values('lucknow')").dump(primaryDbName, withClause);
    // The boootstrap stage, 2 directories for which snapshot is enabled, the database directory and the one table
    // as part of the config. This would be initial copy stage, so only 1 snapshot per directory and none to be deleted.
    assertIncrementalMetricsValues(BOOTSTRAP, collector, 2, 0);
    Iterator<ReplicationMetric> itr = collector.getMetrics().iterator();
    while (itr.hasNext()) {
        ReplicationMetric elem = itr.next();
        assertEquals(BOOTSTRAP, elem.getMetadata().getReplicationType());
        List<Stage> stages = elem.getProgress().getStages();
        for (Stage stage : stages) {
            SnapshotUtils.ReplSnapshotCount counts = stage.getReplSnapshotCount();
            assertEquals(2, counts.getNumCreated());
            assertEquals(0, counts.getNumDeleted());
        }
    }
    // Load and check if the data and table are there.
    replica.load(replicatedDbName, primaryDbName, withClause).run("use " + replicatedDbName).run("show tables like 'emp1'").verifyResults(new String[] { "emp1" }).run("select id from emp1").verifyResults(new String[] { "1", "2" }).run("show tables like 'exttab'").verifyResults(new String[] { "exttab" }).run("select place from exttab").verifyResults(new String[] { "lucknow" }).verifyReplTargetProperty(replicatedDbName);
    // Add some data and try incremental dump.
    tuple = primary.run("use " + primaryDbName).run("insert into emp1 values(3),(4)").run("insert into exttab values('agra')").dump(primaryDbName, withClause);
    // This is from the diff stage, 2 Directories where snapshots were enabled, 1 old snapshots got deleted and 1
    // got created, so 2 created and 2 deleted.
    assertIncrementalMetricsValues(INCREMENTAL, collector, 2, 2);
    // Do a load
    replica.load(replicatedDbName, primaryDbName, withClause);
    // Remove the with clause, hence the external table specified as part of the config.
    tuple = primary.run("use " + primaryDbName).run("insert into exttab values('lucknow')").dump(primaryDbName, null);
    // Only one directory, i.e the database directory is going through snapshot based replication, so only 1 created
    // for it and 1 old deleted for it, 2 deleted for the table removed from the snapshot based replication scope.
    assertIncrementalMetricsValues(INCREMENTAL, collector, 1, 3);
}

Example 13 with Stage

use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage in project hive by apache.

the class TestScheduledReplicationScenarios method generateExpectedMetric.

private ReplicationMetric generateExpectedMetric(String policy, long dumpExecId, String dbName, Metadata.ReplicationType replicationType, String staging, long lastReplId, Status status, List<Stage> stages) {
    Metadata metadata = new Metadata(dbName, replicationType, staging);
    metadata.setLastReplId(lastReplId);
    ReplicationMetric replicationMetric = new ReplicationMetric(0, policy, dumpExecId, metadata);
    Progress progress = new Progress();
    progress.setStatus(status);
    for (Stage stage : stages) {
        progress.addStage(stage);
    }
    replicationMetric.setProgress(progress);
    return replicationMetric;
}

Example 14 with Stage

use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage in project hive by apache.

the class ReplicationMetricCollector method reportFailoverStart.

public void reportFailoverStart(String stageName, Map<String, Long> metricMap, FailoverMetaData failoverMd) throws SemanticException {
    if (isEnabled) {
        LOG.info("Failover Stage Started {}, {}, {}", stageName, metricMap.size(), metricMap);
        Progress progress = replicationMetric.getProgress();
        progress.setStatus(Status.FAILOVER_IN_PROGRESS);
        Stage stage = new Stage(stageName, Status.IN_PROGRESS, getCurrentTimeInMillis());
        for (Map.Entry<String, Long> metric : metricMap.entrySet()) {
            stage.addMetric(new Metric(metric.getKey(), metric.getValue()));
        }
        progress.addStage(stage);
        replicationMetric.setProgress(progress);
        Metadata metadata = replicationMetric.getMetadata();
        metadata.setFailoverMetadataLoc(failoverMd.getFilePath());
        metadata.setFailoverEventId(failoverMd.getFailoverEventId());
        replicationMetric.setMetadata(metadata);
        metricCollector.addMetric(replicationMetric);
    }
}

Example 15 with Stage

use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage in project hive by apache.

the class TestReplicationMetricCollector method testSuccessIncrLoadMetrics.

@Test
public void testSuccessIncrLoadMetrics() throws Exception {
    ReplicationMetricCollector incrLoadMetricCollector = new IncrementalLoadMetricCollector("db", "dummyDir", 1, conf);
    Map<String, Long> metricMap = new HashMap<>();
    metricMap.put(ReplUtils.MetricName.TABLES.name(), (long) 10);
    metricMap.put(ReplUtils.MetricName.FUNCTIONS.name(), (long) 1);
    incrLoadMetricCollector.reportStageStart("dump", metricMap);
    incrLoadMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.TABLES.name(), 1);
    List<ReplicationMetric> actualMetrics = MetricCollector.getInstance().getMetrics();
    Assert.assertEquals(1, actualMetrics.size());
    incrLoadMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.TABLES.name(), 2);
    incrLoadMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.FUNCTIONS.name(), 1);
    actualMetrics = MetricCollector.getInstance().getMetrics();
    Assert.assertEquals(1, actualMetrics.size());
    incrLoadMetricCollector.reportStageEnd("dump", Status.SUCCESS, 10, new SnapshotUtils.ReplSnapshotCount(), new ReplStatsTracker(0));
    incrLoadMetricCollector.reportEnd(Status.SUCCESS);
    actualMetrics = MetricCollector.getInstance().getMetrics();
    Assert.assertEquals(1, actualMetrics.size());
    Metadata expectedMetadata = new Metadata("db", Metadata.ReplicationType.INCREMENTAL, "dummyDir");
    expectedMetadata.setLastReplId(10);
    Progress expectedProgress = new Progress();
    expectedProgress.setStatus(Status.SUCCESS);
    Stage dumpStage = new Stage("dump", Status.SUCCESS, 0);
    dumpStage.setEndTime(0);
    Metric expectedTableMetric = new Metric(ReplUtils.MetricName.TABLES.name(), 10);
    expectedTableMetric.setCurrentCount(3);
    Metric expectedFuncMetric = new Metric(ReplUtils.MetricName.FUNCTIONS.name(), 1);
    expectedFuncMetric.setCurrentCount(1);
    dumpStage.addMetric(expectedTableMetric);
    dumpStage.addMetric(expectedFuncMetric);
    expectedProgress.addStage(dumpStage);
    ReplicationMetric expectedMetric = new ReplicationMetric(1, "repl", 1, expectedMetadata);
    expectedMetric.setProgress(expectedProgress);
    checkSuccess(actualMetrics.get(0), expectedMetric, "dump", Arrays.asList(ReplUtils.MetricName.TABLES.name(), ReplUtils.MetricName.FUNCTIONS.name()));
}