use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage in project hive by apache.
the class TestReplicationScenariosUsingSnapshots method assertIncrementalMetricsValues.
private void assertIncrementalMetricsValues(Metadata.ReplicationType replicationType, MetricCollector collector, int numCreated, int numDeleted) {
Iterator<ReplicationMetric> itr;
itr = collector.getMetrics().iterator();
while (itr.hasNext()) {
ReplicationMetric elem = itr.next();
assertEquals(replicationType, elem.getMetadata().getReplicationType());
List<Stage> stages = elem.getProgress().getStages();
for (Stage stage : stages) {
SnapshotUtils.ReplSnapshotCount count = stage.getReplSnapshotCount();
assertEquals(numCreated, count.getNumCreated());
assertEquals(numDeleted, count.getNumDeleted());
}
}
}
use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage in project hive by apache.
the class TestReplicationScenariosUsingSnapshots method testSnapshotMetrics.
@Test
public void testSnapshotMetrics() throws Throwable {
conf.set(Constants.SCHEDULED_QUERY_SCHEDULENAME, "metrics_test");
List<String> withClause = ReplicationTestUtils.includeExternalTableClause(true);
MetricCollector collector = MetricCollector.getInstance();
Path externalDatabaseLocation = new Path("/" + testName.getMethodName() + "/externalDatabase/");
DistributedFileSystem fs = primary.miniDFSCluster.getFileSystem();
fs.mkdirs(externalDatabaseLocation, new FsPermission("777"));
Path externalTableLocation1 = new Path("/" + testName.getMethodName() + "/t1/");
fs = primary.miniDFSCluster.getFileSystem();
fs.mkdirs(externalTableLocation1, new FsPermission("777"));
withClause.add("'hive.repl.external.warehouse.single.copy.task.paths'='" + externalTableLocation1.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString() + "'");
WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName).run("create external table emp1 (id int)").run("insert into emp1 values(1),(2)").run("create external table exttab (place string) row format delimited fields terminated by ','" + " location '" + externalTableLocation1.toString() + "'").run("insert into exttab values('lucknow')").dump(primaryDbName, withClause);
// The boootstrap stage, 2 directories for which snapshot is enabled, the database directory and the one table
// as part of the config. This would be initial copy stage, so only 1 snapshot per directory and none to be deleted.
assertIncrementalMetricsValues(BOOTSTRAP, collector, 2, 0);
Iterator<ReplicationMetric> itr = collector.getMetrics().iterator();
while (itr.hasNext()) {
ReplicationMetric elem = itr.next();
assertEquals(BOOTSTRAP, elem.getMetadata().getReplicationType());
List<Stage> stages = elem.getProgress().getStages();
for (Stage stage : stages) {
SnapshotUtils.ReplSnapshotCount counts = stage.getReplSnapshotCount();
assertEquals(2, counts.getNumCreated());
assertEquals(0, counts.getNumDeleted());
}
}
// Load and check if the data and table are there.
replica.load(replicatedDbName, primaryDbName, withClause).run("use " + replicatedDbName).run("show tables like 'emp1'").verifyResults(new String[] { "emp1" }).run("select id from emp1").verifyResults(new String[] { "1", "2" }).run("show tables like 'exttab'").verifyResults(new String[] { "exttab" }).run("select place from exttab").verifyResults(new String[] { "lucknow" }).verifyReplTargetProperty(replicatedDbName);
// Add some data and try incremental dump.
tuple = primary.run("use " + primaryDbName).run("insert into emp1 values(3),(4)").run("insert into exttab values('agra')").dump(primaryDbName, withClause);
// This is from the diff stage, 2 Directories where snapshots were enabled, 1 old snapshots got deleted and 1
// got created, so 2 created and 2 deleted.
assertIncrementalMetricsValues(INCREMENTAL, collector, 2, 2);
// Do a load
replica.load(replicatedDbName, primaryDbName, withClause);
// Remove the with clause, hence the external table specified as part of the config.
tuple = primary.run("use " + primaryDbName).run("insert into exttab values('lucknow')").dump(primaryDbName, null);
// Only one directory, i.e the database directory is going through snapshot based replication, so only 1 created
// for it and 1 old deleted for it, 2 deleted for the table removed from the snapshot based replication scope.
assertIncrementalMetricsValues(INCREMENTAL, collector, 1, 3);
}
use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage in project hive by apache.
the class TestScheduledReplicationScenarios method generateExpectedMetric.
private ReplicationMetric generateExpectedMetric(String policy, long dumpExecId, String dbName, Metadata.ReplicationType replicationType, String staging, long lastReplId, Status status, List<Stage> stages) {
Metadata metadata = new Metadata(dbName, replicationType, staging);
metadata.setLastReplId(lastReplId);
ReplicationMetric replicationMetric = new ReplicationMetric(0, policy, dumpExecId, metadata);
Progress progress = new Progress();
progress.setStatus(status);
for (Stage stage : stages) {
progress.addStage(stage);
}
replicationMetric.setProgress(progress);
return replicationMetric;
}
use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage in project hive by apache.
the class ReplicationMetricCollector method reportFailoverStart.
public void reportFailoverStart(String stageName, Map<String, Long> metricMap, FailoverMetaData failoverMd) throws SemanticException {
if (isEnabled) {
LOG.info("Failover Stage Started {}, {}, {}", stageName, metricMap.size(), metricMap);
Progress progress = replicationMetric.getProgress();
progress.setStatus(Status.FAILOVER_IN_PROGRESS);
Stage stage = new Stage(stageName, Status.IN_PROGRESS, getCurrentTimeInMillis());
for (Map.Entry<String, Long> metric : metricMap.entrySet()) {
stage.addMetric(new Metric(metric.getKey(), metric.getValue()));
}
progress.addStage(stage);
replicationMetric.setProgress(progress);
Metadata metadata = replicationMetric.getMetadata();
metadata.setFailoverMetadataLoc(failoverMd.getFilePath());
metadata.setFailoverEventId(failoverMd.getFailoverEventId());
replicationMetric.setMetadata(metadata);
metricCollector.addMetric(replicationMetric);
}
}
use of org.apache.hadoop.hive.ql.parse.repl.metric.event.Stage in project hive by apache.
the class TestReplicationMetricCollector method testSuccessIncrLoadMetrics.
@Test
public void testSuccessIncrLoadMetrics() throws Exception {
ReplicationMetricCollector incrLoadMetricCollector = new IncrementalLoadMetricCollector("db", "dummyDir", 1, conf);
Map<String, Long> metricMap = new HashMap<>();
metricMap.put(ReplUtils.MetricName.TABLES.name(), (long) 10);
metricMap.put(ReplUtils.MetricName.FUNCTIONS.name(), (long) 1);
incrLoadMetricCollector.reportStageStart("dump", metricMap);
incrLoadMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.TABLES.name(), 1);
List<ReplicationMetric> actualMetrics = MetricCollector.getInstance().getMetrics();
Assert.assertEquals(1, actualMetrics.size());
incrLoadMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.TABLES.name(), 2);
incrLoadMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.FUNCTIONS.name(), 1);
actualMetrics = MetricCollector.getInstance().getMetrics();
Assert.assertEquals(1, actualMetrics.size());
incrLoadMetricCollector.reportStageEnd("dump", Status.SUCCESS, 10, new SnapshotUtils.ReplSnapshotCount(), new ReplStatsTracker(0));
incrLoadMetricCollector.reportEnd(Status.SUCCESS);
actualMetrics = MetricCollector.getInstance().getMetrics();
Assert.assertEquals(1, actualMetrics.size());
Metadata expectedMetadata = new Metadata("db", Metadata.ReplicationType.INCREMENTAL, "dummyDir");
expectedMetadata.setLastReplId(10);
Progress expectedProgress = new Progress();
expectedProgress.setStatus(Status.SUCCESS);
Stage dumpStage = new Stage("dump", Status.SUCCESS, 0);
dumpStage.setEndTime(0);
Metric expectedTableMetric = new Metric(ReplUtils.MetricName.TABLES.name(), 10);
expectedTableMetric.setCurrentCount(3);
Metric expectedFuncMetric = new Metric(ReplUtils.MetricName.FUNCTIONS.name(), 1);
expectedFuncMetric.setCurrentCount(1);
dumpStage.addMetric(expectedTableMetric);
dumpStage.addMetric(expectedFuncMetric);
expectedProgress.addStage(dumpStage);
ReplicationMetric expectedMetric = new ReplicationMetric(1, "repl", 1, expectedMetadata);
expectedMetric.setProgress(expectedProgress);
checkSuccess(actualMetrics.get(0), expectedMetric, "dump", Arrays.asList(ReplUtils.MetricName.TABLES.name(), ReplUtils.MetricName.FUNCTIONS.name()));
}
Aggregations