Examples with IncrementalLoadMetricCollector org.apache.hadoop.hive.ql.parse.repl.load.metric.IncrementalLoadMetricCollector used on opensource projects

Example 1 with IncrementalLoadMetricCollector

use of org.apache.hadoop.hive.ql.parse.repl.load.metric.IncrementalLoadMetricCollector in project hive by apache.

the class TestReplicationScenarios method getReplLoadRootTask.

private Task getReplLoadRootTask(String sourceDb, String replicadb, boolean isIncrementalDump, Tuple tuple) throws Throwable {
    HiveConf confTemp = driverMirror.getConf();
    Path loadPath = new Path(tuple.dumpLocation, ReplUtils.REPL_HIVE_BASE_DIR);
    ReplicationMetricCollector metricCollector;
    if (isIncrementalDump) {
        metricCollector = new IncrementalLoadMetricCollector(replicadb, tuple.dumpLocation, 0, confTemp);
    } else {
        metricCollector = new BootstrapLoadMetricCollector(replicadb, tuple.dumpLocation, 0, confTemp);
    }
    /* When 'hive.repl.retain.custom.db.locations.on.target' is enabled, the first iteration of repl load would
       run only database creation task, and only in next iteration of Repl Load Task execution, remaining tasks will be
       executed. Hence disabling this to perform the test on task optimization.  */
    confTemp.setBoolVar(HiveConf.ConfVars.REPL_RETAIN_CUSTOM_LOCATIONS_FOR_DB_ON_TARGET, false);
    ReplLoadWork replLoadWork = new ReplLoadWork(confTemp, loadPath.toString(), sourceDb, replicadb, null, null, isIncrementalDump, Long.valueOf(tuple.lastReplId), 0L, metricCollector, false);
    Task replLoadTask = TaskFactory.get(replLoadWork, confTemp);
    replLoadTask.initialize(null, null, new TaskQueue(driver.getContext()), driver.getContext());
    replLoadTask.executeTask(null);
    Hive.closeCurrent();
    return replLoadWork.getRootTask();
}

Example 2 with IncrementalLoadMetricCollector

use of org.apache.hadoop.hive.ql.parse.repl.load.metric.IncrementalLoadMetricCollector in project hive by apache.

the class TestReplicationMetricUpdateOnFailure method testReplLoadFailure.

@Test
public void testReplLoadFailure() throws Exception {
    String dumpDir = TEST_PATH + Path.SEPARATOR + testName.getMethodName();
    MetricCollector.getInstance().deinit();
    IncrementalLoadMetricCollector metricCollector = new IncrementalLoadMetricCollector(null, TEST_PATH, 0, conf);
    ReplLoadWork replLoadWork = Mockito.mock(ReplLoadWork.class);
    Mockito.when(replLoadWork.getTargetDatabase()).thenReturn("dummy");
    Mockito.when(replLoadWork.getDumpDirectory()).thenReturn(new Path(dumpDir + Path.SEPARATOR + "test").toString());
    Mockito.when(replLoadWork.getMetricCollector()).thenReturn(metricCollector);
    Mockito.when(replLoadWork.getRootTask()).thenThrow(recoverableException, nonRecoverableException);
    Task replLoadTask = TaskFactory.get(replLoadWork, conf);
    String stageName = "REPL_LOAD";
    metricCollector.reportStageStart(stageName, new HashMap<>());
    Assert.assertThrows(RuntimeException.class, () -> replLoadTask.execute());
    performRecoverableChecks(stageName);
    metricCollector.reportStageStart(stageName, new HashMap<>());
    Assert.assertThrows(RuntimeException.class, () -> replLoadTask.execute());
    performNonRecoverableChecks(dumpDir, stageName);
}

Example 3 with IncrementalLoadMetricCollector

use of org.apache.hadoop.hive.ql.parse.repl.load.metric.IncrementalLoadMetricCollector in project hive by apache.

the class TestReplicationMetricUpdateOnFailure method testDDLTaskFailure.

/*
   * Check update on metrics upon intermediate task failures(not repl-dump / repl-load).
   * Here, DDLTask is used as the intermediate task, other task failures should behave in similar fashion.
   */
@Test
public void testDDLTaskFailure() throws Exception {
    // task-setup for DDL-Task
    DDLWork ddlWork = Mockito.mock(DDLWork.class);
    Context context = Mockito.mock(Context.class);
    Mockito.when(context.getExplainAnalyze()).thenReturn(ExplainConfiguration.AnalyzeState.ANALYZING);
    Mockito.when(ddlWork.isReplication()).thenReturn(true);
    String dumpDir = TEST_PATH + Path.SEPARATOR + testName.getMethodName();
    Mockito.when(ddlWork.getDumpDirectory()).thenReturn(dumpDir);
    Task<DDLWork> ddlTask = TaskFactory.get(ddlWork, conf);
    ddlTask.initialize(null, null, null, context);
    IncrementalLoadMetricCollector metricCollector = new IncrementalLoadMetricCollector(null, TEST_PATH, 1, conf);
    Mockito.when(ddlWork.getMetricCollector()).thenReturn(metricCollector);
    // setup for 2 runs - first recoverable and second non-recoverable
    Mockito.when(ddlWork.getDDLDesc()).thenThrow(recoverableException, nonRecoverableException);
    String stageName = "REPL_LOAD";
    // test recoverable error during DDL-Task
    metricCollector.reportStageStart(stageName, new HashMap<>());
    ddlTask.execute();
    performRecoverableChecks(stageName);
    // test non-recoverable error during DDL-Task
    metricCollector.reportStageStart(stageName, new HashMap<>());
    ddlTask.execute();
    performNonRecoverableChecks(dumpDir, stageName);
}

Example 4 with IncrementalLoadMetricCollector

use of org.apache.hadoop.hive.ql.parse.repl.load.metric.IncrementalLoadMetricCollector in project hive by apache.

the class TestReplicationMetricCollector method testSuccessIncrLoadMetrics.

@Test
public void testSuccessIncrLoadMetrics() throws Exception {
    ReplicationMetricCollector incrLoadMetricCollector = new IncrementalLoadMetricCollector("db", "dummyDir", 1, conf);
    Map<String, Long> metricMap = new HashMap<>();
    metricMap.put(ReplUtils.MetricName.TABLES.name(), (long) 10);
    metricMap.put(ReplUtils.MetricName.FUNCTIONS.name(), (long) 1);
    incrLoadMetricCollector.reportStageStart("dump", metricMap);
    incrLoadMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.TABLES.name(), 1);
    List<ReplicationMetric> actualMetrics = MetricCollector.getInstance().getMetrics();
    Assert.assertEquals(1, actualMetrics.size());
    incrLoadMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.TABLES.name(), 2);
    incrLoadMetricCollector.reportStageProgress("dump", ReplUtils.MetricName.FUNCTIONS.name(), 1);
    actualMetrics = MetricCollector.getInstance().getMetrics();
    Assert.assertEquals(1, actualMetrics.size());
    incrLoadMetricCollector.reportStageEnd("dump", Status.SUCCESS, 10, new SnapshotUtils.ReplSnapshotCount(), new ReplStatsTracker(0));
    incrLoadMetricCollector.reportEnd(Status.SUCCESS);
    actualMetrics = MetricCollector.getInstance().getMetrics();
    Assert.assertEquals(1, actualMetrics.size());
    Metadata expectedMetadata = new Metadata("db", Metadata.ReplicationType.INCREMENTAL, "dummyDir");
    expectedMetadata.setLastReplId(10);
    Progress expectedProgress = new Progress();
    expectedProgress.setStatus(Status.SUCCESS);
    Stage dumpStage = new Stage("dump", Status.SUCCESS, 0);
    dumpStage.setEndTime(0);
    Metric expectedTableMetric = new Metric(ReplUtils.MetricName.TABLES.name(), 10);
    expectedTableMetric.setCurrentCount(3);
    Metric expectedFuncMetric = new Metric(ReplUtils.MetricName.FUNCTIONS.name(), 1);
    expectedFuncMetric.setCurrentCount(1);
    dumpStage.addMetric(expectedTableMetric);
    dumpStage.addMetric(expectedFuncMetric);
    expectedProgress.addStage(dumpStage);
    ReplicationMetric expectedMetric = new ReplicationMetric(1, "repl", 1, expectedMetadata);
    expectedMetric.setProgress(expectedProgress);
    checkSuccess(actualMetrics.get(0), expectedMetric, "dump", Arrays.asList(ReplUtils.MetricName.TABLES.name(), ReplUtils.MetricName.FUNCTIONS.name()));
}

Example 5 with IncrementalLoadMetricCollector

use of org.apache.hadoop.hive.ql.parse.repl.load.metric.IncrementalLoadMetricCollector in project hive by apache.

the class TestReplicationMetricUpdateOnFailure method testReplLoadNonRecoverableMissingStage.

@Test
public void testReplLoadNonRecoverableMissingStage() throws Exception {
    String dumpDir = TEST_PATH + Path.SEPARATOR + testName.getMethodName();
    MetricCollector.getInstance().deinit();
    IncrementalLoadMetricCollector metricCollector = new IncrementalLoadMetricCollector(null, TEST_PATH, 0, conf);
    ReplLoadWork replLoadWork = Mockito.mock(ReplLoadWork.class);
    Mockito.when(replLoadWork.getTargetDatabase()).thenReturn("dummy");
    Mockito.when(replLoadWork.getDumpDirectory()).thenReturn(new Path(dumpDir + Path.SEPARATOR + "test").toString());
    Mockito.when(replLoadWork.getMetricCollector()).thenReturn(metricCollector);
    Mockito.when(replLoadWork.getRootTask()).thenThrow(nonRecoverableException);
    Task replLoadTask = TaskFactory.get(replLoadWork, conf);
    // ensure stages are missing initially and execute without reporting start metrics
    Assert.assertEquals(0, MetricCollector.getInstance().getMetrics().size());
    Assert.assertThrows(RuntimeException.class, () -> replLoadTask.execute());
    performNonRecoverableChecks(dumpDir, "REPL_LOAD");
}