Examples with MetricCollector org.apache.hadoop.hive.ql.parse.repl.metric.MetricCollector used on opensource projects

Example 1 with MetricCollector

use of org.apache.hadoop.hive.ql.parse.repl.metric.MetricCollector in project hive by apache.

the class TestReplicationScenarios method testIncrementalStatisticsMetrics.

@Test
public void testIncrementalStatisticsMetrics() throws Throwable {
    isMetricsEnabledForTests(true);
    ReplLoadWork.setMbeansParamsForTesting(true, false);
    MetricCollector collector = MetricCollector.getInstance();
    String testName = "testIncrementalStatisticsMetrics";
    String dbName = createDB(testName, driver);
    String replDbName = dbName + "_dupe";
    String nameStri = "Hadoop:" + "service=HiveServer2" + "," + "name=" + "Database-" + replDbName + " Policy-pol";
    // Do a bootstrap dump & load
    Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName);
    ReplLoadWork.setMbeansParamsForTesting(true, true);
    // 10 create table
    for (int i = 0; i < 10; i++) {
        run("CREATE TABLE " + dbName + ".ptned" + i + "(a string) partitioned by (b int) STORED AS TEXTFILE", driver);
        for (int j = 0; j < 5; j++) {
            // Create 5 partitoins per table.
            run("ALTER TABLE " + dbName + ".ptned" + i + " ADD PARTITION(b=" + j + ")", driver);
        }
    }
    verifyRun("SHOW PARTITIONS " + dbName + ".ptned1", new String[] { "b=0", "b=1", "b=2", "b=3", "b=4" }, driver);
    // Do an incremental load & verify the metrics.
    Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDbName);
    String[] events = new String[] { "[[Event Name: EVENT_CREATE_TABLE; " + "Total Number: 10;", "[[Event Name: EVENT_ADD_PARTITION; Total Number: 50;" };
    Iterator<ReplicationMetric> itr = collector.getMetrics().iterator();
    while (itr.hasNext()) {
        ReplicationMetric elem = itr.next();
        assertEquals(Metadata.ReplicationType.INCREMENTAL, elem.getMetadata().getReplicationType());
        List<Stage> stages = elem.getProgress().getStages();
        assertTrue(stages.size() != 0);
        for (Stage stage : stages) {
            for (String event : events) {
                assertTrue(stage.getReplStats(), stage.getReplStats().contains(event));
            }
        }
    }
    verifyMBeanStatistics(testName, replDbName, nameStri, events, incrementalDump);
    // Do some drop table/drop partition & rename table operations.
    for (int i = 0; i < 3; i++) {
        // Drop 3 tables
        run("DROP TABLE " + dbName + ".ptned" + i, driver);
    }
    for (int i = 3; i < 6; i++) {
        // Rename 3 tables
        run("ALTER TABLE " + dbName + ".ptned" + i + " RENAME TO " + dbName + ".ptned" + i + "_renamed", driver);
    }
    for (int i = 6; i < 10; i++) {
        // Drop partitions from 4 tables
        run("ALTER TABLE " + dbName + ".ptned" + i + " DROP PARTITION(b=1)", driver);
    }
    for (int i = 10; i < 12; i++) {
        // Create 2 tables
        run("CREATE TABLE " + dbName + ".ptned" + i + "(a string) partitioned by (b int) STORED AS TEXTFILE", driver);
    }
    incrementalDump = incrementalLoadAndVerify(dbName, replDbName);
    events = new String[] { "[[Event Name: EVENT_CREATE_TABLE; " + "Total Number: 2;", "[[Event Name: EVENT_DROP_TABLE; " + "Total Number: 3;", "[[Event Name: EVENT_RENAME_TABLE; " + "Total Number: 3;", "[[Event Name: EVENT_DROP_PARTITION; Total Number: 4;" };
    itr = collector.getMetrics().iterator();
    while (itr.hasNext()) {
        ReplicationMetric elem = itr.next();
        assertEquals(Metadata.ReplicationType.INCREMENTAL, elem.getMetadata().getReplicationType());
        List<Stage> stages = elem.getProgress().getStages();
        assertTrue(stages.size() != 0);
        for (Stage stage : stages) {
            for (String event : events) {
                assertTrue(stage.getReplStats(), stage.getReplStats().contains(event));
            }
        }
    }
    verifyMBeanStatistics(testName, replDbName, nameStri, events, incrementalDump);
    // Clean up the test setup.
    ReplLoadWork.setMbeansParamsForTesting(false, false);
    MBeans.unregister(ObjectName.getInstance(nameStri));
}

Example 2 with MetricCollector

use of org.apache.hadoop.hive.ql.parse.repl.metric.MetricCollector in project hive by apache.

the class TestReplicationScenariosUsingSnapshots method testSnapshotMetrics.

@Test
public void testSnapshotMetrics() throws Throwable {
    conf.set(Constants.SCHEDULED_QUERY_SCHEDULENAME, "metrics_test");
    List<String> withClause = ReplicationTestUtils.includeExternalTableClause(true);
    MetricCollector collector = MetricCollector.getInstance();
    Path externalDatabaseLocation = new Path("/" + testName.getMethodName() + "/externalDatabase/");
    DistributedFileSystem fs = primary.miniDFSCluster.getFileSystem();
    fs.mkdirs(externalDatabaseLocation, new FsPermission("777"));
    Path externalTableLocation1 = new Path("/" + testName.getMethodName() + "/t1/");
    fs = primary.miniDFSCluster.getFileSystem();
    fs.mkdirs(externalTableLocation1, new FsPermission("777"));
    withClause.add("'hive.repl.external.warehouse.single.copy.task.paths'='" + externalTableLocation1.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString() + "'");
    WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName).run("create external table emp1 (id int)").run("insert into emp1 values(1),(2)").run("create external table exttab (place string) row format delimited fields terminated by ','" + " location '" + externalTableLocation1.toString() + "'").run("insert into exttab values('lucknow')").dump(primaryDbName, withClause);
    // The boootstrap stage, 2 directories for which snapshot is enabled, the database directory and the one table
    // as part of the config. This would be initial copy stage, so only 1 snapshot per directory and none to be deleted.
    assertIncrementalMetricsValues(BOOTSTRAP, collector, 2, 0);
    Iterator<ReplicationMetric> itr = collector.getMetrics().iterator();
    while (itr.hasNext()) {
        ReplicationMetric elem = itr.next();
        assertEquals(BOOTSTRAP, elem.getMetadata().getReplicationType());
        List<Stage> stages = elem.getProgress().getStages();
        for (Stage stage : stages) {
            SnapshotUtils.ReplSnapshotCount counts = stage.getReplSnapshotCount();
            assertEquals(2, counts.getNumCreated());
            assertEquals(0, counts.getNumDeleted());
        }
    }
    // Load and check if the data and table are there.
    replica.load(replicatedDbName, primaryDbName, withClause).run("use " + replicatedDbName).run("show tables like 'emp1'").verifyResults(new String[] { "emp1" }).run("select id from emp1").verifyResults(new String[] { "1", "2" }).run("show tables like 'exttab'").verifyResults(new String[] { "exttab" }).run("select place from exttab").verifyResults(new String[] { "lucknow" }).verifyReplTargetProperty(replicatedDbName);
    // Add some data and try incremental dump.
    tuple = primary.run("use " + primaryDbName).run("insert into emp1 values(3),(4)").run("insert into exttab values('agra')").dump(primaryDbName, withClause);
    // This is from the diff stage, 2 Directories where snapshots were enabled, 1 old snapshots got deleted and 1
    // got created, so 2 created and 2 deleted.
    assertIncrementalMetricsValues(INCREMENTAL, collector, 2, 2);
    // Do a load
    replica.load(replicatedDbName, primaryDbName, withClause);
    // Remove the with clause, hence the external table specified as part of the config.
    tuple = primary.run("use " + primaryDbName).run("insert into exttab values('lucknow')").dump(primaryDbName, null);
    // Only one directory, i.e the database directory is going through snapshot based replication, so only 1 created
    // for it and 1 old deleted for it, 2 deleted for the table removed from the snapshot based replication scope.
    assertIncrementalMetricsValues(INCREMENTAL, collector, 1, 3);
}