Examples with BehaviourInjection org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.BehaviourInjection used on opensource projects

Example 11 with BehaviourInjection

use of org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.BehaviourInjection in project hive by apache.

the class TestReplicationScenariosAcidTables method testAcidTablesBootstrapWithConcurrentDropTable.

@Test
public void testAcidTablesBootstrapWithConcurrentDropTable() throws Throwable {
    HiveConf primaryConf = primary.getConf();
    primary.run("use " + primaryDbName).run("create table t1 (id int) clustered by(id) into 3 buckets stored as orc " + "tblproperties (\"transactional\"=\"true\")").run("insert into t1 values(1)");
    // Perform concurrent write + drop on the acid table t1 when bootstrap dump in progress. Bootstrap
    // won't dump the table but the subsequent incremental repl with new table with same name should be seen.
    BehaviourInjection<CallerArguments, Boolean> callerInjectedBehavior = new BehaviourInjection<CallerArguments, Boolean>() {

        @Nullable
        @Override
        public Boolean apply(@Nullable CallerArguments args) {
            if (injectionPathCalled) {
                nonInjectedPathCalled = true;
            } else {
                // Insert another row to t1 and drop the table from another txn when bootstrap dump in progress.
                injectionPathCalled = true;
                Thread t = new Thread(new Runnable() {

                    @Override
                    public void run() {
                        LOG.info("Entered new thread");
                        IDriver driver = DriverFactory.newDriver(primaryConf);
                        SessionState.start(new CliSessionState(primaryConf));
                        try {
                            driver.run("insert into " + primaryDbName + ".t1 values(2)");
                            driver.run("drop table " + primaryDbName + ".t1");
                        } catch (CommandProcessorException e) {
                            throw new RuntimeException(e);
                        }
                        LOG.info("Exit new thread success");
                    }
                });
                t.start();
                LOG.info("Created new thread {}", t.getName());
                try {
                    t.join();
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }
            }
            return true;
        }
    };
    InjectableBehaviourObjectStore.setCallerVerifier(callerInjectedBehavior);
    WarehouseInstance.Tuple bootstrapDump = null;
    try {
        bootstrapDump = primary.dump(primaryDbName);
        callerInjectedBehavior.assertInjectionsPerformed(true, true);
    } finally {
        // reset the behaviour
        InjectableBehaviourObjectStore.resetCallerVerifier();
    }
    // Bootstrap dump has taken latest list of tables and hence won't see table t1 as it is dropped.
    replica.load(replicatedDbName, primaryDbName).run("use " + replicatedDbName).run("repl status " + replicatedDbName).verifyResult(bootstrapDump.lastReplicationId).run("show tables").verifyResult(null);
    // Create another ACID table with same name and insert a row. It should be properly replicated.
    WarehouseInstance.Tuple incrementalDump = primary.run("use " + primaryDbName).run("create table t1 (id int) clustered by(id) into 3 buckets stored as orc " + "tblproperties (\"transactional\"=\"true\")").run("insert into t1 values(100)").dump(primaryDbName);
    replica.load(replicatedDbName, primaryDbName).run("use " + replicatedDbName).run("repl status " + replicatedDbName).verifyResult(incrementalDump.lastReplicationId).run("select id from t1 order by id").verifyResult("100");
}

Example 12 with BehaviourInjection

use of org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.BehaviourInjection in project hive by apache.

the class TestReplicationScenariosAcidTablesBootstrap method testRetryAcidTablesBootstrapFromDifferentDump.

@Test
public void testRetryAcidTablesBootstrapFromDifferentDump() throws Throwable {
    WarehouseInstance.Tuple bootstrapDump = prepareDataAndDump(primaryDbName, dumpWithoutAcidClause);
    LOG.info(testName.getMethodName() + ": loading dump without acid tables.");
    replica.load(replicatedDbName, primaryDbName);
    verifyLoadExecution(replicatedDbName, bootstrapDump.lastReplicationId, false);
    prepareIncAcidData(primaryDbName);
    prepareIncNonAcidData(primaryDbName);
    LOG.info(testName.getMethodName() + ": first incremental dump with acid table bootstrap.");
    WarehouseInstance.Tuple incDump = primary.run("use " + primaryDbName).dump(primaryDbName, dumpWithAcidBootstrapClause);
    // Fail setting ckpt property for table t5 but success for earlier tables
    BehaviourInjection<CallerArguments, Boolean> callerVerifier = new BehaviourInjection<CallerArguments, Boolean>() {

        @Nullable
        @Override
        public Boolean apply(@Nullable CallerArguments args) {
            if (args.tblName.equalsIgnoreCase("t5") && args.dbName.equalsIgnoreCase(replicatedDbName)) {
                injectionPathCalled = true;
                LOG.warn("Verifier - DB : " + args.dbName + " TABLE : " + args.tblName);
                return false;
            }
            return true;
        }
    };
    // Fail repl load before the ckpt property is set for t4 and after it is set for t2.
    // In the retry, these half baked tables should be dropped and bootstrap should be successful.
    InjectableBehaviourObjectStore.setAlterTableModifier(callerVerifier);
    try {
        LOG.info(testName.getMethodName() + ": loading first incremental dump with acid table bootstrap (will fail)");
        replica.loadFailure(replicatedDbName, primaryDbName);
        callerVerifier.assertInjectionsPerformed(true, false);
    } finally {
        InjectableBehaviourObjectStore.resetAlterTableModifier();
    }
    Path baseDumpDir = new Path(primary.hiveConf.getVar(HiveConf.ConfVars.REPLDIR));
    Path nonRecoverablePath = TestReplicationScenarios.getNonRecoverablePath(baseDumpDir, primaryDbName, primary.hiveConf);
    if (nonRecoverablePath != null) {
        baseDumpDir.getFileSystem(primary.hiveConf).delete(nonRecoverablePath, true);
    }
    // Load again should succeed as checkpointing is in place
    replica.load(replicatedDbName, primaryDbName);
    verifyIncLoad(replicatedDbName, incDump.lastReplicationId);
    prepareInc2AcidData(primaryDbName, primary.hiveConf);
    prepareInc2NonAcidData(primaryDbName, primary.hiveConf);
    LOG.info(testName.getMethodName() + ": second incremental dump with acid table bootstrap");
    WarehouseInstance.Tuple inc2Dump = primary.run("use " + primaryDbName).dump(primaryDbName, dumpWithAcidClause);
    LOG.info(testName.getMethodName() + ": trying to load second incremental dump (with acid bootstrap) again." + " Should succeed.");
    replica.load(replicatedDbName, primaryDbName);
    verifyInc2Load(replicatedDbName, inc2Dump.lastReplicationId);
}