Examples with BehaviourInjection org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.BehaviourInjection used on opensource projects

Example 6 with BehaviourInjection

use of org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.BehaviourInjection in project hive by apache.

the class TestReplicationScenariosAcrossInstances method testBootstrapLoadRetryAfterFailureForAlterTable.

// This requires the tables are loaded in a fixed sorted order.
@Test
public void testBootstrapLoadRetryAfterFailureForAlterTable() throws Throwable {
    WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName).run("create table t1 (place string)").run("insert into table t1 values ('testCheck')").run("create table t2 (place string) partitioned by (country string)").run("insert into table t2 partition(country='china') values ('shenzhen')").run("insert into table t2 partition(country='india') values ('banaglore')").dump(primaryDbName);
    // fail setting ckpt directory property for table t1.
    BehaviourInjection<CallerArguments, Boolean> callerVerifier = new BehaviourInjection<CallerArguments, Boolean>() {

        @Nullable
        @Override
        public Boolean apply(@Nullable CallerArguments args) {
            if (args.tblName.equalsIgnoreCase("t1") && args.dbName.equalsIgnoreCase(replicatedDbName)) {
                injectionPathCalled = true;
                LOG.warn("Verifier - DB : " + args.dbName + " TABLE : " + args.tblName);
                return false;
            }
            return true;
        }
    };
    // Fail repl load before the ckpt proeprty is set for t1 and after it is set for t2. So in the next run, for
    // t2 it goes directly to partion load with no task for table tracker and for t1 it loads the table
    // again from start.
    InjectableBehaviourObjectStore.setAlterTableModifier(callerVerifier);
    try {
        replica.loadFailure(replicatedDbName, primaryDbName);
        callerVerifier.assertInjectionsPerformed(true, false);
    } finally {
        InjectableBehaviourObjectStore.resetAlterTableModifier();
    }
    // Retry with same dump with which it was already loaded should resume the bootstrap load. Make sure that table t1,
    // is loaded before t2. So that scope is set to table in first iteration for table t1. In the next iteration, it
    // loads only remaining partitions of t2, so that the table tracker has no tasks.
    Path baseDumpDir = new Path(primary.hiveConf.getVar(HiveConf.ConfVars.REPLDIR));
    Path nonRecoverablePath = getNonRecoverablePath(baseDumpDir, primaryDbName);
    if (nonRecoverablePath != null) {
        baseDumpDir.getFileSystem(primary.hiveConf).delete(nonRecoverablePath, true);
    }
    List<String> withConfigs = Arrays.asList("'hive.in.repl.test.files.sorted'='true'");
    replica.load(replicatedDbName, primaryDbName, withConfigs);
    replica.run("use " + replicatedDbName).run("repl status " + replicatedDbName).verifyResult(tuple.lastReplicationId).run("select country from t2 order by country").verifyResults(Arrays.asList("china", "india"));
}

Example 7 with BehaviourInjection

use of org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.BehaviourInjection in project hive by apache.

the class TestReplicationScenariosAcrossInstances method testBootstrapReplLoadRetryAfterFailureForFunctions.

@Test
public void testBootstrapReplLoadRetryAfterFailureForFunctions() throws Throwable {
    String funcName1 = "f1";
    String funcName2 = "f2";
    WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName).run("CREATE FUNCTION " + primaryDbName + "." + funcName1 + " as 'hivemall.tools.string.StopwordUDF' " + "using jar  'ivy://io.github.myui:hivemall:0.4.0-2'").run("CREATE FUNCTION " + primaryDbName + "." + funcName2 + " as 'hivemall.tools.string.SplitWordsUDF' " + "using jar  'ivy://io.github.myui:hivemall:0.4.0-1'").dump(primaryDbName);
    // Allow create function only on f1. Create should fail for the second function.
    BehaviourInjection<CallerArguments, Boolean> callerVerifier = new BehaviourInjection<CallerArguments, Boolean>() {

        @Override
        public Boolean apply(CallerArguments args) {
            injectionPathCalled = true;
            if (!args.dbName.equalsIgnoreCase(replicatedDbName)) {
                LOG.warn("Verifier - DB: " + String.valueOf(args.dbName));
                return false;
            }
            if (args.funcName != null) {
                LOG.debug("Verifier - Function: " + String.valueOf(args.funcName));
                return args.funcName.equals(funcName1);
            }
            return true;
        }
    };
    InjectableBehaviourObjectStore.setCallerVerifier(callerVerifier);
    // Trigger bootstrap dump which just creates function f1 but not f2
    List<String> withConfigs = Arrays.asList("'hive.repl.approx.max.load.tasks'='1'", "'hive.in.repl.test.files.sorted'='true'");
    try {
        replica.loadFailure(replicatedDbName, primaryDbName, withConfigs);
        callerVerifier.assertInjectionsPerformed(true, false);
    } finally {
        // reset the behaviour
        InjectableBehaviourObjectStore.resetCallerVerifier();
    }
    // Verify that only f1 got loaded
    replica.run("use " + replicatedDbName).run("repl status " + replicatedDbName).verifyResult("null").run("show functions like '" + replicatedDbName + "%'").verifyResult(replicatedDbName + "." + funcName1);
    // Verify no calls to load f1 only f2.
    callerVerifier = new BehaviourInjection<CallerArguments, Boolean>() {

        @Override
        public Boolean apply(CallerArguments args) {
            injectionPathCalled = true;
            if (!args.dbName.equalsIgnoreCase(replicatedDbName)) {
                LOG.warn("Verifier - DB: " + String.valueOf(args.dbName));
                return false;
            }
            if (args.funcName != null) {
                LOG.debug("Verifier - Function: " + String.valueOf(args.funcName));
                return args.funcName.equals(funcName2);
            }
            return true;
        }
    };
    InjectableBehaviourObjectStore.setCallerVerifier(callerVerifier);
    try {
        // Retry with same dump with which it was already loaded should resume the bootstrap load.
        // This time, it completes by adding just the function f2
        replica.load(replicatedDbName, primaryDbName);
        callerVerifier.assertInjectionsPerformed(true, false);
    } finally {
        // reset the behaviour
        InjectableBehaviourObjectStore.resetCallerVerifier();
    }
    // Verify that both the functions are available.
    replica.run("use " + replicatedDbName).run("repl status " + replicatedDbName).verifyResult(tuple.lastReplicationId).run("show functions like '" + replicatedDbName + "%'").verifyResults(new String[] { replicatedDbName + "." + funcName1, replicatedDbName + "." + funcName2 });
}

Example 8 with BehaviourInjection

use of org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.BehaviourInjection in project hive by apache.

the class TestReplicationScenariosAcrossInstances method testBootstrapReplLoadRetryAfterFailureForTablesAndConstraints.

@Test
public void testBootstrapReplLoadRetryAfterFailureForTablesAndConstraints() throws Throwable {
    WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName).run("create table t1(a string, b string, primary key (a, b) disable novalidate rely)").run("create table t2(a string, b string, foreign key (a, b) references t1(a, b) disable novalidate)").run("create table t3(a string, b string not null disable, unique (a) disable)").dump(primaryDbName);
    // Need to drop the primary DB as metastore is shared by both primary/replica. So, constraints
    // conflict when loaded. Some issue with framework which needs to be relook into later.
    primary.run("drop database if exists " + primaryDbName + " cascade");
    // Allow create table only on t1. Create should fail for rest of the tables and hence constraints
    // also not loaded.
    BehaviourInjection<CallerArguments, Boolean> callerVerifier = new BehaviourInjection<CallerArguments, Boolean>() {

        @Override
        public Boolean apply(CallerArguments args) {
            injectionPathCalled = true;
            if (!args.dbName.equalsIgnoreCase(replicatedDbName) || (args.constraintTblName != null)) {
                LOG.warn("Verifier - DB: " + String.valueOf(args.dbName) + " Constraint Table: " + String.valueOf(args.constraintTblName));
                return false;
            }
            if (args.tblName != null) {
                LOG.warn("Verifier - Table: " + String.valueOf(args.tblName));
                return args.tblName.equals("t1");
            }
            return true;
        }
    };
    InjectableBehaviourObjectStore.setCallerVerifier(callerVerifier);
    // Trigger bootstrap dump which just creates table t1 and other tables (t2, t3) and constraints not loaded.
    List<String> withConfigs = Arrays.asList("'hive.repl.approx.max.load.tasks'='1'");
    try {
        replica.loadFailure(replicatedDbName, primaryDbName, withConfigs);
        callerVerifier.assertInjectionsPerformed(true, false);
    } finally {
        // reset the behaviour
        InjectableBehaviourObjectStore.resetCallerVerifier();
    }
    replica.run("use " + replicatedDbName).run("repl status " + replicatedDbName).verifyResult("null");
    assertEquals(0, replica.getPrimaryKeyList(replicatedDbName, "t1").size());
    assertEquals(0, replica.getUniqueConstraintList(replicatedDbName, "t3").size());
    assertEquals(0, replica.getNotNullConstraintList(replicatedDbName, "t3").size());
    assertEquals(0, replica.getForeignKeyList(replicatedDbName, "t2").size());
    // Verify if create table is not called on table t1 but called for t2 and t3.
    // Also, allow constraint creation only on t1 and t3. Foreign key creation on t2 fails.
    callerVerifier = new BehaviourInjection<CallerArguments, Boolean>() {

        @Override
        public Boolean apply(CallerArguments args) {
            injectionPathCalled = true;
            if (!args.dbName.equalsIgnoreCase(replicatedDbName) || (args.funcName != null)) {
                LOG.warn("Verifier - DB: " + String.valueOf(args.dbName) + " Func: " + String.valueOf(args.funcName));
                return false;
            }
            if (args.constraintTblName != null) {
                LOG.warn("Verifier - Constraint Table: " + String.valueOf(args.constraintTblName));
                return (args.constraintTblName.equals("t1") || args.constraintTblName.equals("t3"));
            }
            return true;
        }
    };
    InjectableBehaviourObjectStore.setCallerVerifier(callerVerifier);
    try {
        // Retry with same dump with which it was already loaded should resume the bootstrap load.
        // This time, it fails when try to load the foreign key constraints. All other constraints are loaded.
        replica.loadFailure(replicatedDbName, primaryDbName, withConfigs);
        callerVerifier.assertInjectionsPerformed(true, false);
    } finally {
        // reset the behaviour
        InjectableBehaviourObjectStore.resetCallerVerifier();
    }
    replica.run("use " + replicatedDbName).run("repl status " + replicatedDbName).verifyResult("null").run("show tables").verifyResults(new String[] { "t1", "t2", "t3" });
    assertEquals(2, replica.getPrimaryKeyList(replicatedDbName, "t1").size());
    assertEquals(1, replica.getUniqueConstraintList(replicatedDbName, "t3").size());
    assertEquals(1, replica.getNotNullConstraintList(replicatedDbName, "t3").size());
    assertEquals(0, replica.getForeignKeyList(replicatedDbName, "t2").size());
    // Verify if no create table/function calls. Only add foreign key constraints on table t2.
    callerVerifier = new BehaviourInjection<CallerArguments, Boolean>() {

        @Override
        public Boolean apply(CallerArguments args) {
            injectionPathCalled = true;
            if (!args.dbName.equalsIgnoreCase(replicatedDbName) || (args.tblName != null)) {
                LOG.warn("Verifier - DB: " + String.valueOf(args.dbName) + " Table: " + String.valueOf(args.tblName));
                return false;
            }
            if (args.constraintTblName != null) {
                LOG.warn("Verifier - Constraint Table: " + String.valueOf(args.constraintTblName));
                return args.constraintTblName.equals("t2");
            }
            return true;
        }
    };
    InjectableBehaviourObjectStore.setCallerVerifier(callerVerifier);
    try {
        // Retry with same dump with which it was already loaded should resume the bootstrap load.
        // This time, it completes by adding just foreign key constraints for table t2.
        replica.load(replicatedDbName, primaryDbName);
        callerVerifier.assertInjectionsPerformed(true, false);
    } finally {
        // reset the behaviour
        InjectableBehaviourObjectStore.resetCallerVerifier();
    }
    replica.run("use " + replicatedDbName).run("repl status " + replicatedDbName).verifyResult(tuple.lastReplicationId).run("show tables").verifyResults(new String[] { "t1", "t2", "t3" });
    assertEquals(2, replica.getPrimaryKeyList(replicatedDbName, "t1").size());
    assertEquals(1, replica.getUniqueConstraintList(replicatedDbName, "t3").size());
    assertEquals(1, replica.getNotNullConstraintList(replicatedDbName, "t3").size());
    assertEquals(2, replica.getForeignKeyList(replicatedDbName, "t2").size());
}

Example 9 with BehaviourInjection

use of org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.BehaviourInjection in project hive by apache.

the class TestReplicationScenarios method testIdempotentMoveTaskForInsertFiles.

@Test
public void testIdempotentMoveTaskForInsertFiles() throws IOException {
    String name = testName.getMethodName();
    final String dbName = createDB(name, driver);
    String replDbName = dbName + "_dupe";
    run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
    Tuple bootstrap = bootstrapLoadAndVerify(dbName, replDbName);
    String[] unptn_data = new String[] { "ten" };
    run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[0] + "')", driver);
    // Inject a behaviour where it repeats the INSERT event twice with different event IDs
    BehaviourInjection<NotificationEventResponse, NotificationEventResponse> insertEventRepeater = new BehaviourInjection<NotificationEventResponse, NotificationEventResponse>() {

        @Nullable
        @Override
        public NotificationEventResponse apply(@Nullable NotificationEventResponse eventsList) {
            if (null != eventsList) {
                List<NotificationEvent> events = eventsList.getEvents();
                List<NotificationEvent> outEvents = new ArrayList<>();
                long insertEventId = -1;
                for (int i = 0; i < events.size(); i++) {
                    NotificationEvent event = events.get(i);
                    // Skip all the events belong to other DBs/tables.
                    if (event.getDbName().equalsIgnoreCase(dbName)) {
                        if (event.getEventType().equalsIgnoreCase("INSERT")) {
                            // Add insert event twice with different event ID to allow apply of both events.
                            NotificationEvent newEvent = new NotificationEvent(event);
                            outEvents.add(newEvent);
                            insertEventId = newEvent.getEventId();
                        }
                    }
                    NotificationEvent newEvent = new NotificationEvent(event);
                    if (insertEventId != -1) {
                        insertEventId++;
                        newEvent.setEventId(insertEventId);
                    }
                    outEvents.add(newEvent);
                }
                eventsList.setEvents(outEvents);
                injectionPathCalled = true;
            }
            return eventsList;
        }
    };
    InjectableBehaviourObjectStore.setGetNextNotificationBehaviour(insertEventRepeater);
    try {
        incrementalLoadAndVerify(dbName, replDbName);
        insertEventRepeater.assertInjectionsPerformed(true, false);
    } finally {
        // reset the behaviour
        InjectableBehaviourObjectStore.resetGetNextNotificationBehaviour();
    }
    verifyRun("SELECT a from " + replDbName + ".unptned", unptn_data[0], driverMirror);
}

Example 10 with BehaviourInjection

use of org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.BehaviourInjection in project hive by apache.

the class TestReplicationScenarios method testIncrementalReplWithEventsMissing.

@Test
public void testIncrementalReplWithEventsMissing() throws IOException, TException {
    String testName = "incrementalReplWithEventsMissing";
    String dbName = createDB(testName, driver);
    String replDbName = dbName + "_dupe";
    Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName);
    String replDumpId = bootstrapDump.lastReplId;
    // CREATE_TABLE - INSERT - TRUNCATE - INSERT - The result is just one record.
    String[] unptn_data = new String[] { "eleven" };
    run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
    run("INSERT INTO TABLE " + dbName + ".unptned values('ten')", driver);
    run("TRUNCATE TABLE " + dbName + ".unptned", driver);
    run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[0] + "')", driver);
    // Inject a behaviour where some events missing from notification_log table.
    // This ensures the incremental dump doesn't get all events for replication.
    BehaviourInjection<NotificationEventResponse, NotificationEventResponse> eventIdSkipper = new BehaviourInjection<NotificationEventResponse, NotificationEventResponse>() {

        @Nullable
        @Override
        public NotificationEventResponse apply(@Nullable NotificationEventResponse eventIdList) {
            if (null != eventIdList) {
                List<NotificationEvent> eventIds = eventIdList.getEvents();
                List<NotificationEvent> outEventIds = new ArrayList<NotificationEvent>();
                for (int i = 0; i < eventIds.size(); i++) {
                    NotificationEvent event = eventIds.get(i);
                    // Skip all the INSERT events
                    if (event.getDbName().equalsIgnoreCase(dbName) && event.getEventType().equalsIgnoreCase("INSERT")) {
                        injectionPathCalled = true;
                        continue;
                    }
                    outEventIds.add(event);
                }
                // Return the new list
                return new NotificationEventResponse(outEventIds);
            } else {
                return null;
            }
        }
    };
    InjectableBehaviourObjectStore.setGetNextNotificationBehaviour(eventIdSkipper);
    try {
        advanceDumpDir();
        try {
            driver.run("REPL DUMP " + dbName);
            assert false;
        } catch (CommandProcessorException e) {
            assertTrue(e.getCauseMessage() == ErrorMsg.REPL_EVENTS_MISSING_IN_METASTORE.getMsg());
        }
        eventIdSkipper.assertInjectionsPerformed(true, false);
    } finally {
        // reset the behaviour
        InjectableBehaviourObjectStore.resetGetNextNotificationBehaviour();
    }
}