Examples with ReplScope org.apache.hadoop.hive.common.repl.ReplScope used on opensource projects

Example 1 with ReplScope

use of org.apache.hadoop.hive.common.repl.ReplScope in project hive by apache.

the class TestReplicationScenarios method deleteNewMetadataFields.

private void deleteNewMetadataFields(Tuple dump) throws SemanticException {
    Path dumpHiveDir = new Path(dump.dumpLocation, ReplUtils.REPL_HIVE_BASE_DIR);
    DumpMetaData dmd = new DumpMetaData(dumpHiveDir, hconf);
    Path dumpMetaPath = new Path(dumpHiveDir, DUMP_METADATA);
    List<List<String>> listValues = new ArrayList<>();
    DumpType dumpType = dmd.getDumpType();
    Long eventFrom = dmd.getEventFrom();
    Long eventTo = dmd.getEventTo();
    String cmRoot = "testCmRoot";
    String payload = dmd.getPayload();
    Long dumpExecutionId = dmd.getDumpExecutionId();
    ReplScope replScope = dmd.getReplScope();
    listValues.add(Arrays.asList(dumpType.toString(), eventFrom.toString(), eventTo.toString(), cmRoot, dumpExecutionId.toString(), payload));
    if (replScope != null) {
        listValues.add(dmd.prepareReplScopeValues());
    }
    org.apache.hadoop.hive.ql.parse.repl.dump.Utils.writeOutput(listValues, dumpMetaPath, hconf, true);
}

Example 2 with ReplScope

use of org.apache.hadoop.hive.common.repl.ReplScope in project hive by apache.

the class DumpMetaData method readReplScope.

private void readReplScope(String line) throws IOException {
    if (line == null) {
        return;
    }
    String[] lineContents = line.split("\t");
    replScope = new ReplScope();
    for (int idx = 0; idx < lineContents.length; idx++) {
        String value = lineContents[idx];
        switch(idx) {
            case 0:
                LOG.info("Read ReplScope: Set Db Name: {}.", value);
                replScope.setDbName(value);
                break;
            case 1:
                LOG.info("Read ReplScope: Include table name list: {}.", value);
                replScope.setIncludedTablePatterns(value);
                break;
            case 2:
                LOG.info("Read ReplScope: Exclude table name list: {}.", value);
                replScope.setExcludedTablePatterns(value);
                break;
            default:
                throw new IOException("Invalid repl tables list data in dump metadata file");
        }
    }
}

Example 3 with ReplScope

use of org.apache.hadoop.hive.common.repl.ReplScope in project hive by apache.

the class TestReplDumpTask method removeDBPropertyToPreventRenameWhenBootstrapDumpOfTableFails.

@Test(expected = TestException.class)
public void removeDBPropertyToPreventRenameWhenBootstrapDumpOfTableFails() throws Exception {
    List<String> tableList = Arrays.asList("a1", "a2");
    String dbRandomKey = "akeytoberandom";
    ReplScope replScope = new ReplScope("default");
    mockStatic(Utils.class);
    when(Utils.matchesDb(same(hive), eq("default"))).thenReturn(Collections.singletonList("default"));
    when(Utils.getAllTables(same(hive), eq("default"), eq(replScope))).thenReturn(tableList);
    when(Utils.setDbBootstrapDumpState(same(hive), eq("default"))).thenReturn(dbRandomKey);
    when(Utils.matchesTbl(same(hive), eq("default"), eq(replScope))).thenReturn(tableList);
    when(hive.getAllFunctions()).thenReturn(Collections.emptyList());
    when(queryState.getConf()).thenReturn(conf);
    when(conf.getLong("hive.repl.last.repl.id", -1L)).thenReturn(1L);
    when(conf.getBoolVar(HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES)).thenReturn(false);
    when(HiveConf.getVar(conf, HiveConf.ConfVars.REPL_BOOTSTRAP_DUMP_OPEN_TXN_TIMEOUT)).thenReturn("1h");
    whenNew(HiveWrapper.class).withAnyArguments().thenReturn(mock(HiveWrapper.class));
    ReplDumpTask task = new StubReplDumpTask() {

        private int tableDumpCount = 0;

        @Override
        void dumpTable(String dbName, String tblName, String validTxnList, Path dbRootMetadata, Path dbRootData, long lastReplId, Hive hiveDb, HiveWrapper.Tuple<Table> tuple, FileList managedTableDirFileList, boolean dataCopyAtLoad) throws Exception {
            tableDumpCount++;
            if (tableDumpCount > 1) {
                throw new TestException();
            }
        }
    };
    task.initialize(queryState, null, null, null);
    ReplDumpWork replDumpWork = new ReplDumpWork(replScope, "", "");
    replDumpWork.setMetricCollector(metricCollector);
    task.setWork(replDumpWork);
    try {
        task.bootStrapDump(new Path("mock"), new DumpMetaData(new Path("mock"), conf), mock(Path.class), hive);
    } finally {
        Utils.resetDbBootstrapDumpState(same(hive), eq("default"), eq(dbRandomKey));
    }
}

Example 4 with ReplScope

use of org.apache.hadoop.hive.common.repl.ReplScope in project hive by apache.

the class AlterTableHandler method handle.

@Override
public void handle(Context withinContext) throws Exception {
    LOG.info("Processing#{} ALTER_TABLE message : {}", fromEventId(), eventMessageAsJSON);
    Table qlMdTableBefore = new Table(before);
    Set<String> bootstrapTableList;
    ReplScope oldReplScope;
    if (Scenario.RENAME == scenario) {
        // Handling for table level replication is not done in shouldReplicate method for rename events. Its done in
        // handleRenameForReplacePolicy and handleRenameForTableLevelReplication method.
        bootstrapTableList = null;
        oldReplScope = null;
    } else {
        // This check was ignored for alter table event during event filter.
        if (!ReplUtils.tableIncludedInReplScope(withinContext.replScope, before.getTableName())) {
            LOG.debug("Table " + before.getTableName() + " does not satisfy the policy");
            return;
        }
        bootstrapTableList = withinContext.getTablesForBootstrap();
        oldReplScope = withinContext.oldReplScope;
    }
    if (!Utils.shouldReplicate(withinContext.replicationSpec, qlMdTableBefore, true, bootstrapTableList, oldReplScope, withinContext.hiveConf)) {
        return;
    }
    if (withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES)) {
        if (!AcidUtils.isTransactionalTable(before) && AcidUtils.isTransactionalTable(after)) {
            LOG.info("The table " + after.getTableName() + " is converted to ACID table." + " It will be replicated with bootstrap load as hive.repl.bootstrap.acid.tables is set to true.");
            return;
        }
    }
    if (Scenario.RENAME == scenario) {
        String oldName = before.getTableName();
        String newName = after.getTableName();
        boolean needDump = true;
        if (withinContext.oldReplScope != null && !withinContext.oldReplScope.equals(withinContext.replScope)) {
            needDump = handleRenameForReplacePolicy(withinContext, oldName, newName);
        } else if (!withinContext.replScope.includeAllTables()) {
            needDump = handleRenameForTableLevelReplication(withinContext, oldName, newName);
        }
        if (!needDump) {
            LOG.info("Rename event for table " + oldName + " to " + newName + " is skipped from dumping");
            return;
        }
    }
    if (Scenario.ALTER == scenario) {
        withinContext.replicationSpec.setIsMetadataOnly(true);
        Table qlMdTableAfter = new Table(after);
        Path metaDataPath = new Path(withinContext.eventRoot, EximUtil.METADATA_NAME);
        // empty table.
        if (Utils.shouldDumpMetaDataOnly(withinContext.hiveConf) || Utils.shouldDumpMetaDataOnlyForExternalTables(qlMdTableAfter, withinContext.hiveConf)) {
            qlMdTableAfter.setStatsStateLikeNewTable();
        }
        EximUtil.createExportDump(metaDataPath.getFileSystem(withinContext.hiveConf), metaDataPath, qlMdTableAfter, null, withinContext.replicationSpec, withinContext.hiveConf);
    }
    DumpMetaData dmd = withinContext.createDmd(this);
    dmd.setPayload(eventMessageAsJSON);
    dmd.write();
}