Examples with WriteEventInfo org.apache.hadoop.hive.metastore.api.WriteEventInfo used on opensource projects

Example 1 with WriteEventInfo

use of org.apache.hadoop.hive.metastore.api.WriteEventInfo in project hive by apache.

the class CommitTxnHandler method handle.

@Override
public void handle(Context withinContext) throws Exception {
    if (!ReplUtils.includeAcidTableInDump(withinContext.hiveConf)) {
        return;
    }
    LOG.info("Processing#{} COMMIT_TXN message : {}", fromEventId(), eventMessageAsJSON);
    String payload = eventMessageAsJSON;
    if (!withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY)) {
        boolean replicatingAcidEvents = true;
        if (withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES)) {
            // We do not dump ACID table related events when taking a bootstrap dump of ACID tables as
            // part of an incremental dump. So we shouldn't be dumping any changes to ACID table as
            // part of the commit. At the same time we need to dump the commit transaction event so
            // that replication can end a transaction opened when replaying open transaction event.
            LOG.debug("writeEventsInfoList will be removed from commit message because we are " + "bootstrapping acid tables.");
            replicatingAcidEvents = false;
        } else if (!ReplUtils.includeAcidTableInDump(withinContext.hiveConf)) {
            // Similar to the above condition, only for testing purposes, if the config doesn't allow
            // ACID tables to be replicated, we don't dump any changes to the ACID tables as part of
            // commit.
            LOG.debug("writeEventsInfoList will be removed from commit message because we are " + "not dumping acid tables.");
            replicatingAcidEvents = false;
        }
        List<WriteEventInfo> writeEventInfoList = null;
        if (replicatingAcidEvents) {
            writeEventInfoList = getAllWriteEventInfo(withinContext);
        }
        int numEntry = (writeEventInfoList != null ? writeEventInfoList.size() : 0);
        if (numEntry != 0) {
            eventMessage.addWriteEventInfo(writeEventInfoList);
            payload = jsonMessageEncoder.getSerializer().serialize(eventMessage);
            LOG.debug("payload for commit txn event : " + eventMessageAsJSON);
        }
        org.apache.hadoop.hive.ql.metadata.Table qlMdTablePrev = null;
        org.apache.hadoop.hive.ql.metadata.Table qlMdTable = null;
        List<Partition> qlPtns = new ArrayList<>();
        List<List<String>> filesTobeAdded = new ArrayList<>();
        // used during import, so we need not dump the latest table metadata.
        for (int idx = 0; idx < numEntry; idx++) {
            qlMdTable = new org.apache.hadoop.hive.ql.metadata.Table(eventMessage.getTableObj(idx));
            if (qlMdTablePrev == null) {
                qlMdTablePrev = qlMdTable;
            }
            // one dump directory per table
            if (!qlMdTablePrev.getCompleteName().equals(qlMdTable.getCompleteName())) {
                createDumpFileForTable(withinContext, qlMdTablePrev, qlPtns, filesTobeAdded);
                qlPtns = new ArrayList<>();
                filesTobeAdded = new ArrayList<>();
                qlMdTablePrev = qlMdTable;
            }
            if (qlMdTable.isPartitioned() && (null != eventMessage.getPartitionObj(idx))) {
                qlPtns.add(new org.apache.hadoop.hive.ql.metadata.Partition(qlMdTable, eventMessage.getPartitionObj(idx)));
            }
            filesTobeAdded.add(Lists.newArrayList(ReplChangeManager.getListFromSeparatedString(eventMessage.getFiles(idx))));
        }
        // Dump last table in the list
        if (qlMdTablePrev != null) {
            createDumpFileForTable(withinContext, qlMdTablePrev, qlPtns, filesTobeAdded);
        }
    }
    DumpMetaData dmd = withinContext.createDmd(this);
    dmd.setPayload(payload);
    dmd.write();
}

Example 2 with WriteEventInfo

use of org.apache.hadoop.hive.metastore.api.WriteEventInfo in project hive by apache.

the class CommitTxnHandler method getAllWriteEventInfo.

private List<WriteEventInfo> getAllWriteEventInfo(Context withinContext) throws Exception {
    String contextDbName = StringUtils.normalizeIdentifier(withinContext.replScope.getDbName());
    GetAllWriteEventInfoRequest request = new GetAllWriteEventInfoRequest(eventMessage.getTxnId());
    request.setDbName(contextDbName);
    List<WriteEventInfo> writeEventInfoList = withinContext.db.getMSC().getAllWriteEventInfo(request);
    return ((writeEventInfoList == null) ? null : new ArrayList<>(Collections2.filter(writeEventInfoList, writeEventInfo -> {
        assert (writeEventInfo != null);
        // it should be skipped.
        return (ReplUtils.tableIncludedInReplScope(withinContext.replScope, writeEventInfo.getTable()) && ReplUtils.tableIncludedInReplScope(withinContext.oldReplScope, writeEventInfo.getTable()) && !withinContext.getTablesForBootstrap().contains(writeEventInfo.getTable().toLowerCase()));
    })));
}

Example 3 with WriteEventInfo

use of org.apache.hadoop.hive.metastore.api.WriteEventInfo in project hive by apache.

the class JSONCommitTxnMessage method addWriteEventInfo.

@Override
public void addWriteEventInfo(List<WriteEventInfo> writeEventInfoList) {
    if (this.databases == null) {
        this.databases = Lists.newArrayList();
    }
    if (this.tables == null) {
        this.tables = Lists.newArrayList();
    }
    if (this.writeIds == null) {
        this.writeIds = Lists.newArrayList();
    }
    if (this.tableObjs == null) {
        this.tableObjs = Lists.newArrayList();
    }
    if (this.partitions == null) {
        this.partitions = Lists.newArrayList();
    }
    if (this.partitionObjs == null) {
        this.partitionObjs = Lists.newArrayList();
    }
    if (this.files == null) {
        this.files = Lists.newArrayList();
    }
    for (WriteEventInfo writeEventInfo : writeEventInfoList) {
        this.databases.add(writeEventInfo.getDatabase());
        this.tables.add(writeEventInfo.getTable());
        this.writeIds.add(writeEventInfo.getWriteId());
        this.partitions.add(writeEventInfo.getPartition());
        this.tableObjs.add(writeEventInfo.getTableObj());
        this.partitionObjs.add(writeEventInfo.getPartitionObj());
        this.files.add(writeEventInfo.getFiles());
    }
}

Example 4 with WriteEventInfo

use of org.apache.hadoop.hive.metastore.api.WriteEventInfo in project hive by apache.

the class TestGetAllWriteEventInfo method testGetByWrongTable.

@Test
public void testGetByWrongTable() throws Exception {
    GetAllWriteEventInfoRequest req = new GetAllWriteEventInfoRequest();
    req.setTxnId(TXN_ID);
    req.setDbName(DB_NAME);
    req.setTableName("wrong_table");
    List<WriteEventInfo> writeEventInfoList = client.getAllWriteEventInfo(req);
    Assert.assertTrue(writeEventInfoList.isEmpty());
}

Example 5 with WriteEventInfo

use of org.apache.hadoop.hive.metastore.api.WriteEventInfo in project hive by apache.

the class TestGetAllWriteEventInfo method testGetByTxnId.

@Test
public void testGetByTxnId() throws Exception {
    GetAllWriteEventInfoRequest req = new GetAllWriteEventInfoRequest();
    req.setTxnId(TXN_ID);
    List<WriteEventInfo> writeEventInfoList = client.getAllWriteEventInfo(req);
    Assert.assertEquals(1, writeEventInfoList.size());
    WriteEventInfo writeEventInfo = writeEventInfoList.get(0);
    Assert.assertEquals(TXN_ID, writeEventInfo.getWriteId());
    Assert.assertEquals(DB_NAME, writeEventInfo.getDatabase());
    Assert.assertEquals(TABLE_NAME, writeEventInfo.getTable());
}