Examples with ReplicationState org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.ReplicationState used on opensource projects

Example 1 with ReplicationState

use of org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.ReplicationState in project hive by apache.

the class LoadPartitions method addPartition.

private void addPartition(boolean hasMorePartitions, AddPartitionDesc addPartitionDesc) throws Exception {
    tracker.addTask(tasksForAddPartition(table, addPartitionDesc));
    if (hasMorePartitions && !tracker.canAddMoreTasks()) {
        ReplicationState currentReplicationState = new ReplicationState(new PartitionState(table.getTableName(), addPartitionDesc));
        updateReplicationState(currentReplicationState);
    }
}

Example 2 with ReplicationState

use of org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.ReplicationState in project hive by apache.

the class LoadPartitions method forExistingTable.

private TaskTracker forExistingTable(AddPartitionDesc lastPartitionReplicated) throws Exception {
    boolean encounteredTheLastReplicatedPartition = (lastPartitionReplicated == null);
    Map<String, String> lastReplicatedPartSpec = null;
    if (!encounteredTheLastReplicatedPartition) {
        lastReplicatedPartSpec = lastPartitionReplicated.getPartition(0).getPartSpec();
        LOG.info("Start processing from partition info spec : {}", StringUtils.mapToString(lastReplicatedPartSpec));
    }
    ReplicationSpec replicationSpec = event.replicationSpec();
    Iterator<AddPartitionDesc> partitionIterator = event.partitionDescriptions(tableDesc).iterator();
    while (!encounteredTheLastReplicatedPartition && partitionIterator.hasNext()) {
        AddPartitionDesc addPartitionDesc = partitionIterator.next();
        Map<String, String> currentSpec = addPartitionDesc.getPartition(0).getPartSpec();
        encounteredTheLastReplicatedPartition = lastReplicatedPartSpec.equals(currentSpec);
    }
    while (partitionIterator.hasNext() && tracker.canAddMoreTasks()) {
        AddPartitionDesc addPartitionDesc = partitionIterator.next();
        Map<String, String> partSpec = addPartitionDesc.getPartition(0).getPartSpec();
        Partition ptn = context.hiveDb.getPartition(table, partSpec, false);
        if (ptn == null) {
            if (!replicationSpec.isMetadataOnly()) {
                addPartition(partitionIterator.hasNext(), addPartitionDesc);
            }
        } else {
            // the destination ptn's repl.last.id is older than the replacement's.
            if (replicationSpec.allowReplacementInto(ptn.getParameters())) {
                if (replicationSpec.isMetadataOnly()) {
                    tracker.addTask(alterSinglePartition(addPartitionDesc, replicationSpec, ptn));
                    if (!tracker.canAddMoreTasks()) {
                        tracker.setReplicationState(new ReplicationState(new PartitionState(table.getTableName(), addPartitionDesc)));
                    }
                } else {
                    addPartition(partitionIterator.hasNext(), addPartitionDesc);
                }
            } else {
            // ignore this ptn, do nothing, not an error.
            }
        }
    }
    return tracker;
}