Examples with ReplicationTableOfflineException org.apache.accumulo.core.replication.ReplicationTableOfflineException used on opensource projects

Example 6 with ReplicationTableOfflineException

use of org.apache.accumulo.core.replication.ReplicationTableOfflineException in project accumulo by apache.

the class ReplicationUtil method getProgress.

/**
 * Compute a progress string for the replication of the given WAL
 *
 * @param conn
 *          Accumulo Connector
 * @param path
 *          Absolute path to a WAL, or null
 * @param target
 *          ReplicationTarget the WAL is being replicated to
 * @return A status message for a file being replicated
 */
public String getProgress(Connector conn, String path, ReplicationTarget target) {
    // We could try to grep over the table, but without knowing the full file path, we
    // can't find the status quickly
    String status = "Unknown";
    if (null != path) {
        Scanner s;
        try {
            s = ReplicationTable.getScanner(conn);
        } catch (ReplicationTableOfflineException e) {
            log.debug("Replication table no longer online", e);
            return status;
        }
        s.setRange(Range.exact(path));
        s.fetchColumn(WorkSection.NAME, target.toText());
        // Fetch the work entry for this item
        Entry<Key, Value> kv = null;
        try {
            kv = Iterables.getOnlyElement(s);
        } catch (NoSuchElementException e) {
            log.trace("Could not find status of {} replicating to {}", path, target);
            status = "Unknown";
        } finally {
            s.close();
        }
        // If we found the work entry for it, try to compute some progress
        if (null != kv) {
            try {
                Status stat = Status.parseFrom(kv.getValue().get());
                if (StatusUtil.isFullyReplicated(stat)) {
                    status = "Finished";
                } else {
                    if (stat.getInfiniteEnd()) {
                        status = stat.getBegin() + "/&infin; records";
                    } else {
                        status = stat.getBegin() + "/" + stat.getEnd() + " records";
                    }
                }
            } catch (InvalidProtocolBufferException e) {
                log.warn("Could not deserialize protobuf for {}", kv.getKey(), e);
                status = "Unknown";
            }
        }
    }
    return status;
}

Example 7 with ReplicationTableOfflineException

use of org.apache.accumulo.core.replication.ReplicationTableOfflineException in project accumulo by apache.

the class FinishedWorkUpdater method run.

@Override
public void run() {
    log.debug("Looking for finished replication work");
    if (!ReplicationTable.isOnline(conn)) {
        log.debug("Replication table is not yet online, will retry");
        return;
    }
    BatchScanner bs;
    BatchWriter replBw;
    try {
        bs = ReplicationTable.getBatchScanner(conn, 4);
        replBw = ReplicationTable.getBatchWriter(conn);
    } catch (ReplicationTableOfflineException e) {
        log.debug("Table is no longer online, will retry");
        return;
    }
    IteratorSetting cfg = new IteratorSetting(50, WholeRowIterator.class);
    bs.addScanIterator(cfg);
    WorkSection.limit(bs);
    bs.setRanges(Collections.singleton(new Range()));
    try {
        for (Entry<Key, Value> serializedRow : bs) {
            SortedMap<Key, Value> wholeRow;
            try {
                wholeRow = WholeRowIterator.decodeRow(serializedRow.getKey(), serializedRow.getValue());
            } catch (IOException e) {
                log.warn("Could not deserialize whole row with key {}", serializedRow.getKey().toStringNoTruncate(), e);
                continue;
            }
            log.debug("Processing work progress for {} with {} columns", serializedRow.getKey().getRow(), wholeRow.size());
            Map<Table.ID, Long> tableIdToProgress = new HashMap<>();
            boolean error = false;
            Text buffer = new Text();
            // We want to determine what the minimum point that all Work entries have replicated to
            for (Entry<Key, Value> entry : wholeRow.entrySet()) {
                Status status;
                try {
                    status = Status.parseFrom(entry.getValue().get());
                } catch (InvalidProtocolBufferException e) {
                    log.warn("Could not deserialize protobuf for {}", entry.getKey(), e);
                    error = true;
                    break;
                }
                // Get the replication target for the work record
                entry.getKey().getColumnQualifier(buffer);
                ReplicationTarget target = ReplicationTarget.from(buffer);
                // Initialize the value in the map if we don't have one
                if (!tableIdToProgress.containsKey(target.getSourceTableId())) {
                    tableIdToProgress.put(target.getSourceTableId(), Long.MAX_VALUE);
                }
                // Find the minimum value for begin (everyone has replicated up to this offset in the file)
                tableIdToProgress.put(target.getSourceTableId(), Math.min(tableIdToProgress.get(target.getSourceTableId()), status.getBegin()));
            }
            if (error) {
                continue;
            }
            // Update the replication table for each source table we found work records for
            for (Entry<Table.ID, Long> entry : tableIdToProgress.entrySet()) {
                // If the progress is 0, then no one has replicated anything, and we don't need to update anything
                if (0 == entry.getValue()) {
                    continue;
                }
                serializedRow.getKey().getRow(buffer);
                log.debug("For {}, source table ID {} has replicated through {}", serializedRow.getKey().getRow(), entry.getKey(), entry.getValue());
                Mutation replMutation = new Mutation(buffer);
                // Set that we replicated at least this much data, ignoring the other fields
                Status updatedStatus = StatusUtil.replicated(entry.getValue());
                Value serializedUpdatedStatus = ProtobufUtil.toValue(updatedStatus);
                // Pull the sourceTableId into a Text
                Table.ID srcTableId = entry.getKey();
                // Make the mutation
                StatusSection.add(replMutation, srcTableId, serializedUpdatedStatus);
                log.debug("Updating replication status entry for {} with {}", serializedRow.getKey().getRow(), ProtobufUtil.toString(updatedStatus));
                try {
                    replBw.addMutation(replMutation);
                } catch (MutationsRejectedException e) {
                    log.error("Error writing mutations to update replication Status messages in StatusSection, will retry", e);
                    return;
                }
            }
        }
    } finally {
        log.debug("Finished updating files with completed replication work");
        bs.close();
        try {
            replBw.close();
        } catch (MutationsRejectedException e) {
            log.error("Error writing mutations to update replication Status messages in StatusSection, will retry", e);
        }
    }
}

Example 8 with ReplicationTableOfflineException

use of org.apache.accumulo.core.replication.ReplicationTableOfflineException in project accumulo by apache.

the class ReplicationProcessor method process.

@Override
public void process(String workID, byte[] data) {
    ReplicationTarget target = DistributedWorkQueueWorkAssignerHelper.fromQueueKey(workID).getValue();
    String file = new String(data, UTF_8);
    log.debug("Received replication work for {} to {}", file, target);
    ReplicaSystem replica;
    try {
        replica = getReplicaSystem(target);
    } catch (Exception e) {
        log.error("Could not instantiate ReplicaSystem for {}, waiting before returning the work", target, e);
        try {
            // TODO configurable
            Thread.sleep(5000);
        } catch (InterruptedException ie) {
            Thread.currentThread().interrupt();
        }
        return;
    }
    Status status;
    try {
        status = getStatus(file, target);
    } catch (ReplicationTableOfflineException | AccumuloException | AccumuloSecurityException e) {
        log.error("Could not look for replication record", e);
        throw new IllegalStateException("Could not look for replication record", e);
    } catch (InvalidProtocolBufferException e) {
        log.error("Could not deserialize Status from Work section for {} and ", file, target);
        throw new RuntimeException("Could not parse Status for work record", e);
    } catch (NoSuchElementException e) {
        log.error("Assigned work for {} to {} but could not find work record", file, target);
        return;
    }
    log.debug("Current status for {} replicating to {}: {}", file, target, ProtobufUtil.toString(status));
    // We don't need to do anything (shouldn't have gotten this work record in the first place)
    if (!StatusUtil.isWorkRequired(status)) {
        log.info("Received work request for {} and {}, but it does not need replication. Ignoring...", file, target);
        return;
    }
    // Sanity check that nothing bad happened and our replication source still exists
    Path filePath = new Path(file);
    try {
        if (!doesFileExist(filePath, target)) {
            return;
        }
    } catch (IOException e) {
        log.error("Could not determine if file exists {}", filePath, e);
        throw new RuntimeException(e);
    }
    log.debug("Replicating {} to {} using {}", filePath, target, replica.getClass().getName());
    Status newStatus = replica.replicate(filePath, status, target, getHelper());
    log.debug("Finished replicating {}. Original status: {}, New status: {}", filePath, status, newStatus);
}