Examples with LogHeaderIncompleteException org.apache.accumulo.tserver.log.DfsLogger.LogHeaderIncompleteException used on opensource projects

Example 1 with LogHeaderIncompleteException

use of org.apache.accumulo.tserver.log.DfsLogger.LogHeaderIncompleteException in project accumulo by apache.

the class AccumuloReplicaSystem method replicateLogs.

protected Status replicateLogs(ClientContext peerContext, final HostAndPort peerTserver, final ReplicationTarget target, final Path p, final Status status, final long sizeLimit, final String remoteTableId, final TCredentials tcreds, final ReplicaSystemHelper helper, final UserGroupInformation accumuloUgi, long timeout) throws TTransportException, AccumuloException, AccumuloSecurityException {
    log.debug("Replication WAL to peer tserver");
    final Set<Integer> tids;
    try (final FSDataInputStream fsinput = fs.open(p);
        final DataInputStream input = getWalStream(p, fsinput)) {
        log.debug("Skipping unwanted data in WAL");
        Span span = Trace.start("Consume WAL prefix");
        span.data("file", p.toString());
        try {
            // We want to read all records in the WAL up to the "begin" offset contained in the Status message,
            // building a Set of tids from DEFINE_TABLET events which correspond to table ids for future mutations
            tids = consumeWalPrefix(target, input, p, status, sizeLimit);
        } catch (IOException e) {
            log.warn("Unexpected error consuming file.");
            return status;
        } finally {
            span.stop();
        }
        log.debug("Sending batches of data to peer tserver");
        Status lastStatus = status, currentStatus = status;
        final AtomicReference<Exception> exceptionRef = new AtomicReference<>();
        while (true) {
            // Set some trace info
            span = Trace.start("Replicate WAL batch");
            span.data("Batch size (bytes)", Long.toString(sizeLimit));
            span.data("File", p.toString());
            span.data("Peer instance name", peerContext.getInstance().getInstanceName());
            span.data("Peer tserver", peerTserver.toString());
            span.data("Remote table ID", remoteTableId);
            ReplicationStats replResult;
            try {
                // Read and send a batch of mutations
                replResult = ReplicationClient.executeServicerWithReturn(peerContext, peerTserver, new WalClientExecReturn(target, input, p, currentStatus, sizeLimit, remoteTableId, tcreds, tids), timeout);
            } catch (Exception e) {
                log.error("Caught exception replicating data to {} at {}", peerContext.getInstance().getInstanceName(), peerTserver, e);
                throw e;
            } finally {
                span.stop();
            }
            // Catch the overflow
            long newBegin = currentStatus.getBegin() + replResult.entriesConsumed;
            if (newBegin < 0) {
                newBegin = Long.MAX_VALUE;
            }
            currentStatus = Status.newBuilder(currentStatus).setBegin(newBegin).build();
            log.debug("Sent batch for replication of {} to {}, with new Status {}", p, target, ProtobufUtil.toString(currentStatus));
            // If we got a different status
            if (!currentStatus.equals(lastStatus)) {
                span = Trace.start("Update replication table");
                try {
                    if (null != accumuloUgi) {
                        final Status copy = currentStatus;
                        accumuloUgi.doAs(new PrivilegedAction<Void>() {

                            @Override
                            public Void run() {
                                try {
                                    helper.recordNewStatus(p, copy, target);
                                } catch (Exception e) {
                                    exceptionRef.set(e);
                                }
                                return null;
                            }
                        });
                        Exception e = exceptionRef.get();
                        if (null != e) {
                            if (e instanceof TableNotFoundException) {
                                throw (TableNotFoundException) e;
                            } else if (e instanceof AccumuloSecurityException) {
                                throw (AccumuloSecurityException) e;
                            } else if (e instanceof AccumuloException) {
                                throw (AccumuloException) e;
                            } else {
                                throw new RuntimeException("Received unexpected exception", e);
                            }
                        }
                    } else {
                        helper.recordNewStatus(p, currentStatus, target);
                    }
                } catch (TableNotFoundException e) {
                    log.error("Tried to update status in replication table for {} as {}, but the table did not exist", p, ProtobufUtil.toString(currentStatus), e);
                    throw new RuntimeException("Replication table did not exist, will retry", e);
                } finally {
                    span.stop();
                }
                log.debug("Recorded updated status for {}: {}", p, ProtobufUtil.toString(currentStatus));
                // If we don't have any more work, just quit
                if (!StatusUtil.isWorkRequired(currentStatus)) {
                    return currentStatus;
                } else {
                    // Otherwise, let it loop and replicate some more data
                    lastStatus = currentStatus;
                }
            } else {
                log.debug("Did not replicate any new data for {} to {}, (state was {})", p, target, ProtobufUtil.toString(lastStatus));
                // we can just not record any updates, and it will be picked up again by the work assigner
                return status;
            }
        }
    } catch (LogHeaderIncompleteException e) {
        log.warn("Could not read header from {}, assuming that there is no data present in the WAL, therefore replication is complete", p);
        Status newStatus;
        // Bump up the begin to the (infinite) end, trying to be accurate
        if (status.getInfiniteEnd()) {
            newStatus = Status.newBuilder(status).setBegin(Long.MAX_VALUE).build();
        } else {
            newStatus = Status.newBuilder(status).setBegin(status.getEnd()).build();
        }
        Span span = Trace.start("Update replication table");
        try {
            helper.recordNewStatus(p, newStatus, target);
        } catch (TableNotFoundException tnfe) {
            log.error("Tried to update status in replication table for {} as {}, but the table did not exist", p, ProtobufUtil.toString(newStatus), e);
            throw new RuntimeException("Replication table did not exist, will retry", e);
        } finally {
            span.stop();
        }
        return newStatus;
    } catch (IOException e) {
        log.error("Could not create stream for WAL", e);
        // No data sent (bytes nor records) and no progress made
        return status;
    }
}

Example 2 with LogHeaderIncompleteException

use of org.apache.accumulo.tserver.log.DfsLogger.LogHeaderIncompleteException in project accumulo by apache.

the class LogReader method main.

/**
 * Dump a Log File (Map or Sequence) to stdout. Will read from HDFS or local file system.
 *
 * @param args
 *          - first argument is the file to print
 */
public static void main(String[] args) throws IOException {
    Opts opts = new Opts();
    opts.parseArgs(LogReader.class.getName(), args);
    VolumeManager fs = VolumeManagerImpl.get();
    Matcher rowMatcher = null;
    KeyExtent ke = null;
    Text row = null;
    if (opts.files.isEmpty()) {
        new JCommander(opts).usage();
        return;
    }
    if (opts.row != null)
        row = new Text(opts.row);
    if (opts.extent != null) {
        String[] sa = opts.extent.split(";");
        ke = new KeyExtent(Table.ID.of(sa[0]), new Text(sa[1]), new Text(sa[2]));
    }
    if (opts.regexp != null) {
        Pattern pattern = Pattern.compile(opts.regexp);
        rowMatcher = pattern.matcher("");
    }
    Set<Integer> tabletIds = new HashSet<>();
    for (String file : opts.files) {
        Path path = new Path(file);
        LogFileKey key = new LogFileKey();
        LogFileValue value = new LogFileValue();
        if (fs.isFile(path)) {
            try (final FSDataInputStream fsinput = fs.open(path)) {
                // read log entries from a simple hdfs file
                DFSLoggerInputStreams streams;
                try {
                    streams = DfsLogger.readHeaderAndReturnStream(fsinput, SiteConfiguration.getInstance());
                } catch (LogHeaderIncompleteException e) {
                    log.warn("Could not read header for {} . Ignoring...", path);
                    continue;
                }
                try (DataInputStream input = streams.getDecryptingInputStream()) {
                    while (true) {
                        try {
                            key.readFields(input);
                            value.readFields(input);
                        } catch (EOFException ex) {
                            break;
                        }
                        printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
                    }
                }
            }
        } else {
            // read the log entries sorted in a map file
            MultiReader input = new MultiReader(fs, path);
            while (input.next(key, value)) {
                printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations);
            }
        }
    }
}