Examples with Client org.apache.accumulo.core.replication.thrift.ReplicationServicer.Client used on opensource projects

Example 1 with Client

use of org.apache.accumulo.core.replication.thrift.ReplicationServicer.Client in project accumulo by apache.

the class AccumuloReplicaSystem method _replicate.

/**
 * Perform replication, making a few attempts when an exception is returned.
 *
 * @param p
 *          Path of WAL to replicate
 * @param status
 *          Current status for the WAL
 * @param target
 *          Where we're replicating to
 * @param helper
 *          A helper for replication
 * @param localConf
 *          The local instance's configuration
 * @param peerContext
 *          The ClientContext to connect to the peer
 * @return The new (or unchanged) Status for the WAL
 */
private Status _replicate(final Path p, final Status status, final ReplicationTarget target, final ReplicaSystemHelper helper, final AccumuloConfiguration localConf, final ClientContext peerContext, final UserGroupInformation accumuloUgi) {
    try {
        double tracePercent = localConf.getFraction(Property.REPLICATION_TRACE_PERCENT);
        ProbabilitySampler sampler = new ProbabilitySampler(tracePercent);
        Trace.on("AccumuloReplicaSystem", sampler);
        // Remote identifier is an integer (table id) in this case.
        final String remoteTableId = target.getRemoteIdentifier();
        // Attempt the replication of this status a number of times before giving up and
        // trying to replicate it again later some other time.
        int numAttempts = localConf.getCount(Property.REPLICATION_WORK_ATTEMPTS);
        for (int i = 0; i < numAttempts; i++) {
            log.debug("Attempt {}", i);
            String peerTserverStr;
            log.debug("Fetching peer tserver address");
            Span span = Trace.start("Fetch peer tserver");
            try {
                // Ask the master on the remote what TServer we should talk with to replicate the data
                peerTserverStr = ReplicationClient.executeCoordinatorWithReturn(peerContext, new ClientExecReturn<String, ReplicationCoordinator.Client>() {

                    @Override
                    public String execute(ReplicationCoordinator.Client client) throws Exception {
                        return client.getServicerAddress(remoteTableId, peerContext.rpcCreds());
                    }
                });
            } catch (AccumuloException | AccumuloSecurityException e) {
                // No progress is made
                log.error("Could not connect to master at {}, cannot proceed with replication. Will retry", target, e);
                continue;
            } finally {
                span.stop();
            }
            if (null == peerTserverStr) {
                // Something went wrong, and we didn't get a valid tserver from the remote for some reason
                log.warn("Did not receive tserver from master at {}, cannot proceed with replication. Will retry.", target);
                continue;
            }
            final HostAndPort peerTserver = HostAndPort.fromString(peerTserverStr);
            final long timeout = localConf.getTimeInMillis(Property.REPLICATION_RPC_TIMEOUT);
            // We have a tserver on the remote -- send the data its way.
            Status finalStatus;
            final long sizeLimit = conf.getAsBytes(Property.REPLICATION_MAX_UNIT_SIZE);
            try {
                if (p.getName().endsWith(RFILE_SUFFIX)) {
                    span = Trace.start("RFile replication");
                    try {
                        finalStatus = replicateRFiles(peerContext, peerTserver, target, p, status, sizeLimit, remoteTableId, peerContext.rpcCreds(), helper, timeout);
                    } finally {
                        span.stop();
                    }
                } else {
                    span = Trace.start("WAL replication");
                    try {
                        finalStatus = replicateLogs(peerContext, peerTserver, target, p, status, sizeLimit, remoteTableId, peerContext.rpcCreds(), helper, accumuloUgi, timeout);
                    } finally {
                        span.stop();
                    }
                }
                log.debug("New status for {} after replicating to {} is {}", p, peerContext.getInstance(), ProtobufUtil.toString(finalStatus));
                return finalStatus;
            } catch (TTransportException | AccumuloException | AccumuloSecurityException e) {
                log.warn("Could not connect to remote server {}, will retry", peerTserverStr, e);
                sleepUninterruptibly(1, TimeUnit.SECONDS);
            }
        }
        log.info("No progress was made after {} attempts to replicate {}, returning so file can be re-queued", numAttempts, p);
        // We made no status, punt on it for now, and let it re-queue itself for work
        return status;
    } finally {
        Trace.off();
    }
}

Example 2 with Client

use of org.apache.accumulo.core.replication.thrift.ReplicationServicer.Client in project accumulo by apache.

the class AccumuloReplicaSystemTest method dontSendEmptyDataToPeer.

@Test
public void dontSendEmptyDataToPeer() throws Exception {
    Client replClient = createMock(Client.class);
    AccumuloReplicaSystem ars = createMock(AccumuloReplicaSystem.class);
    WalEdits edits = new WalEdits(Collections.emptyList());
    WalReplication walReplication = new WalReplication(edits, 0, 0, 0);
    ReplicationTarget target = new ReplicationTarget("peer", "2", Table.ID.of("1"));
    DataInputStream input = null;
    Path p = new Path("/accumulo/wals/tserver+port/" + UUID.randomUUID().toString());
    Status status = null;
    long sizeLimit = Long.MAX_VALUE;
    String remoteTableId = target.getRemoteIdentifier();
    TCredentials tcreds = null;
    Set<Integer> tids = new HashSet<>();
    WalClientExecReturn walClientExec = ars.new WalClientExecReturn(target, input, p, status, sizeLimit, remoteTableId, tcreds, tids);
    expect(ars.getWalEdits(target, input, p, status, sizeLimit, tids)).andReturn(walReplication);
    replay(replClient, ars);
    ReplicationStats stats = walClientExec.execute(replClient);
    verify(replClient, ars);
    Assert.assertEquals(new ReplicationStats(0l, 0l, 0l), stats);
}

Example 3 with Client

use of org.apache.accumulo.core.replication.thrift.ReplicationServicer.Client in project accumulo by apache.

the class AccumuloReplicaSystemTest method consumedButNotSentDataShouldBeRecorded.

@Test
public void consumedButNotSentDataShouldBeRecorded() throws Exception {
    Client replClient = createMock(Client.class);
    AccumuloReplicaSystem ars = createMock(AccumuloReplicaSystem.class);
    WalEdits edits = new WalEdits(Collections.emptyList());
    WalReplication walReplication = new WalReplication(edits, 0, 5, 0);
    ReplicationTarget target = new ReplicationTarget("peer", "2", Table.ID.of("1"));
    DataInputStream input = null;
    Path p = new Path("/accumulo/wals/tserver+port/" + UUID.randomUUID().toString());
    Status status = null;
    long sizeLimit = Long.MAX_VALUE;
    String remoteTableId = target.getRemoteIdentifier();
    TCredentials tcreds = null;
    Set<Integer> tids = new HashSet<>();
    WalClientExecReturn walClientExec = ars.new WalClientExecReturn(target, input, p, status, sizeLimit, remoteTableId, tcreds, tids);
    expect(ars.getWalEdits(target, input, p, status, sizeLimit, tids)).andReturn(walReplication);
    replay(replClient, ars);
    ReplicationStats stats = walClientExec.execute(replClient);
    verify(replClient, ars);
    Assert.assertEquals(new ReplicationStats(0l, 0l, 5l), stats);
}