Examples with ClientContext org.apache.accumulo.core.clientImpl.ClientContext used on opensource projects

Example 26 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class AccumuloReplicaSystem method _replicate.

/**
 * Perform replication, making a few attempts when an exception is returned.
 *
 * @param p
 *          Path of WAL to replicate
 * @param status
 *          Current status for the WAL
 * @param target
 *          Where we're replicating to
 * @param helper
 *          A helper for replication
 * @param localConf
 *          The local instance's configuration
 * @param peerContext
 *          The ClientContext to connect to the peer
 * @return The new (or unchanged) Status for the WAL
 */
private Status _replicate(final Path p, final Status status, final ReplicationTarget target, final ReplicaSystemHelper helper, final AccumuloConfiguration localConf, final ClientContext peerContext, final UserGroupInformation accumuloUgi) {
    Span span = TraceUtil.startSpan(this.getClass(), "_replicate");
    try (Scope replicaScope = span.makeCurrent()) {
        // Remote identifier is an integer (table id) in this case.
        final String remoteTableId = target.getRemoteIdentifier();
        // Attempt the replication of this status a number of times before giving up and
        // trying to replicate it again later some other time.
        int numAttempts = localConf.getCount(Property.REPLICATION_WORK_ATTEMPTS);
        for (int i = 0; i < numAttempts; i++) {
            log.debug("Attempt {}", i);
            String peerTserverStr;
            log.debug("Fetching peer tserver address");
            Span span2 = TraceUtil.startSpan(this.getClass(), "_replicate::Fetch peer tserver");
            try (Scope scope = span2.makeCurrent()) {
                // Ask the manager on the remote what TServer we should talk with to replicate the data
                peerTserverStr = ReplicationClient.executeCoordinatorWithReturn(peerContext, client -> client.getServicerAddress(remoteTableId, peerContext.rpcCreds()));
            } catch (AccumuloException | AccumuloSecurityException e) {
                // No progress is made
                log.error("Could not connect to manager at {}, cannot proceed with replication. Will retry", target, e);
                TraceUtil.setException(span2, e, false);
                continue;
            } catch (Exception e) {
                TraceUtil.setException(span2, e, true);
                throw e;
            } finally {
                span2.end();
            }
            if (peerTserverStr == null) {
                // Something went wrong, and we didn't get a valid tserver from the remote for some reason
                log.warn("Did not receive tserver from manager at {}, cannot proceed" + " with replication. Will retry.", target);
                continue;
            }
            final HostAndPort peerTserver = HostAndPort.fromString(peerTserverStr);
            final long timeout = localConf.getTimeInMillis(Property.REPLICATION_RPC_TIMEOUT);
            // We have a tserver on the remote -- send the data its way.
            Status finalStatus;
            final long sizeLimit = conf.getAsBytes(Property.REPLICATION_MAX_UNIT_SIZE);
            try {
                if (p.getName().endsWith(RFILE_SUFFIX)) {
                    Span span3 = TraceUtil.startSpan(this.getClass(), "_replicate::RFile replication");
                    try (Scope scope = span3.makeCurrent()) {
                        finalStatus = replicateRFiles(peerContext, peerTserver, target, p, status, timeout);
                    } catch (Exception e) {
                        TraceUtil.setException(span3, e, true);
                        throw e;
                    } finally {
                        span3.end();
                    }
                } else {
                    Span span4 = TraceUtil.startSpan(this.getClass(), "_replicate::WAL replication");
                    try (Scope scope = span4.makeCurrent()) {
                        finalStatus = replicateLogs(peerContext, peerTserver, target, p, status, sizeLimit, remoteTableId, peerContext.rpcCreds(), helper, accumuloUgi, timeout);
                    } catch (Exception e) {
                        TraceUtil.setException(span4, e, true);
                        throw e;
                    } finally {
                        span4.end();
                    }
                }
                log.debug("New status for {} after replicating to {} is {}", p, peerContext.getInstanceName(), ProtobufUtil.toString(finalStatus));
                return finalStatus;
            } catch (TTransportException | AccumuloException | AccumuloSecurityException e) {
                log.warn("Could not connect to remote server {}, will retry", peerTserverStr, e);
                TraceUtil.setException(span, e, false);
                sleepUninterruptibly(1, TimeUnit.SECONDS);
            }
        }
        log.info("No progress was made after {} attempts to replicate {}," + " returning so file can be re-queued", numAttempts, p);
        // We made no status, punt on it for now, and let it re-queue itself for work
        return status;
    } catch (Exception e) {
        TraceUtil.setException(span, e, true);
        throw e;
    } finally {
        span.end();
    }
}

Example 27 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class TestIngest method ingest.

public static void ingest(AccumuloClient accumuloClient, FileSystem fs, IngestParams params) throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException, TableExistsException {
    long stopTime;
    byte[][] bytevals = generateValues(params.dataSize);
    byte[] randomValue = new byte[params.dataSize];
    long bytesWritten = 0;
    createTable(accumuloClient, params);
    BatchWriter bw = null;
    FileSKVWriter writer = null;
    if (params.outputFile != null) {
        ClientContext cc = (ClientContext) accumuloClient;
        writer = FileOperations.getInstance().newWriterBuilder().forFile(params.outputFile + "." + RFile.EXTENSION, fs, cc.getHadoopConf(), CryptoServiceFactory.newDefaultInstance()).withTableConfiguration(DefaultConfiguration.getInstance()).build();
        writer.startDefaultLocalityGroup();
    } else {
        bw = accumuloClient.createBatchWriter(params.tableName);
        String principal = ClientProperty.AUTH_PRINCIPAL.getValue(params.clientProps);
        accumuloClient.securityOperations().changeUserAuthorizations(principal, AUTHS);
    }
    Text labBA = new Text(params.columnVisibility.getExpression());
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < params.rows; i++) {
        int rowid;
        if (params.stride > 0) {
            rowid = ((i % params.stride) * (params.rows / params.stride)) + (i / params.stride);
        } else {
            rowid = i;
        }
        Text row = generateRow(rowid, params.startRow);
        Mutation m = new Mutation(row);
        for (int j = 0; j < params.cols; j++) {
            Text colf = new Text(params.columnFamily);
            Text colq = new Text(FastFormat.toZeroPaddedString(j, 7, 10, COL_PREFIX));
            if (writer != null) {
                Key key = new Key(row, colf, colq, labBA);
                if (params.timestamp >= 0) {
                    key.setTimestamp(params.timestamp);
                } else {
                    key.setTimestamp(startTime);
                }
                if (params.delete) {
                    key.setDeleted(true);
                } else {
                    key.setDeleted(false);
                }
                bytesWritten += key.getSize();
                if (params.delete) {
                    writer.append(key, new Value());
                } else {
                    byte[] value;
                    if (params.random != null) {
                        value = genRandomValue(randomValue, params.random, rowid + params.startRow, j);
                    } else {
                        value = bytevals[j % bytevals.length];
                    }
                    Value v = new Value(value);
                    writer.append(key, v);
                    bytesWritten += v.getSize();
                }
            } else {
                Key key = new Key(row, colf, colq, labBA);
                bytesWritten += key.getSize();
                if (params.delete) {
                    if (params.timestamp >= 0) {
                        m.putDelete(colf, colq, params.columnVisibility, params.timestamp);
                    } else {
                        m.putDelete(colf, colq, params.columnVisibility);
                    }
                } else {
                    byte[] value;
                    if (params.random != null) {
                        value = genRandomValue(randomValue, params.random, rowid + params.startRow, j);
                    } else {
                        value = bytevals[j % bytevals.length];
                    }
                    bytesWritten += value.length;
                    if (params.timestamp >= 0) {
                        m.put(colf, colq, params.columnVisibility, params.timestamp, new Value(value, true));
                    } else {
                        m.put(colf, colq, params.columnVisibility, new Value(value, true));
                    }
                }
            }
        }
        if (bw != null) {
            bw.addMutation(m);
        }
    }
    if (writer != null) {
        writer.close();
    } else if (bw != null) {
        try {
            bw.close();
        } catch (MutationsRejectedException e) {
            if (!e.getSecurityErrorCodes().isEmpty()) {
                for (Entry<TabletId, Set<SecurityErrorCode>> entry : e.getSecurityErrorCodes().entrySet()) {
                    System.err.println("ERROR : Not authorized to write to : " + entry.getKey() + " due to " + entry.getValue());
                }
            }
            if (!e.getConstraintViolationSummaries().isEmpty()) {
                for (ConstraintViolationSummary cvs : e.getConstraintViolationSummaries()) {
                    System.err.println("ERROR : Constraint violates : " + cvs);
                }
            }
            throw e;
        }
    }
    stopTime = System.currentTimeMillis();
    int totalValues = params.rows * params.cols;
    double elapsed = (stopTime - startTime) / 1000.0;
    System.out.printf("%,12d records written | %,8d records/sec | %,12d bytes written" + " | %,8d bytes/sec | %6.3f secs   %n", totalValues, (int) (totalValues / elapsed), bytesWritten, (int) (bytesWritten / elapsed), elapsed);
}

Example 28 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class DetectDeadTabletServersIT method getStats.

private ManagerMonitorInfo getStats(AccumuloClient c) throws Exception {
    ClientContext context = (ClientContext) c;
    Client client = null;
    while (true) {
        try {
            client = ManagerClient.getConnectionWithRetry(context);
            log.info("Fetching manager stats");
            return client.getManagerStats(TraceUtil.traceInfo(), context.rpcCreds());
        } catch (ThriftNotActiveServiceException e) {
            // Let it loop, fetching a new location
            sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
        } finally {
            if (client != null) {
                ManagerClient.close(client, context);
            }
        }
    }
}

Example 29 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class VolumeIT method testRemoveVolumes.

@Test
public void testRemoveVolumes() throws Exception {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        String[] tableNames = getUniqueNames(2);
        verifyVolumesUsed(client, tableNames[0], false, v1, v2);
        assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor());
        cluster.stop();
        updateConfig(config -> config.setProperty(Property.INSTANCE_VOLUMES.getKey(), v2.toString()));
        // start cluster and verify that volume was decommissioned
        cluster.start();
        client.tableOperations().compact(tableNames[0], null, null, true, true);
        verifyVolumesUsed(client, tableNames[0], true, v2);
        client.tableOperations().compact(RootTable.NAME, new CompactionConfig().setWait(true));
        // check that root tablet is not on volume 1
        int count = 0;
        for (StoredTabletFile file : ((ClientContext) client).getAmple().readTablet(RootTable.EXTENT).getFiles()) {
            assertTrue(file.getMetaUpdateDelete().startsWith(v2.toString()));
            count++;
        }
        assertTrue(count > 0);
        client.tableOperations().clone(tableNames[0], tableNames[1], true, new HashMap<>(), new HashSet<>());
        client.tableOperations().flush(MetadataTable.NAME, null, null, true);
        client.tableOperations().flush(RootTable.NAME, null, null, true);
        verifyVolumesUsed(client, tableNames[0], true, v2);
        verifyVolumesUsed(client, tableNames[1], true, v2);
    }
}

Example 30 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class ListBulkCommand method execute.

@Override
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception {
    List<String> tservers;
    ManagerMonitorInfo stats;
    ManagerClientService.Iface client = null;
    ClientContext context = shellState.getContext();
    while (true) {
        try {
            client = ManagerClient.getConnectionWithRetry(context);
            stats = client.getManagerStats(TraceUtil.traceInfo(), context.rpcCreds());
            break;
        } catch (ThriftNotActiveServiceException e) {
            // Let it loop, fetching a new location
            sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
        } finally {
            if (client != null)
                ManagerClient.close(client, context);
        }
    }
    final boolean paginate = !cl.hasOption(disablePaginationOpt.getOpt());
    if (cl.hasOption(tserverOption.getOpt())) {
        tservers = new ArrayList<>();
        tservers.add(cl.getOptionValue(tserverOption.getOpt()));
    } else {
        tservers = Collections.emptyList();
    }
    shellState.printLines(new BulkImportListIterator(tservers, stats), paginate);
    return 0;
}