Examples with ClientContext org.apache.accumulo.core.clientImpl.ClientContext used on opensource projects

Example 36 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class SuspendedTabletsIT method suspensionTestBody.

/**
 * Main test body for suspension tests.
 *
 * @param serverStopper
 *          callback which shuts down some tablet servers.
 */
private void suspensionTestBody(TServerKiller serverStopper) throws Exception {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        ClientContext ctx = (ClientContext) client;
        String tableName = getUniqueNames(1)[0];
        SortedSet<Text> splitPoints = new TreeSet<>();
        for (int i = 1; i < TABLETS; ++i) {
            splitPoints.add(new Text("" + i));
        }
        log.info("Creating table " + tableName);
        NewTableConfiguration ntc = new NewTableConfiguration().withSplits(splitPoints);
        ctx.tableOperations().create(tableName, ntc);
        // Wait for all of the tablets to hosted ...
        log.info("Waiting on hosting and balance");
        TabletLocations ds;
        for (ds = TabletLocations.retrieve(ctx, tableName); ds.hostedCount != TABLETS; ds = TabletLocations.retrieve(ctx, tableName)) {
            Thread.sleep(1000);
        }
        // ... and balanced.
        ctx.instanceOperations().waitForBalance();
        do {
            // Keep checking until all tablets are hosted and spread out across the tablet servers
            Thread.sleep(1000);
            ds = TabletLocations.retrieve(ctx, tableName);
        } while (ds.hostedCount != TABLETS || ds.hosted.keySet().size() != (TSERVERS - 1));
        // Given the loop exit condition above, at this point we're sure that all tablets are hosted
        // and some are hosted on each of the tablet servers other than the one reserved for hosting
        // the metadata table.
        assertEquals(TSERVERS - 1, ds.hosted.keySet().size());
        // Kill two tablet servers hosting our tablets. This should put tablets into suspended state,
        // and thus halt balancing.
        TabletLocations beforeDeathState = ds;
        log.info("Eliminating tablet servers");
        serverStopper.eliminateTabletServers(ctx, beforeDeathState, TSERVERS - 1);
        // All tablets should be either hosted or suspended.
        log.info("Waiting on suspended tablets");
        do {
            Thread.sleep(1000);
            ds = TabletLocations.retrieve(ctx, tableName);
        } while (ds.suspended.keySet().size() != (TSERVERS - 1) || (ds.suspendedCount + ds.hostedCount) != TABLETS);
        SetMultimap<HostAndPort, KeyExtent> deadTabletsByServer = ds.suspended;
        // the same place as before any tserver death.
        for (HostAndPort server : deadTabletsByServer.keySet()) {
            // Comparing pre-death, hosted tablets to suspended tablets on a server
            assertEquals(beforeDeathState.hosted.get(server), deadTabletsByServer.get(server));
        }
        assertEquals(TABLETS, ds.hostedCount + ds.suspendedCount);
        // Restart the first tablet server, making sure it ends up on the same port
        HostAndPort restartedServer = deadTabletsByServer.keySet().iterator().next();
        log.info("Restarting " + restartedServer);
        getCluster().getClusterControl().start(ServerType.TABLET_SERVER, Map.of(Property.TSERV_CLIENTPORT.getKey(), "" + restartedServer.getPort(), Property.TSERV_PORTSEARCH.getKey(), "false"), 1);
        // Eventually, the suspended tablets should be reassigned to the newly alive tserver.
        log.info("Awaiting tablet unsuspension for tablets belonging to " + restartedServer);
        while (ds.suspended.containsKey(restartedServer) || ds.assignedCount != 0) {
            Thread.sleep(1000);
            ds = TabletLocations.retrieve(ctx, tableName);
        }
        assertEquals(deadTabletsByServer.get(restartedServer), ds.hosted.get(restartedServer));
        // Finally, after much longer, remaining suspended tablets should be reassigned.
        log.info("Awaiting tablet reassignment for remaining tablets");
        while (ds.hostedCount != TABLETS) {
            Thread.sleep(1000);
            ds = TabletLocations.retrieve(ctx, tableName);
        }
    }
}

Example 37 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class TransportCachingIT method testCachedTransport.

@Test
public void testCachedTransport() throws InterruptedException {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
        List<String> tservers;
        while ((tservers = client.instanceOperations().getTabletServers()).isEmpty()) {
            // sleep until a tablet server is up
            Thread.sleep(50);
        }
        ClientContext context = (ClientContext) client;
        long rpcTimeout = ConfigurationTypeHelper.getTimeInMillis(Property.GENERAL_RPC_TIMEOUT.getDefaultValue());
        List<ThriftTransportKey> servers = tservers.stream().map(serverStr -> {
            return new ThriftTransportKey(HostAndPort.fromString(serverStr), rpcTimeout, context);
        }).collect(Collectors.toList());
        // only want to use one server for all subsequent test
        servers = servers.subList(0, 1);
        ThriftTransportPool pool = context.getTransportPool();
        TTransport first = getAnyTransport(servers, pool, true);
        assertNotNull(first);
        // Return it to unreserve it
        pool.returnTransport(first);
        TTransport second = getAnyTransport(servers, pool, true);
        // We should get the same transport
        assertSame("Expected the first and second to be the same instance", first, second);
        pool.returnTransport(second);
        // Ensure does not get cached connection just returned
        TTransport third = getAnyTransport(servers, pool, false);
        assertNotSame("Expected second and third transport to be different instances", second, third);
        TTransport fourth = getAnyTransport(servers, pool, false);
        assertNotSame("Expected third and fourth transport to be different instances", third, fourth);
        pool.returnTransport(third);
        pool.returnTransport(fourth);
        // The following three asserts ensure the per server queue is LIFO
        TTransport fifth = getAnyTransport(servers, pool, true);
        assertSame("Expected fourth and fifth transport to be the same instance", fourth, fifth);
        TTransport sixth = getAnyTransport(servers, pool, true);
        assertSame("Expected third and sixth transport to be the same instance", third, sixth);
        TTransport seventh = getAnyTransport(servers, pool, true);
        assertSame("Expected second and seventh transport to be the same instance", second, seventh);
        pool.returnTransport(fifth);
        pool.returnTransport(sixth);
        pool.returnTransport(seventh);
    }
}

Example 38 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class VolumeIT method testReplaceVolume.

private void testReplaceVolume(AccumuloClient client, boolean cleanShutdown) throws Exception {
    String[] tableNames = getUniqueNames(3);
    verifyVolumesUsed(client, tableNames[0], false, v1, v2);
    // write to 2nd table, but do not flush data to disk before shutdown
    try (AccumuloClient c2 = cluster.createAccumuloClient("root", new PasswordToken(ROOT_PASSWORD))) {
        writeData(tableNames[1], c2);
    }
    if (cleanShutdown)
        assertEquals(0, cluster.exec(Admin.class, "stopAll").getProcess().waitFor());
    cluster.stop();
    File v1f = new File(v1.toUri());
    File v8f = new File(new File(v1.getParent().toUri()), "v8");
    assertTrue("Failed to rename " + v1f + " to " + v8f, v1f.renameTo(v8f));
    Path v8 = new Path(v8f.toURI());
    File v2f = new File(v2.toUri());
    File v9f = new File(new File(v2.getParent().toUri()), "v9");
    assertTrue("Failed to rename " + v2f + " to " + v9f, v2f.renameTo(v9f));
    Path v9 = new Path(v9f.toURI());
    updateConfig(config -> {
        config.setProperty(Property.INSTANCE_VOLUMES.getKey(), v8 + "," + v9);
        config.setProperty(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey(), v1 + " " + v8 + "," + v2 + " " + v9);
    });
    // start cluster and verify that volumes were replaced
    cluster.start();
    verifyVolumesUsed(client, tableNames[0], true, v8, v9);
    verifyVolumesUsed(client, tableNames[1], true, v8, v9);
    // verify writes to new dir
    client.tableOperations().compact(tableNames[0], null, null, true, true);
    client.tableOperations().compact(tableNames[1], null, null, true, true);
    verifyVolumesUsed(client, tableNames[0], true, v8, v9);
    verifyVolumesUsed(client, tableNames[1], true, v8, v9);
    client.tableOperations().compact(RootTable.NAME, new CompactionConfig().setWait(true));
    // check that root tablet is not on volume 1 or 2
    int count = 0;
    for (StoredTabletFile file : ((ClientContext) client).getAmple().readTablet(RootTable.EXTENT).getFiles()) {
        assertTrue(file.getMetaUpdateDelete().startsWith(v8.toString()) || file.getMetaUpdateDelete().startsWith(v9.toString()));
        count++;
    }
    assertTrue(count > 0);
    client.tableOperations().clone(tableNames[1], tableNames[2], true, new HashMap<>(), new HashSet<>());
    client.tableOperations().flush(MetadataTable.NAME, null, null, true);
    client.tableOperations().flush(RootTable.NAME, null, null, true);
    verifyVolumesUsed(client, tableNames[0], true, v8, v9);
    verifyVolumesUsed(client, tableNames[1], true, v8, v9);
    verifyVolumesUsed(client, tableNames[2], true, v8, v9);
}

Example 39 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class TestIngest method ingest.

public static void ingest(AccumuloClient c, IngestParams params) throws MutationsRejectedException, IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException {
    ClientContext cc = (ClientContext) c;
    ingest(c, FileSystem.get(cc.getHadoopConf()), params);
}

Example 40 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class BalanceAfterCommsFailureIT method checkBalance.

private void checkBalance(AccumuloClient c) throws Exception {
    ClientContext context = (ClientContext) c;
    ManagerMonitorInfo stats = null;
    int unassignedTablets = 1;
    for (int i = 0; unassignedTablets > 0 && i < 10; i++) {
        ManagerClientService.Iface client = null;
        while (true) {
            try {
                client = ManagerClient.getConnectionWithRetry(context);
                stats = client.getManagerStats(TraceUtil.traceInfo(), context.rpcCreds());
                break;
            } catch (ThriftNotActiveServiceException e) {
                // Let it loop, fetching a new location
                log.debug("Contacted a Manager which is no longer active, retrying");
                sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
            } finally {
                if (client != null)
                    ManagerClient.close(client, context);
            }
        }
        unassignedTablets = stats.getUnassignedTablets();
        if (unassignedTablets > 0) {
            log.info("Found {} unassigned tablets, sleeping 3 seconds for tablet assignment", unassignedTablets);
            Thread.sleep(3000);
        }
    }
    assertEquals("Unassigned tablets were not assigned within 30 seconds", 0, unassignedTablets);
    List<Integer> counts = new ArrayList<>();
    for (TabletServerStatus server : stats.tServerInfo) {
        int count = 0;
        for (TableInfo table : server.tableMap.values()) {
            count += table.onlineTablets;
        }
        counts.add(count);
    }
    assertTrue("Expected to have at least two TabletServers", counts.size() > 1);
    for (int i = 1; i < counts.size(); i++) {
        int diff = Math.abs(counts.get(0) - counts.get(i));
        assertTrue("Expected difference in tablets to be less than or equal to " + counts.size() + " but was " + diff + ". Counts " + counts, diff <= counts.size());
    }
}