Examples with ClientContext org.apache.accumulo.core.clientImpl.ClientContext used on opensource projects

Example 6 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class FateCommand method execute.

@Override
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws ParseException, KeeperException, InterruptedException, IOException {
    ClientContext context = shellState.getContext();
    var siteConfig = SiteConfiguration.auto();
    String[] args = cl.getArgs();
    if (args.length <= 0) {
        throw new ParseException("Must provide a command to execute");
    }
    String cmd = args[0];
    boolean failedCommand = false;
    AdminUtil<FateCommand> admin = new AdminUtil<>(false);
    String path = context.getZooKeeperRoot() + Constants.ZFATE;
    var managerLockPath = ServiceLock.path(context.getZooKeeperRoot() + Constants.ZMANAGER_LOCK);
    ZooReaderWriter zk = getZooReaderWriter(context, siteConfig, cl.getOptionValue(secretOption.getOpt()));
    ZooStore<FateCommand> zs = new ZooStore<>(path, zk);
    if ("fail".equals(cmd)) {
        if (args.length <= 1) {
            throw new ParseException("Must provide transaction ID");
        }
        for (int i = 1; i < args.length; i++) {
            if (!admin.prepFail(zs, zk, managerLockPath, args[i])) {
                System.out.printf("Could not fail transaction: %s%n", args[i]);
                failedCommand = true;
            }
        }
    } else if ("delete".equals(cmd)) {
        if (args.length <= 1) {
            throw new ParseException("Must provide transaction ID");
        }
        for (int i = 1; i < args.length; i++) {
            if (admin.prepDelete(zs, zk, managerLockPath, args[i])) {
                admin.deleteLocks(zk, context.getZooKeeperRoot() + Constants.ZTABLE_LOCKS, args[i]);
            } else {
                System.out.printf("Could not delete transaction: %s%n", args[i]);
                failedCommand = true;
            }
        }
    } else if ("list".equals(cmd) || "print".equals(cmd)) {
        // Parse transaction ID filters for print display
        Set<Long> filterTxid = null;
        if (args.length >= 2) {
            filterTxid = new HashSet<>(args.length);
            for (int i = 1; i < args.length; i++) {
                try {
                    Long val = parseTxid(args[i]);
                    filterTxid.add(val);
                } catch (NumberFormatException nfe) {
                    // Failed to parse, will exit instead of displaying everything since the intention was
                    // to potentially filter some data
                    System.out.printf("Invalid transaction ID format: %s%n", args[i]);
                    return 1;
                }
            }
        }
        // Parse TStatus filters for print display
        EnumSet<TStatus> filterStatus = null;
        if (cl.hasOption(statusOption.getOpt())) {
            filterStatus = EnumSet.noneOf(TStatus.class);
            String[] tstat = cl.getOptionValues(statusOption.getOpt());
            for (String element : tstat) {
                try {
                    filterStatus.add(TStatus.valueOf(element));
                } catch (IllegalArgumentException iae) {
                    System.out.printf("Invalid transaction status name: %s%n", element);
                    return 1;
                }
            }
        }
        StringBuilder buf = new StringBuilder(8096);
        Formatter fmt = new Formatter(buf);
        admin.print(zs, zk, context.getZooKeeperRoot() + Constants.ZTABLE_LOCKS, fmt, filterTxid, filterStatus);
        shellState.printLines(Collections.singletonList(buf.toString()).iterator(), !cl.hasOption(disablePaginationOpt.getOpt()));
    } else if ("dump".equals(cmd)) {
        List<Long> txids;
        if (args.length == 1) {
            txids = zs.list();
        } else {
            txids = new ArrayList<>();
            for (int i = 1; i < args.length; i++) {
                txids.add(parseTxid(args[i]));
            }
        }
        Gson gson = new GsonBuilder().registerTypeAdapter(ReadOnlyRepo.class, new InterfaceSerializer<>()).registerTypeAdapter(Repo.class, new InterfaceSerializer<>()).registerTypeAdapter(byte[].class, new ByteArraySerializer()).setPrettyPrinting().create();
        List<FateStack> txStacks = new ArrayList<>();
        for (Long txid : txids) {
            List<ReadOnlyRepo<FateCommand>> repoStack = zs.getStack(txid);
            txStacks.add(new FateStack(txid, repoStack));
        }
        System.out.println(gson.toJson(txStacks));
    } else {
        throw new ParseException("Invalid command option");
    }
    return failedCommand ? 1 : 0;
}

Example 7 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class BalanceInPresenceOfOfflineTableIT method test.

@Test
public void test() throws Exception {
    log.info("Test that balancing is not stopped by an offline table with outstanding migrations.");
    log.debug("starting test ingestion");
    VerifyParams params = new VerifyParams(getClientProps(), TEST_TABLE, 200_000);
    TestIngest.ingest(accumuloClient, params);
    accumuloClient.tableOperations().flush(TEST_TABLE, null, null, true);
    VerifyIngest.verifyIngest(accumuloClient, params);
    log.debug("waiting for balancing, up to ~5 minutes to allow for migration cleanup.");
    final long startTime = System.currentTimeMillis();
    long currentWait = 10_000;
    boolean balancingWorked = false;
    Credentials creds = new Credentials(getAdminPrincipal(), getAdminToken());
    while (!balancingWorked && (System.currentTimeMillis() - startTime) < ((5 * 60 + 15) * 1000)) {
        Thread.sleep(currentWait);
        currentWait *= 2;
        log.debug("fetch the list of tablets assigned to each tserver.");
        ManagerClientService.Iface client = null;
        ManagerMonitorInfo stats;
        while (true) {
            try {
                client = ManagerClient.getConnectionWithRetry((ClientContext) accumuloClient);
                stats = client.getManagerStats(TraceUtil.traceInfo(), creds.toThrift(accumuloClient.instanceOperations().getInstanceId()));
                break;
            } catch (ThriftSecurityException exception) {
                throw new AccumuloSecurityException(exception);
            } catch (ThriftNotActiveServiceException e) {
                // Let it loop, fetching a new location
                log.debug("Contacted a Manager which is no longer active, retrying");
                sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
            } catch (TException exception) {
                throw new AccumuloException(exception);
            } finally {
                if (client != null) {
                    ManagerClient.close(client, (ClientContext) accumuloClient);
                }
            }
        }
        if (stats.getTServerInfoSize() < 2) {
            log.debug("we need >= 2 servers. sleeping for {}ms", currentWait);
            continue;
        }
        if (stats.getUnassignedTablets() != 0) {
            log.debug("We shouldn't have unassigned tablets. sleeping for {}ms", currentWait);
            continue;
        }
        long[] tabletsPerServer = new long[stats.getTServerInfoSize()];
        Arrays.fill(tabletsPerServer, 0L);
        for (int i = 0; i < stats.getTServerInfoSize(); i++) {
            for (Map.Entry<String, TableInfo> entry : stats.getTServerInfo().get(i).getTableMap().entrySet()) {
                tabletsPerServer[i] += entry.getValue().getTablets();
            }
        }
        if (tabletsPerServer[0] <= 10) {
            log.debug("We should have > 10 tablets. sleeping for {}ms", currentWait);
            continue;
        }
        long min = NumberUtils.min(tabletsPerServer), max = NumberUtils.max(tabletsPerServer);
        log.debug("Min={}, Max={}", min, max);
        if ((min / ((double) max)) < 0.5) {
            log.debug("ratio of min to max tablets per server should be roughly even. sleeping for {}ms", currentWait);
            continue;
        }
        balancingWorked = true;
    }
    assertTrue("did not properly balance", balancingWorked);
}

Example 8 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class GarbageCollectorCommunicatesWithTServersIT method testUnreferencedWalInTserverIsClosed.

@Test
public void testUnreferencedWalInTserverIsClosed() throws Exception {
    final String[] names = getUniqueNames(2);
    // `table` will be replicated, `otherTable` is only used to roll the WAL on the tserver
    final String table = names[0], otherTable = names[1];
    final AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build();
    // Bring the replication table online first and foremost
    ReplicationTable.setOnline(client);
    log.info("Creating {}", table);
    client.tableOperations().create(table);
    client.tableOperations().setProperty(table, Property.TABLE_REPLICATION.getKey(), "true");
    log.info("Writing a few mutations to the table");
    byte[] empty = new byte[0];
    try (BatchWriter bw = client.createBatchWriter(table)) {
        for (int i = 0; i < 5; i++) {
            Mutation m = new Mutation(Integer.toString(i));
            m.put(empty, empty, empty);
            bw.addMutation(m);
        }
        log.info("Flushing mutations to the server");
    }
    log.info("Checking that metadata only has one WAL recorded for this table");
    Set<String> wals = getWalsForTable(table);
    assertEquals("Expected to only find two WAL for the table", 2, wals.size());
    log.info("Compacting the table which will remove all WALs from the tablets");
    // Flush our test table to remove the WAL references in it
    client.tableOperations().flush(table, null, null, true);
    // Flush the metadata table too because it will have a reference to the WAL
    client.tableOperations().flush(MetadataTable.NAME, null, null, true);
    log.info("Fetching replication statuses from metadata table");
    Map<String, Status> fileToStatus = getMetadataStatusForTable(table);
    assertEquals("Expected to only find one replication status message", 1, fileToStatus.size());
    String walName = fileToStatus.keySet().iterator().next();
    assertTrue("Expected log file name from tablet to equal replication entry", wals.contains(walName));
    Status status = fileToStatus.get(walName);
    assertFalse("Expected Status for file to not be closed", status.getClosed());
    Set<String> filesForTable = getFilesForTable(table);
    assertEquals("Expected to only find one rfile for table", 1, filesForTable.size());
    log.info("Files for table before MajC: {}", filesForTable);
    // Issue a MajC to roll a new file in HDFS
    client.tableOperations().compact(table, null, null, false, true);
    Set<String> filesForTableAfterCompaction = getFilesForTable(table);
    log.info("Files for table after MajC: {}", filesForTableAfterCompaction);
    assertEquals("Expected to only find one rfile for table", 1, filesForTableAfterCompaction.size());
    assertNotEquals("Expected the files before and after compaction to differ", filesForTableAfterCompaction, filesForTable);
    // Use the rfile which was just replaced by the MajC to determine when the GC has ran
    Path fileToBeDeleted = new Path(filesForTable.iterator().next());
    FileSystem fs = getCluster().getFileSystem();
    boolean fileExists = fs.exists(fileToBeDeleted);
    while (fileExists) {
        log.info("File which should get deleted still exists: {}", fileToBeDeleted);
        Thread.sleep(2000);
        fileExists = fs.exists(fileToBeDeleted);
    }
    // At this point in time, we *know* that the GarbageCollector has run which means that the
    // Status
    // for our WAL should not be altered.
    Map<String, Status> fileToStatusAfterMinc = getMetadataStatusForTable(table);
    assertEquals("Expected to still find only one replication status message: " + fileToStatusAfterMinc, 1, fileToStatusAfterMinc.size());
    /*
     * To verify that the WALs is still getting closed, we have to force the tserver to close the
     * existing WAL and open a new one instead. The easiest way to do this is to write a load of
     * data that will exceed the 1.33% full threshold that the logger keeps track of
     */
    client.tableOperations().create(otherTable);
    try (BatchWriter bw = client.createBatchWriter(otherTable)) {
        // 500k
        byte[] bigValue = new byte[1024 * 500];
        Arrays.fill(bigValue, (byte) 1);
        // 500k * 50
        for (int i = 0; i < 50; i++) {
            Mutation m = new Mutation(Integer.toString(i));
            m.put(empty, empty, bigValue);
            bw.addMutation(m);
            if (i % 10 == 0) {
                bw.flush();
            }
        }
    }
    client.tableOperations().flush(otherTable, null, null, true);
    // Get the tservers which the manager deems as active
    final ClientContext context = (ClientContext) client;
    List<String> tservers = ManagerClient.execute(context, cli -> cli.getActiveTservers(TraceUtil.traceInfo(), context.rpcCreds()));
    assertEquals("Expected only one active tservers", 1, tservers.size());
    HostAndPort tserver = HostAndPort.fromString(tservers.get(0));
    // Get the active WALs from that server
    log.info("Fetching active WALs from {}", tserver);
    Client cli = ThriftUtil.getTServerClient(tserver, context);
    List<String> activeWalsForTserver = cli.getActiveLogs(TraceUtil.traceInfo(), context.rpcCreds());
    log.info("Active wals: {}", activeWalsForTserver);
    assertEquals("Expected to find only one active WAL", 1, activeWalsForTserver.size());
    String activeWal = new Path(activeWalsForTserver.get(0)).toString();
    assertNotEquals("Current active WAL on tserver should not be the original WAL we saw", walName, activeWal);
    log.info("Ensuring that replication status does get closed after WAL is no" + " longer in use by Tserver");
    do {
        Map<String, Status> replicationStatuses = getMetadataStatusForTable(table);
        log.info("Got replication status messages {}", replicationStatuses);
        assertEquals("Did not expect to find additional status records", 1, replicationStatuses.size());
        status = replicationStatuses.values().iterator().next();
        log.info("Current status: {}", ProtobufUtil.toString(status));
        if (status.getClosed()) {
            return;
        }
        log.info("Status is not yet closed, waiting for garbage collector to close it");
        Thread.sleep(2000);
    } while (true);
}

Example 9 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class MultiTserverReplicationIT method managerReplicationServicePortsAreAdvertised.

@Test
public void managerReplicationServicePortsAreAdvertised() throws Exception {
    // Wait for the cluster to be up
    AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build();
    ClientContext context = (ClientContext) client;
    // Wait for a tserver to come up to fulfill this request
    client.tableOperations().create("foo");
    try (Scanner s = client.createScanner("foo", Authorizations.EMPTY)) {
        assertEquals(0, Iterables.size(s));
        ZooReader zreader = new ZooReader(context.getZooKeepers(), context.getZooKeepersSessionTimeOut());
        // Should have one manager instance
        assertEquals(1, context.getManagerLocations().size());
        // Get the manager thrift service addr
        String managerAddr = Iterables.getOnlyElement(context.getManagerLocations());
        // Get the manager replication coordinator addr
        String replCoordAddr = new String(zreader.getData(ZooUtil.getRoot(client.instanceOperations().getInstanceId()) + Constants.ZMANAGER_REPLICATION_COORDINATOR_ADDR), UTF_8);
        // They shouldn't be the same
        assertNotEquals(managerAddr, replCoordAddr);
        // Neither should be zero as the port
        assertNotEquals(0, HostAndPort.fromString(managerAddr).getPort());
        assertNotEquals(0, HostAndPort.fromString(replCoordAddr).getPort());
    }
}

Example 10 with ClientContext

use of org.apache.accumulo.core.clientImpl.ClientContext in project accumulo by apache.

the class MultiTserverReplicationIT method tserverReplicationServicePortsAreAdvertised.

@Test
public void tserverReplicationServicePortsAreAdvertised() throws Exception {
    // Wait for the cluster to be up
    AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build();
    ClientContext context = (ClientContext) client;
    // Wait for a tserver to come up to fulfill this request
    client.tableOperations().create("foo");
    try (Scanner s = client.createScanner("foo", Authorizations.EMPTY)) {
        assertEquals(0, Iterables.size(s));
        ZooReader zreader = new ZooReader(context.getZooKeepers(), context.getZooKeepersSessionTimeOut());
        Set<String> tserverHost = new HashSet<>();
        tserverHost.addAll(zreader.getChildren(ZooUtil.getRoot(client.instanceOperations().getInstanceId()) + Constants.ZTSERVERS));
        Set<HostAndPort> replicationServices = new HashSet<>();
        for (String tserver : tserverHost) {
            try {
                byte[] portData = zreader.getData(ZooUtil.getRoot(client.instanceOperations().getInstanceId()) + ReplicationConstants.ZOO_TSERVERS + "/" + tserver);
                HostAndPort replAddress = HostAndPort.fromString(new String(portData, UTF_8));
                replicationServices.add(replAddress);
            } catch (Exception e) {
                log.error("Could not find port for {}", tserver, e);
                fail("Did not find replication port advertisement for " + tserver);
            }
        }
        // Each tserver should also have equal replication services running internally
        assertEquals("Expected an equal number of replication servicers and tservers", tserverHost.size(), replicationServices.size());
    }
}