Examples with Client org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client used on opensource projects

Example 1 with Client

use of org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client in project accumulo by apache.

the class GarbageCollectorCommunicatesWithTServersIT method testUnreferencedWalInTserverIsClosed.

@Test(timeout = 2 * 60 * 1000)
public void testUnreferencedWalInTserverIsClosed() throws Exception {
    final String[] names = getUniqueNames(2);
    // `table` will be replicated, `otherTable` is only used to roll the WAL on the tserver
    final String table = names[0], otherTable = names[1];
    final Connector conn = getConnector();
    // Bring the replication table online first and foremost
    ReplicationTable.setOnline(conn);
    log.info("Creating {}", table);
    conn.tableOperations().create(table);
    conn.tableOperations().setProperty(table, Property.TABLE_REPLICATION.getKey(), "true");
    log.info("Writing a few mutations to the table");
    BatchWriter bw = conn.createBatchWriter(table, null);
    byte[] empty = new byte[0];
    for (int i = 0; i < 5; i++) {
        Mutation m = new Mutation(Integer.toString(i));
        m.put(empty, empty, empty);
        bw.addMutation(m);
    }
    log.info("Flushing mutations to the server");
    bw.close();
    log.info("Checking that metadata only has one WAL recorded for this table");
    Set<String> wals = getWalsForTable(table);
    Assert.assertEquals("Expected to only find two WAL for the table", 2, wals.size());
    log.info("Compacting the table which will remove all WALs from the tablets");
    // Flush our test table to remove the WAL references in it
    conn.tableOperations().flush(table, null, null, true);
    // Flush the metadata table too because it will have a reference to the WAL
    conn.tableOperations().flush(MetadataTable.NAME, null, null, true);
    log.info("Fetching replication statuses from metadata table");
    Map<String, Status> fileToStatus = getMetadataStatusForTable(table);
    Assert.assertEquals("Expected to only find one replication status message", 1, fileToStatus.size());
    String walName = fileToStatus.keySet().iterator().next();
    Assert.assertTrue("Expected log file name from tablet to equal replication entry", wals.contains(walName));
    Status status = fileToStatus.get(walName);
    Assert.assertEquals("Expected Status for file to not be closed", false, status.getClosed());
    Set<String> filesForTable = getFilesForTable(table);
    Assert.assertEquals("Expected to only find one rfile for table", 1, filesForTable.size());
    log.info("Files for table before MajC: {}", filesForTable);
    // Issue a MajC to roll a new file in HDFS
    conn.tableOperations().compact(table, null, null, false, true);
    Set<String> filesForTableAfterCompaction = getFilesForTable(table);
    log.info("Files for table after MajC: {}", filesForTableAfterCompaction);
    Assert.assertEquals("Expected to only find one rfile for table", 1, filesForTableAfterCompaction.size());
    Assert.assertNotEquals("Expected the files before and after compaction to differ", filesForTableAfterCompaction, filesForTable);
    // Use the rfile which was just replaced by the MajC to determine when the GC has ran
    Path fileToBeDeleted = new Path(filesForTable.iterator().next());
    FileSystem fs = getCluster().getFileSystem();
    boolean fileExists = fs.exists(fileToBeDeleted);
    while (fileExists) {
        log.info("File which should get deleted still exists: {}", fileToBeDeleted);
        Thread.sleep(2000);
        fileExists = fs.exists(fileToBeDeleted);
    }
    // At this point in time, we *know* that the GarbageCollector has run which means that the Status
    // for our WAL should not be altered.
    Map<String, Status> fileToStatusAfterMinc = getMetadataStatusForTable(table);
    Assert.assertEquals("Expected to still find only one replication status message: " + fileToStatusAfterMinc, 1, fileToStatusAfterMinc.size());
    /*
     * To verify that the WALs is still getting closed, we have to force the tserver to close the existing WAL and open a new one instead. The easiest way to do
     * this is to write a load of data that will exceed the 1.33% full threshold that the logger keeps track of
     */
    conn.tableOperations().create(otherTable);
    bw = conn.createBatchWriter(otherTable, null);
    // 500k
    byte[] bigValue = new byte[1024 * 500];
    Arrays.fill(bigValue, (byte) 1);
    // 500k * 50
    for (int i = 0; i < 50; i++) {
        Mutation m = new Mutation(Integer.toString(i));
        m.put(empty, empty, bigValue);
        bw.addMutation(m);
        if (i % 10 == 0) {
            bw.flush();
        }
    }
    bw.close();
    conn.tableOperations().flush(otherTable, null, null, true);
    // Get the tservers which the master deems as active
    final ClientContext context = new ClientContext(conn.getInstance(), new Credentials("root", new PasswordToken(ConfigurableMacBase.ROOT_PASSWORD)), getClientConfig());
    List<String> tservers = MasterClient.execute(context, new ClientExecReturn<List<String>, MasterClientService.Client>() {

        @Override
        public List<String> execute(MasterClientService.Client client) throws Exception {
            return client.getActiveTservers(Tracer.traceInfo(), context.rpcCreds());
        }
    });
    Assert.assertEquals("Expected only one active tservers", 1, tservers.size());
    HostAndPort tserver = HostAndPort.fromString(tservers.get(0));
    // Get the active WALs from that server
    log.info("Fetching active WALs from {}", tserver);
    Client client = ThriftUtil.getTServerClient(tserver, context);
    List<String> activeWalsForTserver = client.getActiveLogs(Tracer.traceInfo(), context.rpcCreds());
    log.info("Active wals: {}", activeWalsForTserver);
    Assert.assertEquals("Expected to find only one active WAL", 1, activeWalsForTserver.size());
    String activeWal = new Path(activeWalsForTserver.get(0)).toString();
    Assert.assertNotEquals("Current active WAL on tserver should not be the original WAL we saw", walName, activeWal);
    log.info("Ensuring that replication status does get closed after WAL is no longer in use by Tserver");
    do {
        Map<String, Status> replicationStatuses = getMetadataStatusForTable(table);
        log.info("Got replication status messages {}", replicationStatuses);
        Assert.assertEquals("Did not expect to find additional status records", 1, replicationStatuses.size());
        status = replicationStatuses.values().iterator().next();
        log.info("Current status: {}", ProtobufUtil.toString(status));
        if (status.getClosed()) {
            return;
        }
        log.info("Status is not yet closed, waiting for garbage collector to close it");
        Thread.sleep(2000);
    } while (true);
}

Example 2 with Client

use of org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client in project accumulo by apache.

the class Monitor method fetchGcStatus.

private static GCStatus fetchGcStatus() {
    GCStatus result = null;
    HostAndPort address = null;
    try {
        // Read the gc location from its lock
        ZooReaderWriter zk = ZooReaderWriter.getInstance();
        String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
        List<String> locks = zk.getChildren(path, null);
        if (locks != null && locks.size() > 0) {
            Collections.sort(locks);
            address = new ServerServices(new String(zk.getData(path + "/" + locks.get(0), null), UTF_8)).getAddress(Service.GC_CLIENT);
            GCMonitorService.Client client = ThriftUtil.getClient(new GCMonitorService.Client.Factory(), address, new AccumuloServerContext(instance, config));
            try {
                result = client.getStatus(Tracer.traceInfo(), getContext().rpcCreds());
            } finally {
                ThriftUtil.returnClient(client);
            }
        }
    } catch (Exception ex) {
        log.warn("Unable to contact the garbage collector at " + address, ex);
    }
    return result;
}

Example 3 with Client

use of org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client in project accumulo by apache.

the class TabletBalancer method getOnlineTabletsForTable.

/**
 * Fetch the tablets for the given table by asking the tablet server. Useful if your balance strategy needs details at the tablet level to decide what tablets
 * to move.
 *
 * @param tserver
 *          The tablet server to ask.
 * @param tableId
 *          The table id
 * @return a list of tablet statistics
 * @throws ThriftSecurityException
 *           tablet server disapproves of your internal System password.
 * @throws TException
 *           any other problem
 */
public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, Table.ID tableId) throws ThriftSecurityException, TException {
    log.debug("Scanning tablet server {} for table {}", tserver, tableId);
    Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), tserver.getLocation(), context);
    try {
        return client.getTabletStats(Tracer.traceInfo(), context.rpcCreds(), tableId.canonicalID());
    } catch (TTransportException e) {
        log.error("Unable to connect to {}: ", tserver, e);
    } finally {
        ThriftUtil.returnClient(client);
    }
    return null;
}

Example 4 with Client

use of org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client in project accumulo by apache.

the class Monitor method fetchScans.

public static void fetchScans() throws Exception {
    if (instance == null)
        return;
    Connector c = context.getConnector();
    for (String server : c.instanceOperations().getTabletServers()) {
        final HostAndPort parsedServer = HostAndPort.fromString(server);
        Client tserver = ThriftUtil.getTServerClient(parsedServer, context);
        try {
            List<ActiveScan> scans = tserver.getActiveScans(null, context.rpcCreds());
            synchronized (allScans) {
                allScans.put(parsedServer, new ScanStats(scans));
            }
        } catch (Exception ex) {
            log.debug("Failed to get active scans from {}", server, ex);
        } finally {
            ThriftUtil.returnClient(tserver);
        }
    }
    // Age off old scan information
    Iterator<Entry<HostAndPort, ScanStats>> entryIter = allScans.entrySet().iterator();
    long now = System.currentTimeMillis();
    while (entryIter.hasNext()) {
        Entry<HostAndPort, ScanStats> entry = entryIter.next();
        if (now - entry.getValue().fetched > 5 * 60 * 1000) {
            entryIter.remove();
        }
    }
}

Example 5 with Client

use of org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client in project accumulo by apache.

the class InstanceOperationsImpl method getActiveCompactions.

@Override
public List<ActiveCompaction> getActiveCompactions(String tserver) throws AccumuloException, AccumuloSecurityException {
    final HostAndPort parsedTserver = HostAndPort.fromString(tserver);
    Client client = null;
    try {
        client = ThriftUtil.getTServerClient(parsedTserver, context);
        List<ActiveCompaction> as = new ArrayList<>();
        for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction activeCompaction : client.getActiveCompactions(Tracer.traceInfo(), context.rpcCreds())) {
            as.add(new ActiveCompactionImpl(context.getInstance(), activeCompaction));
        }
        return as;
    } catch (TTransportException e) {
        throw new AccumuloException(e);
    } catch (ThriftSecurityException e) {
        throw new AccumuloSecurityException(e.user, e.code, e);
    } catch (TException e) {
        throw new AccumuloException(e);
    } finally {
        if (client != null)
            ThriftUtil.returnClient(client);
    }
}