Examples with TExternalCompactionList org.apache.accumulo.core.compaction.thrift.TExternalCompactionList used on opensource projects

Example 1 with TExternalCompactionList

use of org.apache.accumulo.core.compaction.thrift.TExternalCompactionList in project accumulo by apache.

the class ExternalCompactionTestUtils method confirmCompactionCompleted.

public static void confirmCompactionCompleted(ServerContext ctx, Set<ExternalCompactionId> ecids, TCompactionState expectedState) throws Exception {
    // The running compaction should be removed
    TExternalCompactionList running = ExternalCompactionTestUtils.getRunningCompactions(ctx);
    while (running.getCompactions() != null) {
        running = ExternalCompactionTestUtils.getRunningCompactions(ctx);
        if (running.getCompactions() == null) {
            UtilWaitThread.sleep(250);
        }
    }
    // The compaction should be in the completed list with the expected state
    TExternalCompactionList completed = ExternalCompactionTestUtils.getCompletedCompactions(ctx);
    while (completed.getCompactions() == null) {
        completed = ExternalCompactionTestUtils.getCompletedCompactions(ctx);
        if (completed.getCompactions() == null) {
            UtilWaitThread.sleep(50);
        }
    }
    for (ExternalCompactionId e : ecids) {
        TExternalCompaction tec = completed.getCompactions().get(e.canonical());
        assertNotNull(tec);
        assertEquals(expectedState, ExternalCompactionTestUtils.getLastState(tec));
    }
}

Example 2 with TExternalCompactionList

use of org.apache.accumulo.core.compaction.thrift.TExternalCompactionList in project accumulo by apache.

the class ExternalCompactionTestUtils method getCompletedCompactions.

private static TExternalCompactionList getCompletedCompactions(ClientContext context) throws Exception {
    Optional<HostAndPort> coordinatorHost = ExternalCompactionUtil.findCompactionCoordinator(context);
    if (coordinatorHost.isEmpty()) {
        throw new TTransportException("Unable to get CompactionCoordinator address from ZooKeeper");
    }
    CompactionCoordinatorService.Client client = ThriftUtil.getClient(new CompactionCoordinatorService.Client.Factory(), coordinatorHost.get(), context);
    try {
        TExternalCompactionList completed = client.getCompletedCompactions(TraceUtil.traceInfo(), context.rpcCreds());
        return completed;
    } finally {
        ThriftUtil.returnClient(client, context);
    }
}

Example 3 with TExternalCompactionList

use of org.apache.accumulo.core.compaction.thrift.TExternalCompactionList in project accumulo by apache.

the class ExternalCompaction_3_IT method testCoordinatorRestartsDuringCompaction.

@Test
public void testCoordinatorRestartsDuringCompaction() throws Exception {
    getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class);
    getCluster().getClusterControl().startCompactors(ExternalDoNothingCompactor.class, 1, QUEUE2);
    String table1 = this.getUniqueNames(1)[0];
    try (AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
        createTable(client, table1, "cs2", 2);
        writeData(client, table1);
        compact(client, table1, 2, QUEUE2, false);
        TableId tid = getCluster().getServerContext().getTableId(table1);
        // Wait for the compaction to start by waiting for 1 external compaction column
        Set<ExternalCompactionId> ecids = waitForCompactionStartAndReturnEcids(getCluster().getServerContext(), tid);
        // Stop the Coordinator
        getCluster().getClusterControl().stop(ServerType.COMPACTION_COORDINATOR);
        // Restart the coordinator while the compaction is running
        getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class);
        // Confirm compaction is still running
        int matches = 0;
        while (matches == 0) {
            TExternalCompactionList running = getRunningCompactions(getCluster().getServerContext());
            if (running.getCompactions() != null) {
                for (ExternalCompactionId ecid : ecids) {
                    TExternalCompaction tec = running.getCompactions().get(ecid.canonical());
                    if (tec != null && tec.getUpdates() != null && !tec.getUpdates().isEmpty()) {
                        matches++;
                        assertEquals(TCompactionState.IN_PROGRESS, getLastState(tec));
                    }
                }
            }
            UtilWaitThread.sleep(250);
        }
        assertTrue(matches > 0);
        // We need to cancel the compaction or delete the table here because we initiate a user
        // compaction above in the test. Even though the external compaction was cancelled
        // because we split the table, FaTE will continue to queue up a compaction
        client.tableOperations().cancelCompaction(table1);
    }
}

Example 4 with TExternalCompactionList

use of org.apache.accumulo.core.compaction.thrift.TExternalCompactionList in project accumulo by apache.

the class Monitor method fetchRunningInfo.

/**
 * Fetch running compactions from Compaction Coordinator. Chose not to restrict the frequency of
 * user fetches since RPC calls are going to the coordinator. This allows for fine grain updates
 * of external compaction progress.
 */
public synchronized Map<String, TExternalCompaction> fetchRunningInfo() {
    if (coordinatorHost.isEmpty()) {
        throw new IllegalStateException(coordinatorMissingMsg);
    }
    var ccHost = coordinatorHost.get();
    log.info("User initiated fetch of running External Compactions from " + ccHost);
    var client = getCoordinator(ccHost);
    TExternalCompactionList running;
    try {
        running = client.getRunningCompactions(TraceUtil.traceInfo(), getContext().rpcCreds());
    } catch (Exception e) {
        throw new IllegalStateException("Unable to get running compactions from " + ccHost, e);
    }
    ecRunningMap.clear();
    if (running.getCompactions() != null) {
        ecRunningMap.putAll(running.getCompactions());
    }
    return ecRunningMap;
}

Example 5 with TExternalCompactionList

use of org.apache.accumulo.core.compaction.thrift.TExternalCompactionList in project accumulo by apache.

the class CompactionCoordinator method getRunningCompactions.

/**
 * Return information about running compactions
 *
 * @param tinfo
 *          trace info
 * @param credentials
 *          tcredentials object
 * @return map of ECID to TExternalCompaction objects
 * @throws ThriftSecurityException
 *           permission error
 */
@Override
public TExternalCompactionList getRunningCompactions(TInfo tinfo, TCredentials credentials) throws ThriftSecurityException {
    // do not expect users to call this directly, expect other tservers to call this method
    if (!security.canPerformSystemActions(credentials)) {
        throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED).asThriftException();
    }
    final TExternalCompactionList result = new TExternalCompactionList();
    RUNNING.forEach((ecid, rc) -> {
        TExternalCompaction trc = new TExternalCompaction();
        trc.setQueueName(rc.getQueueName());
        trc.setCompactor(rc.getCompactorAddress());
        trc.setUpdates(rc.getUpdates());
        trc.setJob(rc.getJob());
        result.putToCompactions(ecid.canonical(), trc);
    });
    return result;
}