Examples with TExternalCompactionJob org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob used on opensource projects

Example 1 with TExternalCompactionJob

use of org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob in project accumulo by apache.

the class ExternalCompactionUtil method getRunningCompaction.

/**
 * Get the compaction currently running on the Compactor
 *
 * @param compactorAddr
 *          compactor address
 * @param context
 *          context
 * @return external compaction job or null if none running
 */
public static TExternalCompactionJob getRunningCompaction(HostAndPort compactorAddr, ClientContext context) {
    CompactorService.Client client = null;
    try {
        client = ThriftUtil.getClient(new CompactorService.Client.Factory(), compactorAddr, context);
        TExternalCompactionJob job = client.getRunningCompaction(TraceUtil.traceInfo(), context.rpcCreds());
        if (job.getExternalCompactionId() != null) {
            LOG.debug("Compactor {} is running {}", compactorAddr, job.getExternalCompactionId());
            return job;
        }
    } catch (TException e) {
        LOG.debug("Failed to contact compactor {}", compactorAddr, e);
    } finally {
        ThriftUtil.returnClient(client, context);
    }
    return null;
}

Example 2 with TExternalCompactionJob

use of org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob in project accumulo by apache.

the class ExternalCompactionUtil method getCompactionsRunningOnCompactors.

/**
 * This method returns information from the Compactor about the job that is currently running. The
 * RunningCompactions are not fully populated. This method is used from the CompactionCoordinator
 * on a restart to re-populate the set of running compactions on the compactors.
 *
 * @param context
 *          server context
 * @return map of compactor and external compaction jobs
 */
public static List<RunningCompaction> getCompactionsRunningOnCompactors(ClientContext context) {
    final List<RunningCompactionFuture> rcFutures = new ArrayList<>();
    final ExecutorService executor = ThreadPools.createFixedThreadPool(16, "CompactorRunningCompactions", false);
    getCompactorAddrs(context).forEach((q, hp) -> {
        hp.forEach(hostAndPort -> {
            rcFutures.add(new RunningCompactionFuture(q, hostAndPort, executor.submit(() -> getRunningCompaction(hostAndPort, context))));
        });
    });
    executor.shutdown();
    final List<RunningCompaction> results = new ArrayList<>();
    rcFutures.forEach(rcf -> {
        try {
            TExternalCompactionJob job = rcf.getFuture().get();
            if (null != job && null != job.getExternalCompactionId()) {
                var compactorAddress = getHostPortString(rcf.getCompactor());
                results.add(new RunningCompaction(job, compactorAddress, rcf.getQueue()));
            }
        } catch (InterruptedException | ExecutionException e) {
            throw new RuntimeException(e);
        }
    });
    return results;
}

Example 3 with TExternalCompactionJob

use of org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob in project accumulo by apache.

the class CompactorTest method testCompactionSucceeds.

@Test
public void testCompactionSucceeds() throws Exception {
    UUID uuid = UUID.randomUUID();
    Supplier<UUID> supplier = () -> uuid;
    ExternalCompactionId eci = ExternalCompactionId.generate(supplier.get());
    PowerMock.resetAll();
    PowerMock.suppress(PowerMock.methods(Halt.class, "halt"));
    PowerMock.suppress(PowerMock.constructor(AbstractServer.class));
    ServerAddress client = PowerMock.createNiceMock(ServerAddress.class);
    HostAndPort address = HostAndPort.fromString("localhost:10240");
    EasyMock.expect(client.getAddress()).andReturn(address);
    TExternalCompactionJob job = PowerMock.createNiceMock(TExternalCompactionJob.class);
    TKeyExtent extent = PowerMock.createNiceMock(TKeyExtent.class);
    EasyMock.expect(job.isSetExternalCompactionId()).andReturn(true).anyTimes();
    EasyMock.expect(job.getExternalCompactionId()).andReturn(eci.toString()).anyTimes();
    EasyMock.expect(job.getExtent()).andReturn(extent).anyTimes();
    EasyMock.expect(extent.getTable()).andReturn("testTable".getBytes()).anyTimes();
    AccumuloConfiguration conf = PowerMock.createNiceMock(AccumuloConfiguration.class);
    EasyMock.expect(conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT)).andReturn(86400000L);
    ServerContext context = PowerMock.createNiceMock(ServerContext.class);
    EasyMock.expect(context.getConfiguration()).andReturn(conf);
    ZooReaderWriter zrw = PowerMock.createNiceMock(ZooReaderWriter.class);
    ZooKeeper zk = PowerMock.createNiceMock(ZooKeeper.class);
    EasyMock.expect(context.getZooReaderWriter()).andReturn(zrw).anyTimes();
    EasyMock.expect(zrw.getZooKeeper()).andReturn(zk).anyTimes();
    VolumeManagerImpl vm = PowerMock.createNiceMock(VolumeManagerImpl.class);
    EasyMock.expect(context.getVolumeManager()).andReturn(vm);
    vm.close();
    PowerMock.replayAll();
    SuccessfulCompactor c = new SuccessfulCompactor(supplier, client, job, conf, context, eci);
    c.run();
    PowerMock.verifyAll();
    c.close();
    assertTrue(c.isCompletedCalled());
    assertFalse(c.isFailedCalled());
}

Example 4 with TExternalCompactionJob

use of org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob in project accumulo by apache.

the class CompactionCoordinator method getCompactionJob.

/**
 * Return the next compaction job from the queue to a Compactor
 *
 * @param queueName
 *          queue
 * @param compactorAddress
 *          compactor address
 * @throws ThriftSecurityException
 *           when permission error
 * @return compaction job
 */
@Override
public TExternalCompactionJob getCompactionJob(TInfo tinfo, TCredentials credentials, String queueName, String compactorAddress, String externalCompactionId) throws ThriftSecurityException {
    // do not expect users to call this directly, expect compactors to call this method
    if (!security.canPerformSystemActions(credentials)) {
        throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED).asThriftException();
    }
    final String queue = queueName.intern();
    LOG.trace("getCompactionJob called for queue {} by compactor {}", queue, compactorAddress);
    TIME_COMPACTOR_LAST_CHECKED.put(queue, System.currentTimeMillis());
    TExternalCompactionJob result = null;
    PrioTserver prioTserver = QUEUE_SUMMARIES.getNextTserver(queue);
    while (prioTserver != null) {
        TServerInstance tserver = prioTserver.tserver;
        LOG.trace("Getting compaction for queue {} from tserver {}", queue, tserver.getHostAndPort());
        // Get a compaction from the tserver
        TabletClientService.Client client = null;
        try {
            client = getTabletServerConnection(tserver);
            TExternalCompactionJob job = client.reserveCompactionJob(TraceUtil.traceInfo(), getContext().rpcCreds(), queue, prioTserver.prio, compactorAddress, externalCompactionId);
            if (null == job.getExternalCompactionId()) {
                LOG.trace("No compactions found for queue {} on tserver {}, trying next tserver", queue, tserver.getHostAndPort(), compactorAddress);
                QUEUE_SUMMARIES.removeSummary(tserver, queue, prioTserver.prio);
                prioTserver = QUEUE_SUMMARIES.getNextTserver(queue);
                continue;
            }
            RUNNING.put(ExternalCompactionId.of(job.getExternalCompactionId()), new RunningCompaction(job, compactorAddress, queue));
            LOG.debug("Returning external job {} to {}", job.externalCompactionId, compactorAddress);
            result = job;
            break;
        } catch (TException e) {
            LOG.warn("Error from tserver {} while trying to reserve compaction, trying next tserver", ExternalCompactionUtil.getHostPortString(tserver.getHostAndPort()), e);
            QUEUE_SUMMARIES.removeSummary(tserver, queue, prioTserver.prio);
            prioTserver = QUEUE_SUMMARIES.getNextTserver(queue);
        } finally {
            ThriftUtil.returnClient(client, getContext());
        }
    }
    if (result == null) {
        LOG.trace("No tservers found for queue {}, returning empty job to compactor {}", queue, compactorAddress);
        result = new TExternalCompactionJob();
    }
    return result;
}

Example 5 with TExternalCompactionJob

use of org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob in project accumulo by apache.

the class ThriftClientHandler method reserveCompactionJob.

@Override
public TExternalCompactionJob reserveCompactionJob(TInfo tinfo, TCredentials credentials, String queueName, long priority, String compactor, String externalCompactionId) throws ThriftSecurityException, TException {
    if (!security.canPerformSystemActions(credentials)) {
        throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED).asThriftException();
    }
    ExternalCompactionId eci = ExternalCompactionId.of(externalCompactionId);
    var extCompaction = server.getCompactionManager().reserveExternalCompaction(queueName, priority, compactor, eci);
    if (extCompaction != null) {
        return extCompaction.toThrift();
    }
    return new TExternalCompactionJob();
}