Examples with RunningCompaction org.apache.accumulo.core.util.compaction.RunningCompaction used on opensource projects

Example 1 with RunningCompaction

use of org.apache.accumulo.core.util.compaction.RunningCompaction in project accumulo by apache.

the class CompactionCoordinator method getCompactionJob.

/**
 * Return the next compaction job from the queue to a Compactor
 *
 * @param queueName
 *          queue
 * @param compactorAddress
 *          compactor address
 * @throws ThriftSecurityException
 *           when permission error
 * @return compaction job
 */
@Override
public TExternalCompactionJob getCompactionJob(TInfo tinfo, TCredentials credentials, String queueName, String compactorAddress, String externalCompactionId) throws ThriftSecurityException {
    // do not expect users to call this directly, expect compactors to call this method
    if (!security.canPerformSystemActions(credentials)) {
        throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED).asThriftException();
    }
    final String queue = queueName.intern();
    LOG.trace("getCompactionJob called for queue {} by compactor {}", queue, compactorAddress);
    TIME_COMPACTOR_LAST_CHECKED.put(queue, System.currentTimeMillis());
    TExternalCompactionJob result = null;
    PrioTserver prioTserver = QUEUE_SUMMARIES.getNextTserver(queue);
    while (prioTserver != null) {
        TServerInstance tserver = prioTserver.tserver;
        LOG.trace("Getting compaction for queue {} from tserver {}", queue, tserver.getHostAndPort());
        // Get a compaction from the tserver
        TabletClientService.Client client = null;
        try {
            client = getTabletServerConnection(tserver);
            TExternalCompactionJob job = client.reserveCompactionJob(TraceUtil.traceInfo(), getContext().rpcCreds(), queue, prioTserver.prio, compactorAddress, externalCompactionId);
            if (null == job.getExternalCompactionId()) {
                LOG.trace("No compactions found for queue {} on tserver {}, trying next tserver", queue, tserver.getHostAndPort(), compactorAddress);
                QUEUE_SUMMARIES.removeSummary(tserver, queue, prioTserver.prio);
                prioTserver = QUEUE_SUMMARIES.getNextTserver(queue);
                continue;
            }
            RUNNING.put(ExternalCompactionId.of(job.getExternalCompactionId()), new RunningCompaction(job, compactorAddress, queue));
            LOG.debug("Returning external job {} to {}", job.externalCompactionId, compactorAddress);
            result = job;
            break;
        } catch (TException e) {
            LOG.warn("Error from tserver {} while trying to reserve compaction, trying next tserver", ExternalCompactionUtil.getHostPortString(tserver.getHostAndPort()), e);
            QUEUE_SUMMARIES.removeSummary(tserver, queue, prioTserver.prio);
            prioTserver = QUEUE_SUMMARIES.getNextTserver(queue);
        } finally {
            ThriftUtil.returnClient(client, getContext());
        }
    }
    if (result == null) {
        LOG.trace("No tservers found for queue {}, returning empty job to compactor {}", queue, compactorAddress);
        result = new TExternalCompactionJob();
    }
    return result;
}

Example 2 with RunningCompaction

use of org.apache.accumulo.core.util.compaction.RunningCompaction in project accumulo by apache.

the class CompactionCoordinator method run.

@Override
public void run() {
    ServerAddress coordinatorAddress = null;
    try {
        coordinatorAddress = startCoordinatorClientService();
    } catch (UnknownHostException e1) {
        throw new RuntimeException("Failed to start the coordinator service", e1);
    }
    final HostAndPort clientAddress = coordinatorAddress.address;
    try {
        getCoordinatorLock(clientAddress);
    } catch (KeeperException | InterruptedException e) {
        throw new IllegalStateException("Exception getting Coordinator lock", e);
    }
    try {
        MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName, clientAddress);
    } catch (Exception e1) {
        LOG.error("Error initializing metrics, metrics will not be emitted.", e1);
    }
    // On a re-start of the coordinator it's possible that external compactions are in-progress.
    // Attempt to get the running compactions on the compactors and then resolve which tserver
    // the external compaction came from to re-populate the RUNNING collection.
    LOG.info("Checking for running external compactions");
    // On re-start contact the running Compactors to try and seed the list of running compactions
    List<RunningCompaction> running = ExternalCompactionUtil.getCompactionsRunningOnCompactors(getContext());
    if (running.isEmpty()) {
        LOG.info("No running external compactions found");
    } else {
        LOG.info("Found {} running external compactions", running.size());
        running.forEach(rc -> {
            TCompactionStatusUpdate update = new TCompactionStatusUpdate();
            update.setState(TCompactionState.IN_PROGRESS);
            update.setMessage("Coordinator restarted, compaction found in progress");
            rc.addUpdate(System.currentTimeMillis(), update);
            RUNNING.put(ExternalCompactionId.of(rc.getJob().getExternalCompactionId()), rc);
        });
    }
    tserverSet.startListeningForTabletServerChanges();
    startDeadCompactionDetector();
    LOG.info("Starting loop to check tservers for compaction summaries");
    while (!shutdown) {
        long start = System.currentTimeMillis();
        updateSummaries();
        long now = System.currentTimeMillis();
        TIME_COMPACTOR_LAST_CHECKED.forEach((k, v) -> {
            if ((now - v) > getMissingCompactorWarningTime()) {
                LOG.warn("No compactors have checked in with coordinator for queue {} in {}ms", k, getMissingCompactorWarningTime());
            }
        });
        long checkInterval = getTServerCheckInterval();
        long duration = (System.currentTimeMillis() - start);
        if (checkInterval - duration > 0) {
            LOG.debug("Waiting {}ms for next tserver check", (checkInterval - duration));
            UtilWaitThread.sleep(checkInterval - duration);
        }
    }
    LOG.info("Shutting down");
}

Example 3 with RunningCompaction

use of org.apache.accumulo.core.util.compaction.RunningCompaction in project accumulo by apache.

the class CompactionCoordinator method updateCompactionStatus.

/**
 * Compactor calls to update the status of the assigned compaction
 *
 * @param tinfo
 *          trace info
 * @param credentials
 *          tcredentials object
 * @param externalCompactionId
 *          compaction id
 * @param update
 *          compaction status update
 * @param timestamp
 *          timestamp of the message
 * @throws ThriftSecurityException
 *           when permission error
 */
@Override
public void updateCompactionStatus(TInfo tinfo, TCredentials credentials, String externalCompactionId, TCompactionStatusUpdate update, long timestamp) throws ThriftSecurityException {
    // do not expect users to call this directly, expect other tservers to call this method
    if (!security.canPerformSystemActions(credentials)) {
        throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED).asThriftException();
    }
    LOG.debug("Compaction status update, id: {}, timestamp: {}, update: {}", externalCompactionId, timestamp, update);
    final RunningCompaction rc = RUNNING.get(ExternalCompactionId.of(externalCompactionId));
    if (null != rc) {
        rc.addUpdate(timestamp, update);
    }
}

Example 4 with RunningCompaction

use of org.apache.accumulo.core.util.compaction.RunningCompaction in project accumulo by apache.

the class CompactionCoordinatorTest method testCoordinatorColdStartNoCompactions.

@Test
public void testCoordinatorColdStartNoCompactions() throws Exception {
    PowerMock.resetAll();
    PowerMock.suppress(PowerMock.constructor(AbstractServer.class));
    PowerMock.suppress(PowerMock.methods(ThriftUtil.class, "returnClient"));
    PowerMock.suppress(PowerMock.methods(DeadCompactionDetector.class, "detectDeadCompactions", "detectDanglingFinalStateMarkers"));
    AccumuloConfiguration conf = PowerMock.createNiceMock(AccumuloConfiguration.class);
    ServerContext context = PowerMock.createNiceMock(ServerContext.class);
    PowerMock.mockStatic(ExternalCompactionUtil.class);
    List<RunningCompaction> runningCompactions = new ArrayList<>();
    EasyMock.expect(ExternalCompactionUtil.getCompactionsRunningOnCompactors(context)).andReturn(runningCompactions);
    CompactionFinalizer finalizer = PowerMock.createNiceMock(CompactionFinalizer.class);
    LiveTServerSet tservers = PowerMock.createNiceMock(LiveTServerSet.class);
    EasyMock.expect(tservers.getCurrentServers()).andReturn(Collections.emptySet()).anyTimes();
    ServerAddress client = PowerMock.createNiceMock(ServerAddress.class);
    HostAndPort address = HostAndPort.fromString("localhost:10240");
    EasyMock.expect(client.getAddress()).andReturn(address).anyTimes();
    TServerInstance tsi = PowerMock.createNiceMock(TServerInstance.class);
    EasyMock.expect(tsi.getHostPort()).andReturn("localhost:9997").anyTimes();
    TabletClientService.Client tsc = PowerMock.createNiceMock(TabletClientService.Client.class);
    EasyMock.expect(tsc.getCompactionQueueInfo(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Collections.emptyList()).anyTimes();
    AuditedSecurityOperation security = PowerMock.createNiceMock(AuditedSecurityOperation.class);
    PowerMock.replayAll();
    TestCoordinator coordinator = new TestCoordinator(conf, finalizer, tservers, client, tsc, context, security);
    coordinator.resetInternals();
    assertEquals(0, coordinator.getQueues().size());
    assertEquals(0, coordinator.getIndex().size());
    assertEquals(0, coordinator.getRunning().size());
    coordinator.run();
    assertEquals(0, coordinator.getQueues().size());
    assertEquals(0, coordinator.getIndex().size());
    assertEquals(0, coordinator.getRunning().size());
    PowerMock.verifyAll();
    coordinator.resetInternals();
    coordinator.close();
}

Example 5 with RunningCompaction

use of org.apache.accumulo.core.util.compaction.RunningCompaction in project accumulo by apache.

the class CompactionCoordinatorTest method testCoordinatorRestartNoRunningCompactions.

@Test
public void testCoordinatorRestartNoRunningCompactions() throws Exception {
    PowerMock.resetAll();
    PowerMock.suppress(PowerMock.constructor(AbstractServer.class));
    PowerMock.suppress(PowerMock.methods(ThriftUtil.class, "returnClient"));
    PowerMock.suppress(PowerMock.methods(DeadCompactionDetector.class, "detectDeadCompactions", "detectDanglingFinalStateMarkers"));
    AccumuloConfiguration conf = PowerMock.createNiceMock(AccumuloConfiguration.class);
    ServerContext context = PowerMock.createNiceMock(ServerContext.class);
    TCredentials creds = PowerMock.createNiceMock(TCredentials.class);
    EasyMock.expect(context.rpcCreds()).andReturn(creds);
    CompactionFinalizer finalizer = PowerMock.createNiceMock(CompactionFinalizer.class);
    LiveTServerSet tservers = PowerMock.createNiceMock(LiveTServerSet.class);
    TServerInstance instance = PowerMock.createNiceMock(TServerInstance.class);
    HostAndPort tserverAddress = HostAndPort.fromString("localhost:9997");
    EasyMock.expect(instance.getHostAndPort()).andReturn(tserverAddress).anyTimes();
    EasyMock.expect(tservers.getCurrentServers()).andReturn(Sets.newHashSet(instance)).once();
    tservers.startListeningForTabletServerChanges();
    PowerMock.mockStatic(ExternalCompactionUtil.class);
    List<RunningCompaction> runningCompactions = new ArrayList<>();
    EasyMock.expect(ExternalCompactionUtil.getCompactionsRunningOnCompactors(context)).andReturn(runningCompactions);
    ServerAddress client = PowerMock.createNiceMock(ServerAddress.class);
    HostAndPort address = HostAndPort.fromString("localhost:10240");
    EasyMock.expect(client.getAddress()).andReturn(address).anyTimes();
    EasyMock.expect(instance.getHostPort()).andReturn("localhost:9997").anyTimes();
    TabletClientService.Client tsc = PowerMock.createNiceMock(TabletClientService.Client.class);
    TCompactionQueueSummary queueSummary = PowerMock.createNiceMock(TCompactionQueueSummary.class);
    EasyMock.expect(tsc.getCompactionQueueInfo(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Collections.singletonList(queueSummary)).anyTimes();
    EasyMock.expect(queueSummary.getQueue()).andReturn("R2DQ").anyTimes();
    EasyMock.expect(queueSummary.getPriority()).andReturn((short) 1).anyTimes();
    AuditedSecurityOperation security = PowerMock.createNiceMock(AuditedSecurityOperation.class);
    PowerMock.replayAll();
    TestCoordinator coordinator = new TestCoordinator(conf, finalizer, tservers, client, tsc, context, security);
    coordinator.resetInternals();
    assertEquals(0, coordinator.getQueues().size());
    assertEquals(0, coordinator.getIndex().size());
    assertEquals(0, coordinator.getRunning().size());
    coordinator.run();
    assertEquals(1, coordinator.getQueues().size());
    QueueAndPriority qp = QueueAndPriority.get("R2DQ".intern(), (short) 1);
    Map<Short, TreeSet<TServerInstance>> m = coordinator.getQueues().get("R2DQ".intern());
    assertNotNull(m);
    assertEquals(1, m.size());
    assertTrue(m.containsKey((short) 1));
    Set<TServerInstance> t = m.get((short) 1);
    assertNotNull(t);
    assertEquals(1, t.size());
    TServerInstance queuedTsi = t.iterator().next();
    assertEquals(instance.getHostPortSession(), queuedTsi.getHostPortSession());
    assertEquals(1, coordinator.getIndex().size());
    assertTrue(coordinator.getIndex().containsKey(queuedTsi));
    Set<QueueAndPriority> i = coordinator.getIndex().get(queuedTsi);
    assertEquals(1, i.size());
    assertEquals(qp, i.iterator().next());
    assertEquals(0, coordinator.getRunning().size());
    PowerMock.verifyAll();
    coordinator.resetInternals();
    coordinator.close();
}