Examples with LiveTServerSet org.apache.accumulo.server.manager.LiveTServerSet used on opensource projects

Example 6 with LiveTServerSet

use of org.apache.accumulo.server.manager.LiveTServerSet in project accumulo by apache.

the class CompactionCoordinatorTest method testCoordinatorRestartOneRunningCompaction.

@Test
public void testCoordinatorRestartOneRunningCompaction() throws Exception {
    PowerMock.resetAll();
    PowerMock.suppress(PowerMock.constructor(AbstractServer.class));
    PowerMock.suppress(PowerMock.methods(ThriftUtil.class, "returnClient"));
    PowerMock.suppress(PowerMock.methods(DeadCompactionDetector.class, "detectDeadCompactions", "detectDanglingFinalStateMarkers"));
    AccumuloConfiguration conf = PowerMock.createNiceMock(AccumuloConfiguration.class);
    ServerContext context = PowerMock.createNiceMock(ServerContext.class);
    TCredentials creds = PowerMock.createNiceMock(TCredentials.class);
    EasyMock.expect(context.rpcCreds()).andReturn(creds);
    CompactionFinalizer finalizer = PowerMock.createNiceMock(CompactionFinalizer.class);
    LiveTServerSet tservers = PowerMock.createNiceMock(LiveTServerSet.class);
    TServerInstance instance = PowerMock.createNiceMock(TServerInstance.class);
    HostAndPort tserverAddress = HostAndPort.fromString("localhost:9997");
    EasyMock.expect(instance.getHostAndPort()).andReturn(tserverAddress).anyTimes();
    EasyMock.expect(tservers.getCurrentServers()).andReturn(Sets.newHashSet(instance)).once();
    tservers.startListeningForTabletServerChanges();
    PowerMock.mockStatic(ExternalCompactionUtil.class);
    List<RunningCompaction> runningCompactions = new ArrayList<>();
    ExternalCompactionId eci = ExternalCompactionId.generate(UUID.randomUUID());
    TExternalCompactionJob job = PowerMock.createNiceMock(TExternalCompactionJob.class);
    EasyMock.expect(job.getExternalCompactionId()).andReturn(eci.toString()).anyTimes();
    TKeyExtent extent = new TKeyExtent();
    extent.setTable("1".getBytes());
    runningCompactions.add(new RunningCompaction(job, tserverAddress.toString(), "queue"));
    EasyMock.expect(ExternalCompactionUtil.getCompactionsRunningOnCompactors(context)).andReturn(runningCompactions);
    ServerAddress client = PowerMock.createNiceMock(ServerAddress.class);
    HostAndPort address = HostAndPort.fromString("localhost:10240");
    EasyMock.expect(client.getAddress()).andReturn(address).anyTimes();
    EasyMock.expect(instance.getHostPort()).andReturn("localhost:9997").anyTimes();
    TabletClientService.Client tsc = PowerMock.createNiceMock(TabletClientService.Client.class);
    TCompactionQueueSummary queueSummary = PowerMock.createNiceMock(TCompactionQueueSummary.class);
    EasyMock.expect(tsc.getCompactionQueueInfo(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Collections.singletonList(queueSummary)).anyTimes();
    EasyMock.expect(queueSummary.getQueue()).andReturn("R2DQ").anyTimes();
    EasyMock.expect(queueSummary.getPriority()).andReturn((short) 1).anyTimes();
    AuditedSecurityOperation security = PowerMock.createNiceMock(AuditedSecurityOperation.class);
    PowerMock.replayAll();
    TestCoordinator coordinator = new TestCoordinator(conf, finalizer, tservers, client, tsc, context, security);
    coordinator.resetInternals();
    assertEquals(0, coordinator.getQueues().size());
    assertEquals(0, coordinator.getIndex().size());
    assertEquals(0, coordinator.getRunning().size());
    coordinator.run();
    assertEquals(1, coordinator.getQueues().size());
    QueueAndPriority qp = QueueAndPriority.get("R2DQ".intern(), (short) 1);
    Map<Short, TreeSet<TServerInstance>> m = coordinator.getQueues().get("R2DQ".intern());
    assertNotNull(m);
    assertEquals(1, m.size());
    assertTrue(m.containsKey((short) 1));
    Set<TServerInstance> t = m.get((short) 1);
    assertNotNull(t);
    assertEquals(1, t.size());
    TServerInstance queuedTsi = t.iterator().next();
    assertEquals(instance.getHostPortSession(), queuedTsi.getHostPortSession());
    assertEquals(1, coordinator.getIndex().size());
    assertTrue(coordinator.getIndex().containsKey(queuedTsi));
    Set<QueueAndPriority> i = coordinator.getIndex().get(queuedTsi);
    assertEquals(1, i.size());
    assertEquals(qp, i.iterator().next());
    assertEquals(1, coordinator.getRunning().size());
    PowerMock.verifyAll();
    coordinator.resetInternals();
    coordinator.close();
}

Example 7 with LiveTServerSet

use of org.apache.accumulo.server.manager.LiveTServerSet in project accumulo by apache.

the class CompactionCoordinatorTest method testGetCompactionJobNoJobs.

@Test
public void testGetCompactionJobNoJobs() throws Exception {
    PowerMock.resetAll();
    PowerMock.suppress(PowerMock.constructor(AbstractServer.class));
    AccumuloConfiguration conf = PowerMock.createNiceMock(AccumuloConfiguration.class);
    ServerContext context = PowerMock.createNiceMock(ServerContext.class);
    TCredentials creds = PowerMock.createNiceMock(TCredentials.class);
    CompactionFinalizer finalizer = PowerMock.createNiceMock(CompactionFinalizer.class);
    LiveTServerSet tservers = PowerMock.createNiceMock(LiveTServerSet.class);
    ServerAddress client = PowerMock.createNiceMock(ServerAddress.class);
    HostAndPort address = HostAndPort.fromString("localhost:10240");
    EasyMock.expect(client.getAddress()).andReturn(address).anyTimes();
    TabletClientService.Client tsc = PowerMock.createNiceMock(TabletClientService.Client.class);
    AuditedSecurityOperation security = PowerMock.createNiceMock(AuditedSecurityOperation.class);
    EasyMock.expect(security.canPerformSystemActions(creds)).andReturn(true);
    PowerMock.replayAll();
    TestCoordinator coordinator = new TestCoordinator(conf, finalizer, tservers, client, tsc, context, security);
    coordinator.resetInternals();
    TExternalCompactionJob job = coordinator.getCompactionJob(TraceUtil.traceInfo(), creds, "R2DQ", "localhost:10240", UUID.randomUUID().toString());
    assertNull(job.getExternalCompactionId());
    PowerMock.verifyAll();
    coordinator.resetInternals();
    coordinator.close();
}

Example 8 with LiveTServerSet

use of org.apache.accumulo.server.manager.LiveTServerSet in project accumulo by apache.

the class CompactionCoordinatorTest method testGetCompactionJob.

@Test
public void testGetCompactionJob() throws Exception {
    PowerMock.resetAll();
    PowerMock.suppress(PowerMock.constructor(AbstractServer.class));
    PowerMock.suppress(PowerMock.methods(ThriftUtil.class, "returnClient"));
    PowerMock.suppress(PowerMock.methods(DeadCompactionDetector.class, "detectDeadCompactions", "detectDanglingFinalStateMarkers"));
    AccumuloConfiguration conf = PowerMock.createNiceMock(AccumuloConfiguration.class);
    ServerContext context = PowerMock.createNiceMock(ServerContext.class);
    TCredentials creds = PowerMock.createNiceMock(TCredentials.class);
    EasyMock.expect(context.rpcCreds()).andReturn(creds).anyTimes();
    PowerMock.mockStatic(ExternalCompactionUtil.class);
    List<RunningCompaction> runningCompactions = new ArrayList<>();
    EasyMock.expect(ExternalCompactionUtil.getCompactionsRunningOnCompactors(context)).andReturn(runningCompactions);
    CompactionFinalizer finalizer = PowerMock.createNiceMock(CompactionFinalizer.class);
    LiveTServerSet tservers = PowerMock.createNiceMock(LiveTServerSet.class);
    TServerInstance instance = PowerMock.createNiceMock(TServerInstance.class);
    EasyMock.expect(tservers.getCurrentServers()).andReturn(Collections.singleton(instance)).once();
    HostAndPort tserverAddress = HostAndPort.fromString("localhost:9997");
    EasyMock.expect(instance.getHostAndPort()).andReturn(tserverAddress).anyTimes();
    ServerAddress client = PowerMock.createNiceMock(ServerAddress.class);
    HostAndPort address = HostAndPort.fromString("localhost:10240");
    EasyMock.expect(client.getAddress()).andReturn(address).anyTimes();
    TServerInstance tsi = PowerMock.createNiceMock(TServerInstance.class);
    EasyMock.expect(tsi.getHostPort()).andReturn("localhost:9997").anyTimes();
    TabletClientService.Client tsc = PowerMock.createNiceMock(TabletClientService.Client.class);
    TCompactionQueueSummary queueSummary = PowerMock.createNiceMock(TCompactionQueueSummary.class);
    EasyMock.expect(tsc.getCompactionQueueInfo(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Collections.singletonList(queueSummary)).anyTimes();
    EasyMock.expect(queueSummary.getQueue()).andReturn("R2DQ").anyTimes();
    EasyMock.expect(queueSummary.getPriority()).andReturn((short) 1).anyTimes();
    ExternalCompactionId eci = ExternalCompactionId.generate(UUID.randomUUID());
    TExternalCompactionJob job = PowerMock.createNiceMock(TExternalCompactionJob.class);
    EasyMock.expect(job.getExternalCompactionId()).andReturn(eci.toString()).anyTimes();
    TInfo trace = TraceUtil.traceInfo();
    EasyMock.expect(tsc.reserveCompactionJob(trace, creds, "R2DQ", 1, "localhost:10241", eci.toString())).andReturn(job).anyTimes();
    AuditedSecurityOperation security = PowerMock.createNiceMock(AuditedSecurityOperation.class);
    EasyMock.expect(security.canPerformSystemActions(creds)).andReturn(true);
    PowerMock.replayAll();
    TestCoordinator coordinator = new TestCoordinator(conf, finalizer, tservers, client, tsc, context, security);
    coordinator.resetInternals();
    assertEquals(0, coordinator.getQueues().size());
    assertEquals(0, coordinator.getIndex().size());
    assertEquals(0, coordinator.getRunning().size());
    // Use coordinator.run() to populate the internal data structures. This is tested in a different
    // test.
    coordinator.run();
    assertEquals(1, coordinator.getQueues().size());
    QueueAndPriority qp = QueueAndPriority.get("R2DQ".intern(), (short) 1);
    Map<Short, TreeSet<TServerInstance>> m = coordinator.getQueues().get("R2DQ".intern());
    assertNotNull(m);
    assertEquals(1, m.size());
    assertTrue(m.containsKey((short) 1));
    Set<TServerInstance> t = m.get((short) 1);
    assertNotNull(t);
    assertEquals(1, t.size());
    TServerInstance queuedTsi = t.iterator().next();
    assertEquals(tsi.getHostPortSession(), queuedTsi.getHostPortSession());
    assertEquals(1, coordinator.getIndex().size());
    assertTrue(coordinator.getIndex().containsKey(queuedTsi));
    Set<QueueAndPriority> i = coordinator.getIndex().get(queuedTsi);
    assertEquals(1, i.size());
    assertEquals(qp, i.iterator().next());
    assertEquals(0, coordinator.getRunning().size());
    // Get the next job
    TExternalCompactionJob createdJob = coordinator.getCompactionJob(trace, creds, "R2DQ", "localhost:10241", eci.toString());
    assertEquals(eci.toString(), createdJob.getExternalCompactionId());
    assertEquals(1, coordinator.getQueues().size());
    assertEquals(1, coordinator.getIndex().size());
    assertEquals(1, coordinator.getRunning().size());
    Entry<ExternalCompactionId, RunningCompaction> entry = coordinator.getRunning().entrySet().iterator().next();
    assertEquals(eci.toString(), entry.getKey().toString());
    assertEquals("localhost:10241", entry.getValue().getCompactorAddress());
    assertEquals(eci.toString(), entry.getValue().getJob().getExternalCompactionId());
    PowerMock.verifyAll();
    coordinator.resetInternals();
    coordinator.close();
}

Example 9 with LiveTServerSet

use of org.apache.accumulo.server.manager.LiveTServerSet in project accumulo by apache.

the class CompactionCoordinatorTest method testCoordinatorColdStart.

@Test
public void testCoordinatorColdStart() throws Exception {
    PowerMock.resetAll();
    PowerMock.suppress(PowerMock.constructor(AbstractServer.class));
    PowerMock.suppress(PowerMock.methods(ThriftUtil.class, "returnClient"));
    PowerMock.suppress(PowerMock.methods(DeadCompactionDetector.class, "detectDeadCompactions", "detectDanglingFinalStateMarkers"));
    AccumuloConfiguration conf = PowerMock.createNiceMock(AccumuloConfiguration.class);
    ServerContext context = PowerMock.createNiceMock(ServerContext.class);
    TCredentials creds = PowerMock.createNiceMock(TCredentials.class);
    EasyMock.expect(context.rpcCreds()).andReturn(creds);
    PowerMock.mockStatic(ExternalCompactionUtil.class);
    List<RunningCompaction> runningCompactions = new ArrayList<>();
    EasyMock.expect(ExternalCompactionUtil.getCompactionsRunningOnCompactors(context)).andReturn(runningCompactions);
    CompactionFinalizer finalizer = PowerMock.createNiceMock(CompactionFinalizer.class);
    LiveTServerSet tservers = PowerMock.createNiceMock(LiveTServerSet.class);
    TServerInstance instance = PowerMock.createNiceMock(TServerInstance.class);
    EasyMock.expect(tservers.getCurrentServers()).andReturn(Collections.singleton(instance)).once();
    ServerAddress client = PowerMock.createNiceMock(ServerAddress.class);
    HostAndPort address = HostAndPort.fromString("localhost:10240");
    EasyMock.expect(client.getAddress()).andReturn(address).anyTimes();
    TServerInstance tsi = PowerMock.createNiceMock(TServerInstance.class);
    EasyMock.expect(tsi.getHostPort()).andReturn("localhost:9997").anyTimes();
    TabletClientService.Client tsc = PowerMock.createNiceMock(TabletClientService.Client.class);
    TCompactionQueueSummary queueSummary = PowerMock.createNiceMock(TCompactionQueueSummary.class);
    EasyMock.expect(tsc.getCompactionQueueInfo(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Collections.singletonList(queueSummary)).anyTimes();
    EasyMock.expect(queueSummary.getQueue()).andReturn("R2DQ").anyTimes();
    EasyMock.expect(queueSummary.getPriority()).andReturn((short) 1).anyTimes();
    AuditedSecurityOperation security = PowerMock.createNiceMock(AuditedSecurityOperation.class);
    PowerMock.replayAll();
    TestCoordinator coordinator = new TestCoordinator(conf, finalizer, tservers, client, tsc, context, security);
    coordinator.resetInternals();
    assertEquals(0, coordinator.getQueues().size());
    assertEquals(0, coordinator.getIndex().size());
    assertEquals(0, coordinator.getRunning().size());
    coordinator.run();
    assertEquals(1, coordinator.getQueues().size());
    QueueAndPriority qp = QueueAndPriority.get("R2DQ".intern(), (short) 1);
    Map<Short, TreeSet<TServerInstance>> m = coordinator.getQueues().get("R2DQ".intern());
    assertNotNull(m);
    assertEquals(1, m.size());
    assertTrue(m.containsKey((short) 1));
    Set<TServerInstance> t = m.get((short) 1);
    assertNotNull(t);
    assertEquals(1, t.size());
    TServerInstance queuedTsi = t.iterator().next();
    assertEquals(tsi.getHostPortSession(), queuedTsi.getHostPortSession());
    assertEquals(1, coordinator.getIndex().size());
    assertTrue(coordinator.getIndex().containsKey(queuedTsi));
    Set<QueueAndPriority> i = coordinator.getIndex().get(queuedTsi);
    assertEquals(1, i.size());
    assertEquals(qp, i.iterator().next());
    assertEquals(0, coordinator.getRunning().size());
    PowerMock.verifyAll();
    coordinator.resetInternals();
    coordinator.close();
}

Example 10 with LiveTServerSet

use of org.apache.accumulo.server.manager.LiveTServerSet in project accumulo by apache.

the class SimpleGarbageCollector method run.

@Override
@SuppressFBWarnings(value = "DM_EXIT", justification = "main class can call System.exit")
public void run() {
    final VolumeManager fs = getContext().getVolumeManager();
    // Sleep for an initial period, giving the manager time to start up and
    // old data files to be unused
    log.info("Trying to acquire ZooKeeper lock for garbage collector");
    HostAndPort address = startStatsService();
    try {
        getZooLock(address);
    } catch (Exception ex) {
        log.error("{}", ex.getMessage(), ex);
        System.exit(1);
    }
    try {
        MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName, address);
        MetricsUtil.initializeProducers(new GcMetrics(this));
    } catch (Exception e1) {
        log.error("Error initializing metrics, metrics will not be emitted.", e1);
    }
    try {
        long delay = getStartDelay();
        log.debug("Sleeping for {} milliseconds before beginning garbage collection cycles", delay);
        Thread.sleep(delay);
    } catch (InterruptedException e) {
        log.warn("{}", e.getMessage(), e);
        return;
    }
    // This is created outside of the run loop and passed to the walogCollector so that
    // only a single timed task is created (internal to LiveTServerSet) using SimpleTimer.
    final LiveTServerSet liveTServerSet = new LiveTServerSet(getContext(), (current, deleted, added) -> {
        log.debug("Number of current servers {}, tservers added {}, removed {}", current == null ? -1 : current.size(), added, deleted);
        if (log.isTraceEnabled()) {
            log.trace("Current servers: {}\nAdded: {}\n Removed: {}", current, added, deleted);
        }
    });
    while (true) {
        Span outerSpan = TraceUtil.startSpan(this.getClass(), "gc");
        try (Scope outerScope = outerSpan.makeCurrent()) {
            Span innerSpan = TraceUtil.startSpan(this.getClass(), "loop");
            try (Scope innerScope = innerSpan.makeCurrent()) {
                final long tStart = System.nanoTime();
                try {
                    // make room
                    System.gc();
                    status.current.started = System.currentTimeMillis();
                    new GarbageCollectionAlgorithm().collect(new GCEnv(DataLevel.ROOT));
                    new GarbageCollectionAlgorithm().collect(new GCEnv(DataLevel.METADATA));
                    new GarbageCollectionAlgorithm().collect(new GCEnv(DataLevel.USER));
                    log.info("Number of data file candidates for deletion: {}", status.current.candidates);
                    log.info("Number of data file candidates still in use: {}", status.current.inUse);
                    log.info("Number of successfully deleted data files: {}", status.current.deleted);
                    log.info("Number of data files delete failures: {}", status.current.errors);
                    status.current.finished = System.currentTimeMillis();
                    status.last = status.current;
                    gcCycleMetrics.setLastCollect(status.current);
                    status.current = new GcCycleStats();
                } catch (Exception e) {
                    TraceUtil.setException(innerSpan, e, false);
                    log.error("{}", e.getMessage(), e);
                }
                final long tStop = System.nanoTime();
                log.info(String.format("Collect cycle took %.2f seconds", (TimeUnit.NANOSECONDS.toMillis(tStop - tStart) / 1000.0)));
                /*
           * We want to prune references to fully-replicated WALs from the replication table which
           * are no longer referenced in the metadata table before running
           * GarbageCollectWriteAheadLogs to ensure we delete as many files as possible.
           */
                Span replSpan = TraceUtil.startSpan(this.getClass(), "replicationClose");
                try (Scope replScope = replSpan.makeCurrent()) {
                    @SuppressWarnings("deprecation") Runnable closeWals = new org.apache.accumulo.gc.replication.CloseWriteAheadLogReferences(getContext());
                    closeWals.run();
                } catch (Exception e) {
                    TraceUtil.setException(replSpan, e, false);
                    log.error("Error trying to close write-ahead logs for replication table", e);
                } finally {
                    replSpan.end();
                }
                // Clean up any unused write-ahead logs
                Span walSpan = TraceUtil.startSpan(this.getClass(), "walogs");
                try (Scope walScope = walSpan.makeCurrent()) {
                    GarbageCollectWriteAheadLogs walogCollector = new GarbageCollectWriteAheadLogs(getContext(), fs, liveTServerSet, isUsingTrash());
                    log.info("Beginning garbage collection of write-ahead logs");
                    walogCollector.collect(status);
                    gcCycleMetrics.setLastWalCollect(status.lastLog);
                } catch (Exception e) {
                    TraceUtil.setException(walSpan, e, false);
                    log.error("{}", e.getMessage(), e);
                } finally {
                    walSpan.end();
                }
            } catch (Exception e) {
                TraceUtil.setException(innerSpan, e, true);
                throw e;
            } finally {
                innerSpan.end();
            }
            // we just made a lot of metadata changes: flush them out
            try {
                AccumuloClient accumuloClient = getContext();
                final long actionStart = System.nanoTime();
                String action = getConfiguration().get(Property.GC_USE_FULL_COMPACTION);
                log.debug("gc post action {} started", action);
                switch(action) {
                    case "compact":
                        accumuloClient.tableOperations().compact(MetadataTable.NAME, null, null, true, true);
                        accumuloClient.tableOperations().compact(RootTable.NAME, null, null, true, true);
                        break;
                    case "flush":
                        accumuloClient.tableOperations().flush(MetadataTable.NAME, null, null, true);
                        accumuloClient.tableOperations().flush(RootTable.NAME, null, null, true);
                        break;
                    default:
                        log.trace("'none - no action' or invalid value provided: {}", action);
                }
                final long actionComplete = System.nanoTime();
                gcCycleMetrics.setPostOpDurationNanos(actionComplete - actionStart);
                log.info("gc post action {} completed in {} seconds", action, String.format("%.2f", (TimeUnit.NANOSECONDS.toMillis(actionComplete - actionStart) / 1000.0)));
            } catch (Exception e) {
                TraceUtil.setException(outerSpan, e, false);
                log.warn("{}", e.getMessage(), e);
            }
        } catch (Exception e) {
            TraceUtil.setException(outerSpan, e, true);
            throw e;
        } finally {
            outerSpan.end();
        }
        try {
            gcCycleMetrics.incrementRunCycleCount();
            long gcDelay = getConfiguration().getTimeInMillis(Property.GC_CYCLE_DELAY);
            log.debug("Sleeping for {} milliseconds", gcDelay);
            Thread.sleep(gcDelay);
        } catch (InterruptedException e) {
            log.warn("{}", e.getMessage(), e);
            return;
        }
    }
}