Search in sources :

Example 1 with LiveTServerSet

use of org.apache.accumulo.server.manager.LiveTServerSet in project accumulo by apache.

the class FindOfflineTablets method findOffline.

static int findOffline(ServerContext context, String tableName) throws TableNotFoundException {
    final AtomicBoolean scanning = new AtomicBoolean(false);
    LiveTServerSet tservers = new LiveTServerSet(context, new Listener() {

        @Override
        public void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance> added) {
            if (!deleted.isEmpty() && scanning.get())
                log.warn("Tablet servers deleted while scanning: {}", deleted);
            if (!added.isEmpty() && scanning.get())
                log.warn("Tablet servers added while scanning: {}", added);
        }
    });
    tservers.startListeningForTabletServerChanges();
    scanning.set(true);
    Iterator<TabletLocationState> zooScanner = TabletStateStore.getStoreForLevel(DataLevel.ROOT, context).iterator();
    int offline = 0;
    System.out.println("Scanning zookeeper");
    if ((offline = checkTablets(context, zooScanner, tservers)) > 0)
        return offline;
    if (RootTable.NAME.equals(tableName))
        return 0;
    System.out.println("Scanning " + RootTable.NAME);
    Iterator<TabletLocationState> rootScanner = new MetaDataTableScanner(context, TabletsSection.getRange(), RootTable.NAME);
    if ((offline = checkTablets(context, rootScanner, tservers)) > 0)
        return offline;
    if (MetadataTable.NAME.equals(tableName))
        return 0;
    System.out.println("Scanning " + MetadataTable.NAME);
    Range range = TabletsSection.getRange();
    if (tableName != null) {
        TableId tableId = context.getTableId(tableName);
        range = new KeyExtent(tableId, null, null).toMetaRange();
    }
    try (MetaDataTableScanner metaScanner = new MetaDataTableScanner(context, range, MetadataTable.NAME)) {
        return checkTablets(context, metaScanner, tservers);
    }
}
Also used : TableId(org.apache.accumulo.core.data.TableId) Listener(org.apache.accumulo.server.manager.LiveTServerSet.Listener) Range(org.apache.accumulo.core.data.Range) KeyExtent(org.apache.accumulo.core.dataImpl.KeyExtent) TServerInstance(org.apache.accumulo.core.metadata.TServerInstance) LiveTServerSet(org.apache.accumulo.server.manager.LiveTServerSet) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) MetaDataTableScanner(org.apache.accumulo.server.manager.state.MetaDataTableScanner) TabletLocationState(org.apache.accumulo.core.metadata.TabletLocationState)

Example 2 with LiveTServerSet

use of org.apache.accumulo.server.manager.LiveTServerSet in project accumulo by apache.

the class CompactionCoordinatorTest method testCoordinatorColdStartNoCompactions.

@Test
public void testCoordinatorColdStartNoCompactions() throws Exception {
    PowerMock.resetAll();
    PowerMock.suppress(PowerMock.constructor(AbstractServer.class));
    PowerMock.suppress(PowerMock.methods(ThriftUtil.class, "returnClient"));
    PowerMock.suppress(PowerMock.methods(DeadCompactionDetector.class, "detectDeadCompactions", "detectDanglingFinalStateMarkers"));
    AccumuloConfiguration conf = PowerMock.createNiceMock(AccumuloConfiguration.class);
    ServerContext context = PowerMock.createNiceMock(ServerContext.class);
    PowerMock.mockStatic(ExternalCompactionUtil.class);
    List<RunningCompaction> runningCompactions = new ArrayList<>();
    EasyMock.expect(ExternalCompactionUtil.getCompactionsRunningOnCompactors(context)).andReturn(runningCompactions);
    CompactionFinalizer finalizer = PowerMock.createNiceMock(CompactionFinalizer.class);
    LiveTServerSet tservers = PowerMock.createNiceMock(LiveTServerSet.class);
    EasyMock.expect(tservers.getCurrentServers()).andReturn(Collections.emptySet()).anyTimes();
    ServerAddress client = PowerMock.createNiceMock(ServerAddress.class);
    HostAndPort address = HostAndPort.fromString("localhost:10240");
    EasyMock.expect(client.getAddress()).andReturn(address).anyTimes();
    TServerInstance tsi = PowerMock.createNiceMock(TServerInstance.class);
    EasyMock.expect(tsi.getHostPort()).andReturn("localhost:9997").anyTimes();
    TabletClientService.Client tsc = PowerMock.createNiceMock(TabletClientService.Client.class);
    EasyMock.expect(tsc.getCompactionQueueInfo(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Collections.emptyList()).anyTimes();
    AuditedSecurityOperation security = PowerMock.createNiceMock(AuditedSecurityOperation.class);
    PowerMock.replayAll();
    TestCoordinator coordinator = new TestCoordinator(conf, finalizer, tservers, client, tsc, context, security);
    coordinator.resetInternals();
    assertEquals(0, coordinator.getQueues().size());
    assertEquals(0, coordinator.getIndex().size());
    assertEquals(0, coordinator.getRunning().size());
    coordinator.run();
    assertEquals(0, coordinator.getQueues().size());
    assertEquals(0, coordinator.getIndex().size());
    assertEquals(0, coordinator.getRunning().size());
    PowerMock.verifyAll();
    coordinator.resetInternals();
    coordinator.close();
}
Also used : ThriftUtil(org.apache.accumulo.core.rpc.ThriftUtil) RunningCompaction(org.apache.accumulo.core.util.compaction.RunningCompaction) ArrayList(java.util.ArrayList) ServerAddress(org.apache.accumulo.server.rpc.ServerAddress) AuditedSecurityOperation(org.apache.accumulo.server.security.AuditedSecurityOperation) TServerInstance(org.apache.accumulo.core.metadata.TServerInstance) LiveTServerSet(org.apache.accumulo.server.manager.LiveTServerSet) HostAndPort(org.apache.accumulo.core.util.HostAndPort) AbstractServer(org.apache.accumulo.server.AbstractServer) Client(org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client) ServerContext(org.apache.accumulo.server.ServerContext) TabletClientService(org.apache.accumulo.core.tabletserver.thrift.TabletClientService) AccumuloConfiguration(org.apache.accumulo.core.conf.AccumuloConfiguration) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 3 with LiveTServerSet

use of org.apache.accumulo.server.manager.LiveTServerSet in project accumulo by apache.

the class CompactionCoordinatorTest method testCoordinatorRestartNoRunningCompactions.

@Test
public void testCoordinatorRestartNoRunningCompactions() throws Exception {
    PowerMock.resetAll();
    PowerMock.suppress(PowerMock.constructor(AbstractServer.class));
    PowerMock.suppress(PowerMock.methods(ThriftUtil.class, "returnClient"));
    PowerMock.suppress(PowerMock.methods(DeadCompactionDetector.class, "detectDeadCompactions", "detectDanglingFinalStateMarkers"));
    AccumuloConfiguration conf = PowerMock.createNiceMock(AccumuloConfiguration.class);
    ServerContext context = PowerMock.createNiceMock(ServerContext.class);
    TCredentials creds = PowerMock.createNiceMock(TCredentials.class);
    EasyMock.expect(context.rpcCreds()).andReturn(creds);
    CompactionFinalizer finalizer = PowerMock.createNiceMock(CompactionFinalizer.class);
    LiveTServerSet tservers = PowerMock.createNiceMock(LiveTServerSet.class);
    TServerInstance instance = PowerMock.createNiceMock(TServerInstance.class);
    HostAndPort tserverAddress = HostAndPort.fromString("localhost:9997");
    EasyMock.expect(instance.getHostAndPort()).andReturn(tserverAddress).anyTimes();
    EasyMock.expect(tservers.getCurrentServers()).andReturn(Sets.newHashSet(instance)).once();
    tservers.startListeningForTabletServerChanges();
    PowerMock.mockStatic(ExternalCompactionUtil.class);
    List<RunningCompaction> runningCompactions = new ArrayList<>();
    EasyMock.expect(ExternalCompactionUtil.getCompactionsRunningOnCompactors(context)).andReturn(runningCompactions);
    ServerAddress client = PowerMock.createNiceMock(ServerAddress.class);
    HostAndPort address = HostAndPort.fromString("localhost:10240");
    EasyMock.expect(client.getAddress()).andReturn(address).anyTimes();
    EasyMock.expect(instance.getHostPort()).andReturn("localhost:9997").anyTimes();
    TabletClientService.Client tsc = PowerMock.createNiceMock(TabletClientService.Client.class);
    TCompactionQueueSummary queueSummary = PowerMock.createNiceMock(TCompactionQueueSummary.class);
    EasyMock.expect(tsc.getCompactionQueueInfo(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Collections.singletonList(queueSummary)).anyTimes();
    EasyMock.expect(queueSummary.getQueue()).andReturn("R2DQ").anyTimes();
    EasyMock.expect(queueSummary.getPriority()).andReturn((short) 1).anyTimes();
    AuditedSecurityOperation security = PowerMock.createNiceMock(AuditedSecurityOperation.class);
    PowerMock.replayAll();
    TestCoordinator coordinator = new TestCoordinator(conf, finalizer, tservers, client, tsc, context, security);
    coordinator.resetInternals();
    assertEquals(0, coordinator.getQueues().size());
    assertEquals(0, coordinator.getIndex().size());
    assertEquals(0, coordinator.getRunning().size());
    coordinator.run();
    assertEquals(1, coordinator.getQueues().size());
    QueueAndPriority qp = QueueAndPriority.get("R2DQ".intern(), (short) 1);
    Map<Short, TreeSet<TServerInstance>> m = coordinator.getQueues().get("R2DQ".intern());
    assertNotNull(m);
    assertEquals(1, m.size());
    assertTrue(m.containsKey((short) 1));
    Set<TServerInstance> t = m.get((short) 1);
    assertNotNull(t);
    assertEquals(1, t.size());
    TServerInstance queuedTsi = t.iterator().next();
    assertEquals(instance.getHostPortSession(), queuedTsi.getHostPortSession());
    assertEquals(1, coordinator.getIndex().size());
    assertTrue(coordinator.getIndex().containsKey(queuedTsi));
    Set<QueueAndPriority> i = coordinator.getIndex().get(queuedTsi);
    assertEquals(1, i.size());
    assertEquals(qp, i.iterator().next());
    assertEquals(0, coordinator.getRunning().size());
    PowerMock.verifyAll();
    coordinator.resetInternals();
    coordinator.close();
}
Also used : ThriftUtil(org.apache.accumulo.core.rpc.ThriftUtil) ArrayList(java.util.ArrayList) ServerAddress(org.apache.accumulo.server.rpc.ServerAddress) AuditedSecurityOperation(org.apache.accumulo.server.security.AuditedSecurityOperation) TCompactionQueueSummary(org.apache.accumulo.core.tabletserver.thrift.TCompactionQueueSummary) LiveTServerSet(org.apache.accumulo.server.manager.LiveTServerSet) HostAndPort(org.apache.accumulo.core.util.HostAndPort) Client(org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client) TreeSet(java.util.TreeSet) AccumuloConfiguration(org.apache.accumulo.core.conf.AccumuloConfiguration) TCredentials(org.apache.accumulo.core.securityImpl.thrift.TCredentials) RunningCompaction(org.apache.accumulo.core.util.compaction.RunningCompaction) TServerInstance(org.apache.accumulo.core.metadata.TServerInstance) AbstractServer(org.apache.accumulo.server.AbstractServer) ServerContext(org.apache.accumulo.server.ServerContext) TabletClientService(org.apache.accumulo.core.tabletserver.thrift.TabletClientService) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 4 with LiveTServerSet

use of org.apache.accumulo.server.manager.LiveTServerSet in project accumulo by apache.

the class GarbageCollectWriteAheadLogsTest method testRemoveUnusedLog.

@Test
public void testRemoveUnusedLog() throws Exception {
    ServerContext context = EasyMock.createMock(ServerContext.class);
    VolumeManager fs = EasyMock.createMock(VolumeManager.class);
    WalStateManager marker = EasyMock.createMock(WalStateManager.class);
    LiveTServerSet tserverSet = EasyMock.createMock(LiveTServerSet.class);
    GCStatus status = new GCStatus(null, null, null, new GcCycleStats());
    tserverSet.scanServers();
    EasyMock.expectLastCall();
    EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1));
    EasyMock.expect(marker.getAllMarkers()).andReturn(markers).once();
    EasyMock.expect(marker.state(server1, id)).andReturn(new Pair<>(WalState.UNREFERENCED, path));
    EasyMock.expect(fs.deleteRecursively(path)).andReturn(true).once();
    marker.removeWalMarker(server1, id);
    EasyMock.expectLastCall().once();
    EasyMock.replay(context, fs, marker, tserverSet);
    GarbageCollectWriteAheadLogs gc = new GarbageCollectWriteAheadLogs(context, fs, false, tserverSet, marker, tabletOnServer1List) {

        @Override
        @Deprecated
        protected int removeReplicationEntries(Map<UUID, TServerInstance> candidates) {
            return 0;
        }

        @Override
        protected Map<UUID, Path> getSortedWALogs() {
            return Collections.emptyMap();
        }
    };
    gc.collect(status);
    EasyMock.verify(context, fs, marker, tserverSet);
}
Also used : Path(org.apache.hadoop.fs.Path) VolumeManager(org.apache.accumulo.server.fs.VolumeManager) ServerContext(org.apache.accumulo.server.ServerContext) WalStateManager(org.apache.accumulo.server.log.WalStateManager) GcCycleStats(org.apache.accumulo.core.gc.thrift.GcCycleStats) GCStatus(org.apache.accumulo.core.gc.thrift.GCStatus) UUID(java.util.UUID) Map(java.util.Map) LiveTServerSet(org.apache.accumulo.server.manager.LiveTServerSet) Test(org.junit.Test)

Example 5 with LiveTServerSet

use of org.apache.accumulo.server.manager.LiveTServerSet in project accumulo by apache.

the class GarbageCollectWriteAheadLogsTest method deleteUnreferenceLogOnDeadServer.

@Test
public void deleteUnreferenceLogOnDeadServer() throws Exception {
    ServerContext context = EasyMock.createMock(ServerContext.class);
    VolumeManager fs = EasyMock.createMock(VolumeManager.class);
    WalStateManager marker = EasyMock.createMock(WalStateManager.class);
    LiveTServerSet tserverSet = EasyMock.createMock(LiveTServerSet.class);
    Scanner mscanner = EasyMock.createMock(Scanner.class);
    Scanner rscanner = EasyMock.createMock(Scanner.class);
    GCStatus status = new GCStatus(null, null, null, new GcCycleStats());
    tserverSet.scanServers();
    EasyMock.expectLastCall();
    EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1));
    EasyMock.expect(marker.getAllMarkers()).andReturn(markers2).once();
    EasyMock.expect(marker.state(server2, id)).andReturn(new Pair<>(WalState.OPEN, path));
    EasyMock.expect(context.createScanner(REPL_TABLE_NAME, Authorizations.EMPTY)).andReturn(rscanner);
    rscanner.fetchColumnFamily(STATUS_SECTION_NAME);
    EasyMock.expectLastCall().once();
    EasyMock.expect(rscanner.iterator()).andReturn(emptyKV);
    EasyMock.expect(context.createScanner(MetadataTable.NAME, Authorizations.EMPTY)).andReturn(mscanner);
    mscanner.fetchColumnFamily(ReplicationSection.COLF);
    EasyMock.expectLastCall().once();
    mscanner.setRange(ReplicationSection.getRange());
    EasyMock.expectLastCall().once();
    EasyMock.expect(mscanner.iterator()).andReturn(emptyKV);
    EasyMock.expect(fs.deleteRecursively(path)).andReturn(true).once();
    marker.removeWalMarker(server2, id);
    EasyMock.expectLastCall().once();
    marker.forget(server2);
    EasyMock.expectLastCall().once();
    EasyMock.replay(context, fs, marker, tserverSet, rscanner, mscanner);
    GarbageCollectWriteAheadLogs gc = new GarbageCollectWriteAheadLogs(context, fs, false, tserverSet, marker, tabletOnServer1List) {

        @Override
        protected Map<UUID, Path> getSortedWALogs() {
            return Collections.emptyMap();
        }
    };
    gc.collect(status);
    EasyMock.verify(context, fs, marker, tserverSet, rscanner, mscanner);
}
Also used : Path(org.apache.hadoop.fs.Path) VolumeManager(org.apache.accumulo.server.fs.VolumeManager) Scanner(org.apache.accumulo.core.client.Scanner) ServerContext(org.apache.accumulo.server.ServerContext) WalStateManager(org.apache.accumulo.server.log.WalStateManager) GcCycleStats(org.apache.accumulo.core.gc.thrift.GcCycleStats) GCStatus(org.apache.accumulo.core.gc.thrift.GCStatus) UUID(java.util.UUID) LiveTServerSet(org.apache.accumulo.server.manager.LiveTServerSet) Test(org.junit.Test)

Aggregations

LiveTServerSet (org.apache.accumulo.server.manager.LiveTServerSet)13 ServerContext (org.apache.accumulo.server.ServerContext)11 Test (org.junit.Test)11 HostAndPort (org.apache.accumulo.core.util.HostAndPort)7 AccumuloConfiguration (org.apache.accumulo.core.conf.AccumuloConfiguration)6 GcCycleStats (org.apache.accumulo.core.gc.thrift.GcCycleStats)6 TServerInstance (org.apache.accumulo.core.metadata.TServerInstance)6 TabletClientService (org.apache.accumulo.core.tabletserver.thrift.TabletClientService)6 Client (org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client)6 AbstractServer (org.apache.accumulo.server.AbstractServer)6 VolumeManager (org.apache.accumulo.server.fs.VolumeManager)6 ServerAddress (org.apache.accumulo.server.rpc.ServerAddress)6 AuditedSecurityOperation (org.apache.accumulo.server.security.AuditedSecurityOperation)6 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)6 ArrayList (java.util.ArrayList)5 UUID (java.util.UUID)5 GCStatus (org.apache.accumulo.core.gc.thrift.GCStatus)5 ThriftUtil (org.apache.accumulo.core.rpc.ThriftUtil)5 TCredentials (org.apache.accumulo.core.securityImpl.thrift.TCredentials)5 RunningCompaction (org.apache.accumulo.core.util.compaction.RunningCompaction)5