Search in sources :

Example 1 with LiveTServerSet

use of org.apache.accumulo.server.master.LiveTServerSet in project accumulo by apache.

the class GarbageCollectWriteAheadLogsTest method deleteUnreferenceLogOnDeadServer.

@Test
public void deleteUnreferenceLogOnDeadServer() throws Exception {
    AccumuloServerContext context = EasyMock.createMock(AccumuloServerContext.class);
    VolumeManager fs = EasyMock.createMock(VolumeManager.class);
    WalStateManager marker = EasyMock.createMock(WalStateManager.class);
    LiveTServerSet tserverSet = EasyMock.createMock(LiveTServerSet.class);
    Connector conn = EasyMock.createMock(Connector.class);
    Scanner mscanner = EasyMock.createMock(Scanner.class);
    Scanner rscanner = EasyMock.createMock(Scanner.class);
    GCStatus status = new GCStatus(null, null, null, new GcCycleStats());
    EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1));
    EasyMock.expect(marker.getAllMarkers()).andReturn(markers2).once();
    EasyMock.expect(marker.state(server2, id)).andReturn(new Pair<>(WalState.OPEN, path));
    EasyMock.expect(context.getConnector()).andReturn(conn);
    EasyMock.expect(conn.createScanner(ReplicationTable.NAME, Authorizations.EMPTY)).andReturn(rscanner);
    rscanner.fetchColumnFamily(ReplicationSchema.StatusSection.NAME);
    EasyMock.expectLastCall().once();
    EasyMock.expect(rscanner.iterator()).andReturn(emptyKV);
    EasyMock.expect(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)).andReturn(mscanner);
    mscanner.fetchColumnFamily(MetadataSchema.ReplicationSection.COLF);
    EasyMock.expectLastCall().once();
    mscanner.setRange(MetadataSchema.ReplicationSection.getRange());
    EasyMock.expectLastCall().once();
    EasyMock.expect(mscanner.iterator()).andReturn(emptyKV);
    EasyMock.expect(fs.deleteRecursively(path)).andReturn(true).once();
    marker.removeWalMarker(server2, id);
    EasyMock.expectLastCall().once();
    marker.forget(server2);
    EasyMock.expectLastCall().once();
    EasyMock.replay(context, fs, marker, tserverSet, conn, rscanner, mscanner);
    GarbageCollectWriteAheadLogs gc = new GarbageCollectWriteAheadLogs(context, fs, false, tserverSet, marker, tabletOnServer1List);
    gc.collect(status);
    EasyMock.verify(context, fs, marker, tserverSet, conn, rscanner, mscanner);
}
Also used : VolumeManager(org.apache.accumulo.server.fs.VolumeManager) Connector(org.apache.accumulo.core.client.Connector) Scanner(org.apache.accumulo.core.client.Scanner) AccumuloServerContext(org.apache.accumulo.server.AccumuloServerContext) WalStateManager(org.apache.accumulo.server.log.WalStateManager) GcCycleStats(org.apache.accumulo.core.gc.thrift.GcCycleStats) GCStatus(org.apache.accumulo.core.gc.thrift.GCStatus) LiveTServerSet(org.apache.accumulo.server.master.LiveTServerSet) Test(org.junit.Test)

Example 2 with LiveTServerSet

use of org.apache.accumulo.server.master.LiveTServerSet in project accumulo by apache.

the class GarbageCollectWriteAheadLogsTest method testRemoveUnusedLog.

@Test
public void testRemoveUnusedLog() throws Exception {
    AccumuloServerContext context = EasyMock.createMock(AccumuloServerContext.class);
    VolumeManager fs = EasyMock.createMock(VolumeManager.class);
    WalStateManager marker = EasyMock.createMock(WalStateManager.class);
    LiveTServerSet tserverSet = EasyMock.createMock(LiveTServerSet.class);
    GCStatus status = new GCStatus(null, null, null, new GcCycleStats());
    EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1));
    EasyMock.expect(marker.getAllMarkers()).andReturn(markers).once();
    EasyMock.expect(marker.state(server1, id)).andReturn(new Pair<>(WalState.UNREFERENCED, path));
    EasyMock.expect(fs.deleteRecursively(path)).andReturn(true).once();
    marker.removeWalMarker(server1, id);
    EasyMock.expectLastCall().once();
    EasyMock.replay(context, fs, marker, tserverSet);
    GarbageCollectWriteAheadLogs gc = new GarbageCollectWriteAheadLogs(context, fs, false, tserverSet, marker, tabletOnServer1List) {

        @Override
        protected int removeReplicationEntries(Map<UUID, TServerInstance> candidates) throws IOException, KeeperException, InterruptedException {
            return 0;
        }
    };
    gc.collect(status);
    EasyMock.verify(context, fs, marker, tserverSet);
}
Also used : VolumeManager(org.apache.accumulo.server.fs.VolumeManager) AccumuloServerContext(org.apache.accumulo.server.AccumuloServerContext) WalStateManager(org.apache.accumulo.server.log.WalStateManager) GcCycleStats(org.apache.accumulo.core.gc.thrift.GcCycleStats) GCStatus(org.apache.accumulo.core.gc.thrift.GCStatus) Map(java.util.Map) LiveTServerSet(org.apache.accumulo.server.master.LiveTServerSet) Test(org.junit.Test)

Example 3 with LiveTServerSet

use of org.apache.accumulo.server.master.LiveTServerSet in project accumulo by apache.

the class FindOfflineTablets method findOffline.

static int findOffline(ClientContext context, String tableName) throws AccumuloException, TableNotFoundException {
    final AtomicBoolean scanning = new AtomicBoolean(false);
    LiveTServerSet tservers = new LiveTServerSet(context, new Listener() {

        @Override
        public void update(LiveTServerSet current, Set<TServerInstance> deleted, Set<TServerInstance> added) {
            if (!deleted.isEmpty() && scanning.get())
                log.warn("Tablet servers deleted while scanning: {}", deleted);
            if (!added.isEmpty() && scanning.get())
                log.warn("Tablet servers added while scanning: {}", added);
        }
    });
    tservers.startListeningForTabletServerChanges();
    scanning.set(true);
    Iterator<TabletLocationState> zooScanner;
    try {
        zooScanner = new ZooTabletStateStore().iterator();
    } catch (DistributedStoreException e) {
        throw new AccumuloException(e);
    }
    int offline = 0;
    System.out.println("Scanning zookeeper");
    if ((offline = checkTablets(zooScanner, tservers)) > 0)
        return offline;
    if (RootTable.NAME.equals(tableName))
        return 0;
    System.out.println("Scanning " + RootTable.NAME);
    Iterator<TabletLocationState> rootScanner = new MetaDataTableScanner(context, MetadataSchema.TabletsSection.getRange(), RootTable.NAME);
    if ((offline = checkTablets(rootScanner, tservers)) > 0)
        return offline;
    if (MetadataTable.NAME.equals(tableName))
        return 0;
    System.out.println("Scanning " + MetadataTable.NAME);
    Range range = MetadataSchema.TabletsSection.getRange();
    if (tableName != null) {
        Table.ID tableId = Tables.getTableId(context.getInstance(), tableName);
        range = new KeyExtent(tableId, null, null).toMetadataRange();
    }
    try (MetaDataTableScanner metaScanner = new MetaDataTableScanner(context, range, MetadataTable.NAME)) {
        return checkTablets(metaScanner, tservers);
    }
}
Also used : AccumuloException(org.apache.accumulo.core.client.AccumuloException) Listener(org.apache.accumulo.server.master.LiveTServerSet.Listener) MetadataTable(org.apache.accumulo.core.metadata.MetadataTable) RootTable(org.apache.accumulo.core.metadata.RootTable) Table(org.apache.accumulo.core.client.impl.Table) DistributedStoreException(org.apache.accumulo.server.master.state.DistributedStoreException) Range(org.apache.accumulo.core.data.Range) KeyExtent(org.apache.accumulo.core.data.impl.KeyExtent) TServerInstance(org.apache.accumulo.server.master.state.TServerInstance) LiveTServerSet(org.apache.accumulo.server.master.LiveTServerSet) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) MetaDataTableScanner(org.apache.accumulo.server.master.state.MetaDataTableScanner) TabletLocationState(org.apache.accumulo.server.master.state.TabletLocationState) ZooTabletStateStore(org.apache.accumulo.server.master.state.ZooTabletStateStore)

Example 4 with LiveTServerSet

use of org.apache.accumulo.server.master.LiveTServerSet in project accumulo by apache.

the class GarbageCollectWriteAheadLogsTest method replicationDelaysFileCollection.

@Test
public void replicationDelaysFileCollection() throws Exception {
    AccumuloServerContext context = EasyMock.createMock(AccumuloServerContext.class);
    VolumeManager fs = EasyMock.createMock(VolumeManager.class);
    WalStateManager marker = EasyMock.createMock(WalStateManager.class);
    LiveTServerSet tserverSet = EasyMock.createMock(LiveTServerSet.class);
    Connector conn = EasyMock.createMock(Connector.class);
    Scanner mscanner = EasyMock.createMock(Scanner.class);
    Scanner rscanner = EasyMock.createMock(Scanner.class);
    String row = MetadataSchema.ReplicationSection.getRowPrefix() + path.toString();
    String colf = MetadataSchema.ReplicationSection.COLF.toString();
    String colq = "1";
    Map<Key, Value> replicationWork = Collections.singletonMap(new Key(row, colf, colq), new Value(new byte[0]));
    GCStatus status = new GCStatus(null, null, null, new GcCycleStats());
    EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1));
    EasyMock.expect(marker.getAllMarkers()).andReturn(markers).once();
    EasyMock.expect(marker.state(server1, id)).andReturn(new Pair<>(WalState.UNREFERENCED, path));
    EasyMock.expect(context.getConnector()).andReturn(conn);
    EasyMock.expect(conn.createScanner(ReplicationTable.NAME, Authorizations.EMPTY)).andReturn(rscanner);
    rscanner.fetchColumnFamily(ReplicationSchema.StatusSection.NAME);
    EasyMock.expectLastCall().once();
    EasyMock.expect(rscanner.iterator()).andReturn(emptyKV);
    EasyMock.expect(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)).andReturn(mscanner);
    mscanner.fetchColumnFamily(MetadataSchema.ReplicationSection.COLF);
    EasyMock.expectLastCall().once();
    mscanner.setRange(MetadataSchema.ReplicationSection.getRange());
    EasyMock.expectLastCall().once();
    EasyMock.expect(mscanner.iterator()).andReturn(replicationWork.entrySet().iterator());
    EasyMock.replay(context, fs, marker, tserverSet, conn, rscanner, mscanner);
    GarbageCollectWriteAheadLogs gc = new GarbageCollectWriteAheadLogs(context, fs, false, tserverSet, marker, tabletOnServer1List);
    gc.collect(status);
    EasyMock.verify(context, fs, marker, tserverSet, conn, rscanner, mscanner);
}
Also used : VolumeManager(org.apache.accumulo.server.fs.VolumeManager) Connector(org.apache.accumulo.core.client.Connector) Scanner(org.apache.accumulo.core.client.Scanner) AccumuloServerContext(org.apache.accumulo.server.AccumuloServerContext) GcCycleStats(org.apache.accumulo.core.gc.thrift.GcCycleStats) GCStatus(org.apache.accumulo.core.gc.thrift.GCStatus) LiveTServerSet(org.apache.accumulo.server.master.LiveTServerSet) WalStateManager(org.apache.accumulo.server.log.WalStateManager) Value(org.apache.accumulo.core.data.Value) Key(org.apache.accumulo.core.data.Key) Test(org.junit.Test)

Example 5 with LiveTServerSet

use of org.apache.accumulo.server.master.LiveTServerSet in project accumulo by apache.

the class GarbageCollectWriteAheadLogsTest method ignoreReferenceLogOnDeadServer.

@Test
public void ignoreReferenceLogOnDeadServer() throws Exception {
    AccumuloServerContext context = EasyMock.createMock(AccumuloServerContext.class);
    VolumeManager fs = EasyMock.createMock(VolumeManager.class);
    WalStateManager marker = EasyMock.createMock(WalStateManager.class);
    LiveTServerSet tserverSet = EasyMock.createMock(LiveTServerSet.class);
    Connector conn = EasyMock.createMock(Connector.class);
    Scanner mscanner = EasyMock.createMock(Scanner.class);
    Scanner rscanner = EasyMock.createMock(Scanner.class);
    GCStatus status = new GCStatus(null, null, null, new GcCycleStats());
    EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1));
    EasyMock.expect(marker.getAllMarkers()).andReturn(markers2).once();
    EasyMock.expect(marker.state(server2, id)).andReturn(new Pair<>(WalState.OPEN, path));
    EasyMock.expect(context.getConnector()).andReturn(conn);
    EasyMock.expect(conn.createScanner(ReplicationTable.NAME, Authorizations.EMPTY)).andReturn(rscanner);
    rscanner.fetchColumnFamily(ReplicationSchema.StatusSection.NAME);
    EasyMock.expectLastCall().once();
    EasyMock.expect(rscanner.iterator()).andReturn(emptyKV);
    EasyMock.expect(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)).andReturn(mscanner);
    mscanner.fetchColumnFamily(MetadataSchema.ReplicationSection.COLF);
    EasyMock.expectLastCall().once();
    mscanner.setRange(MetadataSchema.ReplicationSection.getRange());
    EasyMock.expectLastCall().once();
    EasyMock.expect(mscanner.iterator()).andReturn(emptyKV);
    EasyMock.replay(context, fs, marker, tserverSet, conn, rscanner, mscanner);
    GarbageCollectWriteAheadLogs gc = new GarbageCollectWriteAheadLogs(context, fs, false, tserverSet, marker, tabletOnServer2List);
    gc.collect(status);
    EasyMock.verify(context, fs, marker, tserverSet, conn, rscanner, mscanner);
}
Also used : VolumeManager(org.apache.accumulo.server.fs.VolumeManager) Connector(org.apache.accumulo.core.client.Connector) Scanner(org.apache.accumulo.core.client.Scanner) AccumuloServerContext(org.apache.accumulo.server.AccumuloServerContext) WalStateManager(org.apache.accumulo.server.log.WalStateManager) GcCycleStats(org.apache.accumulo.core.gc.thrift.GcCycleStats) GCStatus(org.apache.accumulo.core.gc.thrift.GCStatus) LiveTServerSet(org.apache.accumulo.server.master.LiveTServerSet) Test(org.junit.Test)

Aggregations

LiveTServerSet (org.apache.accumulo.server.master.LiveTServerSet)6 GCStatus (org.apache.accumulo.core.gc.thrift.GCStatus)5 GcCycleStats (org.apache.accumulo.core.gc.thrift.GcCycleStats)5 AccumuloServerContext (org.apache.accumulo.server.AccumuloServerContext)5 VolumeManager (org.apache.accumulo.server.fs.VolumeManager)5 WalStateManager (org.apache.accumulo.server.log.WalStateManager)5 Test (org.junit.Test)5 Connector (org.apache.accumulo.core.client.Connector)3 Scanner (org.apache.accumulo.core.client.Scanner)3 Map (java.util.Map)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1 AccumuloException (org.apache.accumulo.core.client.AccumuloException)1 Table (org.apache.accumulo.core.client.impl.Table)1 Key (org.apache.accumulo.core.data.Key)1 Range (org.apache.accumulo.core.data.Range)1 Value (org.apache.accumulo.core.data.Value)1 KeyExtent (org.apache.accumulo.core.data.impl.KeyExtent)1 MetadataTable (org.apache.accumulo.core.metadata.MetadataTable)1 RootTable (org.apache.accumulo.core.metadata.RootTable)1 Listener (org.apache.accumulo.server.master.LiveTServerSet.Listener)1