Examples with ExternalCompactionId org.apache.accumulo.core.metadata.schema.ExternalCompactionId used on opensource projects

Example 16 with ExternalCompactionId

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionId in project accumulo by apache.

the class CompactorTest method testCompactionFails.

@Test
public void testCompactionFails() throws Exception {
    UUID uuid = UUID.randomUUID();
    Supplier<UUID> supplier = () -> uuid;
    ExternalCompactionId eci = ExternalCompactionId.generate(supplier.get());
    PowerMock.resetAll();
    PowerMock.suppress(PowerMock.methods(Halt.class, "halt"));
    PowerMock.suppress(PowerMock.constructor(AbstractServer.class));
    ServerAddress client = PowerMock.createNiceMock(ServerAddress.class);
    HostAndPort address = HostAndPort.fromString("localhost:10240");
    EasyMock.expect(client.getAddress()).andReturn(address);
    TExternalCompactionJob job = PowerMock.createNiceMock(TExternalCompactionJob.class);
    TKeyExtent extent = PowerMock.createNiceMock(TKeyExtent.class);
    EasyMock.expect(extent.getTable()).andReturn("testTable".getBytes()).anyTimes();
    EasyMock.expect(job.isSetExternalCompactionId()).andReturn(true).anyTimes();
    EasyMock.expect(job.getExternalCompactionId()).andReturn(eci.toString()).anyTimes();
    EasyMock.expect(job.getExtent()).andReturn(extent).anyTimes();
    AccumuloConfiguration conf = PowerMock.createNiceMock(AccumuloConfiguration.class);
    EasyMock.expect(conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT)).andReturn(86400000L);
    ServerContext context = PowerMock.createNiceMock(ServerContext.class);
    EasyMock.expect(context.getConfiguration()).andReturn(conf);
    ZooReaderWriter zrw = PowerMock.createNiceMock(ZooReaderWriter.class);
    ZooKeeper zk = PowerMock.createNiceMock(ZooKeeper.class);
    EasyMock.expect(context.getZooReaderWriter()).andReturn(zrw).anyTimes();
    EasyMock.expect(zrw.getZooKeeper()).andReturn(zk).anyTimes();
    VolumeManagerImpl vm = PowerMock.createNiceMock(VolumeManagerImpl.class);
    EasyMock.expect(context.getVolumeManager()).andReturn(vm);
    vm.close();
    PowerMock.replayAll();
    FailedCompactor c = new FailedCompactor(supplier, client, job, conf, context, eci);
    c.run();
    PowerMock.verifyAll();
    c.close();
    assertFalse(c.isCompletedCalled());
    assertTrue(c.isFailedCalled());
    assertEquals(TCompactionState.FAILED, c.getLatestState());
}

Example 17 with ExternalCompactionId

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionId in project accumulo by apache.

the class CompactorTest method testCompactionInterrupted.

@Test
public void testCompactionInterrupted() throws Exception {
    UUID uuid = UUID.randomUUID();
    Supplier<UUID> supplier = () -> uuid;
    ExternalCompactionId eci = ExternalCompactionId.generate(supplier.get());
    PowerMock.resetAll();
    PowerMock.suppress(PowerMock.methods(Halt.class, "halt"));
    PowerMock.suppress(PowerMock.constructor(AbstractServer.class));
    ServerAddress client = PowerMock.createNiceMock(ServerAddress.class);
    HostAndPort address = HostAndPort.fromString("localhost:10240");
    EasyMock.expect(client.getAddress()).andReturn(address);
    TExternalCompactionJob job = PowerMock.createNiceMock(TExternalCompactionJob.class);
    TKeyExtent extent = PowerMock.createNiceMock(TKeyExtent.class);
    EasyMock.expect(job.isSetExternalCompactionId()).andReturn(true).anyTimes();
    EasyMock.expect(job.getExternalCompactionId()).andReturn(eci.toString()).anyTimes();
    EasyMock.expect(job.getExtent()).andReturn(extent).anyTimes();
    EasyMock.expect(extent.getTable()).andReturn("testTable".getBytes()).anyTimes();
    AccumuloConfiguration conf = PowerMock.createNiceMock(AccumuloConfiguration.class);
    EasyMock.expect(conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT)).andReturn(86400000L);
    ServerContext context = PowerMock.createNiceMock(ServerContext.class);
    EasyMock.expect(context.getConfiguration()).andReturn(conf);
    ZooReaderWriter zrw = PowerMock.createNiceMock(ZooReaderWriter.class);
    ZooKeeper zk = PowerMock.createNiceMock(ZooKeeper.class);
    EasyMock.expect(context.getZooReaderWriter()).andReturn(zrw).anyTimes();
    EasyMock.expect(zrw.getZooKeeper()).andReturn(zk).anyTimes();
    VolumeManagerImpl vm = PowerMock.createNiceMock(VolumeManagerImpl.class);
    EasyMock.expect(context.getVolumeManager()).andReturn(vm);
    vm.close();
    PowerMock.replayAll();
    InterruptedCompactor c = new InterruptedCompactor(supplier, client, job, conf, context, eci);
    c.run();
    PowerMock.verifyAll();
    c.close();
    assertFalse(c.isCompletedCalled());
    assertTrue(c.isFailedCalled());
    assertEquals(TCompactionState.CANCELLED, c.getLatestState());
}

Example 18 with ExternalCompactionId

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionId in project accumulo by apache.

the class ExternalCompaction_1_IT method testExternalCompactionDeadTServer.

@Test
public void testExternalCompactionDeadTServer() throws Exception {
    // Shut down the normal TServers
    getCluster().getProcesses().get(TABLET_SERVER).forEach(p -> {
        try {
            getCluster().killProcess(TABLET_SERVER, p);
        } catch (Exception e) {
            fail("Failed to shutdown tablet server");
        }
    });
    // Start our TServer that will not commit the compaction
    ProcessInfo tserverProcess = getCluster().exec(ExternalCompactionTServer.class);
    final String table3 = this.getUniqueNames(1)[0];
    try (final AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
        createTable(client, table3, "cs7");
        writeData(client, table3);
        getCluster().getClusterControl().startCompactors(Compactor.class, 1, QUEUE7);
        getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class);
        compact(client, table3, 2, QUEUE7, false);
        // ExternalCompactionTServer will not commit the compaction. Wait for the
        // metadata table entries to show up.
        LOG.info("Waiting for external compaction to complete.");
        TableId tid = getCluster().getServerContext().getTableId(table3);
        Stream<ExternalCompactionFinalState> fs = getFinalStatesForTable(getCluster(), tid);
        while (fs.count() == 0) {
            LOG.info("Waiting for compaction completed marker to appear");
            UtilWaitThread.sleep(250);
            fs = getFinalStatesForTable(getCluster(), tid);
        }
        LOG.info("Validating metadata table contents.");
        TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets().forTable(tid).fetch(ColumnType.ECOMP).build();
        List<TabletMetadata> md = new ArrayList<>();
        tm.forEach(t -> md.add(t));
        assertEquals(1, md.size());
        TabletMetadata m = md.get(0);
        Map<ExternalCompactionId, ExternalCompactionMetadata> em = m.getExternalCompactions();
        assertEquals(1, em.size());
        List<ExternalCompactionFinalState> finished = new ArrayList<>();
        getFinalStatesForTable(getCluster(), tid).forEach(f -> finished.add(f));
        assertEquals(1, finished.size());
        assertEquals(em.entrySet().iterator().next().getKey(), finished.get(0).getExternalCompactionId());
        tm.close();
        // Force a flush on the metadata table before killing our tserver
        client.tableOperations().flush("accumulo.metadata");
        // Stop our TabletServer. Need to perform a normal shutdown so that the WAL is closed
        // normally.
        LOG.info("Stopping our tablet server");
        getCluster().stopProcessWithTimeout(tserverProcess.getProcess(), 30, TimeUnit.SECONDS);
        getCluster().getClusterControl().stop(ServerType.TABLET_SERVER);
        // Start a TabletServer to commit the compaction.
        LOG.info("Starting normal tablet server");
        getCluster().getClusterControl().start(ServerType.TABLET_SERVER);
        // Wait for the compaction to be committed.
        LOG.info("Waiting for compaction completed marker to disappear");
        Stream<ExternalCompactionFinalState> fs2 = getFinalStatesForTable(getCluster(), tid);
        while (fs2.count() != 0) {
            LOG.info("Waiting for compaction completed marker to disappear");
            UtilWaitThread.sleep(500);
            fs2 = getFinalStatesForTable(getCluster(), tid);
        }
        verify(client, table3, 2);
        // We need to cancel the compaction or delete the table here because we initiate a user
        // compaction above in the test. Even though the external compaction was cancelled
        // because we split the table, FaTE will continue to queue up a compaction
        client.tableOperations().cancelCompaction(table3);
    }
}

Example 19 with ExternalCompactionId

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionId in project accumulo by apache.

the class ExternalCompactionTestUtils method confirmCompactionRunning.

public static int confirmCompactionRunning(ServerContext ctx, Set<ExternalCompactionId> ecids) throws Exception {
    int matches = 0;
    while (matches == 0) {
        TExternalCompactionList running = ExternalCompactionTestUtils.getRunningCompactions(ctx);
        if (running.getCompactions() != null) {
            for (ExternalCompactionId ecid : ecids) {
                TExternalCompaction tec = running.getCompactions().get(ecid.canonical());
                if (tec != null && tec.getUpdates() != null && !tec.getUpdates().isEmpty()) {
                    matches++;
                    assertEquals(TCompactionState.STARTED, ExternalCompactionTestUtils.getLastState(tec));
                }
            }
        }
        if (matches == 0) {
            UtilWaitThread.sleep(50);
        }
    }
    return matches;
}

Example 20 with ExternalCompactionId

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionId in project accumulo by apache.

the class ExternalCompaction_2_IT method testDeleteTableCancelsUserExternalCompaction.

@Test
public void testDeleteTableCancelsUserExternalCompaction() throws Exception {
    getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class);
    getCluster().getClusterControl().startCompactors(ExternalDoNothingCompactor.class, 1, QUEUE4);
    String table1 = this.getUniqueNames(1)[0];
    try (AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
        createTable(client, table1, "cs4");
        TableId tid = getCluster().getServerContext().getTableId(table1);
        writeData(client, table1);
        compact(client, table1, 2, QUEUE4, false);
        // Wait for the compaction to start by waiting for 1 external compaction column
        Set<ExternalCompactionId> ecids = waitForCompactionStartAndReturnEcids(getCluster().getServerContext(), tid);
        // Confirm that this ECID shows up in RUNNING set
        int matches = confirmCompactionRunning(getCluster().getServerContext(), ecids);
        assertTrue(matches > 0);
        client.tableOperations().delete(table1);
        confirmCompactionCompleted(getCluster().getServerContext(), ecids, TCompactionState.CANCELLED);
    }
}