Examples with ExternalCompactionFinalState org.apache.accumulo.core.metadata.schema.ExternalCompactionFinalState used on opensource projects

Example 1 with ExternalCompactionFinalState

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionFinalState in project accumulo by apache.

the class ServerAmpleImpl method putExternalCompactionFinalStates.

@Override
public void putExternalCompactionFinalStates(Collection<ExternalCompactionFinalState> finalStates) {
    try (BatchWriter writer = context.createBatchWriter(DataLevel.USER.metaTable())) {
        String prefix = ExternalCompactionSection.getRowPrefix();
        for (ExternalCompactionFinalState finalState : finalStates) {
            Mutation m = new Mutation(prefix + finalState.getExternalCompactionId().canonical());
            m.put("", "", finalState.toJson());
            writer.addMutation(m);
        }
    } catch (MutationsRejectedException | TableNotFoundException e) {
        throw new RuntimeException(e);
    }
}

Example 2 with ExternalCompactionFinalState

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionFinalState in project accumulo by apache.

the class CompactionFinalizer method processPending.

private void processPending() {
    while (!Thread.interrupted()) {
        try {
            ArrayList<ExternalCompactionFinalState> batch = new ArrayList<>();
            batch.add(pendingNotifications.take());
            pendingNotifications.drainTo(batch);
            List<Future<?>> futures = new ArrayList<>();
            List<ExternalCompactionId> statusesToDelete = new ArrayList<>();
            Map<KeyExtent, TabletMetadata> tabletsMetadata;
            var extents = batch.stream().map(ExternalCompactionFinalState::getExtent).collect(toList());
            try (TabletsMetadata tablets = context.getAmple().readTablets().forTablets(extents).fetch(ColumnType.LOCATION, ColumnType.PREV_ROW, ColumnType.ECOMP).build()) {
                tabletsMetadata = tablets.stream().collect(toMap(TabletMetadata::getExtent, identity()));
            }
            for (ExternalCompactionFinalState ecfs : batch) {
                TabletMetadata tabletMetadata = tabletsMetadata.get(ecfs.getExtent());
                if (tabletMetadata == null || !tabletMetadata.getExternalCompactions().keySet().contains(ecfs.getExternalCompactionId())) {
                    // there is not per tablet external compaction entry, so delete its final state marker
                    // from metadata table
                    LOG.debug("Unable to find tablets external compaction entry, deleting completion entry {}", ecfs);
                    statusesToDelete.add(ecfs.getExternalCompactionId());
                } else if (tabletMetadata.getLocation() != null && tabletMetadata.getLocation().getType() == LocationType.CURRENT) {
                    futures.add(ntfyExecutor.submit(() -> notifyTserver(tabletMetadata.getLocation(), ecfs)));
                } else {
                    LOG.trace("External compaction {} is completed, but there is no location for tablet.  Unable to notify tablet, will try again later.", ecfs);
                }
            }
            if (!statusesToDelete.isEmpty()) {
                LOG.info("Deleting unresolvable completed external compactions from metadata table, ids: {}", statusesToDelete);
                context.getAmple().deleteExternalCompactionFinalStates(statusesToDelete);
            }
            for (Future<?> future : futures) {
                try {
                    future.get();
                } catch (ExecutionException e) {
                    LOG.debug("Failed to notify tserver", e);
                }
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new RuntimeException(e);
        } catch (RuntimeException e) {
            LOG.warn("Failed to process pending notifications", e);
        }
    }
}

Example 3 with ExternalCompactionFinalState

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionFinalState in project accumulo by apache.

the class CompactionFinalizer method notifyTservers.

private void notifyTservers() {
    try {
        Iterator<ExternalCompactionFinalState> finalStates = context.getAmple().getExternalCompactionFinalStates().iterator();
        while (finalStates.hasNext()) {
            ExternalCompactionFinalState state = finalStates.next();
            LOG.debug("Found external compaction in final state: {}, queueing for tserver notification", state);
            pendingNotifications.put(state);
        }
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new RuntimeException(e);
    } catch (RuntimeException e) {
        LOG.warn("Failed to notify tservers", e);
    }
}

Example 4 with ExternalCompactionFinalState

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionFinalState in project accumulo by apache.

the class ExternalCompaction_1_IT method testExternalCompactionDeadTServer.

@Test
public void testExternalCompactionDeadTServer() throws Exception {
    // Shut down the normal TServers
    getCluster().getProcesses().get(TABLET_SERVER).forEach(p -> {
        try {
            getCluster().killProcess(TABLET_SERVER, p);
        } catch (Exception e) {
            fail("Failed to shutdown tablet server");
        }
    });
    // Start our TServer that will not commit the compaction
    ProcessInfo tserverProcess = getCluster().exec(ExternalCompactionTServer.class);
    final String table3 = this.getUniqueNames(1)[0];
    try (final AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
        createTable(client, table3, "cs7");
        writeData(client, table3);
        getCluster().getClusterControl().startCompactors(Compactor.class, 1, QUEUE7);
        getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class);
        compact(client, table3, 2, QUEUE7, false);
        // ExternalCompactionTServer will not commit the compaction. Wait for the
        // metadata table entries to show up.
        LOG.info("Waiting for external compaction to complete.");
        TableId tid = getCluster().getServerContext().getTableId(table3);
        Stream<ExternalCompactionFinalState> fs = getFinalStatesForTable(getCluster(), tid);
        while (fs.count() == 0) {
            LOG.info("Waiting for compaction completed marker to appear");
            UtilWaitThread.sleep(250);
            fs = getFinalStatesForTable(getCluster(), tid);
        }
        LOG.info("Validating metadata table contents.");
        TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets().forTable(tid).fetch(ColumnType.ECOMP).build();
        List<TabletMetadata> md = new ArrayList<>();
        tm.forEach(t -> md.add(t));
        assertEquals(1, md.size());
        TabletMetadata m = md.get(0);
        Map<ExternalCompactionId, ExternalCompactionMetadata> em = m.getExternalCompactions();
        assertEquals(1, em.size());
        List<ExternalCompactionFinalState> finished = new ArrayList<>();
        getFinalStatesForTable(getCluster(), tid).forEach(f -> finished.add(f));
        assertEquals(1, finished.size());
        assertEquals(em.entrySet().iterator().next().getKey(), finished.get(0).getExternalCompactionId());
        tm.close();
        // Force a flush on the metadata table before killing our tserver
        client.tableOperations().flush("accumulo.metadata");
        // Stop our TabletServer. Need to perform a normal shutdown so that the WAL is closed
        // normally.
        LOG.info("Stopping our tablet server");
        getCluster().stopProcessWithTimeout(tserverProcess.getProcess(), 30, TimeUnit.SECONDS);
        getCluster().getClusterControl().stop(ServerType.TABLET_SERVER);
        // Start a TabletServer to commit the compaction.
        LOG.info("Starting normal tablet server");
        getCluster().getClusterControl().start(ServerType.TABLET_SERVER);
        // Wait for the compaction to be committed.
        LOG.info("Waiting for compaction completed marker to disappear");
        Stream<ExternalCompactionFinalState> fs2 = getFinalStatesForTable(getCluster(), tid);
        while (fs2.count() != 0) {
            LOG.info("Waiting for compaction completed marker to disappear");
            UtilWaitThread.sleep(500);
            fs2 = getFinalStatesForTable(getCluster(), tid);
        }
        verify(client, table3, 2);
        // We need to cancel the compaction or delete the table here because we initiate a user
        // compaction above in the test. Even though the external compaction was cancelled
        // because we split the table, FaTE will continue to queue up a compaction
        client.tableOperations().cancelCompaction(table3);
    }
}