Examples with ExternalCompactionId org.apache.accumulo.core.metadata.schema.ExternalCompactionId used on opensource projects

Example 21 with ExternalCompactionId

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionId in project accumulo by apache.

the class ExternalCompaction_2_IT method testExternalCompactionsSucceedsRunWithTableOffline.

@Test
public void testExternalCompactionsSucceedsRunWithTableOffline() throws Exception {
    getCluster().getClusterControl().stop(ServerType.COMPACTION_COORDINATOR);
    getCluster().getClusterControl().stop(ServerType.COMPACTOR);
    String table1 = this.getUniqueNames(1)[0];
    try (AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
        createTable(client, table1, "cs2");
        // set compaction ratio to 1 so that majc occurs naturally, not user compaction
        // user compaction blocks merge
        client.tableOperations().setProperty(table1, Property.TABLE_MAJC_RATIO.toString(), "1.0");
        // cause multiple rfiles to be created
        writeData(client, table1);
        writeData(client, table1);
        writeData(client, table1);
        writeData(client, table1);
        getCluster().getClusterControl().startCoordinator(TestCompactionCoordinatorForOfflineTable.class);
        TableId tid = getCluster().getServerContext().getTableId(table1);
        // Confirm that no final state is in the metadata table
        assertEquals(0, getFinalStatesForTable(getCluster(), tid).count());
        // Offline the table when the compaction starts
        final AtomicBoolean succeededInTakingOffline = new AtomicBoolean(false);
        Thread t = new Thread(() -> {
            try (AccumuloClient client2 = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
                TExternalCompactionList metrics2 = getRunningCompactions(getCluster().getServerContext());
                while (metrics2.getCompactions() == null) {
                    metrics2 = getRunningCompactions(getCluster().getServerContext());
                    if (metrics2.getCompactions() == null) {
                        UtilWaitThread.sleep(50);
                    }
                }
                LOG.info("Taking table offline");
                client2.tableOperations().offline(table1, false);
                succeededInTakingOffline.set(true);
            } catch (Exception e) {
                LOG.error("Error: ", e);
            }
        });
        t.start();
        // Start the compactor
        getCluster().getClusterControl().startCompactors(Compactor.class, 1, QUEUE2);
        // Wait for the compaction to start by waiting for 1 external compaction column
        Set<ExternalCompactionId> ecids = ExternalCompactionTestUtils.waitForCompactionStartAndReturnEcids(getCluster().getServerContext(), tid);
        // Confirm that this ECID shows up in RUNNING set
        int matches = ExternalCompactionTestUtils.confirmCompactionRunning(getCluster().getServerContext(), ecids);
        assertTrue(matches > 0);
        t.join();
        if (!succeededInTakingOffline.get()) {
            fail("Failed to offline table");
        }
        confirmCompactionCompleted(getCluster().getServerContext(), ecids, TCompactionState.SUCCEEDED);
        // Confirm that final state is in the metadata table
        assertEquals(1, getFinalStatesForTable(getCluster(), tid).count());
        // Online the table
        client.tableOperations().online(table1);
        // wait for compaction to be committed by tserver or test timeout
        long finalStateCount = getFinalStatesForTable(getCluster(), tid).count();
        while (finalStateCount > 0) {
            finalStateCount = getFinalStatesForTable(getCluster(), tid).count();
            if (finalStateCount > 0) {
                UtilWaitThread.sleep(50);
            }
        }
        // We need to cancel the compaction or delete the table here because we initiate a user
        // compaction above in the test. Even though the external compaction was cancelled
        // because we split the table, FaTE will continue to queue up a compaction
        client.tableOperations().delete(table1);
        getCluster().getClusterControl().stop(ServerType.COMPACTION_COORDINATOR);
        getCluster().getClusterControl().stop(ServerType.COMPACTOR);
    }
}

Example 22 with ExternalCompactionId

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionId in project accumulo by apache.

the class CompactableImpl method processExternalMetadata.

/**
 * This method validates metadata about external compactions. It also extracts specific
 * information needed for user and selector compactions.
 */
static Optional<SelectedInfo> processExternalMetadata(Map<ExternalCompactionId, ExternalCompactionMetadata> extCompactions, Supplier<Optional<Long>> tabletCompactionId, Set<StoredTabletFile> tabletFiles, Map<ExternalCompactionId, String> externalCompactionsToRemove) {
    // Check that external compactions have disjoint sets of files. Also check that each external
    // compaction only has files inside a tablet.
    Set<StoredTabletFile> seen = new HashSet<>();
    boolean overlap = false;
    for (var entry : extCompactions.entrySet()) {
        ExternalCompactionMetadata ecMeta = entry.getValue();
        if (!tabletFiles.containsAll(ecMeta.getJobFiles())) {
            externalCompactionsToRemove.putIfAbsent(entry.getKey(), "Has files outside of tablet files");
        } else if (!Collections.disjoint(seen, ecMeta.getJobFiles())) {
            overlap = true;
        }
        seen.addAll(ecMeta.getJobFiles());
    }
    if (overlap) {
        extCompactions.keySet().forEach(ecid -> {
            externalCompactionsToRemove.putIfAbsent(ecid, "Some external compaction files overlap");
        });
        return Optional.empty();
    }
    /*
     * The rest of the code validates user compaction metadata and extracts needed information.
     *
     * For user compactions a set of files is selected. Those files then get compacted by one or
     * more compactions until the set is empty. This method attempts to reconstruct the selected set
     * of files when a tablet is loaded with an external user compaction. It avoids repeating work
     * and when a user compaction completes, files are verified against the selected set. Since the
     * data is coming from persisted storage, lots of checks are done in this method rather than
     * assuming the persisted data is correct.
     */
    CompactionKind extKind = null;
    boolean unexpectedExternal = false;
    Set<StoredTabletFile> tmpSelectedFiles = null;
    Boolean initiallySelAll = null;
    Long cid = null;
    Boolean propDel = null;
    int count = 0;
    ArrayList<String> reasons = new ArrayList<>();
    for (Entry<ExternalCompactionId, ExternalCompactionMetadata> entry : extCompactions.entrySet()) {
        var ecMeta = entry.getValue();
        if (ecMeta.getKind() != CompactionKind.USER && ecMeta.getKind() != CompactionKind.SELECTOR) {
            continue;
        }
        count++;
        if (extKind == null || extKind == ecMeta.getKind()) {
            extKind = ecMeta.getKind();
        } else {
            reasons.add("Saw USER and SELECTOR");
            unexpectedExternal = true;
            break;
        }
        if (tmpSelectedFiles == null) {
            tmpSelectedFiles = Sets.union(ecMeta.getJobFiles(), ecMeta.getNextFiles());
        } else if (!Sets.union(ecMeta.getJobFiles(), ecMeta.getNextFiles()).equals(tmpSelectedFiles)) {
            reasons.add("Selected set of files differs");
            unexpectedExternal = true;
            break;
        }
        if (initiallySelAll == null) {
            initiallySelAll = ecMeta.getInitiallySelecteAll();
        } else if (initiallySelAll != ecMeta.getInitiallySelecteAll()) {
            unexpectedExternal = true;
            reasons.add("Disagreement on selectedAll");
            break;
        }
        if (ecMeta.getKind() == CompactionKind.USER) {
            if (ecMeta.getCompactionId() == null) {
                unexpectedExternal = true;
                reasons.add("Missing compactionId");
                break;
            } else if (cid == null) {
                cid = ecMeta.getCompactionId();
            } else if (!cid.equals(ecMeta.getCompactionId())) {
                unexpectedExternal = true;
                reasons.add("Disagreement on compactionId");
                break;
            }
        } else if (ecMeta.getCompactionId() != null) {
            unexpectedExternal = true;
            reasons.add("Unexpected compactionId");
            break;
        }
        if (propDel == null) {
            propDel = ecMeta.getPropagateDeletes();
        } else if (propDel != ecMeta.getPropagateDeletes()) {
            unexpectedExternal = true;
            reasons.add("Disagreement on propagateDeletes");
            break;
        }
    }
    if (propDel != null && !propDel && count > 1) {
        unexpectedExternal = true;
        reasons.add("Concurrent compactions not propagatingDeletes");
    }
    if (extKind == CompactionKind.USER) {
        Optional<Long> compactionId = tabletCompactionId.get();
        if (compactionId.isEmpty()) {
            unexpectedExternal = true;
            reasons.add("No compaction id in zookeeper");
        } else if (!compactionId.get().equals(cid)) {
            unexpectedExternal = true;
            reasons.add("Compaction id mismatch with zookeeper");
        }
    }
    if (unexpectedExternal) {
        String reason = reasons.toString();
        extCompactions.entrySet().stream().filter(e -> {
            var kind = e.getValue().getKind();
            return kind == CompactionKind.SELECTOR || kind == CompactionKind.USER;
        }).map(Entry::getKey).forEach(ecid -> externalCompactionsToRemove.putIfAbsent(ecid, reason));
        return Optional.empty();
    }
    if (extKind != null)
        return Optional.of(new SelectedInfo(initiallySelAll, tmpSelectedFiles, extKind));
    return Optional.empty();
}