Examples with ExternalCompactionMetadata org.apache.accumulo.core.metadata.schema.ExternalCompactionMetadata used on opensource projects

Example 1 with ExternalCompactionMetadata

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionMetadata in project accumulo by apache.

the class CompactableImplTest method newECM.

private static ExternalCompactionMetadata newECM(Set<StoredTabletFile> jobFiles, Set<StoredTabletFile> nextFiles, CompactionKind kind, boolean propagateDeletes, boolean initiallySelectedAll, Long compactionId) {
    TabletFile compactTmpName = newFile("C00000A.rf_tmp");
    String compactorId = "cid";
    short priority = 9;
    CompactionExecutorId ceid = CompactionExecutorIdImpl.externalId("ecs1");
    return new ExternalCompactionMetadata(jobFiles, nextFiles, compactTmpName, compactorId, kind, priority, ceid, propagateDeletes, initiallySelectedAll, compactionId);
}

Example 2 with ExternalCompactionMetadata

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionMetadata in project accumulo by apache.

the class Tablet method removeOldTemporaryFiles.

private void removeOldTemporaryFiles(Map<ExternalCompactionId, ExternalCompactionMetadata> externalCompactions) {
    // remove any temporary files created by a previous tablet server
    try {
        var extCompactionFiles = externalCompactions.values().stream().map(ecMeta -> ecMeta.getCompactTmpName().getPath()).collect(Collectors.toSet());
        for (Volume volume : getTabletServer().getVolumeManager().getVolumes()) {
            String dirUri = volume.getBasePath() + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + extent.tableId() + Path.SEPARATOR + dirName;
            for (FileStatus tmp : volume.getFileSystem().globStatus(new Path(dirUri, "*_tmp"))) {
                if (extCompactionFiles.contains(tmp.getPath())) {
                    continue;
                }
                try {
                    log.debug("Removing old temp file {}", tmp.getPath());
                    volume.getFileSystem().delete(tmp.getPath(), false);
                } catch (IOException ex) {
                    log.error("Unable to remove old temp file " + tmp.getPath() + ": " + ex);
                }
            }
        }
    } catch (IOException ex) {
        log.error("Error scanning for old temp files", ex);
    }
}

Example 3 with ExternalCompactionMetadata

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionMetadata in project accumulo by apache.

the class CompactableImpl method processExternalMetadata.

/**
 * This method validates metadata about external compactions. It also extracts specific
 * information needed for user and selector compactions.
 */
static Optional<SelectedInfo> processExternalMetadata(Map<ExternalCompactionId, ExternalCompactionMetadata> extCompactions, Supplier<Optional<Long>> tabletCompactionId, Set<StoredTabletFile> tabletFiles, Map<ExternalCompactionId, String> externalCompactionsToRemove) {
    // Check that external compactions have disjoint sets of files. Also check that each external
    // compaction only has files inside a tablet.
    Set<StoredTabletFile> seen = new HashSet<>();
    boolean overlap = false;
    for (var entry : extCompactions.entrySet()) {
        ExternalCompactionMetadata ecMeta = entry.getValue();
        if (!tabletFiles.containsAll(ecMeta.getJobFiles())) {
            externalCompactionsToRemove.putIfAbsent(entry.getKey(), "Has files outside of tablet files");
        } else if (!Collections.disjoint(seen, ecMeta.getJobFiles())) {
            overlap = true;
        }
        seen.addAll(ecMeta.getJobFiles());
    }
    if (overlap) {
        extCompactions.keySet().forEach(ecid -> {
            externalCompactionsToRemove.putIfAbsent(ecid, "Some external compaction files overlap");
        });
        return Optional.empty();
    }
    /*
     * The rest of the code validates user compaction metadata and extracts needed information.
     *
     * For user compactions a set of files is selected. Those files then get compacted by one or
     * more compactions until the set is empty. This method attempts to reconstruct the selected set
     * of files when a tablet is loaded with an external user compaction. It avoids repeating work
     * and when a user compaction completes, files are verified against the selected set. Since the
     * data is coming from persisted storage, lots of checks are done in this method rather than
     * assuming the persisted data is correct.
     */
    CompactionKind extKind = null;
    boolean unexpectedExternal = false;
    Set<StoredTabletFile> tmpSelectedFiles = null;
    Boolean initiallySelAll = null;
    Long cid = null;
    Boolean propDel = null;
    int count = 0;
    ArrayList<String> reasons = new ArrayList<>();
    for (Entry<ExternalCompactionId, ExternalCompactionMetadata> entry : extCompactions.entrySet()) {
        var ecMeta = entry.getValue();
        if (ecMeta.getKind() != CompactionKind.USER && ecMeta.getKind() != CompactionKind.SELECTOR) {
            continue;
        }
        count++;
        if (extKind == null || extKind == ecMeta.getKind()) {
            extKind = ecMeta.getKind();
        } else {
            reasons.add("Saw USER and SELECTOR");
            unexpectedExternal = true;
            break;
        }
        if (tmpSelectedFiles == null) {
            tmpSelectedFiles = Sets.union(ecMeta.getJobFiles(), ecMeta.getNextFiles());
        } else if (!Sets.union(ecMeta.getJobFiles(), ecMeta.getNextFiles()).equals(tmpSelectedFiles)) {
            reasons.add("Selected set of files differs");
            unexpectedExternal = true;
            break;
        }
        if (initiallySelAll == null) {
            initiallySelAll = ecMeta.getInitiallySelecteAll();
        } else if (initiallySelAll != ecMeta.getInitiallySelecteAll()) {
            unexpectedExternal = true;
            reasons.add("Disagreement on selectedAll");
            break;
        }
        if (ecMeta.getKind() == CompactionKind.USER) {
            if (ecMeta.getCompactionId() == null) {
                unexpectedExternal = true;
                reasons.add("Missing compactionId");
                break;
            } else if (cid == null) {
                cid = ecMeta.getCompactionId();
            } else if (!cid.equals(ecMeta.getCompactionId())) {
                unexpectedExternal = true;
                reasons.add("Disagreement on compactionId");
                break;
            }
        } else if (ecMeta.getCompactionId() != null) {
            unexpectedExternal = true;
            reasons.add("Unexpected compactionId");
            break;
        }
        if (propDel == null) {
            propDel = ecMeta.getPropagateDeletes();
        } else if (propDel != ecMeta.getPropagateDeletes()) {
            unexpectedExternal = true;
            reasons.add("Disagreement on propagateDeletes");
            break;
        }
    }
    if (propDel != null && !propDel && count > 1) {
        unexpectedExternal = true;
        reasons.add("Concurrent compactions not propagatingDeletes");
    }
    if (extKind == CompactionKind.USER) {
        Optional<Long> compactionId = tabletCompactionId.get();
        if (compactionId.isEmpty()) {
            unexpectedExternal = true;
            reasons.add("No compaction id in zookeeper");
        } else if (!compactionId.get().equals(cid)) {
            unexpectedExternal = true;
            reasons.add("Compaction id mismatch with zookeeper");
        }
    }
    if (unexpectedExternal) {
        String reason = reasons.toString();
        extCompactions.entrySet().stream().filter(e -> {
            var kind = e.getValue().getKind();
            return kind == CompactionKind.SELECTOR || kind == CompactionKind.USER;
        }).map(Entry::getKey).forEach(ecid -> externalCompactionsToRemove.putIfAbsent(ecid, reason));
        return Optional.empty();
    }
    if (extKind != null)
        return Optional.of(new SelectedInfo(initiallySelAll, tmpSelectedFiles, extKind));
    return Optional.empty();
}

Example 4 with ExternalCompactionMetadata

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionMetadata in project accumulo by apache.

the class CompactableImpl method reserveExternalCompaction.

@Override
public ExternalCompactionJob reserveExternalCompaction(CompactionServiceId service, CompactionJob job, String compactorId, ExternalCompactionId externalCompactionId) {
    Preconditions.checkState(!tablet.getExtent().isMeta());
    Optional<CompactionInfo> ocInfo = reserveFilesForCompaction(service, job);
    if (ocInfo.isEmpty())
        return null;
    var cInfo = ocInfo.get();
    try {
        Map<String, String> overrides = CompactableUtils.getOverrides(job.getKind(), tablet, cInfo.localHelper, job.getFiles());
        TabletFile compactTmpName = tablet.getNextMapFilenameForMajc(cInfo.propagateDeletes);
        ExternalCompactionInfo ecInfo = new ExternalCompactionInfo();
        ecInfo.meta = new ExternalCompactionMetadata(cInfo.jobFiles, Sets.difference(cInfo.selectedFiles, cInfo.jobFiles), compactTmpName, compactorId, job.getKind(), job.getPriority(), job.getExecutor(), cInfo.propagateDeletes, cInfo.initiallySelectedAll, cInfo.checkCompactionId);
        tablet.getContext().getAmple().mutateTablet(getExtent()).putExternalCompaction(externalCompactionId, ecInfo.meta).mutate();
        ecInfo.job = job;
        externalCompactions.put(externalCompactionId, ecInfo);
        SortedMap<StoredTabletFile, DataFileValue> allFiles = tablet.getDatafiles();
        HashMap<StoredTabletFile, DataFileValue> compactFiles = new HashMap<>();
        cInfo.jobFiles.forEach(file -> compactFiles.put(file, allFiles.get(file)));
        TabletLogger.compacting(getExtent(), job, cInfo.localCompactionCfg);
        return new ExternalCompactionJob(compactFiles, cInfo.propagateDeletes, compactTmpName, getExtent(), externalCompactionId, job.getKind(), cInfo.iters, cInfo.checkCompactionId, overrides);
    } catch (Exception e) {
        externalCompactions.remove(externalCompactionId);
        completeCompaction(job, cInfo.jobFiles, null);
        throw new RuntimeException(e);
    }
}

Example 5 with ExternalCompactionMetadata

use of org.apache.accumulo.core.metadata.schema.ExternalCompactionMetadata in project accumulo by apache.

the class ExternalCompaction_1_IT method testExternalCompactionDeadTServer.

@Test
public void testExternalCompactionDeadTServer() throws Exception {
    // Shut down the normal TServers
    getCluster().getProcesses().get(TABLET_SERVER).forEach(p -> {
        try {
            getCluster().killProcess(TABLET_SERVER, p);
        } catch (Exception e) {
            fail("Failed to shutdown tablet server");
        }
    });
    // Start our TServer that will not commit the compaction
    ProcessInfo tserverProcess = getCluster().exec(ExternalCompactionTServer.class);
    final String table3 = this.getUniqueNames(1)[0];
    try (final AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
        createTable(client, table3, "cs7");
        writeData(client, table3);
        getCluster().getClusterControl().startCompactors(Compactor.class, 1, QUEUE7);
        getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class);
        compact(client, table3, 2, QUEUE7, false);
        // ExternalCompactionTServer will not commit the compaction. Wait for the
        // metadata table entries to show up.
        LOG.info("Waiting for external compaction to complete.");
        TableId tid = getCluster().getServerContext().getTableId(table3);
        Stream<ExternalCompactionFinalState> fs = getFinalStatesForTable(getCluster(), tid);
        while (fs.count() == 0) {
            LOG.info("Waiting for compaction completed marker to appear");
            UtilWaitThread.sleep(250);
            fs = getFinalStatesForTable(getCluster(), tid);
        }
        LOG.info("Validating metadata table contents.");
        TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets().forTable(tid).fetch(ColumnType.ECOMP).build();
        List<TabletMetadata> md = new ArrayList<>();
        tm.forEach(t -> md.add(t));
        assertEquals(1, md.size());
        TabletMetadata m = md.get(0);
        Map<ExternalCompactionId, ExternalCompactionMetadata> em = m.getExternalCompactions();
        assertEquals(1, em.size());
        List<ExternalCompactionFinalState> finished = new ArrayList<>();
        getFinalStatesForTable(getCluster(), tid).forEach(f -> finished.add(f));
        assertEquals(1, finished.size());
        assertEquals(em.entrySet().iterator().next().getKey(), finished.get(0).getExternalCompactionId());
        tm.close();
        // Force a flush on the metadata table before killing our tserver
        client.tableOperations().flush("accumulo.metadata");
        // Stop our TabletServer. Need to perform a normal shutdown so that the WAL is closed
        // normally.
        LOG.info("Stopping our tablet server");
        getCluster().stopProcessWithTimeout(tserverProcess.getProcess(), 30, TimeUnit.SECONDS);
        getCluster().getClusterControl().stop(ServerType.TABLET_SERVER);
        // Start a TabletServer to commit the compaction.
        LOG.info("Starting normal tablet server");
        getCluster().getClusterControl().start(ServerType.TABLET_SERVER);
        // Wait for the compaction to be committed.
        LOG.info("Waiting for compaction completed marker to disappear");
        Stream<ExternalCompactionFinalState> fs2 = getFinalStatesForTable(getCluster(), tid);
        while (fs2.count() != 0) {
            LOG.info("Waiting for compaction completed marker to disappear");
            UtilWaitThread.sleep(500);
            fs2 = getFinalStatesForTable(getCluster(), tid);
        }
        verify(client, table3, 2);
        // We need to cancel the compaction or delete the table here because we initiate a user
        // compaction above in the test. Even though the external compaction was cancelled
        // because we split the table, FaTE will continue to queue up a compaction
        client.tableOperations().cancelCompaction(table3);
    }
}