use of org.apache.accumulo.core.spi.compaction.CompactionKind in project accumulo by apache.
the class CompactionManager method submitCompaction.
/**
* Get each configured service for the compactable tablet and submit for compaction
*/
private void submitCompaction(Compactable compactable) {
for (CompactionKind ctype : CompactionKind.values()) {
var csid = compactable.getConfiguredService(ctype);
var service = services.get(csid);
if (service == null) {
checkForConfigChanges(true);
service = services.get(csid);
if (service == null) {
log.error("Tablet {} returned non-existent compaction service {} for compaction type {}. Check" + " the table compaction dispatcher configuration. Attempting to fall back to " + "{} service.", compactable.getExtent(), csid, ctype, DEFAULT_SERVICE);
service = services.get(DEFAULT_SERVICE);
}
}
if (service != null) {
service.submitCompaction(ctype, compactable, compactablesToCheck::add);
}
}
}
use of org.apache.accumulo.core.spi.compaction.CompactionKind in project accumulo by apache.
the class CompactableImpl method processExternalMetadata.
/**
* This method validates metadata about external compactions. It also extracts specific
* information needed for user and selector compactions.
*/
static Optional<SelectedInfo> processExternalMetadata(Map<ExternalCompactionId, ExternalCompactionMetadata> extCompactions, Supplier<Optional<Long>> tabletCompactionId, Set<StoredTabletFile> tabletFiles, Map<ExternalCompactionId, String> externalCompactionsToRemove) {
// Check that external compactions have disjoint sets of files. Also check that each external
// compaction only has files inside a tablet.
Set<StoredTabletFile> seen = new HashSet<>();
boolean overlap = false;
for (var entry : extCompactions.entrySet()) {
ExternalCompactionMetadata ecMeta = entry.getValue();
if (!tabletFiles.containsAll(ecMeta.getJobFiles())) {
externalCompactionsToRemove.putIfAbsent(entry.getKey(), "Has files outside of tablet files");
} else if (!Collections.disjoint(seen, ecMeta.getJobFiles())) {
overlap = true;
}
seen.addAll(ecMeta.getJobFiles());
}
if (overlap) {
extCompactions.keySet().forEach(ecid -> {
externalCompactionsToRemove.putIfAbsent(ecid, "Some external compaction files overlap");
});
return Optional.empty();
}
/*
* The rest of the code validates user compaction metadata and extracts needed information.
*
* For user compactions a set of files is selected. Those files then get compacted by one or
* more compactions until the set is empty. This method attempts to reconstruct the selected set
* of files when a tablet is loaded with an external user compaction. It avoids repeating work
* and when a user compaction completes, files are verified against the selected set. Since the
* data is coming from persisted storage, lots of checks are done in this method rather than
* assuming the persisted data is correct.
*/
CompactionKind extKind = null;
boolean unexpectedExternal = false;
Set<StoredTabletFile> tmpSelectedFiles = null;
Boolean initiallySelAll = null;
Long cid = null;
Boolean propDel = null;
int count = 0;
ArrayList<String> reasons = new ArrayList<>();
for (Entry<ExternalCompactionId, ExternalCompactionMetadata> entry : extCompactions.entrySet()) {
var ecMeta = entry.getValue();
if (ecMeta.getKind() != CompactionKind.USER && ecMeta.getKind() != CompactionKind.SELECTOR) {
continue;
}
count++;
if (extKind == null || extKind == ecMeta.getKind()) {
extKind = ecMeta.getKind();
} else {
reasons.add("Saw USER and SELECTOR");
unexpectedExternal = true;
break;
}
if (tmpSelectedFiles == null) {
tmpSelectedFiles = Sets.union(ecMeta.getJobFiles(), ecMeta.getNextFiles());
} else if (!Sets.union(ecMeta.getJobFiles(), ecMeta.getNextFiles()).equals(tmpSelectedFiles)) {
reasons.add("Selected set of files differs");
unexpectedExternal = true;
break;
}
if (initiallySelAll == null) {
initiallySelAll = ecMeta.getInitiallySelecteAll();
} else if (initiallySelAll != ecMeta.getInitiallySelecteAll()) {
unexpectedExternal = true;
reasons.add("Disagreement on selectedAll");
break;
}
if (ecMeta.getKind() == CompactionKind.USER) {
if (ecMeta.getCompactionId() == null) {
unexpectedExternal = true;
reasons.add("Missing compactionId");
break;
} else if (cid == null) {
cid = ecMeta.getCompactionId();
} else if (!cid.equals(ecMeta.getCompactionId())) {
unexpectedExternal = true;
reasons.add("Disagreement on compactionId");
break;
}
} else if (ecMeta.getCompactionId() != null) {
unexpectedExternal = true;
reasons.add("Unexpected compactionId");
break;
}
if (propDel == null) {
propDel = ecMeta.getPropagateDeletes();
} else if (propDel != ecMeta.getPropagateDeletes()) {
unexpectedExternal = true;
reasons.add("Disagreement on propagateDeletes");
break;
}
}
if (propDel != null && !propDel && count > 1) {
unexpectedExternal = true;
reasons.add("Concurrent compactions not propagatingDeletes");
}
if (extKind == CompactionKind.USER) {
Optional<Long> compactionId = tabletCompactionId.get();
if (compactionId.isEmpty()) {
unexpectedExternal = true;
reasons.add("No compaction id in zookeeper");
} else if (!compactionId.get().equals(cid)) {
unexpectedExternal = true;
reasons.add("Compaction id mismatch with zookeeper");
}
}
if (unexpectedExternal) {
String reason = reasons.toString();
extCompactions.entrySet().stream().filter(e -> {
var kind = e.getValue().getKind();
return kind == CompactionKind.SELECTOR || kind == CompactionKind.USER;
}).map(Entry::getKey).forEach(ecid -> externalCompactionsToRemove.putIfAbsent(ecid, reason));
return Optional.empty();
}
if (extKind != null)
return Optional.of(new SelectedInfo(initiallySelAll, tmpSelectedFiles, extKind));
return Optional.empty();
}
use of org.apache.accumulo.core.spi.compaction.CompactionKind in project accumulo by apache.
the class CompactableImpl method compact.
@Override
public void compact(CompactionServiceId service, CompactionJob job, RateLimiter readLimiter, RateLimiter writeLimiter, long queuedTime) {
Optional<CompactionInfo> ocInfo = reserveFilesForCompaction(service, job);
if (ocInfo.isEmpty())
return;
var cInfo = ocInfo.get();
StoredTabletFile newFile = null;
long startTime = System.currentTimeMillis();
CompactionKind kind = job.getKind();
CompactionStats stats = new CompactionStats();
try {
TabletLogger.compacting(getExtent(), job, cInfo.localCompactionCfg);
tablet.incrementStatusMajor();
var check = new CompactionCheck(service, kind, cInfo.checkCompactionId);
TabletFile tmpFileName = tablet.getNextMapFilenameForMajc(cInfo.propagateDeletes);
var compactEnv = new MajCEnv(kind, check, readLimiter, writeLimiter, cInfo.propagateDeletes);
SortedMap<StoredTabletFile, DataFileValue> allFiles = tablet.getDatafiles();
HashMap<StoredTabletFile, DataFileValue> compactFiles = new HashMap<>();
cInfo.jobFiles.forEach(file -> compactFiles.put(file, allFiles.get(file)));
stats = CompactableUtils.compact(tablet, job, cInfo, compactEnv, compactFiles, tmpFileName);
newFile = CompactableUtils.bringOnline(tablet.getDatafileManager(), cInfo, stats, compactFiles, allFiles, kind, tmpFileName);
TabletLogger.compacted(getExtent(), job, newFile);
} catch (CompactionCanceledException cce) {
log.debug("Compaction canceled {} ", getExtent());
} catch (Exception e) {
newFile = null;
throw new RuntimeException(e);
} finally {
completeCompaction(job, cInfo.jobFiles, newFile);
tablet.updateTimer(MAJOR, queuedTime, startTime, stats.getEntriesRead(), newFile == null);
}
}
use of org.apache.accumulo.core.spi.compaction.CompactionKind in project accumulo by apache.
the class CompactableImpl method getConfiguredService.
@Override
public CompactionServiceId getConfiguredService(CompactionKind kind) {
Map<String, String> debugHints = null;
try {
var dispatcher = tablet.getTableConfiguration().getCompactionDispatcher();
Map<String, String> tmpHints = Map.of();
if (kind == CompactionKind.USER) {
synchronized (this) {
if (fileMgr.getSelectionStatus() != FileSelectionStatus.NOT_ACTIVE && fileMgr.getSelectionStatus() != FileSelectionStatus.CANCELED && fileMgr.getSelectionKind() == CompactionKind.USER) {
tmpHints = compactionConfig.getExecutionHints();
}
}
}
var hints = tmpHints;
debugHints = hints;
var dispatch = dispatcher.dispatch(new DispatchParameters() {
private final ServiceEnvironment senv = new ServiceEnvironmentImpl(tablet.getContext());
@Override
public ServiceEnvironment getServiceEnv() {
return senv;
}
@Override
public Map<String, String> getExecutionHints() {
return hints;
}
@Override
public CompactionKind getCompactionKind() {
return kind;
}
@Override
public CompactionServices getCompactionServices() {
return manager.getServices();
}
});
return dispatch.getService();
} catch (RuntimeException e) {
log.error("Failed to dispatch compaction {} kind:{} hints:{}, falling back to {} service.", getExtent(), kind, debugHints, CompactionServicesConfig.DEFAULT_SERVICE, e);
return CompactionServicesConfig.DEFAULT_SERVICE;
}
}
Aggregations