Examples with StoredTabletFile org.apache.accumulo.core.metadata.StoredTabletFile used on opensource projects

Example 41 with StoredTabletFile

use of org.apache.accumulo.core.metadata.StoredTabletFile in project accumulo by apache.

the class DatafileManager method importMapFiles.

public Collection<StoredTabletFile> importMapFiles(long tid, Map<TabletFile, DataFileValue> paths, boolean setTime) throws IOException {
    String bulkDir = null;
    // once tablet files are inserted into the metadata they will become StoredTabletFiles
    Map<StoredTabletFile, DataFileValue> newFiles = new HashMap<>(paths.size());
    for (TabletFile tpath : paths.keySet()) {
        boolean inTheRightDirectory = false;
        Path parent = tpath.getPath().getParent().getParent();
        for (String tablesDir : tablet.getContext().getTablesDirs()) {
            if (parent.equals(new Path(tablesDir, tablet.getExtent().tableId().canonical()))) {
                inTheRightDirectory = true;
                break;
            }
        }
        if (!inTheRightDirectory) {
            throw new IOException("Data file " + tpath + " not in table dirs");
        }
        if (bulkDir == null)
            bulkDir = tpath.getTabletDir();
        else if (!bulkDir.equals(tpath.getTabletDir()))
            throw new IllegalArgumentException("bulk files in different dirs " + bulkDir + " " + tpath);
    }
    if (tablet.getExtent().isMeta()) {
        throw new IllegalArgumentException("Can not import files to a metadata tablet");
    }
    synchronized (bulkFileImportLock) {
        if (!paths.isEmpty()) {
            long bulkTime = Long.MIN_VALUE;
            if (setTime) {
                for (DataFileValue dfv : paths.values()) {
                    long nextTime = tablet.getAndUpdateTime();
                    if (nextTime < bulkTime)
                        throw new IllegalStateException("Time went backwards unexpectedly " + nextTime + " " + bulkTime);
                    bulkTime = nextTime;
                    dfv.setTime(bulkTime);
                }
            }
            newFiles = tablet.updatePersistedTime(bulkTime, paths, tid);
        }
    }
    synchronized (tablet) {
        for (Entry<StoredTabletFile, DataFileValue> tpath : newFiles.entrySet()) {
            if (datafileSizes.containsKey(tpath.getKey())) {
                log.error("Adding file that is already in set {}", tpath.getKey());
            }
            datafileSizes.put(tpath.getKey(), tpath.getValue());
        }
        updateCount++;
        tablet.getTabletResources().importedMapFiles();
        tablet.computeNumEntries();
    }
    for (Entry<StoredTabletFile, DataFileValue> entry : newFiles.entrySet()) {
        TabletLogger.bulkImported(tablet.getExtent(), entry.getKey());
    }
    return newFiles.keySet();
}

Example 42 with StoredTabletFile

use of org.apache.accumulo.core.metadata.StoredTabletFile in project accumulo by apache.

the class DatafileManager method waitForScansToFinish.

private TreeSet<StoredTabletFile> waitForScansToFinish(Set<StoredTabletFile> pathsToWaitFor) {
    long maxWait = 10000L;
    long startTime = System.currentTimeMillis();
    TreeSet<StoredTabletFile> inUse = new TreeSet<>();
    Span span = TraceUtil.startSpan(this.getClass(), "waitForScans");
    try (Scope scope = span.makeCurrent()) {
        synchronized (tablet) {
            for (StoredTabletFile path : pathsToWaitFor) {
                while (fileScanReferenceCounts.get(path) > 0 && System.currentTimeMillis() - startTime < maxWait) {
                    try {
                        tablet.wait(100);
                    } catch (InterruptedException e) {
                        log.warn("{}", e.getMessage(), e);
                    }
                }
            }
            for (StoredTabletFile path : pathsToWaitFor) {
                if (fileScanReferenceCounts.get(path) > 0)
                    inUse.add(path);
            }
        }
    } catch (Exception e) {
        TraceUtil.setException(span, e, true);
        throw e;
    } finally {
        span.end();
    }
    return inUse;
}

Example 43 with StoredTabletFile

use of org.apache.accumulo.core.metadata.StoredTabletFile in project accumulo by apache.

the class CompactionPlanTest method testOverlappingInputAndDelete.

@Test
public void testOverlappingInputAndDelete() {
    CompactionPlan cp1 = new CompactionPlan();
    StoredTabletFile fr1 = new StoredTabletFile("hdfs://nn1/accumulo/tables/1/t-1/1.rf");
    StoredTabletFile fr2 = new StoredTabletFile("hdfs://nn1/accumulo/tables/1/t-1/2.rf");
    cp1.inputFiles.add(fr1);
    cp1.deleteFiles.add(fr1);
    cp1.deleteFiles.add(fr2);
    Set<StoredTabletFile> allFiles = Set.of(fr1, fr2);
    assertThrows(IllegalStateException.class, () -> cp1.validate(allFiles));
}

Example 44 with StoredTabletFile

use of org.apache.accumulo.core.metadata.StoredTabletFile in project accumulo by apache.

the class CompactableImpl method processExternalMetadata.

/**
 * This method validates metadata about external compactions. It also extracts specific
 * information needed for user and selector compactions.
 */
static Optional<SelectedInfo> processExternalMetadata(Map<ExternalCompactionId, ExternalCompactionMetadata> extCompactions, Supplier<Optional<Long>> tabletCompactionId, Set<StoredTabletFile> tabletFiles, Map<ExternalCompactionId, String> externalCompactionsToRemove) {
    // Check that external compactions have disjoint sets of files. Also check that each external
    // compaction only has files inside a tablet.
    Set<StoredTabletFile> seen = new HashSet<>();
    boolean overlap = false;
    for (var entry : extCompactions.entrySet()) {
        ExternalCompactionMetadata ecMeta = entry.getValue();
        if (!tabletFiles.containsAll(ecMeta.getJobFiles())) {
            externalCompactionsToRemove.putIfAbsent(entry.getKey(), "Has files outside of tablet files");
        } else if (!Collections.disjoint(seen, ecMeta.getJobFiles())) {
            overlap = true;
        }
        seen.addAll(ecMeta.getJobFiles());
    }
    if (overlap) {
        extCompactions.keySet().forEach(ecid -> {
            externalCompactionsToRemove.putIfAbsent(ecid, "Some external compaction files overlap");
        });
        return Optional.empty();
    }
    /*
     * The rest of the code validates user compaction metadata and extracts needed information.
     *
     * For user compactions a set of files is selected. Those files then get compacted by one or
     * more compactions until the set is empty. This method attempts to reconstruct the selected set
     * of files when a tablet is loaded with an external user compaction. It avoids repeating work
     * and when a user compaction completes, files are verified against the selected set. Since the
     * data is coming from persisted storage, lots of checks are done in this method rather than
     * assuming the persisted data is correct.
     */
    CompactionKind extKind = null;
    boolean unexpectedExternal = false;
    Set<StoredTabletFile> tmpSelectedFiles = null;
    Boolean initiallySelAll = null;
    Long cid = null;
    Boolean propDel = null;
    int count = 0;
    ArrayList<String> reasons = new ArrayList<>();
    for (Entry<ExternalCompactionId, ExternalCompactionMetadata> entry : extCompactions.entrySet()) {
        var ecMeta = entry.getValue();
        if (ecMeta.getKind() != CompactionKind.USER && ecMeta.getKind() != CompactionKind.SELECTOR) {
            continue;
        }
        count++;
        if (extKind == null || extKind == ecMeta.getKind()) {
            extKind = ecMeta.getKind();
        } else {
            reasons.add("Saw USER and SELECTOR");
            unexpectedExternal = true;
            break;
        }
        if (tmpSelectedFiles == null) {
            tmpSelectedFiles = Sets.union(ecMeta.getJobFiles(), ecMeta.getNextFiles());
        } else if (!Sets.union(ecMeta.getJobFiles(), ecMeta.getNextFiles()).equals(tmpSelectedFiles)) {
            reasons.add("Selected set of files differs");
            unexpectedExternal = true;
            break;
        }
        if (initiallySelAll == null) {
            initiallySelAll = ecMeta.getInitiallySelecteAll();
        } else if (initiallySelAll != ecMeta.getInitiallySelecteAll()) {
            unexpectedExternal = true;
            reasons.add("Disagreement on selectedAll");
            break;
        }
        if (ecMeta.getKind() == CompactionKind.USER) {
            if (ecMeta.getCompactionId() == null) {
                unexpectedExternal = true;
                reasons.add("Missing compactionId");
                break;
            } else if (cid == null) {
                cid = ecMeta.getCompactionId();
            } else if (!cid.equals(ecMeta.getCompactionId())) {
                unexpectedExternal = true;
                reasons.add("Disagreement on compactionId");
                break;
            }
        } else if (ecMeta.getCompactionId() != null) {
            unexpectedExternal = true;
            reasons.add("Unexpected compactionId");
            break;
        }
        if (propDel == null) {
            propDel = ecMeta.getPropagateDeletes();
        } else if (propDel != ecMeta.getPropagateDeletes()) {
            unexpectedExternal = true;
            reasons.add("Disagreement on propagateDeletes");
            break;
        }
    }
    if (propDel != null && !propDel && count > 1) {
        unexpectedExternal = true;
        reasons.add("Concurrent compactions not propagatingDeletes");
    }
    if (extKind == CompactionKind.USER) {
        Optional<Long> compactionId = tabletCompactionId.get();
        if (compactionId.isEmpty()) {
            unexpectedExternal = true;
            reasons.add("No compaction id in zookeeper");
        } else if (!compactionId.get().equals(cid)) {
            unexpectedExternal = true;
            reasons.add("Compaction id mismatch with zookeeper");
        }
    }
    if (unexpectedExternal) {
        String reason = reasons.toString();
        extCompactions.entrySet().stream().filter(e -> {
            var kind = e.getValue().getKind();
            return kind == CompactionKind.SELECTOR || kind == CompactionKind.USER;
        }).map(Entry::getKey).forEach(ecid -> externalCompactionsToRemove.putIfAbsent(ecid, reason));
        return Optional.empty();
    }
    if (extKind != null)
        return Optional.of(new SelectedInfo(initiallySelAll, tmpSelectedFiles, extKind));
    return Optional.empty();
}

Example 45 with StoredTabletFile

use of org.apache.accumulo.core.metadata.StoredTabletFile in project accumulo by apache.

the class CompactableImpl method compact.

@Override
public void compact(CompactionServiceId service, CompactionJob job, RateLimiter readLimiter, RateLimiter writeLimiter, long queuedTime) {
    Optional<CompactionInfo> ocInfo = reserveFilesForCompaction(service, job);
    if (ocInfo.isEmpty())
        return;
    var cInfo = ocInfo.get();
    StoredTabletFile newFile = null;
    long startTime = System.currentTimeMillis();
    CompactionKind kind = job.getKind();
    CompactionStats stats = new CompactionStats();
    try {
        TabletLogger.compacting(getExtent(), job, cInfo.localCompactionCfg);
        tablet.incrementStatusMajor();
        var check = new CompactionCheck(service, kind, cInfo.checkCompactionId);
        TabletFile tmpFileName = tablet.getNextMapFilenameForMajc(cInfo.propagateDeletes);
        var compactEnv = new MajCEnv(kind, check, readLimiter, writeLimiter, cInfo.propagateDeletes);
        SortedMap<StoredTabletFile, DataFileValue> allFiles = tablet.getDatafiles();
        HashMap<StoredTabletFile, DataFileValue> compactFiles = new HashMap<>();
        cInfo.jobFiles.forEach(file -> compactFiles.put(file, allFiles.get(file)));
        stats = CompactableUtils.compact(tablet, job, cInfo, compactEnv, compactFiles, tmpFileName);
        newFile = CompactableUtils.bringOnline(tablet.getDatafileManager(), cInfo, stats, compactFiles, allFiles, kind, tmpFileName);
        TabletLogger.compacted(getExtent(), job, newFile);
    } catch (CompactionCanceledException cce) {
        log.debug("Compaction canceled {} ", getExtent());
    } catch (Exception e) {
        newFile = null;
        throw new RuntimeException(e);
    } finally {
        completeCompaction(job, cInfo.jobFiles, newFile);
        tablet.updateTimer(MAJOR, queuedTime, startTime, stats.getEntriesRead(), newFile == null);
    }
}