Examples with FileRef org.apache.accumulo.server.fs.FileRef used on opensource projects

Example 36 with FileRef

use of org.apache.accumulo.server.fs.FileRef in project accumulo by apache.

the class RootFiles method prepareReplacement.

public static void prepareReplacement(VolumeManager fs, Path location, Set<FileRef> oldDatafiles, String compactName) throws IOException {
    for (FileRef ref : oldDatafiles) {
        Path path = ref.path();
        DatafileManager.rename(fs, path, new Path(location + "/delete+" + compactName + "+" + path.getName()));
    }
}

Example 37 with FileRef

use of org.apache.accumulo.server.fs.FileRef in project accumulo by apache.

the class RootFiles method finishReplacement.

public static void finishReplacement(AccumuloConfiguration acuTableConf, VolumeManager fs, Path location, Set<FileRef> oldDatafiles, String compactName) throws IOException {
    // up later
    for (FileRef ref : oldDatafiles) {
        Path path = ref.path();
        Path deleteFile = new Path(location + "/delete+" + compactName + "+" + path.getName());
        if (acuTableConf.getBoolean(Property.GC_TRASH_IGNORE) || !fs.moveToTrash(deleteFile))
            fs.deleteRecursively(deleteFile);
    }
}

Example 38 with FileRef

use of org.apache.accumulo.server.fs.FileRef in project accumulo by apache.

the class Tablet method _majorCompact.

private CompactionStats _majorCompact(MajorCompactionReason reason) throws IOException, CompactionCanceledException {
    long t1, t2, t3;
    Pair<Long, UserCompactionConfig> compactionId = null;
    CompactionStrategy strategy = null;
    Map<FileRef, Pair<Key, Key>> firstAndLastKeys = null;
    if (reason == MajorCompactionReason.USER) {
        try {
            compactionId = getCompactionID();
            strategy = createCompactionStrategy(compactionId.getSecond().getCompactionStrategy());
        } catch (NoNodeException e) {
            throw new RuntimeException(e);
        }
    } else if (reason == MajorCompactionReason.NORMAL || reason == MajorCompactionReason.IDLE) {
        strategy = Property.createTableInstanceFromPropertyName(tableConfiguration, Property.TABLE_COMPACTION_STRATEGY, CompactionStrategy.class, new DefaultCompactionStrategy());
        strategy.init(Property.getCompactionStrategyOptions(tableConfiguration));
    } else if (reason == MajorCompactionReason.CHOP) {
        firstAndLastKeys = getFirstAndLastKeys(getDatafileManager().getDatafileSizes());
    } else {
        throw new IllegalArgumentException("Unknown compaction reason " + reason);
    }
    if (strategy != null) {
        BlockCache sc = tabletResources.getTabletServerResourceManager().getSummaryCache();
        BlockCache ic = tabletResources.getTabletServerResourceManager().getIndexCache();
        MajorCompactionRequest request = new MajorCompactionRequest(extent, reason, getTabletServer().getFileSystem(), tableConfiguration, sc, ic);
        request.setFiles(getDatafileManager().getDatafileSizes());
        strategy.gatherInformation(request);
    }
    Map<FileRef, DataFileValue> filesToCompact = null;
    int maxFilesToCompact = tableConfiguration.getCount(Property.TSERV_MAJC_THREAD_MAXOPEN);
    CompactionStats majCStats = new CompactionStats();
    CompactionPlan plan = null;
    boolean propogateDeletes = false;
    boolean updateCompactionID = false;
    synchronized (this) {
        // plan all that work that needs to be done in the sync block... then do the actual work
        // outside the sync block
        t1 = System.currentTimeMillis();
        majorCompactionState = CompactionState.WAITING_TO_START;
        getTabletMemory().waitForMinC();
        t2 = System.currentTimeMillis();
        majorCompactionState = CompactionState.IN_PROGRESS;
        notifyAll();
        VolumeManager fs = getTabletServer().getFileSystem();
        if (extent.isRootTablet()) {
            // very important that we call this before doing major compaction,
            // otherwise deleted compacted files could possible be brought back
            // at some point if the file they were compacted to was legitimately
            // removed by a major compaction
            RootFiles.cleanupReplacement(fs, fs.listStatus(this.location), false);
        }
        SortedMap<FileRef, DataFileValue> allFiles = getDatafileManager().getDatafileSizes();
        List<FileRef> inputFiles = new ArrayList<>();
        if (reason == MajorCompactionReason.CHOP) {
            // enforce rules: files with keys outside our range need to be compacted
            inputFiles.addAll(findChopFiles(extent, firstAndLastKeys, allFiles.keySet()));
        } else {
            MajorCompactionRequest request = new MajorCompactionRequest(extent, reason, tableConfiguration);
            request.setFiles(allFiles);
            plan = strategy.getCompactionPlan(request);
            if (plan != null) {
                plan.validate(allFiles.keySet());
                inputFiles.addAll(plan.inputFiles);
            }
        }
        if (inputFiles.isEmpty()) {
            if (reason == MajorCompactionReason.USER) {
                if (compactionId.getSecond().getIterators().isEmpty()) {
                    log.debug("No-op major compaction by USER on 0 input files because no iterators present.");
                    lastCompactID = compactionId.getFirst();
                    updateCompactionID = true;
                } else {
                    log.debug("Major compaction by USER on 0 input files with iterators.");
                    filesToCompact = new HashMap<>();
                }
            } else {
                return majCStats;
            }
        } else {
            // If no original files will exist at the end of the compaction, we do not have to propogate deletes
            Set<FileRef> droppedFiles = new HashSet<>();
            droppedFiles.addAll(inputFiles);
            if (plan != null)
                droppedFiles.addAll(plan.deleteFiles);
            propogateDeletes = !(droppedFiles.equals(allFiles.keySet()));
            log.debug("Major compaction plan: {} propogate deletes : {}", plan, propogateDeletes);
            filesToCompact = new HashMap<>(allFiles);
            filesToCompact.keySet().retainAll(inputFiles);
            getDatafileManager().reserveMajorCompactingFiles(filesToCompact.keySet());
        }
        t3 = System.currentTimeMillis();
    }
    try {
        log.debug(String.format("MajC initiate lock %.2f secs, wait %.2f secs", (t3 - t2) / 1000.0, (t2 - t1) / 1000.0));
        if (updateCompactionID) {
            MetadataTableUtil.updateTabletCompactID(extent, compactionId.getFirst(), tabletServer, getTabletServer().getLock());
            return majCStats;
        }
        if (!propogateDeletes && compactionId == null) {
            // compacting everything, so update the compaction id in metadata
            try {
                compactionId = getCompactionID();
                if (compactionId.getSecond().getCompactionStrategy() != null) {
                    compactionId = null;
                // TODO maybe return unless chop?
                }
            } catch (NoNodeException e) {
                throw new RuntimeException(e);
            }
        }
        List<IteratorSetting> compactionIterators = new ArrayList<>();
        if (compactionId != null) {
            if (reason == MajorCompactionReason.USER) {
                if (getCompactionCancelID() >= compactionId.getFirst()) {
                    // compaction was canceled
                    return majCStats;
                }
                compactionIterators = compactionId.getSecond().getIterators();
                synchronized (this) {
                    if (lastCompactID >= compactionId.getFirst())
                        // already compacted
                        return majCStats;
                }
            }
        }
        // ACCUMULO-3645 run loop at least once, even if filesToCompact.isEmpty()
        do {
            int numToCompact = maxFilesToCompact;
            if (filesToCompact.size() > maxFilesToCompact && filesToCompact.size() < 2 * maxFilesToCompact) {
                // on the second to last compaction pass, compact the minimum amount of files possible
                numToCompact = filesToCompact.size() - maxFilesToCompact + 1;
            }
            Set<FileRef> smallestFiles = removeSmallest(filesToCompact, numToCompact);
            FileRef fileName = getNextMapFilename((filesToCompact.size() == 0 && !propogateDeletes) ? "A" : "C");
            FileRef compactTmpName = new FileRef(fileName.path().toString() + "_tmp");
            AccumuloConfiguration tableConf = createTableConfiguration(tableConfiguration, plan);
            Span span = Trace.start("compactFiles");
            try {
                CompactionEnv cenv = new CompactionEnv() {

                    @Override
                    public boolean isCompactionEnabled() {
                        return Tablet.this.isCompactionEnabled();
                    }

                    @Override
                    public IteratorScope getIteratorScope() {
                        return IteratorScope.majc;
                    }

                    @Override
                    public RateLimiter getReadLimiter() {
                        return getTabletServer().getMajorCompactionReadLimiter();
                    }

                    @Override
                    public RateLimiter getWriteLimiter() {
                        return getTabletServer().getMajorCompactionWriteLimiter();
                    }
                };
                HashMap<FileRef, DataFileValue> copy = new HashMap<>(getDatafileManager().getDatafileSizes());
                if (!copy.keySet().containsAll(smallestFiles))
                    throw new IllegalStateException("Cannot find data file values for " + smallestFiles);
                copy.keySet().retainAll(smallestFiles);
                log.debug("Starting MajC {} ({}) {} --> {} {}", extent, reason, copy.keySet(), compactTmpName, compactionIterators);
                // always propagate deletes, unless last batch
                boolean lastBatch = filesToCompact.isEmpty();
                Compactor compactor = new Compactor(tabletServer, this, copy, null, compactTmpName, lastBatch ? propogateDeletes : true, cenv, compactionIterators, reason.ordinal(), tableConf);
                CompactionStats mcs = compactor.call();
                span.data("files", "" + smallestFiles.size());
                span.data("read", "" + mcs.getEntriesRead());
                span.data("written", "" + mcs.getEntriesWritten());
                majCStats.add(mcs);
                if (lastBatch && plan != null && plan.deleteFiles != null) {
                    smallestFiles.addAll(plan.deleteFiles);
                }
                getDatafileManager().bringMajorCompactionOnline(smallestFiles, compactTmpName, fileName, filesToCompact.size() == 0 && compactionId != null ? compactionId.getFirst() : null, new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()));
                // to add the deleted file
                if (filesToCompact.size() > 0 && mcs.getEntriesWritten() > 0) {
                    filesToCompact.put(fileName, new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()));
                }
            } finally {
                span.stop();
            }
        } while (filesToCompact.size() > 0);
        return majCStats;
    } finally {
        synchronized (Tablet.this) {
            getDatafileManager().clearMajorCompactingFile();
        }
    }
}

Example 39 with FileRef

use of org.apache.accumulo.server.fs.FileRef in project accumulo by apache.

the class Tablet method findChopFiles.

List<FileRef> findChopFiles(KeyExtent extent, Map<FileRef, Pair<Key, Key>> firstAndLastKeys, Collection<FileRef> allFiles) throws IOException {
    List<FileRef> result = new ArrayList<>();
    if (firstAndLastKeys == null) {
        result.addAll(allFiles);
        return result;
    }
    for (FileRef file : allFiles) {
        Pair<Key, Key> pair = firstAndLastKeys.get(file);
        if (pair == null) {
            // file was created or imported after we obtained the first and last keys... there
            // are a few options here... throw an exception which will cause the compaction to
            // retry and also cause ugly error message that the admin has to ignore... could
            // go get the first and last key, but this code is called while the tablet lock
            // is held... or just compact the file....
            result.add(file);
        } else {
            Key first = pair.getFirst();
            Key last = pair.getSecond();
            // If first and last are null, it's an empty file. Add it to the compact set so it goes away.
            if ((first == null && last == null) || (first != null && !extent.contains(first.getRow())) || (last != null && !extent.contains(last.getRow()))) {
                result.add(file);
            }
        }
    }
    return result;
}

Example 40 with FileRef

use of org.apache.accumulo.server.fs.FileRef in project accumulo by apache.

the class Tablet method importMapFiles.

public void importMapFiles(long tid, Map<FileRef, MapFileInfo> fileMap, boolean setTime) throws IOException {
    Map<FileRef, DataFileValue> entries = new HashMap<>(fileMap.size());
    List<String> files = new ArrayList<>();
    for (Entry<FileRef, MapFileInfo> entry : fileMap.entrySet()) {
        entries.put(entry.getKey(), new DataFileValue(entry.getValue().estimatedSize, 0l));
        files.add(entry.getKey().path().toString());
    }
    // Clients timeout and will think that this operation failed.
    // Don't do it if we spent too long waiting for the lock
    long now = System.currentTimeMillis();
    synchronized (this) {
        if (isClosed()) {
            throw new IOException("tablet " + extent + " is closed");
        }
        // TODO check seems uneeded now - ACCUMULO-1291
        long lockWait = System.currentTimeMillis() - now;
        if (lockWait > getTabletServer().getConfiguration().getTimeInMillis(Property.GENERAL_RPC_TIMEOUT)) {
            throw new IOException("Timeout waiting " + (lockWait / 1000.) + " seconds to get tablet lock");
        }
        List<FileRef> alreadyImported = bulkImported.getIfPresent(tid);
        if (alreadyImported != null) {
            for (FileRef entry : alreadyImported) {
                if (fileMap.remove(entry) != null) {
                    log.info("Ignoring import of bulk file already imported: " + entry);
                }
            }
        }
        if (fileMap.isEmpty()) {
            return;
        }
        if (writesInProgress < 0) {
            throw new IllegalStateException("writesInProgress < 0 " + writesInProgress);
        }
        writesInProgress++;
    }
    tabletServer.updateBulkImportState(files, BulkImportState.LOADING);
    try {
        getDatafileManager().importMapFiles(tid, entries, setTime);
        lastMapFileImportTime = System.currentTimeMillis();
        if (needsSplit()) {
            getTabletServer().executeSplit(this);
        } else {
            initiateMajorCompaction(MajorCompactionReason.NORMAL);
        }
    } finally {
        synchronized (this) {
            if (writesInProgress < 1)
                throw new IllegalStateException("writesInProgress < 1 " + writesInProgress);
            writesInProgress--;
            if (writesInProgress == 0)
                this.notifyAll();
            try {
                bulkImported.get(tid, new Callable<List<FileRef>>() {

                    @Override
                    public List<FileRef> call() throws Exception {
                        return new ArrayList<>();
                    }
                }).addAll(fileMap.keySet());
            } catch (Exception ex) {
                log.info(ex.toString(), ex);
            }
            tabletServer.removeBulkImportState(files);
        }
    }
}