Examples with DataFileValue org.apache.accumulo.core.metadata.schema.DataFileValue used on opensource projects

Example 21 with DataFileValue

use of org.apache.accumulo.core.metadata.schema.DataFileValue in project accumulo by apache.

the class DatafileManager method importMapFiles.

public void importMapFiles(long tid, Map<FileRef, DataFileValue> pathsString, boolean setTime) throws IOException {
    String bulkDir = null;
    Map<FileRef, DataFileValue> paths = new HashMap<>();
    for (Entry<FileRef, DataFileValue> entry : pathsString.entrySet()) paths.put(entry.getKey(), entry.getValue());
    for (FileRef tpath : paths.keySet()) {
        boolean inTheRightDirectory = false;
        Path parent = tpath.path().getParent().getParent();
        for (String tablesDir : ServerConstants.getTablesDirs()) {
            if (parent.equals(new Path(tablesDir, tablet.getExtent().getTableId().canonicalID()))) {
                inTheRightDirectory = true;
                break;
            }
        }
        if (!inTheRightDirectory) {
            throw new IOException("Data file " + tpath + " not in table dirs");
        }
        if (bulkDir == null)
            bulkDir = tpath.path().getParent().toString();
        else if (!bulkDir.equals(tpath.path().getParent().toString()))
            throw new IllegalArgumentException("bulk files in different dirs " + bulkDir + " " + tpath);
    }
    if (tablet.getExtent().isMeta()) {
        throw new IllegalArgumentException("Can not import files to a metadata tablet");
    }
    synchronized (bulkFileImportLock) {
        if (paths.size() > 0) {
            long bulkTime = Long.MIN_VALUE;
            if (setTime) {
                for (DataFileValue dfv : paths.values()) {
                    long nextTime = tablet.getAndUpdateTime();
                    if (nextTime < bulkTime)
                        throw new IllegalStateException("Time went backwards unexpectedly " + nextTime + " " + bulkTime);
                    bulkTime = nextTime;
                    dfv.setTime(bulkTime);
                }
            }
            tablet.updatePersistedTime(bulkTime, paths, tid);
        }
    }
    synchronized (tablet) {
        for (Entry<FileRef, DataFileValue> tpath : paths.entrySet()) {
            if (datafileSizes.containsKey(tpath.getKey())) {
                log.error("Adding file that is already in set {}", tpath.getKey());
            }
            datafileSizes.put(tpath.getKey(), tpath.getValue());
        }
        tablet.getTabletResources().importedMapFiles();
        tablet.computeNumEntries();
    }
    for (Entry<FileRef, DataFileValue> entry : paths.entrySet()) {
        log.debug("TABLET_HIST {} import {} {}", tablet.getExtent(), entry.getKey(), entry.getValue());
    }
}

Example 22 with DataFileValue

use of org.apache.accumulo.core.metadata.schema.DataFileValue in project accumulo by apache.

the class DatafileManager method reserveFilesForScan.

Pair<Long, Map<FileRef, DataFileValue>> reserveFilesForScan() {
    synchronized (tablet) {
        while (reservationsBlocked) {
            try {
                tablet.wait(50);
            } catch (InterruptedException e) {
                log.warn("{}", e.getMessage(), e);
            }
        }
        Set<FileRef> absFilePaths = new HashSet<>(datafileSizes.keySet());
        long rid = nextScanReservationId++;
        scanFileReservations.put(rid, absFilePaths);
        Map<FileRef, DataFileValue> ret = new HashMap<>();
        for (FileRef path : absFilePaths) {
            fileScanReferenceCounts.increment(path, 1);
            ret.put(path, datafileSizes.get(path));
        }
        return new Pair<>(rid, ret);
    }
}

Example 23 with DataFileValue

use of org.apache.accumulo.core.metadata.schema.DataFileValue in project accumulo by apache.

the class Tablet method minorCompact.

DataFileValue minorCompact(VolumeManager fs, InMemoryMap memTable, FileRef tmpDatafile, FileRef newDatafile, FileRef mergeFile, boolean hasQueueTime, long queued, CommitSession commitSession, long flushId, MinorCompactionReason mincReason) {
    boolean failed = false;
    long start = System.currentTimeMillis();
    timer.incrementStatusMinor();
    long count = 0;
    String oldName = Thread.currentThread().getName();
    try {
        Thread.currentThread().setName("Minor compacting " + this.extent);
        Span span = Trace.start("write");
        CompactionStats stats;
        try {
            count = memTable.getNumEntries();
            DataFileValue dfv = null;
            if (mergeFile != null)
                dfv = getDatafileManager().getDatafileSizes().get(mergeFile);
            MinorCompactor compactor = new MinorCompactor(tabletServer, this, memTable, mergeFile, dfv, tmpDatafile, mincReason, tableConfiguration);
            stats = compactor.call();
        } finally {
            span.stop();
        }
        span = Trace.start("bringOnline");
        try {
            getDatafileManager().bringMinorCompactionOnline(tmpDatafile, newDatafile, mergeFile, new DataFileValue(stats.getFileSize(), stats.getEntriesWritten()), commitSession, flushId);
        } finally {
            span.stop();
        }
        return new DataFileValue(stats.getFileSize(), stats.getEntriesWritten());
    } catch (Exception | Error e) {
        failed = true;
        throw new RuntimeException(e);
    } finally {
        Thread.currentThread().setName(oldName);
        try {
            getTabletMemory().finalizeMinC();
        } catch (Throwable t) {
            log.error("Failed to free tablet memory", t);
        }
        if (!failed) {
            lastMinorCompactionFinishTime = System.currentTimeMillis();
        }
        Metrics minCMetrics = getTabletServer().getMinCMetrics();
        if (minCMetrics.isEnabled())
            minCMetrics.add(TabletServerMinCMetrics.MINC, (lastMinorCompactionFinishTime - start));
        if (hasQueueTime) {
            timer.updateTime(Operation.MINOR, queued, start, count, failed);
            if (minCMetrics.isEnabled())
                minCMetrics.add(TabletServerMinCMetrics.QUEUE, (start - queued));
        } else
            timer.updateTime(Operation.MINOR, start, count, failed);
    }
}

Example 24 with DataFileValue

use of org.apache.accumulo.core.metadata.schema.DataFileValue in project accumulo by apache.

the class Tablet method _majorCompact.

private CompactionStats _majorCompact(MajorCompactionReason reason) throws IOException, CompactionCanceledException {
    long t1, t2, t3;
    Pair<Long, UserCompactionConfig> compactionId = null;
    CompactionStrategy strategy = null;
    Map<FileRef, Pair<Key, Key>> firstAndLastKeys = null;
    if (reason == MajorCompactionReason.USER) {
        try {
            compactionId = getCompactionID();
            strategy = createCompactionStrategy(compactionId.getSecond().getCompactionStrategy());
        } catch (NoNodeException e) {
            throw new RuntimeException(e);
        }
    } else if (reason == MajorCompactionReason.NORMAL || reason == MajorCompactionReason.IDLE) {
        strategy = Property.createTableInstanceFromPropertyName(tableConfiguration, Property.TABLE_COMPACTION_STRATEGY, CompactionStrategy.class, new DefaultCompactionStrategy());
        strategy.init(Property.getCompactionStrategyOptions(tableConfiguration));
    } else if (reason == MajorCompactionReason.CHOP) {
        firstAndLastKeys = getFirstAndLastKeys(getDatafileManager().getDatafileSizes());
    } else {
        throw new IllegalArgumentException("Unknown compaction reason " + reason);
    }
    if (strategy != null) {
        BlockCache sc = tabletResources.getTabletServerResourceManager().getSummaryCache();
        BlockCache ic = tabletResources.getTabletServerResourceManager().getIndexCache();
        MajorCompactionRequest request = new MajorCompactionRequest(extent, reason, getTabletServer().getFileSystem(), tableConfiguration, sc, ic);
        request.setFiles(getDatafileManager().getDatafileSizes());
        strategy.gatherInformation(request);
    }
    Map<FileRef, DataFileValue> filesToCompact = null;
    int maxFilesToCompact = tableConfiguration.getCount(Property.TSERV_MAJC_THREAD_MAXOPEN);
    CompactionStats majCStats = new CompactionStats();
    CompactionPlan plan = null;
    boolean propogateDeletes = false;
    boolean updateCompactionID = false;
    synchronized (this) {
        // plan all that work that needs to be done in the sync block... then do the actual work
        // outside the sync block
        t1 = System.currentTimeMillis();
        majorCompactionState = CompactionState.WAITING_TO_START;
        getTabletMemory().waitForMinC();
        t2 = System.currentTimeMillis();
        majorCompactionState = CompactionState.IN_PROGRESS;
        notifyAll();
        VolumeManager fs = getTabletServer().getFileSystem();
        if (extent.isRootTablet()) {
            // very important that we call this before doing major compaction,
            // otherwise deleted compacted files could possible be brought back
            // at some point if the file they were compacted to was legitimately
            // removed by a major compaction
            RootFiles.cleanupReplacement(fs, fs.listStatus(this.location), false);
        }
        SortedMap<FileRef, DataFileValue> allFiles = getDatafileManager().getDatafileSizes();
        List<FileRef> inputFiles = new ArrayList<>();
        if (reason == MajorCompactionReason.CHOP) {
            // enforce rules: files with keys outside our range need to be compacted
            inputFiles.addAll(findChopFiles(extent, firstAndLastKeys, allFiles.keySet()));
        } else {
            MajorCompactionRequest request = new MajorCompactionRequest(extent, reason, tableConfiguration);
            request.setFiles(allFiles);
            plan = strategy.getCompactionPlan(request);
            if (plan != null) {
                plan.validate(allFiles.keySet());
                inputFiles.addAll(plan.inputFiles);
            }
        }
        if (inputFiles.isEmpty()) {
            if (reason == MajorCompactionReason.USER) {
                if (compactionId.getSecond().getIterators().isEmpty()) {
                    log.debug("No-op major compaction by USER on 0 input files because no iterators present.");
                    lastCompactID = compactionId.getFirst();
                    updateCompactionID = true;
                } else {
                    log.debug("Major compaction by USER on 0 input files with iterators.");
                    filesToCompact = new HashMap<>();
                }
            } else {
                return majCStats;
            }
        } else {
            // If no original files will exist at the end of the compaction, we do not have to propogate deletes
            Set<FileRef> droppedFiles = new HashSet<>();
            droppedFiles.addAll(inputFiles);
            if (plan != null)
                droppedFiles.addAll(plan.deleteFiles);
            propogateDeletes = !(droppedFiles.equals(allFiles.keySet()));
            log.debug("Major compaction plan: {} propogate deletes : {}", plan, propogateDeletes);
            filesToCompact = new HashMap<>(allFiles);
            filesToCompact.keySet().retainAll(inputFiles);
            getDatafileManager().reserveMajorCompactingFiles(filesToCompact.keySet());
        }
        t3 = System.currentTimeMillis();
    }
    try {
        log.debug(String.format("MajC initiate lock %.2f secs, wait %.2f secs", (t3 - t2) / 1000.0, (t2 - t1) / 1000.0));
        if (updateCompactionID) {
            MetadataTableUtil.updateTabletCompactID(extent, compactionId.getFirst(), tabletServer, getTabletServer().getLock());
            return majCStats;
        }
        if (!propogateDeletes && compactionId == null) {
            // compacting everything, so update the compaction id in metadata
            try {
                compactionId = getCompactionID();
                if (compactionId.getSecond().getCompactionStrategy() != null) {
                    compactionId = null;
                // TODO maybe return unless chop?
                }
            } catch (NoNodeException e) {
                throw new RuntimeException(e);
            }
        }
        List<IteratorSetting> compactionIterators = new ArrayList<>();
        if (compactionId != null) {
            if (reason == MajorCompactionReason.USER) {
                if (getCompactionCancelID() >= compactionId.getFirst()) {
                    // compaction was canceled
                    return majCStats;
                }
                compactionIterators = compactionId.getSecond().getIterators();
                synchronized (this) {
                    if (lastCompactID >= compactionId.getFirst())
                        // already compacted
                        return majCStats;
                }
            }
        }
        // ACCUMULO-3645 run loop at least once, even if filesToCompact.isEmpty()
        do {
            int numToCompact = maxFilesToCompact;
            if (filesToCompact.size() > maxFilesToCompact && filesToCompact.size() < 2 * maxFilesToCompact) {
                // on the second to last compaction pass, compact the minimum amount of files possible
                numToCompact = filesToCompact.size() - maxFilesToCompact + 1;
            }
            Set<FileRef> smallestFiles = removeSmallest(filesToCompact, numToCompact);
            FileRef fileName = getNextMapFilename((filesToCompact.size() == 0 && !propogateDeletes) ? "A" : "C");
            FileRef compactTmpName = new FileRef(fileName.path().toString() + "_tmp");
            AccumuloConfiguration tableConf = createTableConfiguration(tableConfiguration, plan);
            Span span = Trace.start("compactFiles");
            try {
                CompactionEnv cenv = new CompactionEnv() {

                    @Override
                    public boolean isCompactionEnabled() {
                        return Tablet.this.isCompactionEnabled();
                    }

                    @Override
                    public IteratorScope getIteratorScope() {
                        return IteratorScope.majc;
                    }

                    @Override
                    public RateLimiter getReadLimiter() {
                        return getTabletServer().getMajorCompactionReadLimiter();
                    }

                    @Override
                    public RateLimiter getWriteLimiter() {
                        return getTabletServer().getMajorCompactionWriteLimiter();
                    }
                };
                HashMap<FileRef, DataFileValue> copy = new HashMap<>(getDatafileManager().getDatafileSizes());
                if (!copy.keySet().containsAll(smallestFiles))
                    throw new IllegalStateException("Cannot find data file values for " + smallestFiles);
                copy.keySet().retainAll(smallestFiles);
                log.debug("Starting MajC {} ({}) {} --> {} {}", extent, reason, copy.keySet(), compactTmpName, compactionIterators);
                // always propagate deletes, unless last batch
                boolean lastBatch = filesToCompact.isEmpty();
                Compactor compactor = new Compactor(tabletServer, this, copy, null, compactTmpName, lastBatch ? propogateDeletes : true, cenv, compactionIterators, reason.ordinal(), tableConf);
                CompactionStats mcs = compactor.call();
                span.data("files", "" + smallestFiles.size());
                span.data("read", "" + mcs.getEntriesRead());
                span.data("written", "" + mcs.getEntriesWritten());
                majCStats.add(mcs);
                if (lastBatch && plan != null && plan.deleteFiles != null) {
                    smallestFiles.addAll(plan.deleteFiles);
                }
                getDatafileManager().bringMajorCompactionOnline(smallestFiles, compactTmpName, fileName, filesToCompact.size() == 0 && compactionId != null ? compactionId.getFirst() : null, new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()));
                // to add the deleted file
                if (filesToCompact.size() > 0 && mcs.getEntriesWritten() > 0) {
                    filesToCompact.put(fileName, new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()));
                }
            } finally {
                span.stop();
            }
        } while (filesToCompact.size() > 0);
        return majCStats;
    } finally {
        synchronized (Tablet.this) {
            getDatafileManager().clearMajorCompactingFile();
        }
    }
}

Example 25 with DataFileValue

use of org.apache.accumulo.core.metadata.schema.DataFileValue in project accumulo by apache.

the class Tablet method importMapFiles.

public void importMapFiles(long tid, Map<FileRef, MapFileInfo> fileMap, boolean setTime) throws IOException {
    Map<FileRef, DataFileValue> entries = new HashMap<>(fileMap.size());
    List<String> files = new ArrayList<>();
    for (Entry<FileRef, MapFileInfo> entry : fileMap.entrySet()) {
        entries.put(entry.getKey(), new DataFileValue(entry.getValue().estimatedSize, 0l));
        files.add(entry.getKey().path().toString());
    }
    // Clients timeout and will think that this operation failed.
    // Don't do it if we spent too long waiting for the lock
    long now = System.currentTimeMillis();
    synchronized (this) {
        if (isClosed()) {
            throw new IOException("tablet " + extent + " is closed");
        }
        // TODO check seems uneeded now - ACCUMULO-1291
        long lockWait = System.currentTimeMillis() - now;
        if (lockWait > getTabletServer().getConfiguration().getTimeInMillis(Property.GENERAL_RPC_TIMEOUT)) {
            throw new IOException("Timeout waiting " + (lockWait / 1000.) + " seconds to get tablet lock");
        }
        List<FileRef> alreadyImported = bulkImported.getIfPresent(tid);
        if (alreadyImported != null) {
            for (FileRef entry : alreadyImported) {
                if (fileMap.remove(entry) != null) {
                    log.info("Ignoring import of bulk file already imported: " + entry);
                }
            }
        }
        if (fileMap.isEmpty()) {
            return;
        }
        if (writesInProgress < 0) {
            throw new IllegalStateException("writesInProgress < 0 " + writesInProgress);
        }
        writesInProgress++;
    }
    tabletServer.updateBulkImportState(files, BulkImportState.LOADING);
    try {
        getDatafileManager().importMapFiles(tid, entries, setTime);
        lastMapFileImportTime = System.currentTimeMillis();
        if (needsSplit()) {
            getTabletServer().executeSplit(this);
        } else {
            initiateMajorCompaction(MajorCompactionReason.NORMAL);
        }
    } finally {
        synchronized (this) {
            if (writesInProgress < 1)
                throw new IllegalStateException("writesInProgress < 1 " + writesInProgress);
            writesInProgress--;
            if (writesInProgress == 0)
                this.notifyAll();
            try {
                bulkImported.get(tid, new Callable<List<FileRef>>() {

                    @Override
                    public List<FileRef> call() throws Exception {
                        return new ArrayList<>();
                    }
                }).addAll(fileMap.keySet());
            } catch (Exception ex) {
                log.info(ex.toString(), ex);
            }
            tabletServer.removeBulkImportState(files);
        }
    }
}