Examples with FileRef org.apache.accumulo.server.fs.FileRef used on opensource projects

Example 1 with FileRef

use of org.apache.accumulo.server.fs.FileRef in project accumulo by apache.

the class CollectTabletStats method readFilesUsingIterStack.

private static int readFilesUsingIterStack(VolumeManager fs, ServerConfigurationFactory aconf, List<FileRef> files, Authorizations auths, KeyExtent ke, String[] columns, boolean useTableIterators) throws Exception {
    SortedKeyValueIterator<Key, Value> reader;
    List<SortedKeyValueIterator<Key, Value>> readers = new ArrayList<>(files.size());
    for (FileRef file : files) {
        FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
        readers.add(FileOperations.getInstance().newReaderBuilder().forFile(file.path().toString(), ns, ns.getConf()).withTableConfiguration(aconf.getSystemConfiguration()).build());
    }
    List<IterInfo> emptyIterinfo = Collections.emptyList();
    Map<String, Map<String, String>> emptySsio = Collections.emptyMap();
    TableConfiguration tconf = aconf.getTableConfiguration(ke.getTableId());
    reader = createScanIterator(ke, readers, auths, new byte[] {}, new HashSet<>(), emptyIterinfo, emptySsio, useTableIterators, tconf);
    HashSet<ByteSequence> columnSet = createColumnBSS(columns);
    reader.seek(new Range(ke.getPrevEndRow(), false, ke.getEndRow(), true), columnSet, columnSet.size() == 0 ? false : true);
    int count = 0;
    while (reader.hasTop()) {
        count++;
        reader.next();
    }
    return count;
}

Example 2 with FileRef

use of org.apache.accumulo.server.fs.FileRef in project accumulo by apache.

the class SplitRecoveryIT method verifySame.

private void verifySame(SortedMap<FileRef, DataFileValue> datafileSizes, SortedMap<FileRef, DataFileValue> fixedDatafileSizes) throws Exception {
    if (!datafileSizes.keySet().containsAll(fixedDatafileSizes.keySet()) || !fixedDatafileSizes.keySet().containsAll(datafileSizes.keySet())) {
        throw new Exception("Key sets not the same " + datafileSizes.keySet() + " !=  " + fixedDatafileSizes.keySet());
    }
    for (Entry<FileRef, DataFileValue> entry : datafileSizes.entrySet()) {
        DataFileValue dfv = entry.getValue();
        DataFileValue otherDfv = fixedDatafileSizes.get(entry.getKey());
        if (!dfv.equals(otherDfv)) {
            throw new Exception(entry.getKey() + " dfv not equal  " + dfv + "  " + otherDfv);
        }
    }
}

Example 3 with FileRef

use of org.apache.accumulo.server.fs.FileRef in project accumulo by apache.

the class TwoTierCompactionStrategy method calculateTotalSize.

/**
 * Calculates the total size of input files in the compaction plan
 */
private Long calculateTotalSize(MajorCompactionRequest request, CompactionPlan plan) {
    long totalSize = 0;
    Map<FileRef, DataFileValue> allFiles = request.getFiles();
    for (FileRef fileRef : plan.inputFiles) {
        totalSize += allFiles.get(fileRef).getSize();
    }
    return totalSize;
}

Example 4 with FileRef

use of org.apache.accumulo.server.fs.FileRef in project accumulo by apache.

the class ConfigurableCompactionStrategy method getCompactionPlan.

@Override
public CompactionPlan getCompactionPlan(MajorCompactionRequest request) throws IOException {
    List<FileRef> filesToCompact = getFilesToCompact(request);
    if (filesToCompact.size() >= minFiles) {
        CompactionPlan plan = new CompactionPlan();
        plan.inputFiles.addAll(filesToCompact);
        plan.writeParameters = writeParams;
        return plan;
    }
    return null;
}

Example 5 with FileRef

use of org.apache.accumulo.server.fs.FileRef in project accumulo by apache.

the class TooManyDeletesCompactionStrategy method gatherInformation.

@Override
public void gatherInformation(MajorCompactionRequest request) throws IOException {
    super.gatherInformation(request);
    Predicate<SummarizerConfiguration> summarizerPredicate = conf -> conf.getClassName().equals(DeletesSummarizer.class.getName()) && conf.getOptions().isEmpty();
    long total = 0;
    long deletes = 0;
    for (Entry<FileRef, DataFileValue> entry : request.getFiles().entrySet()) {
        Collection<Summary> summaries = request.getSummaries(Collections.singleton(entry.getKey()), summarizerPredicate);
        if (summaries.size() == 1) {
            Summary summary = summaries.iterator().next();
            total += summary.getStatistics().get(TOTAL_STAT);
            deletes += summary.getStatistics().get(DELETES_STAT);
        } else {
            long numEntries = entry.getValue().getNumEntries();
            if (numEntries == 0 && !proceed_bns) {
                shouldCompact = false;
                return;
            } else {
                // no summary data so use Accumulo's estimate of total entries in file
                total += entry.getValue().getNumEntries();
            }
        }
    }
    long nonDeletes = total - deletes;
    if (nonDeletes >= 0) {
        // check nonDeletes >= 0 because if this is not true then its clear evidence that the estimates are off
        double ratio = deletes / (double) nonDeletes;
        shouldCompact = ratio >= threshold;
    } else {
        shouldCompact = false;
    }
}