Examples with FileOperations org.apache.accumulo.core.file.FileOperations used on opensource projects

Example 1 with FileOperations

use of org.apache.accumulo.core.file.FileOperations in project accumulo by apache.

the class Compactor method call.

@Override
public CompactionStats call() throws IOException, CompactionCanceledException {
    FileSKVWriter mfw = null;
    CompactionStats majCStats = new CompactionStats();
    boolean remove = runningCompactions.add(this);
    clearStats();
    final Path outputFilePath = outputFile.path();
    final String outputFilePathName = outputFilePath.toString();
    String oldThreadName = Thread.currentThread().getName();
    String newThreadName = "MajC compacting " + extent.toString() + " started " + dateFormatter.format(new Date()) + " file: " + outputFile;
    Thread.currentThread().setName(newThreadName);
    thread = Thread.currentThread();
    try {
        FileOperations fileFactory = FileOperations.getInstance();
        FileSystem ns = this.fs.getVolumeByPath(outputFilePath).getFileSystem();
        mfw = fileFactory.newWriterBuilder().forFile(outputFilePathName, ns, ns.getConf()).withTableConfiguration(acuTableConf).withRateLimiter(env.getWriteLimiter()).build();
        Map<String, Set<ByteSequence>> lGroups;
        try {
            lGroups = LocalityGroupUtil.getLocalityGroups(acuTableConf);
        } catch (LocalityGroupConfigurationError e) {
            throw new IOException(e);
        }
        long t1 = System.currentTimeMillis();
        HashSet<ByteSequence> allColumnFamilies = new HashSet<>();
        if (mfw.supportsLocalityGroups()) {
            for (Entry<String, Set<ByteSequence>> entry : lGroups.entrySet()) {
                setLocalityGroup(entry.getKey());
                compactLocalityGroup(entry.getKey(), entry.getValue(), true, mfw, majCStats);
                allColumnFamilies.addAll(entry.getValue());
            }
        }
        setLocalityGroup("");
        compactLocalityGroup(null, allColumnFamilies, false, mfw, majCStats);
        long t2 = System.currentTimeMillis();
        FileSKVWriter mfwTmp = mfw;
        // set this to null so we do not try to close it again in finally if the close fails
        mfw = null;
        try {
            // if the close fails it will cause the compaction to fail
            mfwTmp.close();
        } catch (IOException ex) {
            if (!fs.deleteRecursively(outputFile.path())) {
                if (fs.exists(outputFile.path())) {
                    log.error("Unable to delete {}", outputFile);
                }
            }
            throw ex;
        }
        log.debug(String.format("Compaction %s %,d read | %,d written | %,6d entries/sec | %,6.3f secs | %,12d bytes | %9.3f byte/sec", extent, majCStats.getEntriesRead(), majCStats.getEntriesWritten(), (int) (majCStats.getEntriesRead() / ((t2 - t1) / 1000.0)), (t2 - t1) / 1000.0, mfwTmp.getLength(), mfwTmp.getLength() / ((t2 - t1) / 1000.0)));
        majCStats.setFileSize(mfwTmp.getLength());
        return majCStats;
    } catch (IOException | RuntimeException e) {
        log.error("{}", e.getMessage(), e);
        throw e;
    } finally {
        Thread.currentThread().setName(oldThreadName);
        if (remove) {
            thread = null;
            runningCompactions.remove(this);
        }
        try {
            if (mfw != null) {
                // compaction must not have finished successfully, so close its output file
                try {
                    mfw.close();
                } finally {
                    if (!fs.deleteRecursively(outputFile.path()))
                        if (fs.exists(outputFile.path()))
                            log.error("Unable to delete {}", outputFile);
                }
            }
        } catch (IOException | RuntimeException e) {
            log.warn("{}", e.getMessage(), e);
        }
    }
}

Example 2 with FileOperations

use of org.apache.accumulo.core.file.FileOperations in project accumulo by apache.

the class Compactor method openMapDataFiles.

private List<SortedKeyValueIterator<Key, Value>> openMapDataFiles(String lgName, ArrayList<FileSKVIterator> readers) throws IOException {
    List<SortedKeyValueIterator<Key, Value>> iters = new ArrayList<>(filesToCompact.size());
    for (FileRef mapFile : filesToCompact.keySet()) {
        try {
            FileOperations fileFactory = FileOperations.getInstance();
            FileSystem fs = this.fs.getVolumeByPath(mapFile.path()).getFileSystem();
            FileSKVIterator reader;
            reader = fileFactory.newReaderBuilder().forFile(mapFile.path().toString(), fs, fs.getConf()).withTableConfiguration(acuTableConf).withRateLimiter(env.getReadLimiter()).build();
            readers.add(reader);
            SortedKeyValueIterator<Key, Value> iter = new ProblemReportingIterator(context, extent.getTableId(), mapFile.path().toString(), false, reader);
            if (filesToCompact.get(mapFile).isTimeSet()) {
                iter = new TimeSettingIterator(iter, filesToCompact.get(mapFile).getTime());
            }
            iters.add(iter);
        } catch (Throwable e) {
            ProblemReports.getInstance(context).report(new ProblemReport(extent.getTableId(), ProblemType.FILE_READ, mapFile.path().toString(), e));
            log.warn("Some problem opening map file {} {}", mapFile, e.getMessage(), e);
            // failed to open some map file... close the ones that were opened
            for (FileSKVIterator reader : readers) {
                try {
                    reader.close();
                } catch (Throwable e2) {
                    log.warn("Failed to close map file", e2);
                }
            }
            readers.clear();
            if (e instanceof IOException)
                throw (IOException) e;
            throw new IOException("Failed to open map data files", e);
        }
    }
    return iters;
}

Example 3 with FileOperations

use of org.apache.accumulo.core.file.FileOperations in project accumulo by apache.

the class MajorCompactionRequest method openReader.

public FileSKVIterator openReader(FileRef ref) throws IOException {
    Preconditions.checkState(volumeManager != null, "Opening files is not supported at this time.  Its only supported when CompactionStrategy.gatherInformation() is called.");
    // @TODO verify the file isn't some random file in HDFS
    // @TODO ensure these files are always closed?
    FileOperations fileFactory = FileOperations.getInstance();
    FileSystem ns = volumeManager.getVolumeByPath(ref.path()).getFileSystem();
    FileSKVIterator openReader = fileFactory.newReaderBuilder().forFile(ref.path().toString(), ns, ns.getConf()).withTableConfiguration(tableConfig).seekToBeginning().build();
    return openReader;
}

Example 4 with FileOperations

use of org.apache.accumulo.core.file.FileOperations in project accumulo by apache.

the class Tablet method getFirstAndLastKeys.

private Map<FileRef, Pair<Key, Key>> getFirstAndLastKeys(SortedMap<FileRef, DataFileValue> allFiles) throws IOException {
    Map<FileRef, Pair<Key, Key>> result = new HashMap<>();
    FileOperations fileFactory = FileOperations.getInstance();
    VolumeManager fs = getTabletServer().getFileSystem();
    for (Entry<FileRef, DataFileValue> entry : allFiles.entrySet()) {
        FileRef file = entry.getKey();
        FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
        try (FileSKVIterator openReader = fileFactory.newReaderBuilder().forFile(file.path().toString(), ns, ns.getConf()).withTableConfiguration(this.getTableConfiguration()).seekToBeginning().build()) {
            Key first = openReader.getFirstKey();
            Key last = openReader.getLastKey();
            result.put(file, new Pair<>(first, last));
        }
    }
    return result;
}

Example 5 with FileOperations

use of org.apache.accumulo.core.file.FileOperations in project accumulo by apache.

the class RFileWriterBuilder method build.

@Override
public RFileWriter build() throws IOException {
    FileOperations fileops = FileOperations.getInstance();
    AccumuloConfiguration acuconf = DefaultConfiguration.getInstance();
    HashMap<String, String> userProps = new HashMap<>();
    userProps.putAll(tableConfig);
    userProps.putAll(summarizerProps);
    userProps.putAll(samplerProps);
    if (userProps.size() > 0) {
        acuconf = new ConfigurationCopy(Iterables.concat(acuconf, userProps.entrySet()));
    }
    if (out.getOutputStream() != null) {
        FSDataOutputStream fsdo;
        if (out.getOutputStream() instanceof FSDataOutputStream) {
            fsdo = (FSDataOutputStream) out.getOutputStream();
        } else {
            fsdo = new FSDataOutputStream(out.getOutputStream(), new FileSystem.Statistics("foo"));
        }
        return new RFileWriter(fileops.newWriterBuilder().forOutputStream(".rf", fsdo, out.getConf()).withTableConfiguration(acuconf).setAccumuloStartEnabled(false).build(), visCacheSize);
    } else {
        return new RFileWriter(fileops.newWriterBuilder().forFile(out.path.toString(), out.getFileSystem(), out.getConf()).withTableConfiguration(acuconf).setAccumuloStartEnabled(false).build(), visCacheSize);
    }
}