Examples with AccumuloConfiguration org.apache.accumulo.core.conf.AccumuloConfiguration used on opensource projects

Example 81 with AccumuloConfiguration

use of org.apache.accumulo.core.conf.AccumuloConfiguration in project accumulo by apache.

the class TabletServer method config.

private void config() {
    log.info("Tablet server starting on {}", getHostname());
    Threads.createThread("Split/MajC initiator", new MajorCompactor(context)).start();
    clientAddress = HostAndPort.fromParts(getHostname(), 0);
    final AccumuloConfiguration aconf = getConfiguration();
    FileSystemMonitor.start(aconf, Property.TSERV_MONITOR_FS);
    Runnable gcDebugTask = () -> gcLogger.logGCInfo(getConfiguration());
    context.getScheduledExecutor().scheduleWithFixedDelay(gcDebugTask, 0, TIME_BETWEEN_GC_CHECKS, TimeUnit.MILLISECONDS);
}

Example 82 with AccumuloConfiguration

use of org.apache.accumulo.core.conf.AccumuloConfiguration in project accumulo by apache.

the class AccumuloFileOutputFormat method getRecordWriter.

@Override
public RecordWriter<Key, Value> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress) {
    // get the path of the temporary output file
    final Configuration conf = job;
    final AccumuloConfiguration acuConf = FileOutputConfigurator.getAccumuloConfiguration(AccumuloFileOutputFormat.class, job);
    final String extension = acuConf.get(Property.TABLE_FILE_TYPE);
    final Path file = new Path(getWorkOutputPath(job), getUniqueName(job, "part") + "." + extension);
    final int visCacheSize = ConfiguratorBase.getVisibilityCacheSize(conf);
    return new RecordWriter<>() {

        RFileWriter out = null;

        @Override
        public void close(Reporter reporter) throws IOException {
            if (out != null)
                out.close();
        }

        @Override
        public void write(Key key, Value value) throws IOException {
            if (out == null) {
                out = RFile.newWriter().to(file.toString()).withFileSystem(file.getFileSystem(conf)).withTableProperties(acuConf).withVisibilityCacheSize(visCacheSize).build();
                out.startDefaultLocalityGroup();
            }
            out.append(key, value);
        }
    };
}

Example 83 with AccumuloConfiguration

use of org.apache.accumulo.core.conf.AccumuloConfiguration in project accumulo by apache.

the class AccumuloFileOutputFormat method getRecordWriter.

@Override
public RecordWriter<Key, Value> getRecordWriter(TaskAttemptContext context) throws IOException {
    // get the path of the temporary output file
    final Configuration conf = context.getConfiguration();
    final AccumuloConfiguration acuConf = FileOutputConfigurator.getAccumuloConfiguration(AccumuloFileOutputFormat.class, context.getConfiguration());
    final String extension = acuConf.get(Property.TABLE_FILE_TYPE);
    final Path file = this.getDefaultWorkFile(context, "." + extension);
    final int visCacheSize = ConfiguratorBase.getVisibilityCacheSize(conf);
    return new RecordWriter<>() {

        RFileWriter out = null;

        @Override
        public void close(TaskAttemptContext context) throws IOException {
            if (out != null)
                out.close();
        }

        @Override
        public void write(Key key, Value value) throws IOException {
            if (out == null) {
                out = RFile.newWriter().to(file.toString()).withFileSystem(file.getFileSystem(conf)).withTableProperties(acuConf).withVisibilityCacheSize(visCacheSize).build();
                out.startDefaultLocalityGroup();
            }
            out.append(key, value);
        }
    };
}

Example 84 with AccumuloConfiguration

use of org.apache.accumulo.core.conf.AccumuloConfiguration in project accumulo by apache.

the class FileUtil method countIndexEntries.

private static long countIndexEntries(ServerContext context, Text prevEndRow, Text endRow, Collection<TabletFile> mapFiles, boolean useIndex, ArrayList<FileSKVIterator> readers) throws IOException {
    AccumuloConfiguration acuConf = context.getConfiguration();
    long numKeys = 0;
    // count the total number of index entries
    for (TabletFile file : mapFiles) {
        FileSKVIterator reader = null;
        FileSystem ns = context.getVolumeManager().getFileSystemByPath(file.getPath());
        try {
            if (useIndex)
                reader = FileOperations.getInstance().newIndexReaderBuilder().forFile(file.getPathStr(), ns, ns.getConf(), context.getCryptoService()).withTableConfiguration(acuConf).build();
            else
                reader = FileOperations.getInstance().newScanReaderBuilder().forFile(file.getPathStr(), ns, ns.getConf(), context.getCryptoService()).withTableConfiguration(acuConf).overRange(new Range(prevEndRow, false, null, true), Set.of(), false).build();
            while (reader.hasTop()) {
                Key key = reader.getTopKey();
                if (endRow != null && key.compareRow(endRow) > 0)
                    break;
                else if (prevEndRow == null || key.compareRow(prevEndRow) > 0)
                    numKeys++;
                reader.next();
            }
        } finally {
            try {
                if (reader != null)
                    reader.close();
            } catch (IOException e) {
                log.error("{}", e.getMessage(), e);
            }
        }
        if (useIndex)
            readers.add(FileOperations.getInstance().newIndexReaderBuilder().forFile(file.getPathStr(), ns, ns.getConf(), context.getCryptoService()).withTableConfiguration(acuConf).build());
        else
            readers.add(FileOperations.getInstance().newScanReaderBuilder().forFile(file.getPathStr(), ns, ns.getConf(), context.getCryptoService()).withTableConfiguration(acuConf).overRange(new Range(prevEndRow, false, null, true), Set.of(), false).build());
    }
    return numKeys;
}

Example 85 with AccumuloConfiguration

use of org.apache.accumulo.core.conf.AccumuloConfiguration in project accumulo by apache.

the class FileUtil method reduceFiles.

public static Collection<TabletFile> reduceFiles(ServerContext context, Configuration conf, Text prevEndRow, Text endRow, Collection<TabletFile> mapFiles, int maxFiles, Path tmpDir, int pass) throws IOException {
    AccumuloConfiguration acuConf = context.getConfiguration();
    ArrayList<TabletFile> paths = new ArrayList<>(mapFiles);
    if (paths.size() <= maxFiles)
        return paths;
    String newDir = String.format("%s/pass_%04d", tmpDir, pass);
    int start = 0;
    ArrayList<TabletFile> outFiles = new ArrayList<>();
    int count = 0;
    while (start < paths.size()) {
        int end = Math.min(maxFiles + start, paths.size());
        List<TabletFile> inFiles = paths.subList(start, end);
        start = end;
        TabletFile newMapFile = new TabletFile(new Path(String.format("%s/%04d.%s", newDir, count++, RFile.EXTENSION)));
        outFiles.add(newMapFile);
        FileSystem ns = context.getVolumeManager().getFileSystemByPath(newMapFile.getPath());
        FileSKVWriter writer = new RFileOperations().newWriterBuilder().forFile(newMapFile.getPathStr(), ns, ns.getConf(), context.getCryptoService()).withTableConfiguration(acuConf).build();
        writer.startDefaultLocalityGroup();
        List<SortedKeyValueIterator<Key, Value>> iters = new ArrayList<>(inFiles.size());
        FileSKVIterator reader = null;
        try {
            for (TabletFile file : inFiles) {
                ns = context.getVolumeManager().getFileSystemByPath(file.getPath());
                reader = FileOperations.getInstance().newIndexReaderBuilder().forFile(file.getPathStr(), ns, ns.getConf(), context.getCryptoService()).withTableConfiguration(acuConf).build();
                iters.add(reader);
            }
            MultiIterator mmfi = new MultiIterator(iters, true);
            while (mmfi.hasTop()) {
                Key key = mmfi.getTopKey();
                boolean gtPrevEndRow = prevEndRow == null || key.compareRow(prevEndRow) > 0;
                boolean lteEndRow = endRow == null || key.compareRow(endRow) <= 0;
                if (gtPrevEndRow && lteEndRow)
                    writer.append(key, new Value());
                if (!lteEndRow)
                    break;
                mmfi.next();
            }
        } finally {
            try {
                if (reader != null)
                    reader.close();
            } catch (IOException e) {
                log.error("{}", e.getMessage(), e);
            }
            for (SortedKeyValueIterator<Key, Value> r : iters) try {
                if (r != null)
                    ((FileSKVIterator) r).close();
            } catch (IOException e) {
                // continue closing
                log.error("{}", e.getMessage(), e);
            }
            try {
                writer.close();
            } catch (IOException e) {
                log.error("{}", e.getMessage(), e);
                throw e;
            }
        }
    }
    return reduceFiles(context, conf, prevEndRow, endRow, outFiles, maxFiles, tmpDir, pass + 1);
}