Examples with CompactableFile org.apache.accumulo.core.client.admin.compaction.CompactableFile used on opensource projects

Example 1 with CompactableFile

use of org.apache.accumulo.core.client.admin.compaction.CompactableFile in project accumulo by apache.

the class CompactableUtils method selectFiles.

static Set<StoredTabletFile> selectFiles(Tablet tablet, SortedMap<StoredTabletFile, DataFileValue> datafiles, PluginConfig selectorConfig) {
    CompactionSelector selector = newInstance(tablet.getTableConfiguration(), selectorConfig.getClassName(), CompactionSelector.class);
    final ServiceEnvironment senv = new ServiceEnvironmentImpl(tablet.getContext());
    selector.init(new CompactionSelector.InitParameters() {

        @Override
        public Map<String, String> getOptions() {
            return selectorConfig.getOptions();
        }

        @Override
        public PluginEnvironment getEnvironment() {
            return senv;
        }

        @Override
        public TableId getTableId() {
            return tablet.getExtent().tableId();
        }
    });
    Selection selection = selector.select(new CompactionSelector.SelectionParameters() {

        @Override
        public PluginEnvironment getEnvironment() {
            return senv;
        }

        @Override
        public Collection<CompactableFile> getAvailableFiles() {
            return Collections2.transform(datafiles.entrySet(), e -> new CompactableFileImpl(e.getKey(), e.getValue()));
        }

        @Override
        public Collection<Summary> getSummaries(Collection<CompactableFile> files, Predicate<SummarizerConfiguration> summarySelector) {
            var context = tablet.getContext();
            var tsrm = tablet.getTabletResources().getTabletServerResourceManager();
            SummaryCollection sc = new SummaryCollection();
            SummarizerFactory factory = new SummarizerFactory(tablet.getTableConfiguration());
            for (CompactableFile cf : files) {
                var file = CompactableFileImpl.toStoredTabletFile(cf);
                FileSystem fs = context.getVolumeManager().getFileSystemByPath(file.getPath());
                Configuration conf = context.getHadoopConf();
                SummaryCollection fsc = SummaryReader.load(fs, conf, factory, file.getPath(), summarySelector, tsrm.getSummaryCache(), tsrm.getIndexCache(), tsrm.getFileLenCache(), context.getCryptoService()).getSummaries(Collections.singletonList(new Gatherer.RowRange(tablet.getExtent())));
                sc.merge(fsc, factory);
            }
            return sc.getSummaries();
        }

        @Override
        public TableId getTableId() {
            return tablet.getExtent().tableId();
        }

        @Override
        public Optional<SortedKeyValueIterator<Key, Value>> getSample(CompactableFile file, SamplerConfiguration sc) {
            try {
                FileOperations fileFactory = FileOperations.getInstance();
                Path path = new Path(file.getUri());
                FileSystem ns = tablet.getTabletServer().getVolumeManager().getFileSystemByPath(path);
                var fiter = fileFactory.newReaderBuilder().forFile(path.toString(), ns, ns.getConf(), tablet.getContext().getCryptoService()).withTableConfiguration(tablet.getTableConfiguration()).seekToBeginning().build();
                return Optional.ofNullable(fiter.getSample(new SamplerConfigurationImpl(sc)));
            } catch (IOException e) {
                throw new UncheckedIOException(e);
            }
        }
    });
    return selection.getFilesToCompact().stream().map(CompactableFileImpl::toStoredTabletFile).collect(Collectors.toSet());
}

Example 2 with CompactableFile

use of org.apache.accumulo.core.client.admin.compaction.CompactableFile in project accumulo by apache.

the class ConfigurableCompactionStrategyTest method testOutputOptions.

// file selection options are adequately tested by ShellServerIT
@Test
public void testOutputOptions() throws URISyntaxException {
    Collection<CompactableFile> files = Set.of(CompactableFile.create(new URI("hdfs://nn1/accumulo/tables/1/t-009/F00001.rf"), 50000, 400));
    // test setting no output options
    ConfigurableCompactionStrategy ccs = new ConfigurableCompactionStrategy();
    Map<String, String> opts = new HashMap<>();
    var initParams = new CompactionConfigurer.InitParameters() {

        @Override
        public TableId getTableId() {
            return TableId.of("1");
        }

        @Override
        public Map<String, String> getOptions() {
            return opts;
        }

        @Override
        public PluginEnvironment getEnvironment() {
            return null;
        }
    };
    ccs.init(initParams);
    var inputParams = new CompactionConfigurer.InputParameters() {

        @Override
        public TableId getTableId() {
            return null;
        }

        @Override
        public Collection<CompactableFile> getInputFiles() {
            return files;
        }

        @Override
        public PluginEnvironment getEnvironment() {
            return null;
        }
    };
    Overrides plan = ccs.override(inputParams);
    assertTrue(plan.getOverrides().isEmpty());
    // test setting all output options
    ccs = new ConfigurableCompactionStrategy();
    CompactionSettings.OUTPUT_BLOCK_SIZE_OPT.put(null, opts, "64K");
    CompactionSettings.OUTPUT_COMPRESSION_OPT.put(null, opts, "snappy");
    CompactionSettings.OUTPUT_HDFS_BLOCK_SIZE_OPT.put(null, opts, "256M");
    CompactionSettings.OUTPUT_INDEX_BLOCK_SIZE_OPT.put(null, opts, "32K");
    CompactionSettings.OUTPUT_REPLICATION_OPT.put(null, opts, "5");
    ccs.init(initParams);
    plan = ccs.override(inputParams);
    Map<String, String> expected = Map.of(Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "snappy", Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), getFixedMemoryAsBytes("64K") + "", Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX.getKey(), getFixedMemoryAsBytes("32K") + "", Property.TABLE_FILE_BLOCK_SIZE.getKey(), getFixedMemoryAsBytes("256M") + "", Property.TABLE_FILE_REPLICATION.getKey(), "5");
    assertEquals(expected, plan.getOverrides());
}