Examples with PluginConfig org.apache.accumulo.core.client.admin.PluginConfig used on opensource projects

Example 6 with PluginConfig

use of org.apache.accumulo.core.client.admin.PluginConfig in project accumulo by apache.

the class SummaryIT method compactionSelectorTest.

@Test
public void compactionSelectorTest() throws Exception {
    // Create a compaction config that will filter out foos if there are too many. Uses summary
    // data to know if there are too many foos.
    PluginConfig csc = new PluginConfig(FooSelector.class.getName());
    CompactionConfig compactConfig = new CompactionConfig().setSelector(csc);
    compactionTest(compactConfig);
}

Example 7 with PluginConfig

use of org.apache.accumulo.core.client.admin.PluginConfig in project accumulo by apache.

the class CompactionIT method testConfigurer.

@Test
public void testConfigurer() throws Exception {
    String tableName = this.getUniqueNames(1)[0];
    try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
        Map<String, String> props = Map.of(Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "none");
        NewTableConfiguration ntc = new NewTableConfiguration().setProperties(props);
        client.tableOperations().create(tableName, ntc);
        byte[] data = new byte[100000];
        Arrays.fill(data, (byte) 65);
        try (var writer = client.createBatchWriter(tableName)) {
            for (int row = 0; row < 10; row++) {
                Mutation m = new Mutation(row + "");
                m.at().family("big").qualifier("stuff").put(data);
                writer.addMutation(m);
            }
        }
        client.tableOperations().flush(tableName, null, null, true);
        // without compression, expect file to be large
        long sizes = CompactionExecutorIT.getFileSizes(client, tableName);
        assertTrue("Unexpected files sizes : " + sizes, sizes > data.length * 10 && sizes < data.length * 11);
        client.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setConfigurer(new PluginConfig(CompressionConfigurer.class.getName(), Map.of(CompressionConfigurer.LARGE_FILE_COMPRESSION_TYPE, "gz", CompressionConfigurer.LARGE_FILE_COMPRESSION_THRESHOLD, data.length + ""))));
        // after compacting with compression, expect small file
        sizes = CompactionExecutorIT.getFileSizes(client, tableName);
        assertTrue("Unexpected files sizes: data: " + data.length + ", file:" + sizes, sizes < data.length);
        client.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
        // after compacting without compression, expect big files again
        sizes = CompactionExecutorIT.getFileSizes(client, tableName);
        assertTrue("Unexpected files sizes : " + sizes, sizes > data.length * 10 && sizes < data.length * 11);
    }
}

Example 8 with PluginConfig

use of org.apache.accumulo.core.client.admin.PluginConfig in project accumulo by apache.

the class CompactionIT method testBadSelector.

@Test
public void testBadSelector() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        final String tableName = getUniqueNames(1)[0];
        NewTableConfiguration tc = new NewTableConfiguration();
        // Ensure compactions don't kick off
        tc.setProperties(Map.of(Property.TABLE_MAJC_RATIO.getKey(), "10.0"));
        c.tableOperations().create(tableName, tc);
        // Create multiple RFiles
        try (BatchWriter bw = c.createBatchWriter(tableName)) {
            for (int i = 1; i <= 4; i++) {
                Mutation m = new Mutation(Integer.toString(i));
                m.put("cf", "cq", new Value());
                bw.addMutation(m);
                bw.flush();
                c.tableOperations().flush(tableName, null, null, true);
            }
        }
        List<String> files = FunctionalTestUtils.getRFilePaths(c, tableName);
        assertEquals(4, files.size());
        String subset = files.get(0).substring(files.get(0).lastIndexOf('/') + 1) + "," + files.get(3).substring(files.get(3).lastIndexOf('/') + 1);
        CompactionConfig config = new CompactionConfig().setSelector(new PluginConfig(RandomErrorThrowingSelector.class.getName(), Map.of(RandomErrorThrowingSelector.FILE_LIST_PARAM, subset))).setWait(true);
        c.tableOperations().compact(tableName, config);
        // check that the subset of files selected are compacted, but the others remain untouched
        List<String> filesAfterCompact = FunctionalTestUtils.getRFilePaths(c, tableName);
        assertFalse(filesAfterCompact.contains(files.get(0)));
        assertTrue(filesAfterCompact.contains(files.get(1)));
        assertTrue(filesAfterCompact.contains(files.get(2)));
        assertFalse(filesAfterCompact.contains(files.get(3)));
        List<String> rows = new ArrayList<>();
        c.createScanner(tableName).forEach((k, v) -> rows.add(k.getRow().toString()));
        assertEquals(List.of("1", "2", "3", "4"), rows);
    }
}

Example 9 with PluginConfig

use of org.apache.accumulo.core.client.admin.PluginConfig in project accumulo by apache.

the class CompactableUtils method selectFiles.

static Set<StoredTabletFile> selectFiles(Tablet tablet, SortedMap<StoredTabletFile, DataFileValue> datafiles, PluginConfig selectorConfig) {
    CompactionSelector selector = newInstance(tablet.getTableConfiguration(), selectorConfig.getClassName(), CompactionSelector.class);
    final ServiceEnvironment senv = new ServiceEnvironmentImpl(tablet.getContext());
    selector.init(new CompactionSelector.InitParameters() {

        @Override
        public Map<String, String> getOptions() {
            return selectorConfig.getOptions();
        }

        @Override
        public PluginEnvironment getEnvironment() {
            return senv;
        }

        @Override
        public TableId getTableId() {
            return tablet.getExtent().tableId();
        }
    });
    Selection selection = selector.select(new CompactionSelector.SelectionParameters() {

        @Override
        public PluginEnvironment getEnvironment() {
            return senv;
        }

        @Override
        public Collection<CompactableFile> getAvailableFiles() {
            return Collections2.transform(datafiles.entrySet(), e -> new CompactableFileImpl(e.getKey(), e.getValue()));
        }

        @Override
        public Collection<Summary> getSummaries(Collection<CompactableFile> files, Predicate<SummarizerConfiguration> summarySelector) {
            var context = tablet.getContext();
            var tsrm = tablet.getTabletResources().getTabletServerResourceManager();
            SummaryCollection sc = new SummaryCollection();
            SummarizerFactory factory = new SummarizerFactory(tablet.getTableConfiguration());
            for (CompactableFile cf : files) {
                var file = CompactableFileImpl.toStoredTabletFile(cf);
                FileSystem fs = context.getVolumeManager().getFileSystemByPath(file.getPath());
                Configuration conf = context.getHadoopConf();
                SummaryCollection fsc = SummaryReader.load(fs, conf, factory, file.getPath(), summarySelector, tsrm.getSummaryCache(), tsrm.getIndexCache(), tsrm.getFileLenCache(), context.getCryptoService()).getSummaries(Collections.singletonList(new Gatherer.RowRange(tablet.getExtent())));
                sc.merge(fsc, factory);
            }
            return sc.getSummaries();
        }

        @Override
        public TableId getTableId() {
            return tablet.getExtent().tableId();
        }

        @Override
        public Optional<SortedKeyValueIterator<Key, Value>> getSample(CompactableFile file, SamplerConfiguration sc) {
            try {
                FileOperations fileFactory = FileOperations.getInstance();
                Path path = new Path(file.getUri());
                FileSystem ns = tablet.getTabletServer().getVolumeManager().getFileSystemByPath(path);
                var fiter = fileFactory.newReaderBuilder().forFile(path.toString(), ns, ns.getConf(), tablet.getContext().getCryptoService()).withTableConfiguration(tablet.getTableConfiguration()).seekToBeginning().build();
                return Optional.ofNullable(fiter.getSample(new SamplerConfigurationImpl(sc)));
            } catch (IOException e) {
                throw new UncheckedIOException(e);
            }
        }
    });
    return selection.getFilesToCompact().stream().map(CompactableFileImpl::toStoredTabletFile).collect(Collectors.toSet());
}

Example 10 with PluginConfig

use of org.apache.accumulo.core.client.admin.PluginConfig in project accumulo by apache.

the class CompactableUtils method computeOverrides.

static Map<String, String> computeOverrides(Tablet tablet, Set<CompactableFile> files) {
    var tconf = tablet.getTableConfiguration();
    var configurorClass = tconf.get(Property.TABLE_COMPACTION_CONFIGURER);
    if (configurorClass == null || configurorClass.isBlank()) {
        return null;
    }
    var opts = tconf.getAllPropertiesWithPrefixStripped(Property.TABLE_COMPACTION_CONFIGURER_OPTS);
    return computeOverrides(tablet, files, new PluginConfig(configurorClass, opts));
}