Examples with SamplerConfigurationImpl org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl used on opensource projects

Example 11 with SamplerConfigurationImpl

use of org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl in project accumulo by apache.

the class OfflineIterator method createIterator.

private SortedKeyValueIterator<Key, Value> createIterator(KeyExtent extent, List<String> absFiles) throws TableNotFoundException, AccumuloException, IOException {
    // TODO share code w/ tablet - ACCUMULO-1303
    // possible race condition here, if table is renamed
    String tableName = Tables.getTableName(conn.getInstance(), tableId);
    AccumuloConfiguration acuTableConf = new ConfigurationCopy(conn.tableOperations().getProperties(tableName));
    Configuration conf = CachedConfiguration.getInstance();
    for (SortedKeyValueIterator<Key, Value> reader : readers) {
        ((FileSKVIterator) reader).close();
    }
    readers.clear();
    SamplerConfiguration scannerSamplerConfig = options.getSamplerConfiguration();
    SamplerConfigurationImpl scannerSamplerConfigImpl = scannerSamplerConfig == null ? null : new SamplerConfigurationImpl(scannerSamplerConfig);
    SamplerConfigurationImpl samplerConfImpl = SamplerConfigurationImpl.newSamplerConfig(acuTableConf);
    if (scannerSamplerConfigImpl != null && ((samplerConfImpl != null && !scannerSamplerConfigImpl.equals(samplerConfImpl)) || samplerConfImpl == null)) {
        throw new SampleNotPresentException();
    }
    // TODO need to close files - ACCUMULO-1303
    for (String file : absFiles) {
        FileSystem fs = VolumeConfiguration.getVolume(file, conf, config).getFileSystem();
        FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder().forFile(file, fs, conf).withTableConfiguration(acuTableConf).build();
        if (scannerSamplerConfigImpl != null) {
            reader = reader.getSample(scannerSamplerConfigImpl);
            if (reader == null)
                throw new SampleNotPresentException();
        }
        readers.add(reader);
    }
    MultiIterator multiIter = new MultiIterator(readers, extent);
    OfflineIteratorEnvironment iterEnv = new OfflineIteratorEnvironment(authorizations, acuTableConf, false, samplerConfImpl == null ? null : samplerConfImpl.toSamplerConfiguration());
    byte[] defaultSecurityLabel;
    ColumnVisibility cv = new ColumnVisibility(acuTableConf.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY));
    defaultSecurityLabel = cv.getExpression();
    SortedKeyValueIterator<Key, Value> visFilter = IteratorUtil.setupSystemScanIterators(multiIter, new HashSet<>(options.fetchedColumns), authorizations, defaultSecurityLabel);
    return iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(IteratorScope.scan, visFilter, extent, acuTableConf, options.serverSideIteratorList, options.serverSideIteratorOptions, iterEnv, false));
}

Example 12 with SamplerConfigurationImpl

use of org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl in project accumulo by apache.

the class FileOutputConfigurator method setSampler.

/**
 * @since 1.8.0
 */
public static void setSampler(Class<?> implementingClass, Configuration conf, SamplerConfiguration samplerConfig) {
    Map<String, String> props = new SamplerConfigurationImpl(samplerConfig).toTablePropertiesMap();
    Set<Entry<String, String>> es = props.entrySet();
    for (Entry<String, String> entry : es) {
        conf.set(enumToConfKey(implementingClass, Opts.ACCUMULO_PROPERTIES) + "." + entry.getKey(), entry.getValue());
    }
}

Example 13 with SamplerConfigurationImpl

use of org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl in project accumulo by apache.

the class InputConfigurator method setSamplerConfiguration.

public static void setSamplerConfiguration(Class<?> implementingClass, Configuration conf, SamplerConfiguration samplerConfig) {
    requireNonNull(samplerConfig);
    String key = enumToConfKey(implementingClass, ScanOpts.SAMPLER_CONFIG);
    String val = toBase64(new SamplerConfigurationImpl(samplerConfig));
    conf.set(key, val);
}

Example 14 with SamplerConfigurationImpl

use of org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl in project accumulo by apache.

the class InMemoryMapTest method testSample.

@Test
public void testSample() throws Exception {
    SamplerConfigurationImpl sampleConfig = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "7"));
    Sampler sampler = SamplerFactory.newSampler(sampleConfig, DefaultConfiguration.getInstance());
    ConfigurationCopy config1 = newConfig(tempFolder.newFolder().getAbsolutePath());
    for (Entry<String, String> entry : sampleConfig.toTablePropertiesMap().entrySet()) {
        config1.set(entry.getKey(), entry.getValue());
    }
    ConfigurationCopy config2 = newConfig(tempFolder.newFolder().getAbsolutePath());
    config2.set(Property.TABLE_LOCALITY_GROUP_PREFIX + "lg1", LocalityGroupUtil.encodeColumnFamilies(toTextSet("cf2")));
    config2.set(Property.TABLE_LOCALITY_GROUPS.getKey(), "lg1");
    for (Entry<String, String> entry : sampleConfig.toTablePropertiesMap().entrySet()) {
        config2.set(entry.getKey(), entry.getValue());
    }
    for (ConfigurationCopy config : Arrays.asList(config1, config2)) {
        InMemoryMap imm = new InMemoryMap(config);
        TreeMap<Key, Value> expectedSample = new TreeMap<>();
        TreeMap<Key, Value> expectedAll = new TreeMap<>();
        TreeMap<Key, Value> expectedNone = new TreeMap<>();
        MemoryIterator iter0 = imm.skvIterator(sampleConfig);
        for (int r = 0; r < 100; r++) {
            String row = String.format("r%06d", r);
            mutate(imm, row, "cf1:cq1", 5, "v" + (2 * r), sampler, expectedSample, expectedAll);
            mutate(imm, row, "cf2:cq2", 5, "v" + ((2 * r) + 1), sampler, expectedSample, expectedAll);
        }
        assertTrue(expectedSample.size() > 0);
        MemoryIterator iter1 = imm.skvIterator(sampleConfig);
        MemoryIterator iter2 = imm.skvIterator(null);
        SortedKeyValueIterator<Key, Value> iter0dc1 = iter0.deepCopy(new SampleIE());
        SortedKeyValueIterator<Key, Value> iter0dc2 = iter0.deepCopy(new SampleIE(sampleConfig));
        SortedKeyValueIterator<Key, Value> iter1dc1 = iter1.deepCopy(new SampleIE());
        SortedKeyValueIterator<Key, Value> iter1dc2 = iter1.deepCopy(new SampleIE(sampleConfig));
        SortedKeyValueIterator<Key, Value> iter2dc1 = iter2.deepCopy(new SampleIE());
        SortedKeyValueIterator<Key, Value> iter2dc2 = iter2.deepCopy(new SampleIE(sampleConfig));
        assertEquals(expectedNone, readAll(iter0));
        assertEquals(expectedNone, readAll(iter0dc1));
        assertEquals(expectedNone, readAll(iter0dc2));
        assertEquals(expectedSample, readAll(iter1));
        assertEquals(expectedAll, readAll(iter2));
        assertEquals(expectedAll, readAll(iter1dc1));
        assertEquals(expectedAll, readAll(iter2dc1));
        assertEquals(expectedSample, readAll(iter1dc2));
        assertEquals(expectedSample, readAll(iter2dc2));
        imm.delete(0);
        assertEquals(expectedNone, readAll(iter0));
        assertEquals(expectedNone, readAll(iter0dc1));
        assertEquals(expectedNone, readAll(iter0dc2));
        assertEquals(expectedSample, readAll(iter1));
        assertEquals(expectedAll, readAll(iter2));
        assertEquals(expectedAll, readAll(iter1dc1));
        assertEquals(expectedAll, readAll(iter2dc1));
        assertEquals(expectedSample, readAll(iter1dc2));
        assertEquals(expectedSample, readAll(iter2dc2));
        SortedKeyValueIterator<Key, Value> iter0dc3 = iter0.deepCopy(new SampleIE());
        SortedKeyValueIterator<Key, Value> iter0dc4 = iter0.deepCopy(new SampleIE(sampleConfig));
        SortedKeyValueIterator<Key, Value> iter1dc3 = iter1.deepCopy(new SampleIE());
        SortedKeyValueIterator<Key, Value> iter1dc4 = iter1.deepCopy(new SampleIE(sampleConfig));
        SortedKeyValueIterator<Key, Value> iter2dc3 = iter2.deepCopy(new SampleIE());
        SortedKeyValueIterator<Key, Value> iter2dc4 = iter2.deepCopy(new SampleIE(sampleConfig));
        assertEquals(expectedNone, readAll(iter0dc3));
        assertEquals(expectedNone, readAll(iter0dc4));
        assertEquals(expectedAll, readAll(iter1dc3));
        assertEquals(expectedAll, readAll(iter2dc3));
        assertEquals(expectedSample, readAll(iter1dc4));
        assertEquals(expectedSample, readAll(iter2dc4));
        iter1.close();
        iter2.close();
    }
}

Example 15 with SamplerConfigurationImpl

use of org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl in project accumulo by apache.

the class InMemoryMapTest method testDifferentSampleConfig.

@Test(expected = SampleNotPresentException.class)
public void testDifferentSampleConfig() throws Exception {
    SamplerConfigurationImpl sampleConfig = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "7"));
    ConfigurationCopy config1 = newConfig(tempFolder.newFolder().getAbsolutePath());
    for (Entry<String, String> entry : sampleConfig.toTablePropertiesMap().entrySet()) {
        config1.set(entry.getKey(), entry.getValue());
    }
    InMemoryMap imm = new InMemoryMap(config1);
    mutate(imm, "r", "cf:cq", 5, "b");
    SamplerConfigurationImpl sampleConfig2 = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "9"));
    MemoryIterator iter = imm.skvIterator(sampleConfig2);
    iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false);
}