Examples with SamplerConfigurationImpl org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl used on opensource projects

Example 6 with SamplerConfigurationImpl

use of org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl in project accumulo by apache.

the class InMemoryMapTest method testDeferredSamplerCreation.

@Test
public void testDeferredSamplerCreation() throws Exception {
    SamplerConfigurationImpl sampleConfig1 = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "9"));
    ConfigurationCopy config1 = newConfig(tempFolder.newFolder().getAbsolutePath());
    for (Entry<String, String> entry : sampleConfig1.toTablePropertiesMap().entrySet()) {
        config1.set(entry.getKey(), entry.getValue());
    }
    InMemoryMap imm = new InMemoryMap(config1);
    // change sampler config after creating in mem map.
    SamplerConfigurationImpl sampleConfig2 = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "7"));
    for (Entry<String, String> entry : sampleConfig2.toTablePropertiesMap().entrySet()) {
        config1.set(entry.getKey(), entry.getValue());
    }
    TreeMap<Key, Value> expectedSample = new TreeMap<>();
    TreeMap<Key, Value> expectedAll = new TreeMap<>();
    Sampler sampler = SamplerFactory.newSampler(sampleConfig2, config1);
    for (int i = 0; i < 100; i++) {
        mutate(imm, "r" + i, "cf:cq", 5, "v" + i, sampler, expectedSample, expectedAll);
    }
    MemoryIterator iter = imm.skvIterator(sampleConfig2);
    iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false);
    Assert.assertEquals(expectedSample, readAll(iter));
    SortedKeyValueIterator<Key, Value> dc = iter.deepCopy(new SampleIE(sampleConfig2));
    dc.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false);
    Assert.assertEquals(expectedSample, readAll(dc));
    iter = imm.skvIterator(null);
    iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false);
    Assert.assertEquals(expectedAll, readAll(iter));
    iter = imm.skvIterator(sampleConfig1);
    thrown.expect(SampleNotPresentException.class);
    iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false);
}

Example 7 with SamplerConfigurationImpl

use of org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl in project accumulo by apache.

the class InMemoryMapTest method testEmptyNoSampleConfig.

@Test
public void testEmptyNoSampleConfig() throws Exception {
    InMemoryMap imm = newInMemoryMap(false, tempFolder.newFolder().getAbsolutePath());
    SamplerConfigurationImpl sampleConfig2 = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "9"));
    // when in mem map is empty should be able to get sample iterator with any sample config
    MemoryIterator iter = imm.skvIterator(sampleConfig2);
    iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false);
    Assert.assertFalse(iter.hasTop());
}

Example 8 with SamplerConfigurationImpl

use of org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl in project accumulo by apache.

the class AccumuloFileOutputFormatTest method validateConfiguration.

@Test
public void validateConfiguration() throws IOException, InterruptedException {
    int a = 7;
    long b = 300l;
    long c = 50l;
    long d = 10l;
    String e = "snappy";
    SamplerConfiguration samplerConfig = new SamplerConfiguration(RowSampler.class.getName());
    samplerConfig.addOption("hasher", "murmur3_32");
    samplerConfig.addOption("modulus", "109");
    SummarizerConfiguration sc1 = SummarizerConfiguration.builder(VisibilitySummarizer.class).addOption(CountingSummarizer.MAX_COUNTERS_OPT, 2048).build();
    SummarizerConfiguration sc2 = SummarizerConfiguration.builder(FamilySummarizer.class).addOption(CountingSummarizer.MAX_COUNTERS_OPT, 256).build();
    JobConf job = new JobConf();
    AccumuloFileOutputFormat.setReplication(job, a);
    AccumuloFileOutputFormat.setFileBlockSize(job, b);
    AccumuloFileOutputFormat.setDataBlockSize(job, c);
    AccumuloFileOutputFormat.setIndexBlockSize(job, d);
    AccumuloFileOutputFormat.setCompressionType(job, e);
    AccumuloFileOutputFormat.setSampler(job, samplerConfig);
    AccumuloFileOutputFormat.setSummarizers(job, sc1, sc2);
    AccumuloConfiguration acuconf = FileOutputConfigurator.getAccumuloConfiguration(AccumuloFileOutputFormat.class, job);
    assertEquals(7, acuconf.getCount(Property.TABLE_FILE_REPLICATION));
    assertEquals(300l, acuconf.getAsBytes(Property.TABLE_FILE_BLOCK_SIZE));
    assertEquals(50l, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE));
    assertEquals(10l, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
    assertEquals("snappy", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE));
    assertEquals(new SamplerConfigurationImpl(samplerConfig), SamplerConfigurationImpl.newSamplerConfig(acuconf));
    Collection<SummarizerConfiguration> summarizerConfigs = SummarizerConfiguration.fromTableProperties(acuconf);
    assertEquals(2, summarizerConfigs.size());
    assertTrue(summarizerConfigs.contains(sc1));
    assertTrue(summarizerConfigs.contains(sc2));
    a = 17;
    b = 1300l;
    c = 150l;
    d = 110l;
    e = "lzo";
    samplerConfig = new SamplerConfiguration(RowSampler.class.getName());
    samplerConfig.addOption("hasher", "md5");
    samplerConfig.addOption("modulus", "100003");
    job = new JobConf();
    AccumuloFileOutputFormat.setReplication(job, a);
    AccumuloFileOutputFormat.setFileBlockSize(job, b);
    AccumuloFileOutputFormat.setDataBlockSize(job, c);
    AccumuloFileOutputFormat.setIndexBlockSize(job, d);
    AccumuloFileOutputFormat.setCompressionType(job, e);
    AccumuloFileOutputFormat.setSampler(job, samplerConfig);
    acuconf = FileOutputConfigurator.getAccumuloConfiguration(AccumuloFileOutputFormat.class, job);
    assertEquals(17, acuconf.getCount(Property.TABLE_FILE_REPLICATION));
    assertEquals(1300l, acuconf.getAsBytes(Property.TABLE_FILE_BLOCK_SIZE));
    assertEquals(150l, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE));
    assertEquals(110l, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
    assertEquals("lzo", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE));
    assertEquals(new SamplerConfigurationImpl(samplerConfig), SamplerConfigurationImpl.newSamplerConfig(acuconf));
    summarizerConfigs = SummarizerConfiguration.fromTableProperties(acuconf);
    assertEquals(0, summarizerConfigs.size());
}

Example 9 with SamplerConfigurationImpl

use of org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl in project accumulo by apache.

the class TableOperationsImpl method setSamplerConfiguration.

@Override
public void setSamplerConfiguration(String tableName, SamplerConfiguration samplerConfiguration) throws AccumuloException, TableNotFoundException, AccumuloSecurityException {
    clearSamplerOptions(tableName);
    List<Pair<String, String>> props = new SamplerConfigurationImpl(samplerConfiguration).toTableProperties();
    for (Pair<String, String> pair : props) {
        setProperty(tableName, pair.getFirst(), pair.getSecond());
    }
}

Example 10 with SamplerConfigurationImpl

use of org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl in project accumulo by apache.

the class TableOperationsImpl method getSamplerConfiguration.

@Override
public SamplerConfiguration getSamplerConfiguration(String tableName) throws TableNotFoundException, AccumuloException {
    AccumuloConfiguration conf = new ConfigurationCopy(this.getProperties(tableName));
    SamplerConfigurationImpl sci = SamplerConfigurationImpl.newSamplerConfig(conf);
    if (sci == null) {
        return null;
    }
    return sci.toSamplerConfiguration();
}