Examples with NewTableConfiguration org.apache.accumulo.core.client.admin.NewTableConfiguration used on opensource projects

Example 51 with NewTableConfiguration

use of org.apache.accumulo.core.client.admin.NewTableConfiguration in project accumulo by apache.

the class AccumuloInputFormatIT method testSample.

@Test
public void testSample() throws Exception {
    final String TEST_TABLE_3 = getUniqueNames(1)[0];
    Connector c = getConnector();
    c.tableOperations().create(TEST_TABLE_3, new NewTableConfiguration().enableSampling(SAMPLER_CONFIG));
    BatchWriter bw = c.createBatchWriter(TEST_TABLE_3, new BatchWriterConfig());
    for (int i = 0; i < 100; i++) {
        Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
        m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
        bw.addMutation(m);
    }
    bw.close();
    MRTester.main(TEST_TABLE_3, "False", "True");
    Assert.assertEquals(38, e1Count);
    Assert.assertEquals(1, e2Count);
    e2Count = e1Count = 0;
    MRTester.main(TEST_TABLE_3, "False", "False");
    Assert.assertEquals(0, e1Count);
    Assert.assertEquals(0, e2Count);
    e2Count = e1Count = 0;
    MRTester.main(TEST_TABLE_3, "True", "True");
    Assert.assertEquals(38, e1Count);
    Assert.assertEquals(1, e2Count);
}

Example 52 with NewTableConfiguration

use of org.apache.accumulo.core.client.admin.NewTableConfiguration in project accumulo by apache.

the class AccumuloInputFormatIT method testSample.

@Test
public void testSample() throws Exception {
    final String TEST_TABLE_3 = getUniqueNames(1)[0];
    Connector c = getConnector();
    c.tableOperations().create(TEST_TABLE_3, new NewTableConfiguration().enableSampling(SAMPLER_CONFIG));
    BatchWriter bw = c.createBatchWriter(TEST_TABLE_3, new BatchWriterConfig());
    for (int i = 0; i < 100; i++) {
        Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
        m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
        bw.addMutation(m);
    }
    bw.close();
    Assert.assertEquals(0, MRTester.main(new String[] { TEST_TABLE_3, AccumuloInputFormat.class.getName(), "False", "True" }));
    assertEquals(39, assertionErrors.get(TEST_TABLE_3 + "_map").size());
    assertEquals(2, assertionErrors.get(TEST_TABLE_3 + "_cleanup").size());
    assertionErrors.clear();
    Assert.assertEquals(0, MRTester.main(new String[] { TEST_TABLE_3, AccumuloInputFormat.class.getName(), "False", "False" }));
    assertEquals(1, assertionErrors.get(TEST_TABLE_3 + "_map").size());
    assertEquals(1, assertionErrors.get(TEST_TABLE_3 + "_cleanup").size());
    assertionErrors.clear();
    Assert.assertEquals(0, MRTester.main(new String[] { TEST_TABLE_3, AccumuloInputFormat.class.getName(), "True", "True" }));
    assertEquals(39, assertionErrors.get(TEST_TABLE_3 + "_map").size());
    assertEquals(2, assertionErrors.get(TEST_TABLE_3 + "_cleanup").size());
}

Example 53 with NewTableConfiguration

use of org.apache.accumulo.core.client.admin.NewTableConfiguration in project accumulo by apache.

the class SummaryIT method compactionTest.

@Test
public void compactionTest() throws Exception {
    final String table = getUniqueNames(1)[0];
    Connector c = getConnector();
    NewTableConfiguration ntc = new NewTableConfiguration();
    SummarizerConfiguration sc1 = SummarizerConfiguration.builder(FooCounter.class.getName()).build();
    ntc.enableSummarization(sc1);
    c.tableOperations().create(table, ntc);
    try (BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig())) {
        write(bw, "bar1", "f1", "q1", "v1");
        write(bw, "bar2", "f1", "q1", "v2");
        write(bw, "foo1", "f1", "q1", "v3");
    }
    // Create a compaction config that will filter out foos if there are too many. Uses summary data to know if there are too many foos.
    CompactionStrategyConfig csc = new CompactionStrategyConfig(FooCS.class.getName());
    List<IteratorSetting> iterators = Collections.singletonList(new IteratorSetting(100, FooFilter.class));
    CompactionConfig compactConfig = new CompactionConfig().setFlush(true).setCompactionStrategy(csc).setIterators(iterators).setWait(true);
    // this compaction should make no changes because there are less foos than bars
    c.tableOperations().compact(table, compactConfig);
    try (Scanner scanner = c.createScanner(table, Authorizations.EMPTY)) {
        Stream<Entry<Key, Value>> stream = StreamSupport.stream(scanner.spliterator(), false);
        Map<String, Long> counts = // convert to row
        stream.map(e -> e.getKey().getRowData().toString()).map(// strip numbers off row
        r -> r.replaceAll("[0-9]+", "")).collect(// count different row types
        groupingBy(identity(), counting()));
        Assert.assertEquals(1l, (long) counts.getOrDefault("foo", 0l));
        Assert.assertEquals(2l, (long) counts.getOrDefault("bar", 0l));
        Assert.assertEquals(2, counts.size());
    }
    try (BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig())) {
        write(bw, "foo2", "f1", "q1", "v4");
        write(bw, "foo3", "f1", "q1", "v5");
        write(bw, "foo4", "f1", "q1", "v6");
    }
    // this compaction should remove all foos because there are more foos than bars
    c.tableOperations().compact(table, compactConfig);
    try (Scanner scanner = c.createScanner(table, Authorizations.EMPTY)) {
        Stream<Entry<Key, Value>> stream = StreamSupport.stream(scanner.spliterator(), false);
        Map<String, Long> counts = // convert to row
        stream.map(e -> e.getKey().getRowData().toString()).map(// strip numbers off row
        r -> r.replaceAll("[0-9]+", "")).collect(// count different row types
        groupingBy(identity(), counting()));
        Assert.assertEquals(0l, (long) counts.getOrDefault("foo", 0l));
        Assert.assertEquals(2l, (long) counts.getOrDefault("bar", 0l));
        Assert.assertEquals(1, counts.size());
    }
}

Example 54 with NewTableConfiguration

use of org.apache.accumulo.core.client.admin.NewTableConfiguration in project vertexium by visallo.

the class AccumuloGraph method ensureTableExists.

protected static void ensureTableExists(Connector connector, String tableName, Integer maxVersions, String hdfsContextClasspath, boolean createTable) {
    try {
        if (!connector.tableOperations().exists(tableName)) {
            if (!createTable) {
                throw new VertexiumException("Table '" + tableName + "' does not exist and 'graph." + GraphConfiguration.CREATE_TABLES + "' is set to false");
            }
            NewTableConfiguration ntc = new NewTableConfiguration().setTimeType(TimeType.MILLIS).withoutDefaultIterators();
            connector.tableOperations().create(tableName, ntc);
            if (maxVersions != null) {
                // The following parameters match the Accumulo defaults for the VersioningIterator
                IteratorSetting versioningSettings = new IteratorSetting(ACCUMULO_DEFAULT_VERSIONING_ITERATOR_PRIORITY, ACCUMULO_DEFAULT_VERSIONING_ITERATOR_NAME, VersioningIterator.class);
                VersioningIterator.setMaxVersions(versioningSettings, maxVersions);
                EnumSet<IteratorUtil.IteratorScope> scope = EnumSet.allOf(IteratorUtil.IteratorScope.class);
                connector.tableOperations().attachIterator(tableName, versioningSettings, scope);
            }
        }
        if (hdfsContextClasspath != null) {
            connector.instanceOperations().setProperty("general.vfs.context.classpath." + CLASSPATH_CONTEXT_NAME + "-" + tableName, hdfsContextClasspath);
            connector.tableOperations().setProperty(tableName, "table.classpath.context", CLASSPATH_CONTEXT_NAME + "-" + tableName);
        }
    } catch (Exception e) {
        throw new VertexiumException("Unable to create table " + tableName, e);
    }
}