Examples with BatchWriterConfig org.apache.accumulo.core.client.BatchWriterConfig used on opensource projects

Example 91 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class AccumuloOutputFormatIT method testMapred.

// Prevent regression of ACCUMULO-3709.
@Test
public void testMapred() throws Exception {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
        // create a table and put some data in it
        client.tableOperations().create(testName.getMethodName());
        JobConf job = new JobConf();
        BatchWriterConfig batchConfig = new BatchWriterConfig();
        // no flushes!!!!!
        batchConfig.setMaxLatency(0, TimeUnit.MILLISECONDS);
        // use a single thread to ensure our update session times out
        batchConfig.setMaxWriteThreads(1);
        // set the max memory so that we ensure we don't flush on the write.
        batchConfig.setMaxMemory(Long.MAX_VALUE);
        AccumuloOutputFormat outputFormat = new AccumuloOutputFormat();
        Properties props = Accumulo.newClientProperties().from(getClientProperties()).batchWriterConfig(batchConfig).build();
        AccumuloOutputFormat.configure().clientProperties(props).store(job);
        RecordWriter<Text, Mutation> writer = outputFormat.getRecordWriter(null, job, "Test", null);
        try {
            for (int i = 0; i < 3; i++) {
                Mutation m = new Mutation(new Text(String.format("%08d", i)));
                for (int j = 0; j < 3; j++) {
                    m.put("cf1", "cq" + j, i + "_" + j);
                }
                writer.write(new Text(testName.getMethodName()), m);
            }
        } catch (Exception e) {
            e.printStackTrace();
        // we don't want the exception to come from write
        }
        client.securityOperations().revokeTablePermission("root", testName.getMethodName(), TablePermission.WRITE);
        var ex = assertThrows(IOException.class, () -> writer.close(null));
        log.info(ex.getMessage(), ex);
        assertTrue(ex.getCause() instanceof MutationsRejectedException);
    }
}

Example 92 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class AccumuloOutputFormatTest method testBWSettings.

@Test
public void testBWSettings() throws IOException {
    Job job = Job.getInstance();
    // make sure we aren't testing defaults
    final BatchWriterConfig bwDefaults = new BatchWriterConfig();
    assertNotEquals(7654321L, bwDefaults.getMaxLatency(TimeUnit.MILLISECONDS));
    assertNotEquals(9898989L, bwDefaults.getTimeout(TimeUnit.MILLISECONDS));
    assertNotEquals(42, bwDefaults.getMaxWriteThreads());
    assertNotEquals(1123581321L, bwDefaults.getMaxMemory());
    final BatchWriterConfig bwConfig = new BatchWriterConfig();
    bwConfig.setMaxLatency(7654321L, TimeUnit.MILLISECONDS);
    bwConfig.setTimeout(9898989L, TimeUnit.MILLISECONDS);
    bwConfig.setMaxWriteThreads(42);
    bwConfig.setMaxMemory(1123581321L);
    Properties cp = Accumulo.newClientProperties().to("test", "zk").as("blah", "blah").batchWriterConfig(bwConfig).build();
    AccumuloOutputFormat.configure().clientProperties(cp).store(job);
    AccumuloOutputFormat myAOF = new AccumuloOutputFormat() {

        @Override
        public void checkOutputSpecs(JobContext job) {
            BatchWriterConfig bwOpts = OutputConfigurator.getBatchWriterOptions(AccumuloOutputFormat.class, job.getConfiguration());
            // passive check
            assertEquals(bwConfig.getMaxLatency(TimeUnit.MILLISECONDS), bwOpts.getMaxLatency(TimeUnit.MILLISECONDS));
            assertEquals(bwConfig.getTimeout(TimeUnit.MILLISECONDS), bwOpts.getTimeout(TimeUnit.MILLISECONDS));
            assertEquals(bwConfig.getMaxWriteThreads(), bwOpts.getMaxWriteThreads());
            assertEquals(bwConfig.getMaxMemory(), bwOpts.getMaxMemory());
            // explicit check
            assertEquals(7654321L, bwOpts.getMaxLatency(TimeUnit.MILLISECONDS));
            assertEquals(9898989L, bwOpts.getTimeout(TimeUnit.MILLISECONDS));
            assertEquals(42, bwOpts.getMaxWriteThreads());
            assertEquals(1123581321L, bwOpts.getMaxMemory());
        }
    };
    myAOF.checkOutputSpecs(job);
}

Example 93 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class BatchWriterInTabletServerIT method test.

private void test(String t1, String t2, AccumuloClient c, IteratorSetting itset, int numEntriesToWritePerEntry) throws Exception {
    // Write an entry to t1
    c.tableOperations().create(t1);
    Key k = new Key(new Text("row"), new Text("cf"), new Text("cq"));
    Value v = new Value("1");
    BatchWriterConfig config = new BatchWriterConfig();
    config.setMaxMemory(0);
    try (BatchWriter writer = c.createBatchWriter(t1, config)) {
        Mutation m = new Mutation(k.getRow());
        m.put(k.getColumnFamily(), k.getColumnQualifier(), v);
        writer.addMutation(m);
    }
    // Create t2 with a combiner to count entries written to it
    c.tableOperations().create(t2);
    IteratorSetting summer = new IteratorSetting(2, "summer", SummingCombiner.class);
    LongCombiner.setEncodingType(summer, LongCombiner.Type.STRING);
    LongCombiner.setCombineAllColumns(summer, true);
    c.tableOperations().attachIterator(t2, summer);
    Map.Entry<Key, Value> actual;
    try (Scanner scanner = c.createScanner(t1, Authorizations.EMPTY)) {
        // Scan t1 with an iterator that writes to table t2
        scanner.addScanIterator(itset);
        actual = Iterators.getOnlyElement(scanner.iterator());
        assertTrue(actual.getKey().equals(k, PartialKey.ROW_COLFAM_COLQUAL));
        assertEquals(BatchWriterIterator.SUCCESS_VALUE, actual.getValue());
    }
    try (Scanner scanner = c.createScanner(t2, Authorizations.EMPTY)) {
        // ensure entries correctly wrote to table t2
        actual = Iterators.getOnlyElement(scanner.iterator());
        log.debug("t2 entry is {} -> {}", actual.getKey().toStringNoTime(), actual.getValue());
        assertTrue(actual.getKey().equals(k, PartialKey.ROW_COLFAM_COLQUAL));
        assertEquals(numEntriesToWritePerEntry, Integer.parseInt(actual.getValue().toString()));
    }
    c.tableOperations().delete(t1);
    c.tableOperations().delete(t2);
}

Example 94 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class BatchWriterIT method test.

@Test
public void test() throws Exception {
    // call the batchwriter with buffer of size zero
    String table = getUniqueNames(1)[0];
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        c.tableOperations().create(table);
        BatchWriterConfig config = new BatchWriterConfig();
        config.setMaxMemory(0);
        try (BatchWriter writer = c.createBatchWriter(table, config)) {
            Mutation m = new Mutation("row");
            m.put("cf", "cq", new Value("value"));
            writer.addMutation(m);
        }
    }
}

Example 95 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class IteratorMincClassCastBugIT method test.

@Test
public void test() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        String tableName = getUniqueNames(1)[0];
        NewTableConfiguration ntc = new NewTableConfiguration();
        Map<String, Set<Text>> groups = new HashMap<>();
        groups.put("g1", Set.of(new Text("even")));
        groups.put("g2", Set.of(new Text("odd")));
        ntc.setLocalityGroups(groups);
        IteratorSetting iteratorSetting = new IteratorSetting(20, ClasCastIterator.class);
        ntc.attachIterator(iteratorSetting, EnumSet.of(IteratorUtil.IteratorScope.minc));
        c.tableOperations().create(tableName, ntc);
        try (BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig())) {
            Mutation m1 = new Mutation(new Text("r1"));
            m1.put(new Text("odd"), new Text(), new Value("1"));
            bw.addMutation(m1);
            Mutation m2 = new Mutation(new Text("r2"));
            m2.put(new Text("even"), new Text(), new Value("2"));
            bw.addMutation(m2);
        }
        // class cast exception will happen in tserver and hang on flush
        c.tableOperations().flush(tableName, null, null, true);
    }
}