Examples with BatchWriterConfig org.apache.accumulo.core.client.BatchWriterConfig used on opensource projects

Example 86 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class OutputConfigurator method getBatchWriterOptions.

/**
 * Gets the {@link BatchWriterConfig} settings that were stored with ClientInfo
 */
public static BatchWriterConfig getBatchWriterOptions(Class<?> implementingClass, Configuration conf) {
    BatchWriterConfig bwConfig = new BatchWriterConfig();
    Properties props = getClientProperties(implementingClass, conf);
    String property = props.getProperty(BATCH_WRITER_DURABILITY.getKey());
    if (property != null)
        bwConfig.setDurability(DurabilityImpl.fromString(property));
    Long value = BATCH_WRITER_LATENCY_MAX.getTimeInMillis(props);
    if (value != null)
        bwConfig.setMaxLatency(value, TimeUnit.MILLISECONDS);
    value = BATCH_WRITER_MEMORY_MAX.getBytes(props);
    if (value != null)
        bwConfig.setMaxMemory(value);
    value = BATCH_WRITER_TIMEOUT_MAX.getTimeInMillis(props);
    if (value != null)
        bwConfig.setTimeout(value, TimeUnit.MILLISECONDS);
    Integer intValue = BATCH_WRITER_THREADS_MAX.getInteger(props);
    if (intValue != null)
        bwConfig.setMaxWriteThreads(intValue);
    return bwConfig;
}

Example 87 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class AccumuloOutputFormatIT method testMapred.

// Prevent regression of ACCUMULO-3709.
@Test
public void testMapred() throws Exception {
    Properties props = getClientProperties();
    try (AccumuloClient client = Accumulo.newClient().from(props).build()) {
        // create a table and put some data in it
        client.tableOperations().create(testName.getMethodName());
        JobConf job = new JobConf();
        BatchWriterConfig batchConfig = new BatchWriterConfig();
        // no flushes!!!!!
        batchConfig.setMaxLatency(0, TimeUnit.MILLISECONDS);
        // use a single thread to ensure our update session times out
        batchConfig.setMaxWriteThreads(1);
        // set the max memory so that we ensure we don't flush on the write.
        batchConfig.setMaxMemory(Long.MAX_VALUE);
        org.apache.accumulo.core.client.mapred.AccumuloOutputFormat outputFormat = new org.apache.accumulo.core.client.mapred.AccumuloOutputFormat();
        ClientInfo ci = ClientInfo.from(props);
        org.apache.accumulo.core.client.mapred.AccumuloOutputFormat.setZooKeeperInstance(job, ci.getInstanceName(), ci.getZooKeepers());
        org.apache.accumulo.core.client.mapred.AccumuloOutputFormat.setConnectorInfo(job, ci.getPrincipal(), ci.getAuthenticationToken());
        org.apache.accumulo.core.client.mapred.AccumuloOutputFormat.setBatchWriterOptions(job, batchConfig);
        RecordWriter<Text, Mutation> writer = outputFormat.getRecordWriter(null, job, "Test", null);
        try {
            for (int i = 0; i < 3; i++) {
                Mutation m = new Mutation(new Text(String.format("%08d", i)));
                for (int j = 0; j < 3; j++) {
                    m.put("cf1", "cq" + j, i + "_" + j);
                }
                writer.write(new Text(testName.getMethodName()), m);
            }
        } catch (Exception e) {
            e.printStackTrace();
        // we don't want the exception to come from write
        }
        client.securityOperations().revokeTablePermission("root", testName.getMethodName(), TablePermission.WRITE);
        var ex = assertThrows(IOException.class, () -> writer.close(null));
        log.info(ex.getMessage(), ex);
        assertTrue(ex.getCause() instanceof MutationsRejectedException);
    }
}

Example 88 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class TimeoutIT method testBatchWriterTimeout.

public void testBatchWriterTimeout(AccumuloClient client, String tableName) throws Exception {
    client.tableOperations().create(tableName);
    client.tableOperations().addConstraint(tableName, SlowConstraint.class.getName());
    // give constraint time to propagate through zookeeper
    sleepUninterruptibly(1, TimeUnit.SECONDS);
    BatchWriter bw = client.createBatchWriter(tableName, new BatchWriterConfig().setTimeout(3, TimeUnit.SECONDS));
    Mutation mut = new Mutation("r1");
    mut.put("cf1", "cq1", "v1");
    bw.addMutation(mut);
    var mre = assertThrows("batch writer did not timeout", MutationsRejectedException.class, bw::close);
    if (mre.getCause() instanceof TimedOutException)
        return;
    throw mre;
}

Example 89 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class WriteLotsIT method writeLots.

@Test
public void writeLots() throws Exception {
    BatchWriterConfig bwConfig = new BatchWriterConfig();
    bwConfig.setMaxMemory(1024L * 1024);
    bwConfig.setMaxWriteThreads(2);
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).batchWriterConfig(bwConfig).build()) {
        final String tableName = getUniqueNames(1)[0];
        c.tableOperations().create(tableName);
        final AtomicReference<Exception> ref = new AtomicReference<>();
        final int THREADS = 5;
        ThreadPoolExecutor tpe = new ThreadPoolExecutor(0, THREADS, 0, TimeUnit.SECONDS, new ArrayBlockingQueue<>(THREADS));
        for (int i = 0; i < THREADS; i++) {
            final int index = i;
            Runnable r = () -> {
                try {
                    IngestParams ingestParams = new IngestParams(getClientProps(), tableName, 10_000);
                    ingestParams.startRow = index * 10000;
                    TestIngest.ingest(c, ingestParams);
                } catch (Exception ex) {
                    ref.set(ex);
                }
            };
            tpe.execute(r);
        }
        tpe.shutdown();
        tpe.awaitTermination(90, TimeUnit.SECONDS);
        if (ref.get() != null) {
            throw ref.get();
        }
        VerifyParams params = new VerifyParams(getClientProps(), tableName, 10_000 * THREADS);
        VerifyIngest.verifyIngest(c, params);
    }
}

Example 90 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class SessionDurabilityIT method nondurableTableHasDurableWrites.

@Test
public void nondurableTableHasDurableWrites() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProperties()).build()) {
        String tableName = getUniqueNames(1)[0];
        // table default has no durability
        c.tableOperations().create(tableName, new NewTableConfiguration().setProperties(singletonMap(Property.TABLE_DURABILITY.getKey(), "none")));
        // send durable writes
        BatchWriterConfig cfg = new BatchWriterConfig();
        cfg.setDurability(Durability.SYNC);
        writeSome(c, tableName, 10, cfg);
        assertEquals(10, count(c, tableName));
        // verify writes servive restart
        restartTServer();
        assertEquals(10, count(c, tableName));
    }
}