Examples with BatchWriterConfig org.apache.accumulo.core.client.BatchWriterConfig used on opensource projects

Example 76 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project apex-malhar by apache.

the class AccumuloTestHelper method populateAccumulo.

public static void populateAccumulo() throws IOException {
    BatchWriterConfig config = new BatchWriterConfig();
    BatchWriter batchwriter = null;
    try {
        batchwriter = con.createBatchWriter("tab1", config);
    } catch (TableNotFoundException e) {
        logger.error("error in test helper");
        DTThrowable.rethrow(e);
    }
    try {
        for (int i = 0; i < 500; ++i) {
            String rowstr = "row" + i;
            Mutation mutation = new Mutation(rowstr.getBytes());
            for (int j = 0; j < 500; ++j) {
                String colstr = "col" + "-" + j;
                String valstr = "val" + "-" + i + "-" + j;
                mutation.put(colfam0_bytes, colstr.getBytes(), System.currentTimeMillis(), valstr.getBytes());
            }
            batchwriter.addMutation(mutation);
        }
        batchwriter.close();
    } catch (MutationsRejectedException e) {
        logger.error("error in test helper");
        DTThrowable.rethrow(e);
    }
}

Example 77 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class ClientContextTest method testGetBatchWriterConfigUsingDefaults.

@Test
public void testGetBatchWriterConfigUsingDefaults() {
    Properties props = new Properties();
    BatchWriterConfig batchWriterConfig = ClientContext.getBatchWriterConfig(props);
    assertNotNull(batchWriterConfig);
    long expectedMemory = ConfigurationTypeHelper.getMemoryAsBytes(ClientProperty.BATCH_WRITER_MEMORY_MAX.getDefaultValue());
    assertEquals(expectedMemory, batchWriterConfig.getMaxMemory());
    // If the value of BATCH_WRITE_LATENCY_MAX or BATCH_WRITER_TIMEOUT_MAX, is set to zero,
    // Long.MAX_VALUE is returned. Effectively, this will cause data to be held in memory
    // indefinitely for BATCH_WRITE_LATENCY_MAX and for no timeout, for BATCH_WRITER_TIMEOUT_MAX.
    // Due to this behavior, the test compares the return values differently. If a value of
    // 0 is used, compare the return value using TimeUnit.MILLISECONDS, otherwise the value
    // should be converted to seconds in order to match the value set in ClientProperty.
    long expectedLatency = ConfigurationTypeHelper.getTimeInMillis(ClientProperty.BATCH_WRITER_LATENCY_MAX.getDefaultValue());
    if (expectedLatency == 0) {
        expectedLatency = Long.MAX_VALUE;
        assertEquals(expectedLatency, batchWriterConfig.getMaxLatency(MILLISECONDS));
    } else {
        assertEquals(expectedLatency, batchWriterConfig.getMaxLatency(SECONDS));
    }
    long expectedTimeout = ConfigurationTypeHelper.getTimeInMillis(ClientProperty.BATCH_WRITER_TIMEOUT_MAX.getDefaultValue());
    if (expectedTimeout == 0) {
        expectedTimeout = Long.MAX_VALUE;
        assertEquals(expectedTimeout, batchWriterConfig.getTimeout(MILLISECONDS));
    } else {
        assertEquals(expectedTimeout, batchWriterConfig.getTimeout(SECONDS));
    }
    int expectedThreads = Integer.parseInt(ClientProperty.BATCH_WRITER_THREADS_MAX.getDefaultValue());
    assertEquals(expectedThreads, batchWriterConfig.getMaxWriteThreads());
    Durability expectedDurability = Durability.valueOf(ClientProperty.BATCH_WRITER_DURABILITY.getDefaultValue().toUpperCase());
    assertEquals(expectedDurability, batchWriterConfig.getDurability());
}

Example 78 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class AccumuloOutputFormatTest method testBWSettings.

@Test
public void testBWSettings() throws IOException {
    JobConf job = new JobConf();
    // make sure we aren't testing defaults
    final BatchWriterConfig bwDefaults = new BatchWriterConfig();
    assertNotEquals(7654321L, bwDefaults.getMaxLatency(MILLISECONDS));
    assertNotEquals(9898989L, bwDefaults.getTimeout(MILLISECONDS));
    assertNotEquals(42, bwDefaults.getMaxWriteThreads());
    assertNotEquals(1123581321L, bwDefaults.getMaxMemory());
    final BatchWriterConfig bwConfig = new BatchWriterConfig();
    bwConfig.setMaxLatency(7654321L, MILLISECONDS);
    bwConfig.setTimeout(9898989L, MILLISECONDS);
    bwConfig.setMaxWriteThreads(42);
    bwConfig.setMaxMemory(1123581321L);
    AccumuloOutputFormat.setBatchWriterOptions(job, bwConfig);
    AccumuloOutputFormat myAOF = new AccumuloOutputFormat() {

        @Override
        public void checkOutputSpecs(FileSystem ignored, JobConf job) {
            BatchWriterConfig bwOpts = getBatchWriterOptions(job);
            // passive check
            assertEquals(bwConfig.getMaxLatency(MILLISECONDS), bwOpts.getMaxLatency(MILLISECONDS));
            assertEquals(bwConfig.getTimeout(MILLISECONDS), bwOpts.getTimeout(MILLISECONDS));
            assertEquals(bwConfig.getMaxWriteThreads(), bwOpts.getMaxWriteThreads());
            assertEquals(bwConfig.getMaxMemory(), bwOpts.getMaxMemory());
            // explicit check
            assertEquals(7654321L, bwOpts.getMaxLatency(MILLISECONDS));
            assertEquals(9898989L, bwOpts.getTimeout(MILLISECONDS));
            assertEquals(42, bwOpts.getMaxWriteThreads());
            assertEquals(1123581321L, bwOpts.getMaxMemory());
        }
    };
    myAOF.checkOutputSpecs(null, job);
}

Example 79 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class OutputConfigurator method getBatchWriterOptions.

/**
 * Gets the {@link BatchWriterConfig} settings.
 *
 * @param implementingClass
 *          the class whose name will be used as a prefix for the property configuration key
 * @param conf
 *          the Hadoop configuration object to configure
 * @return the configuration object
 * @since 1.6.0
 * @see #setBatchWriterOptions(Class, Configuration, BatchWriterConfig)
 */
public static BatchWriterConfig getBatchWriterOptions(Class<?> implementingClass, Configuration conf) {
    String serialized = conf.get(enumToConfKey(implementingClass, WriteOpts.BATCH_WRITER_CONFIG));
    BatchWriterConfig bwConfig = new BatchWriterConfig();
    if (serialized == null || serialized.isEmpty()) {
        return bwConfig;
    } else {
        try {
            ByteArrayInputStream bais = new ByteArrayInputStream(serialized.getBytes(UTF_8));
            bwConfig.readFields(new DataInputStream(bais));
            bais.close();
            return bwConfig;
        } catch (IOException e) {
            throw new IllegalArgumentException("unable to serialize " + BatchWriterConfig.class.getName());
        }
    }
}

Example 80 with BatchWriterConfig

use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.

the class DeleteCommand method execute.

@Override
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, IOException, ConstraintViolationException {
    final Mutation m = new Mutation(new Text(cl.getArgs()[0].getBytes(Shell.CHARSET)));
    final Text colf = new Text(cl.getArgs()[1].getBytes(Shell.CHARSET));
    final Text colq = new Text(cl.getArgs()[2].getBytes(Shell.CHARSET));
    final String tableName = OptUtil.getTableOpt(cl, shellState);
    if (cl.hasOption(deleteOptAuths.getOpt())) {
        final ColumnVisibility le = new ColumnVisibility(cl.getOptionValue(deleteOptAuths.getOpt()));
        if (cl.hasOption(timestampOpt.getOpt())) {
            m.putDelete(colf, colq, le, Long.parseLong(cl.getOptionValue(timestampOpt.getOpt())));
        } else {
            m.putDelete(colf, colq, le);
        }
    } else if (cl.hasOption(timestampOpt.getOpt())) {
        m.putDelete(colf, colq, Long.parseLong(cl.getOptionValue(timestampOpt.getOpt())));
    } else {
        m.putDelete(colf, colq);
    }
    final BatchWriter bw = shellState.getAccumuloClient().createBatchWriter(tableName, new BatchWriterConfig().setMaxMemory(Math.max(m.estimatedMemoryUsed(), 1024)).setMaxWriteThreads(1).setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS));
    bw.addMutation(m);
    bw.close();
    return 0;
}