Examples with FormatterBuilder org.voltdb.importer.formatter.FormatterBuilder used on opensource projects

Example 1 with FormatterBuilder

use of org.voltdb.importer.formatter.FormatterBuilder in project voltdb by VoltDB.

the class KafkaTopicTest method main.

/**
     * @param args the command line arguments
     */
public static void main(String[] args) {
    Properties props = new Properties();
    if (args.length < 1) {
        System.out.println("testkafkaimporter: path-to-properties-file - The file should have brokers and topics properties at minimum.");
        System.exit(1);
    }
    String filename = args[0];
    try (InputStream is = new FileInputStream(new File(filename))) {
        props.load(is);
    } catch (Exception ex) {
        ex.printStackTrace();
    }
    String partition = props.getProperty("partition");
    if (args.length > 2) {
        partition = args[2];
        if (partition != null && partition.equals("all")) {
            //Process all partitions.
            partition = null;
        }
    }
    String format = props.getProperty("format");
    if (format == null || format.length() <= 0)
        format = "csv";
    System.out.println("Properties are: " + props);
    props.put("procedure", "fake");
    KafkaStreamImporterFactory factory = new KafkaStreamImporterFactory();
    VoltCSVFormatterFactory ffactory = new VoltCSVFormatterFactory();
    ffactory.create(format, props);
    FormatterBuilder fb = new FormatterBuilder(format, props);
    fb.setFormatterFactory(ffactory);
    Map<URI, ImporterConfig> mmap = factory.createImporterConfigurations(props, fb);
    System.out.println("Number of Partitions for topic are: " + mmap.size() + " Requested partition: " + partition);
    CountDownLatch cdl = new CountDownLatch(mmap.size());
    for (URI uri : mmap.keySet()) {
        if (partition != null && !uri.toString().endsWith(partition)) {
            cdl.countDown();
            continue;
        }
        KafkaTopicPartitionImporter importer = new KafkaTopicPartitionImporter((KafkaStreamImporterConfig) mmap.get(uri));
        Runner runner = new Runner(importer, cdl);
        runner.start();
    }
    try {
        cdl.await();
    } catch (InterruptedException ex) {
        ex.printStackTrace();
    }
    System.out.println("Exiting.");
}

Example 2 with FormatterBuilder

use of org.voltdb.importer.formatter.FormatterBuilder in project voltdb by VoltDB.

the class TestVoltCSVFormatter method testQuoteChar.

//char separator, char quotechar, char escape, boolean strictQuotes, boolean ignoreLeadingWhiteSpace
@Test
public void testQuoteChar() throws Exception {
    ServiceReference[] refs = m_bundle.getRegisteredServices();
    ServiceReference<AbstractFormatterFactory> reference = refs[0];
    AbstractFormatterFactory o = m_bundle.getBundleContext().getService(reference);
    Properties prop = new Properties();
    prop.setProperty("quotechar", "'");
    FormatterBuilder builder = new FormatterBuilder("csv", prop);
    builder.setFormatterFactory(o);
    Formatter formatter = builder.create();
    Object[] results = formatter.transform(ByteBuffer.wrap("12,'10.05,test'".getBytes(StandardCharsets.UTF_8)));
    assertEquals(results.length, 2);
    assertEquals(results[0], "12");
    assertEquals(results[1], "10.05,test");
}

Example 3 with FormatterBuilder

use of org.voltdb.importer.formatter.FormatterBuilder in project voltdb by VoltDB.

the class TestVoltCSVFormatter method testBadFormat.

@Test
public void testBadFormat() throws Exception {
    ServiceReference[] refs = m_bundle.getRegisteredServices();
    ServiceReference<AbstractFormatterFactory> reference = refs[0];
    AbstractFormatterFactory o = m_bundle.getBundleContext().getService(reference);
    Properties prop = new Properties();
    FormatterBuilder builder = new FormatterBuilder("badformat", prop);
    builder.setFormatterFactory(o);
    try {
        builder.create();
        fail();
    } catch (RuntimeException e) {
    }
}

Example 4 with FormatterBuilder

use of org.voltdb.importer.formatter.FormatterBuilder in project voltdb by VoltDB.

the class TestVoltCSVFormatter method testTrimunquoted.

@Test
public void testTrimunquoted() throws Exception {
    ServiceReference[] refs = m_bundle.getRegisteredServices();
    ServiceReference<AbstractFormatterFactory> reference = refs[0];
    AbstractFormatterFactory o = m_bundle.getBundleContext().getService(reference);
    Properties prop = new Properties();
    prop.setProperty("trimunquoted", "false");
    FormatterBuilder builder = new FormatterBuilder("csv", prop);
    builder.setFormatterFactory(o);
    Formatter formatter = builder.create();
    Object[] results = formatter.transform(ByteBuffer.wrap("12,10.05,  test".getBytes(StandardCharsets.UTF_8)));
    assertEquals(results.length, 3);
    assertEquals(results[0], "12");
    assertEquals(results[1], "10.05");
    assertEquals(results[2], "  test");
}

Example 5 with FormatterBuilder

use of org.voltdb.importer.formatter.FormatterBuilder in project voltdb by VoltDB.

the class TestVoltCSVFormatter method testBlankError.

@Test
public void testBlankError() throws Exception {
    ServiceReference[] refs = m_bundle.getRegisteredServices();
    ServiceReference<AbstractFormatterFactory> reference = refs[0];
    AbstractFormatterFactory o = m_bundle.getBundleContext().getService(reference);
    Properties prop = new Properties();
    prop.setProperty("separator", ",");
    prop.setProperty("blank", "error");
    FormatterBuilder builder = new FormatterBuilder("csv", prop);
    builder.setFormatterFactory(o);
    Formatter formatter = builder.create();
    try {
        Object[] results = formatter.transform(ByteBuffer.wrap("12,,test".getBytes(StandardCharsets.UTF_8)));
        fail();
    } catch (RuntimeException e) {
    }
}