Examples with Formatter org.voltdb.importer.formatter.Formatter used on opensource projects

Example 1 with Formatter

use of org.voltdb.importer.formatter.Formatter in project voltdb by VoltDB.

the class PullSocketImporter method susceptibleRun.

private void susceptibleRun() {
    if (m_eos.get())
        return;
    info(null, "Starting socket puller for " + m_config.getResourceID());
    m_thread = Optional.of(Thread.currentThread());
    Optional<BufferedReader> reader = null;
    Formatter formatter = m_config.getFormatterBuilder().create();
    while (!m_eos.get()) {
        try {
            reader = attemptBufferedReader();
            if (!reader.isPresent()) {
                sleep(2_000);
                continue;
            }
            BufferedReader br = reader.get();
            String csv = null;
            while ((csv = br.readLine()) != null) {
                try {
                    Object[] params = formatter.transform(ByteBuffer.wrap(csv.getBytes()));
                    Invocation invocation = new Invocation(m_config.getProcedure(), params);
                    if (!callProcedure(invocation)) {
                        if (isDebugEnabled()) {
                            debug(null, "Failed to process Invocation possibly bad data: " + csv);
                        }
                    }
                } catch (FormatException e) {
                    rateLimitedLog(Level.ERROR, e, "Failed to tranform data: %s", csv);
                    ;
                }
            }
            if (csv == null) {
                warn(null, m_config.getResourceID() + " peer terminated stream");
            }
        } catch (EOFException e) {
            rateLimitedLog(Level.WARN, e, m_config.getResourceID() + " peer terminated stream");
        } catch (InterruptedException e) {
            if (m_eos.get())
                return;
            rateLimitedLog(Level.ERROR, e, "Socket puller %s was interrupted", m_config.getResourceID());
        } catch (InterruptedIOException e) {
            if (m_eos.get())
                return;
            rateLimitedLog(Level.ERROR, e, "Socket puller for %s was interrupted", m_config.getResourceID());
        } catch (IOException e) {
            rateLimitedLog(Level.ERROR, e, "Read fault for %s", m_config.getResourceID());
        }
    }
    info(null, "Stopping socket puller for " + m_config.getResourceID());
}

Example 2 with Formatter

use of org.voltdb.importer.formatter.Formatter in project voltdb by VoltDB.

the class TestVoltCSVFormatter method testBlankError.

@Test
public void testBlankError() throws Exception {
    ServiceReference[] refs = m_bundle.getRegisteredServices();
    ServiceReference<AbstractFormatterFactory> reference = refs[0];
    AbstractFormatterFactory o = m_bundle.getBundleContext().getService(reference);
    Properties prop = new Properties();
    prop.setProperty("separator", ",");
    prop.setProperty("blank", "error");
    FormatterBuilder builder = new FormatterBuilder("csv", prop);
    builder.setFormatterFactory(o);
    Formatter formatter = builder.create();
    try {
        Object[] results = formatter.transform(ByteBuffer.wrap("12,,test".getBytes(StandardCharsets.UTF_8)));
        fail();
    } catch (RuntimeException e) {
    }
}

Example 3 with Formatter

use of org.voltdb.importer.formatter.Formatter in project voltdb by VoltDB.

the class TestVoltCSVFormatter method testQuoteChar.

//char separator, char quotechar, char escape, boolean strictQuotes, boolean ignoreLeadingWhiteSpace
@Test
public void testQuoteChar() throws Exception {
    ServiceReference[] refs = m_bundle.getRegisteredServices();
    ServiceReference<AbstractFormatterFactory> reference = refs[0];
    AbstractFormatterFactory o = m_bundle.getBundleContext().getService(reference);
    Properties prop = new Properties();
    prop.setProperty("quotechar", "'");
    FormatterBuilder builder = new FormatterBuilder("csv", prop);
    builder.setFormatterFactory(o);
    Formatter formatter = builder.create();
    Object[] results = formatter.transform(ByteBuffer.wrap("12,'10.05,test'".getBytes(StandardCharsets.UTF_8)));
    assertEquals(results.length, 2);
    assertEquals(results[0], "12");
    assertEquals(results[1], "10.05,test");
}

Example 4 with Formatter

use of org.voltdb.importer.formatter.Formatter in project voltdb by VoltDB.

the class TestVoltCSVFormatter method testTrimunquoted.

@Test
public void testTrimunquoted() throws Exception {
    ServiceReference[] refs = m_bundle.getRegisteredServices();
    ServiceReference<AbstractFormatterFactory> reference = refs[0];
    AbstractFormatterFactory o = m_bundle.getBundleContext().getService(reference);
    Properties prop = new Properties();
    prop.setProperty("trimunquoted", "false");
    FormatterBuilder builder = new FormatterBuilder("csv", prop);
    builder.setFormatterFactory(o);
    Formatter formatter = builder.create();
    Object[] results = formatter.transform(ByteBuffer.wrap("12,10.05,  test".getBytes(StandardCharsets.UTF_8)));
    assertEquals(results.length, 3);
    assertEquals(results[0], "12");
    assertEquals(results[1], "10.05");
    assertEquals(results[2], "  test");
}

Example 5 with Formatter

use of org.voltdb.importer.formatter.Formatter in project voltdb by VoltDB.

the class TestVoltCSVFormatter method testStrictQuotes.

@Test
public void testStrictQuotes() throws Exception {
    ServiceReference[] refs = m_bundle.getRegisteredServices();
    ServiceReference<AbstractFormatterFactory> reference = refs[0];
    AbstractFormatterFactory o = m_bundle.getBundleContext().getService(reference);
    Properties prop = new Properties();
    prop.setProperty("strictquotes", "true");
    FormatterBuilder builder = new FormatterBuilder("csv", prop);
    builder.setFormatterFactory(o);
    Formatter formatter = builder.create();
    Object[] results = formatter.transform(ByteBuffer.wrap("\"12\",\"10.05\",\"es\"".getBytes(StandardCharsets.UTF_8)));
    assertEquals(results.length, 3);
    assertEquals(results[0], "12");
    assertEquals(results[1], "10.05");
    assertEquals(results[2], "es");
}