Examples with FormatException org.voltdb.importer.formatter.FormatException used on opensource projects

Example 1 with FormatException

use of org.voltdb.importer.formatter.FormatException in project voltdb by VoltDB.

the class PullSocketImporter method susceptibleRun.

private void susceptibleRun() {
    if (m_eos.get())
        return;
    info(null, "Starting socket puller for " + m_config.getResourceID());
    m_thread = Optional.of(Thread.currentThread());
    Optional<BufferedReader> reader = null;
    Formatter formatter = m_config.getFormatterBuilder().create();
    while (!m_eos.get()) {
        try {
            reader = attemptBufferedReader();
            if (!reader.isPresent()) {
                sleep(2_000);
                continue;
            }
            BufferedReader br = reader.get();
            String csv = null;
            while ((csv = br.readLine()) != null) {
                try {
                    Object[] params = formatter.transform(ByteBuffer.wrap(csv.getBytes()));
                    Invocation invocation = new Invocation(m_config.getProcedure(), params);
                    if (!callProcedure(invocation)) {
                        if (isDebugEnabled()) {
                            debug(null, "Failed to process Invocation possibly bad data: " + csv);
                        }
                    }
                } catch (FormatException e) {
                    rateLimitedLog(Level.ERROR, e, "Failed to tranform data: %s", csv);
                    ;
                }
            }
            if (csv == null) {
                warn(null, m_config.getResourceID() + " peer terminated stream");
            }
        } catch (EOFException e) {
            rateLimitedLog(Level.WARN, e, m_config.getResourceID() + " peer terminated stream");
        } catch (InterruptedException e) {
            if (m_eos.get())
                return;
            rateLimitedLog(Level.ERROR, e, "Socket puller %s was interrupted", m_config.getResourceID());
        } catch (InterruptedIOException e) {
            if (m_eos.get())
                return;
            rateLimitedLog(Level.ERROR, e, "Socket puller for %s was interrupted", m_config.getResourceID());
        } catch (IOException e) {
            rateLimitedLog(Level.ERROR, e, "Read fault for %s", m_config.getResourceID());
        }
    }
    info(null, "Stopping socket puller for " + m_config.getResourceID());
}

Example 2 with FormatException

use of org.voltdb.importer.formatter.FormatException in project voltdb by VoltDB.

the class VoltCSVFormatter method transform.

@Override
public Object[] transform(ByteBuffer payload) throws FormatException {
    String line = null;
    try {
        if (payload == null) {
            return null;
        }
        line = new String(payload.array(), payload.arrayOffset(), payload.limit(), StandardCharsets.UTF_8);
        Object[] list = m_parser.parseLine(line);
        if (list != null) {
            for (int i = 0; i < list.length; i++) {
                if ("NULL".equals(list[i]) || Constants.CSV_NULL.equals(list[i]) || Constants.QUOTED_CSV_NULL.equals(list[i])) {
                    list[i] = null;
                }
            }
        }
        return list;
    } catch (IOException e) {
        throw new FormatException("failed to format " + line, e);
    }
}

Example 3 with FormatException

use of org.voltdb.importer.formatter.FormatException in project voltdb by VoltDB.

the class ExampleFormatter method transform.

/**
     * Function that is called by the importer to handle the transformation.
     * For this example it turns a string into an object array by splitting the string into characters.
     * @param sourceData - Data type of this field is designated by the Formatter<?> data type.
     */
@Override
public Object[] transform(ByteBuffer payload) throws FormatException {
    Object[] objs = { "", "", "", "", "" };
    Object[] badobjs1 = { "abc", "def", "123", "", "this one is ok" };
    Object[] badobjs2 = { "", "", "", "", "", "" };
    JSONParser parser = new JSONParser();
    JSONObject jsonObj = null;
    Object seq = null;
    Object instance_id = null;
    Object event_type_id = null;
    Object event_date = null;
    Object trans = null;
    String percentErrors = m_prop.getProperty("errorrate", "1").trim();
    int badinject = 1;
    String sourceData = null;
    try {
        sourceData = new String(payload.array(), payload.arrayOffset(), payload.limit(), StandardCharsets.UTF_8);
        badinject = Integer.parseInt(percentErrors);
    } catch (NumberFormatException e1) {
        e1.printStackTrace();
        badinject = 1;
    }
    try {
        Object obj = parser.parse(sourceData);
        jsonObj = (JSONObject) obj;
        seq = jsonObj.get("seq");
        instance_id = jsonObj.get("instance_id");
        event_type_id = jsonObj.get("event_type_id");
        event_date = jsonObj.get("event_date");
        trans = jsonObj.get("trans");
        objs[0] = seq;
        objs[1] = instance_id;
        objs[2] = event_type_id;
        objs[3] = event_date;
        objs[4] = trans;
    } catch (Exception e) {
        e.printStackTrace();
    }
    double r = Math.random();
    if (r < (badinject / 100.0)) {
        if (// randomize the choice of bad outcomes
        r < 1.0 / 3.0)
            return badobjs1;
        else if (r >= 1.0 / 3.0 && r < 2.0 / 3.0)
            return badobjs2;
        throw new FormatException();
    }
    return objs;
}

Example 4 with FormatException

use of org.voltdb.importer.formatter.FormatException in project voltdb by VoltDB.

the class KafkaTopicPartitionImporter method accept.

@Override
protected void accept() {
    info(null, "Starting partition fetcher for " + m_topicAndPartition);
    long submitCount = 0;
    PendingWorkTracker callbackTracker = new PendingWorkTracker();
    Formatter formatter = m_config.getFormatterBuilder().create();
    try {
        //Start with the starting leader.
        resetLeader();
        int sleepCounter = 1;
        while (shouldRun()) {
            if (m_currentOffset.get() < 0) {
                getOffsetCoordinator();
                if (m_offsetManager.get() == null) {
                    sleepCounter = backoffSleep(sleepCounter);
                    continue;
                }
                long lastOffset = getLastOffset();
                if (lastOffset == -1) {
                    sleepCounter = backoffSleep(sleepCounter);
                    continue;
                }
                m_gapTracker.resetTo(lastOffset);
                m_lastCommittedOffset = lastOffset;
                m_currentOffset.set(lastOffset);
                if (m_currentOffset.get() < 0) {
                    //If we dont know the offset get it backoff if we fail.
                    sleepCounter = backoffSleep(sleepCounter);
                    info(null, "No valid offset found for " + m_topicAndPartition);
                    continue;
                }
                info(null, "Starting offset for " + m_topicAndPartition + " is " + m_currentOffset.get());
            }
            long currentFetchCount = 0;
            //Build fetch request of we have a valid offset and not too many are pending.
            FetchRequest req = m_fetchRequestBuilder.addFetch(m_topicAndPartition.topic(), m_topicAndPartition.partition(), m_currentOffset.get(), m_config.getFetchSize()).build();
            FetchResponse fetchResponse = null;
            try {
                fetchResponse = m_consumer.fetch(req);
                if (fetchResponse == null) {
                    sleepCounter = backoffSleep(sleepCounter);
                    continue;
                }
            } catch (Exception ex) {
                rateLimitedLog(Level.WARN, ex, "Failed to fetch from " + m_topicAndPartition);
                //See if its network error and find new leader for this partition.
                if (ex instanceof IOException) {
                    resetLeader();
                    //find leader in resetLeader would sleep and backoff
                    continue;
                }
                sleepCounter = backoffSleep(sleepCounter);
                continue;
            }
            if (fetchResponse.hasError()) {
                // Something went wrong!
                short code = fetchResponse.errorCode(m_topicAndPartition.topic(), m_topicAndPartition.partition());
                warn(ErrorMapping.exceptionFor(code), "Failed to fetch messages for %s", m_topicAndPartition);
                sleepCounter = backoffSleep(sleepCounter);
                if (code == ErrorMapping.OffsetOutOfRangeCode()) {
                    // We asked for an invalid offset. For simple case ask for the last element to reset
                    info(null, "Invalid offset requested for " + m_topicAndPartition);
                    getOffsetCoordinator();
                    m_currentOffset.set(-1L);
                    continue;
                }
                resetLeader();
                continue;
            }
            sleepCounter = 1;
            for (MessageAndOffset messageAndOffset : fetchResponse.messageSet(m_topicAndPartition.topic(), m_topicAndPartition.partition())) {
                //You may be catchin up so dont sleep.
                currentFetchCount++;
                long currentOffset = messageAndOffset.offset();
                //if currentOffset is less means we have already pushed it and also check pending queue.
                if (currentOffset < m_currentOffset.get()) {
                    continue;
                }
                ByteBuffer payload = messageAndOffset.message().payload();
                Object[] params = null;
                try {
                    m_gapTracker.submit(messageAndOffset.nextOffset());
                    params = formatter.transform(payload);
                    Invocation invocation = new Invocation(m_config.getProcedure(), params);
                    TopicPartitionInvocationCallback cb = new TopicPartitionInvocationCallback(messageAndOffset.offset(), messageAndOffset.nextOffset(), callbackTracker, m_gapTracker, m_dead, m_pauseOffset);
                    if (!noTransaction) {
                        if (callProcedure(invocation, cb)) {
                            callbackTracker.produceWork();
                        } else {
                            if (isDebugEnabled()) {
                                debug(null, "Failed to process Invocation possibly bad data: " + Arrays.toString(params));
                            }
                            m_gapTracker.commit(messageAndOffset.nextOffset());
                        }
                    }
                } catch (FormatException e) {
                    rateLimitedLog(Level.WARN, e, "Failed to tranform data: %s", Arrays.toString(params));
                    m_gapTracker.commit(messageAndOffset.nextOffset());
                }
                submitCount++;
                m_currentOffset.set(messageAndOffset.nextOffset());
                if (!shouldRun()) {
                    break;
                }
            }
            if (!shouldRun()) {
                break;
            }
            //wait to fetch more if we read nothing last time.
            if (currentFetchCount == 0) {
                try {
                    Thread.sleep(m_waitSleepMs);
                } catch (InterruptedException ie) {
                }
            }
            if (shouldCommit()) {
                commitOffset(false);
            }
        }
    } catch (Exception ex) {
        error(ex, "Failed to start topic partition fetcher for " + m_topicAndPartition);
    } finally {
        final boolean usePausedOffset = m_pauseOffset.get() != -1;
        boolean skipCommit = false;
        if (usePausedOffset) {
            // Paused offset is not guaranteed reliable until all callbacks have been called.
            if (callbackTracker.waitForWorkToFinish() == false) {
                if (m_pauseOffset.get() < m_lastCommittedOffset) {
                    warn(null, "Committing paused offset even though a timeout occurred waiting for pending stored procedures to finish.");
                } else {
                    warn(null, "Refusing to commit paused offset because a timeout occurred waiting for pending stored procedures to finish.");
                    skipCommit = true;
                }
            }
        }
        if (skipCommit == false) {
            // Force a commit. Paused offset will be re-acquired if needed.
            commitOffset(usePausedOffset);
        }
        KafkaStreamImporterConfig.closeConsumer(m_consumer);
        m_consumer = null;
        BlockingChannel channel = m_offsetManager.getAndSet(null);
        if (channel != null) {
            try {
                channel.disconnect();
            } catch (Exception ignoreIt) {
            }
        }
    }
    m_dead.compareAndSet(false, true);
    info(null, "Partition fetcher stopped for " + m_topicAndPartition + " Last commit point is: " + m_lastCommittedOffset + " Callback Rcvd: " + callbackTracker.getCallbackCount() + " Submitted: " + submitCount);
}

Example 5 with FormatException

use of org.voltdb.importer.formatter.FormatException in project voltdb by VoltDB.

the class VoltSuperCSVFormatter method transform.

@Override
public Object[] transform(ByteBuffer payload) throws FormatException {
    if (payload == null) {
        return null;
    }
    String line = new String(payload.array(), payload.arrayOffset(), payload.limit(), StandardCharsets.UTF_8);
    m_tokenizer.setSourceString(line);
    List<String> dataList;
    try {
        dataList = m_csvReader.read();
    } catch (IOException | SuperCsvException e) {
        throw new FormatException("Failed to parse csv data", e);
    }
    if (dataList == null)
        return null;
    String[] data = dataList.toArray(new String[0]);
    normalize(data);
    return data;
}