Examples with BigInteger java.math.BigInteger used on opensource projects

Example 26 with BigInteger

use of java.math.BigInteger in project kafka by apache.

the class JsonConverterTest method decimalToConnectWithDefaultValue.

@Test
public void decimalToConnectWithDefaultValue() {
    BigDecimal reference = new BigDecimal(new BigInteger("156"), 2);
    Schema schema = Decimal.builder(2).defaultValue(reference).build();
    String msg = "{ \"schema\": { \"type\": \"bytes\", \"name\": \"org.apache.kafka.connect.data.Decimal\", \"version\": 1, \"default\": \"AJw=\", \"parameters\": { \"scale\": \"2\" } }, \"payload\": null }";
    SchemaAndValue schemaAndValue = converter.toConnectData(TOPIC, msg.getBytes());
    assertEquals(schema, schemaAndValue.schema());
    assertEquals(reference, schemaAndValue.value());
}

Example 27 with BigInteger

use of java.math.BigInteger in project storm by apache.

the class KinesisRecordsManager method next.

void next(SpoutOutputCollector collector) {
    if (shouldCommit()) {
        commit();
    }
    KinesisMessageId failedMessageId = kinesisConfig.getFailedMessageRetryHandler().getNextFailedMessageToRetry();
    if (failedMessageId != null) {
        // if the retry service returns a message that is not in failed set then ignore it. should never happen
        BigInteger failedSequenceNumber = new BigInteger(failedMessageId.getSequenceNumber());
        if (failedPerShard.containsKey(failedMessageId.getShardId()) && failedPerShard.get(failedMessageId.getShardId()).contains(failedSequenceNumber)) {
            if (!failedandFetchedRecords.containsKey(failedMessageId)) {
                fetchFailedRecords(failedMessageId);
            }
            if (emitFailedRecord(collector, failedMessageId)) {
                failedPerShard.get(failedMessageId.getShardId()).remove(failedSequenceNumber);
                kinesisConfig.getFailedMessageRetryHandler().failedMessageEmitted(failedMessageId);
                return;
            } else {
                LOG.warn("failedMessageEmitted not called on retrier for " + failedMessageId + ". This can happen a few times but should not happen " + "infinitely");
            }
        } else {
            LOG.warn("failedPerShard does not contain " + failedMessageId + ". This should never happen.");
        }
    }
    LOG.debug("No failed record to emit for now. Hence will try to emit new records");
    // if maximum uncommitted records count has reached, so dont emit any new records and return
    if (!(getUncommittedRecordsCount() < kinesisConfig.getMaxUncommittedRecords())) {
        LOG.warn("maximum uncommitted records count has reached. so not emitting any new records and returning");
        return;
    }
    // early return as no shard is assigned - probably because number of executors > number of shards
    if (toEmitPerShard.isEmpty()) {
        LOG.warn("No shard is assigned to this task. Hence not emitting any tuple.");
        return;
    }
    if (shouldFetchNewRecords()) {
        fetchNewRecords();
    }
    emitNewRecord(collector);
}

Example 28 with BigInteger

use of java.math.BigInteger in project storm by apache.

the class KinesisRecordsManager method commit.

void commit() {
    // failedPerShard is 3,7 then we can only commit 1 and not 4 because 2 is still pending and 3 has failed
    for (String shardId : toEmitPerShard.keySet()) {
        if (ackedPerShard.containsKey(shardId)) {
            BigInteger commitSequenceNumberBound = null;
            if (failedPerShard.containsKey(shardId) && !failedPerShard.get(shardId).isEmpty()) {
                commitSequenceNumberBound = failedPerShard.get(shardId).first();
            }
            if (emittedPerShard.containsKey(shardId) && !emittedPerShard.get(shardId).isEmpty()) {
                BigInteger smallestEmittedSequenceNumber = emittedPerShard.get(shardId).first();
                if (commitSequenceNumberBound == null || (commitSequenceNumberBound.compareTo(smallestEmittedSequenceNumber) == 1)) {
                    commitSequenceNumberBound = smallestEmittedSequenceNumber;
                }
            }
            Iterator<BigInteger> ackedSequenceNumbers = ackedPerShard.get(shardId).iterator();
            BigInteger ackedSequenceNumberToCommit = null;
            while (ackedSequenceNumbers.hasNext()) {
                BigInteger ackedSequenceNumber = ackedSequenceNumbers.next();
                if (commitSequenceNumberBound == null || (commitSequenceNumberBound.compareTo(ackedSequenceNumber) == 1)) {
                    ackedSequenceNumberToCommit = ackedSequenceNumber;
                    ackedSequenceNumbers.remove();
                } else {
                    break;
                }
            }
            if (ackedSequenceNumberToCommit != null) {
                Map<Object, Object> state = new HashMap<>();
                state.put("committedSequenceNumber", ackedSequenceNumberToCommit.toString());
                LOG.debug("Committing sequence number " + ackedSequenceNumberToCommit.toString() + " for shardId " + shardId);
                zkConnection.commitState(kinesisConfig.getStreamName(), shardId, state);
            }
        }
    }
    lastCommitTime = System.currentTimeMillis();
}

Example 29 with BigInteger

use of java.math.BigInteger in project storm by apache.

the class KinesisRecordsManager method fetchFailedRecords.

// fetch records from kinesis starting at sequence number for message passed as argument. Any other messages fetched and are in the failed queue will also
// be kept in memory to avoid going to kinesis again for retry
private void fetchFailedRecords(KinesisMessageId kinesisMessageId) {
    // if shard iterator not present for this message, get it
    if (!shardIteratorPerFailedMessage.containsKey(kinesisMessageId)) {
        refreshShardIteratorForFailedRecord(kinesisMessageId);
    }
    String shardIterator = shardIteratorPerFailedMessage.get(kinesisMessageId);
    LOG.debug("Fetching failed records for shard id :" + kinesisMessageId.getShardId() + " at sequence number " + kinesisMessageId.getSequenceNumber() + " using shardIterator " + shardIterator);
    try {
        GetRecordsResult getRecordsResult = kinesisConnection.fetchRecords(shardIterator);
        if (getRecordsResult != null) {
            List<Record> records = getRecordsResult.getRecords();
            LOG.debug("Records size from fetchFailedRecords is " + records.size());
            // update the shard iterator to next one in case this fetch does not give the message.
            shardIteratorPerFailedMessage.put(kinesisMessageId, getRecordsResult.getNextShardIterator());
            if (records.size() == 0) {
                LOG.warn("No records returned from kinesis. Hence sleeping for 1 second");
                Thread.sleep(1000);
            } else {
                // add all fetched records to the set of failed records if they are present in failed set
                for (Record record : records) {
                    KinesisMessageId current = new KinesisMessageId(kinesisMessageId.getStreamName(), kinesisMessageId.getShardId(), record.getSequenceNumber());
                    if (failedPerShard.get(kinesisMessageId.getShardId()).contains(new BigInteger(current.getSequenceNumber()))) {
                        failedandFetchedRecords.put(current, record);
                        shardIteratorPerFailedMessage.remove(current);
                    }
                }
            }
        }
    } catch (InterruptedException ie) {
        LOG.warn("Thread interrupted while sleeping", ie);
    } catch (ExpiredIteratorException ex) {
        LOG.warn("shardIterator for failedRecord " + kinesisMessageId + " has expired. Refreshing shardIterator");
        refreshShardIteratorForFailedRecord(kinesisMessageId);
    } catch (ProvisionedThroughputExceededException pe) {
        try {
            LOG.warn("ProvisionedThroughputExceededException occured. Check your limits. Sleeping for 1 second.", pe);
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.warn("Thread interrupted exception", e);
        }
    }
}

Example 30 with BigInteger

use of java.math.BigInteger in project hive by apache.

the class VectorizedColumnReaderTestBase method getDecimal.

protected static HiveDecimal getDecimal(boolean isDictionaryEncoding, int index) {
    int decimalVal = index % 100;
    String decimalStr = (decimalVal < 10) ? "0" + String.valueOf(decimalVal) : String.valueOf(decimalVal);
    int intVal = (isDictionaryEncoding) ? index % UNIQUE_NUM : index / 100;
    String d = String.valueOf(intVal) + decimalStr;
    BigInteger bi = new BigInteger(d);
    BigDecimal bd = new BigDecimal(bi);
    return HiveDecimal.create(bd);
}