Examples with ProcessorStateException org.apache.kafka.streams.errors.ProcessorStateException used on opensource projects

Example 36 with ProcessorStateException

use of org.apache.kafka.streams.errors.ProcessorStateException in project kafka by apache.

the class RocksDBStore method openDB.

@SuppressWarnings("unchecked")
void openDB(final Map<String, Object> configs, final File stateDir) {
    // initialize the default rocksdb options
    final DBOptions dbOptions = new DBOptions();
    final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions();
    userSpecifiedOptions = new RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter(dbOptions, columnFamilyOptions);
    final BlockBasedTableConfigWithAccessibleCache tableConfig = new BlockBasedTableConfigWithAccessibleCache();
    cache = new LRUCache(BLOCK_CACHE_SIZE);
    tableConfig.setBlockCache(cache);
    tableConfig.setBlockSize(BLOCK_SIZE);
    filter = new BloomFilter();
    tableConfig.setFilterPolicy(filter);
    userSpecifiedOptions.optimizeFiltersForHits();
    userSpecifiedOptions.setTableFormatConfig(tableConfig);
    userSpecifiedOptions.setWriteBufferSize(WRITE_BUFFER_SIZE);
    userSpecifiedOptions.setCompressionType(COMPRESSION_TYPE);
    userSpecifiedOptions.setCompactionStyle(COMPACTION_STYLE);
    userSpecifiedOptions.setMaxWriteBufferNumber(MAX_WRITE_BUFFERS);
    userSpecifiedOptions.setCreateIfMissing(true);
    userSpecifiedOptions.setErrorIfExists(false);
    userSpecifiedOptions.setInfoLogLevel(InfoLogLevel.ERROR_LEVEL);
    // this is the recommended way to increase parallelism in RocksDb
    // note that the current implementation of setIncreaseParallelism affects the number
    // of compaction threads but not flush threads (the latter remains one). Also,
    // the parallelism value needs to be at least two because of the code in
    // https://github.com/facebook/rocksdb/blob/62ad0a9b19f0be4cefa70b6b32876e764b7f3c11/util/options.cc#L580
    // subtracts one from the value passed to determine the number of compaction threads
    // (this could be a bug in the RocksDB code and their devs have been contacted).
    userSpecifiedOptions.setIncreaseParallelism(Math.max(Runtime.getRuntime().availableProcessors(), 2));
    wOptions = new WriteOptions();
    wOptions.setDisableWAL(true);
    fOptions = new FlushOptions();
    fOptions.setWaitForFlush(true);
    final Class<RocksDBConfigSetter> configSetterClass = (Class<RocksDBConfigSetter>) configs.get(StreamsConfig.ROCKSDB_CONFIG_SETTER_CLASS_CONFIG);
    if (configSetterClass != null) {
        configSetter = Utils.newInstance(configSetterClass);
        configSetter.setConfig(name, userSpecifiedOptions, configs);
    }
    dbDir = new File(new File(stateDir, parentDir), name);
    try {
        Files.createDirectories(dbDir.getParentFile().toPath());
        Files.createDirectories(dbDir.getAbsoluteFile().toPath());
    } catch (final IOException fatal) {
        throw new ProcessorStateException(fatal);
    }
    // Setup statistics before the database is opened, otherwise the statistics are not updated
    // with the measurements from Rocks DB
    maybeSetUpStatistics(configs);
    openRocksDB(dbOptions, columnFamilyOptions);
    open = true;
    addValueProvidersToMetricsRecorder();
}

Example 37 with ProcessorStateException

use of org.apache.kafka.streams.errors.ProcessorStateException in project kafka by apache.

the class RocksDBStore method approximateNumEntries.

/**
 * Return an approximate count of key-value mappings in this store.
 *
 * <code>RocksDB</code> cannot return an exact entry count without doing a
 * full scan, so this method relies on the <code>rocksdb.estimate-num-keys</code>
 * property to get an approximate count. The returned size also includes
 * a count of dirty keys in the store's in-memory cache, which may lead to some
 * double-counting of entries and inflate the estimate.
 *
 * @return an approximate count of key-value mappings in the store.
 */
@Override
public long approximateNumEntries() {
    validateStoreOpen();
    final long numEntries;
    try {
        numEntries = dbAccessor.approximateNumEntries();
    } catch (final RocksDBException e) {
        throw new ProcessorStateException("Error fetching property from store " + name, e);
    }
    if (isOverflowing(numEntries)) {
        return Long.MAX_VALUE;
    }
    return numEntries;
}

Example 38 with ProcessorStateException

use of org.apache.kafka.streams.errors.ProcessorStateException in project kafka by apache.

the class MeteredWindowStore method put.

@Override
public void put(final K key, final V value, final long windowStartTimestamp) {
    Objects.requireNonNull(key, "key cannot be null");
    try {
        maybeMeasureLatency(() -> wrapped().put(keyBytes(key), serdes.rawValue(value), windowStartTimestamp), time, putSensor);
        maybeRecordE2ELatency();
    } catch (final ProcessorStateException e) {
        final String message = String.format(e.getMessage(), key, value);
        throw new ProcessorStateException(message, e);
    }
}

Example 39 with ProcessorStateException

use of org.apache.kafka.streams.errors.ProcessorStateException in project kafka by apache.

the class MeteredKeyValueStore method put.

@Override
public void put(final K key, final V value) {
    Objects.requireNonNull(key, "key cannot be null");
    try {
        maybeMeasureLatency(() -> wrapped().put(keyBytes(key), serdes.rawValue(value)), time, putSensor);
        maybeRecordE2ELatency();
    } catch (final ProcessorStateException e) {
        final String message = String.format(e.getMessage(), key, value);
        throw new ProcessorStateException(message, e);
    }
}

Example 40 with ProcessorStateException

use of org.apache.kafka.streams.errors.ProcessorStateException in project kafka by apache.

the class MeteredSessionStore method put.

@Override
public void put(final Windowed<K> sessionKey, final V aggregate) {
    Objects.requireNonNull(sessionKey, "sessionKey can't be null");
    Objects.requireNonNull(sessionKey.key(), "sessionKey.key() can't be null");
    Objects.requireNonNull(sessionKey.window(), "sessionKey.window() can't be null");
    try {
        maybeMeasureLatency(() -> {
            final Bytes key = keyBytes(sessionKey.key());
            wrapped().put(new Windowed<>(key, sessionKey.window()), serdes.rawValue(aggregate));
        }, time, putSensor);
        maybeRecordE2ELatency();
    } catch (final ProcessorStateException e) {
        final String message = String.format(e.getMessage(), sessionKey.key(), aggregate);
        throw new ProcessorStateException(message, e);
    }
}