Examples with ColumnFamilyHandle org.rocksdb.ColumnFamilyHandle used on opensource projects

Example 26 with ColumnFamilyHandle

use of org.rocksdb.ColumnFamilyHandle in project jstorm by alibaba.

the class WindowedRocksDbHdfsState method removeWindow.

@Override
public void removeWindow(TimeWindow window) {
    try {
        ColumnFamilyHandle handler = getColumnFamilyHandle(window);
        rocksDb.dropColumnFamily(handler);
        windowToCFHandler.remove(window);
    } catch (Exception e) {
        LOG.error("Failed to remove timeWindow={}", window);
    }
}

Example 27 with ColumnFamilyHandle

use of org.rocksdb.ColumnFamilyHandle in project jstorm by alibaba.

the class WindowedRocksDbHdfsState method getBatch.

@Override
public Map<K, V> getBatch(TimeWindow window) {
    Map<K, V> batch = new HashMap<K, V>();
    try {
        ColumnFamilyHandle handler = getColumnFamilyHandle(window);
        RocksIterator itr = rocksDb.newIterator(handler);
        itr.seekToFirst();
        while (itr.isValid()) {
            byte[] rawKey = itr.key();
            byte[] rawValue = itr.value();
            V value = rawValue != null ? (V) serializer.deserialize(rawValue) : null;
            batch.put((K) serializer.deserialize(rawKey), value);
            itr.next();
        }
        return batch;
    } catch (RocksDBException e) {
        LOG.error("Failed to get batch for window={}", window);
        throw new RuntimeException(e.getMessage());
    }
}

Example 28 with ColumnFamilyHandle

use of org.rocksdb.ColumnFamilyHandle in project jstorm by alibaba.

the class WindowedRocksDbHdfsState method initRocksDb.

@Override
protected void initRocksDb() {
    windowToCFHandler = new HashMap<>();
    RocksDbOptionsFactory optionFactory = new RocksDbOptionsFactory.Defaults();
    Options options = optionFactory.createOptions(null);
    DBOptions dbOptions = optionFactory.createDbOptions(null);
    ColumnFamilyOptions cfOptions = optionFactory.createColumnFamilyOptions(null);
    String optionsFactoryClass = (String) conf.get(ConfigExtension.ROCKSDB_OPTIONS_FACTORY_CLASS);
    if (optionsFactoryClass != null) {
        RocksDbOptionsFactory udfOptionFactory = (RocksDbOptionsFactory) Utils.newInstance(optionsFactoryClass);
        options = udfOptionFactory.createOptions(options);
        dbOptions = udfOptionFactory.createDbOptions(dbOptions);
        cfOptions = udfOptionFactory.createColumnFamilyOptions(cfOptions);
    }
    try {
        ttlTimeSec = ConfigExtension.getStateTtlTime(conf);
        List<Integer> ttlValues = new ArrayList<>();
        List<byte[]> families = RocksDB.listColumnFamilies(options, rocksDbDir);
        List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
        List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
        if (families != null) {
            for (byte[] bytes : families) {
                columnFamilyDescriptors.add(new ColumnFamilyDescriptor(bytes, cfOptions));
                LOG.debug("Load colum family of {}", new String(bytes));
                if (ttlTimeSec > 0)
                    ttlValues.add(ttlTimeSec);
            }
        }
        if (columnFamilyDescriptors.size() > 0) {
            if (ttlTimeSec > 0)
                rocksDb = TtlDB.open(dbOptions, rocksDbDir, columnFamilyDescriptors, columnFamilyHandles, ttlValues, false);
            else
                rocksDb = RocksDB.open(dbOptions, rocksDbDir, columnFamilyDescriptors, columnFamilyHandles);
            int n = Math.min(columnFamilyDescriptors.size(), columnFamilyHandles.size());
            LOG.info("Try to load RocksDB with column family, desc_num={}, handler_num={}", columnFamilyDescriptors.size(), columnFamilyHandles.size());
            // skip default column
            for (int i = 1; i < n; i++) {
                windowToCFHandler.put((TimeWindow) serializer.deserialize(columnFamilyDescriptors.get(i).columnFamilyName()), columnFamilyHandles.get(i));
            }
        } else {
            rocksDb = RocksDB.open(options, rocksDbDir);
        }
        rocksDb.compactRange();
        LOG.info("Finish the initialization of RocksDB");
    } catch (RocksDBException e) {
        LOG.error("Failed to open rocksdb located at " + rocksDbDir, e);
        throw new RuntimeException(e.getMessage());
    }
    lastCheckpointFiles = new HashSet<String>();
    lastCleanTime = System.currentTimeMillis();
    lastSuccessBatchId = -1;
}

Example 29 with ColumnFamilyHandle

use of org.rocksdb.ColumnFamilyHandle in project jstorm by alibaba.

the class WindowedRocksDbHdfsState method putBatch.

@Override
public void putBatch(TimeWindow window, Map<K, V> batch) {
    try {
        ColumnFamilyHandle handler = getColumnFamilyHandle(window);
        WriteBatch writeBatch = new WriteBatch();
        for (Map.Entry<K, V> entry : batch.entrySet()) {
            writeBatch.put(handler, serializer.serialize(entry.getKey()), serializer.serialize(entry.getValue()));
        }
        rocksDb.write(new WriteOptions(), writeBatch);
    } catch (RocksDBException e) {
        LOG.error("Failed to put batch={} for window={}", batch, window);
        throw new RuntimeException(e.getMessage());
    }
}

Example 30 with ColumnFamilyHandle

use of org.rocksdb.ColumnFamilyHandle in project jstorm by alibaba.

the class WindowedRocksDbHdfsState method put.

@Override
public void put(TimeWindow window, Object key, Object value) {
    try {
        ColumnFamilyHandle handler = getColumnFamilyHandle(window);
        rocksDb.put(handler, serializer.serialize(key), serializer.serialize(value));
    } catch (RocksDBException e) {
        LOG.error("Failed to put data, key={}, value={}, for timeWindow={}", key, value, window);
        throw new RuntimeException(e.getMessage());
    }
}