Examples with ColumnFamilyDescriptor org.rocksdb.ColumnFamilyDescriptor used on opensource projects

Example 16 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project jstorm by alibaba.

the class WindowedRocksDbHdfsState method initRocksDb.

@Override
protected void initRocksDb() {
    windowToCFHandler = new HashMap<>();
    RocksDbOptionsFactory optionFactory = new RocksDbOptionsFactory.Defaults();
    Options options = optionFactory.createOptions(null);
    DBOptions dbOptions = optionFactory.createDbOptions(null);
    ColumnFamilyOptions cfOptions = optionFactory.createColumnFamilyOptions(null);
    String optionsFactoryClass = (String) conf.get(ConfigExtension.ROCKSDB_OPTIONS_FACTORY_CLASS);
    if (optionsFactoryClass != null) {
        RocksDbOptionsFactory udfOptionFactory = (RocksDbOptionsFactory) Utils.newInstance(optionsFactoryClass);
        options = udfOptionFactory.createOptions(options);
        dbOptions = udfOptionFactory.createDbOptions(dbOptions);
        cfOptions = udfOptionFactory.createColumnFamilyOptions(cfOptions);
    }
    try {
        ttlTimeSec = ConfigExtension.getStateTtlTime(conf);
        List<Integer> ttlValues = new ArrayList<>();
        List<byte[]> families = RocksDB.listColumnFamilies(options, rocksDbDir);
        List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
        List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
        if (families != null) {
            for (byte[] bytes : families) {
                columnFamilyDescriptors.add(new ColumnFamilyDescriptor(bytes, cfOptions));
                LOG.debug("Load colum family of {}", new String(bytes));
                if (ttlTimeSec > 0)
                    ttlValues.add(ttlTimeSec);
            }
        }
        if (columnFamilyDescriptors.size() > 0) {
            if (ttlTimeSec > 0)
                rocksDb = TtlDB.open(dbOptions, rocksDbDir, columnFamilyDescriptors, columnFamilyHandles, ttlValues, false);
            else
                rocksDb = RocksDB.open(dbOptions, rocksDbDir, columnFamilyDescriptors, columnFamilyHandles);
            int n = Math.min(columnFamilyDescriptors.size(), columnFamilyHandles.size());
            LOG.info("Try to load RocksDB with column family, desc_num={}, handler_num={}", columnFamilyDescriptors.size(), columnFamilyHandles.size());
            // skip default column
            for (int i = 1; i < n; i++) {
                windowToCFHandler.put((TimeWindow) serializer.deserialize(columnFamilyDescriptors.get(i).columnFamilyName()), columnFamilyHandles.get(i));
            }
        } else {
            rocksDb = RocksDB.open(options, rocksDbDir);
        }
        rocksDb.compactRange();
        LOG.info("Finish the initialization of RocksDB");
    } catch (RocksDBException e) {
        LOG.error("Failed to open rocksdb located at " + rocksDbDir, e);
        throw new RuntimeException(e.getMessage());
    }
    lastCheckpointFiles = new HashSet<String>();
    lastCleanTime = System.currentTimeMillis();
    lastSuccessBatchId = -1;
}

Example 17 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project jstorm by alibaba.

the class RocksDBTest method ttlDbOpenWithColumnFamilies.

public void ttlDbOpenWithColumnFamilies() throws RocksDBException, InterruptedException {
    DBOptions dbOptions = null;
    TtlDB ttlDB = null;
    List<ColumnFamilyDescriptor> cfNames = new ArrayList<ColumnFamilyDescriptor>();
    List<ColumnFamilyHandle> columnFamilyHandleList = new ArrayList<ColumnFamilyHandle>();
    cfNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
    cfNames.add(new ColumnFamilyDescriptor("new_cf".getBytes()));
    List<Integer> ttlValues = new ArrayList<Integer>();
    // Default column family with infinite lifetime
    ttlValues.add(0);
    // new column family with 1 second ttl
    ttlValues.add(1);
    try {
        System.out.println("Begin to open db");
        dbOptions = new DBOptions().setCreateMissingColumnFamilies(true).setCreateIfMissing(true);
        ttlDB = TtlDB.open(dbOptions, rootDir, cfNames, columnFamilyHandleList, ttlValues, false);
        System.out.println("Successfully open db " + rootDir);
        ttlDB.put("key".getBytes(), "value".getBytes());
        assertThat(ttlDB.get("key".getBytes())).isEqualTo("value".getBytes());
        ttlDB.put(columnFamilyHandleList.get(1), "key".getBytes(), "value".getBytes());
        assertThat(ttlDB.get(columnFamilyHandleList.get(1), "key".getBytes())).isEqualTo("value".getBytes());
        TimeUnit.SECONDS.sleep(2);
        ttlDB.compactRange();
        ttlDB.compactRange(columnFamilyHandleList.get(1));
        assertThat(ttlDB.get("key".getBytes())).isNotNull();
        assertThat(ttlDB.get(columnFamilyHandleList.get(1), "key".getBytes())).isNull();
    } finally {
        for (ColumnFamilyHandle columnFamilyHandle : columnFamilyHandleList) {
            columnFamilyHandle.dispose();
        }
        if (ttlDB != null) {
            ttlDB.close();
        }
        if (dbOptions != null) {
            dbOptions.dispose();
        }
    }
}

Example 18 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project jstorm by alibaba.

the class WindowedRocksDbHdfsState method getColumnFamilyHandle.

private ColumnFamilyHandle getColumnFamilyHandle(TimeWindow window) throws RocksDBException {
    ColumnFamilyHandle handler = null;
    if (window == null) {
        handler = rocksDb.getDefaultColumnFamily();
    } else {
        handler = windowToCFHandler.get(window);
        if (handler == null) {
            handler = rocksDb.createColumnFamily(new ColumnFamilyDescriptor(serializer.serialize(window)));
            windowToCFHandler.put(window, handler);
        }
    }
    return handler;
}

Example 19 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project jstorm by alibaba.

the class RocksDbUnitTest method main.

public static void main(String[] args) {
    Map conf = JStormHelper.LoadConf(args[0]);
    putNum = JStormUtils.parseInt(conf.get("put.number"), 100);
    isFlush = JStormUtils.parseBoolean(conf.get("is.flush"), true);
    isCheckpoint = JStormUtils.parseBoolean(conf.get("is.checkpoint"), true);
    sleepTime = JStormUtils.parseInt(conf.get("sleep.time"), 5000);
    compactionInterval = JStormUtils.parseInt(conf.get("compaction.interval"), 30000);
    flushInterval = JStormUtils.parseInt(conf.get("flush.interval"), 3000);
    isCompaction = JStormUtils.parseBoolean(conf.get("is.compaction"), true);
    fileSizeBase = JStormUtils.parseLong(conf.get("file.size.base"), 10 * SizeUnit.KB);
    levelNum = JStormUtils.parseInt(conf.get("db.level.num"), 1);
    compactionTriggerNum = JStormUtils.parseInt(conf.get("db.compaction.trigger.num"), 4);
    LOG.info("Conf={}", conf);
    RocksDB db;
    File file = new File(cpPath);
    file.mkdirs();
    List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
    try {
        Options options = new Options();
        options.setCreateMissingColumnFamilies(true);
        options.setCreateIfMissing(true);
        options.setTargetFileSizeBase(fileSizeBase);
        options.setMaxBackgroundFlushes(2);
        options.setMaxBackgroundCompactions(2);
        options.setCompactionStyle(CompactionStyle.LEVEL);
        options.setNumLevels(levelNum);
        options.setLevelZeroFileNumCompactionTrigger(compactionTriggerNum);
        DBOptions dbOptions = new DBOptions();
        dbOptions.setCreateMissingColumnFamilies(true);
        dbOptions.setCreateIfMissing(true);
        dbOptions.setMaxBackgroundFlushes(2);
        dbOptions.setMaxBackgroundCompactions(2);
        ColumnFamilyOptions familyOptions = new ColumnFamilyOptions();
        familyOptions.setTargetFileSizeBase(fileSizeBase);
        familyOptions.setCompactionStyle(CompactionStyle.LEVEL);
        familyOptions.setNumLevels(levelNum);
        familyOptions.setLevelZeroFileNumCompactionTrigger(compactionTriggerNum);
        List<byte[]> families = RocksDB.listColumnFamilies(options, dbPath);
        List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
        if (families != null) {
            for (byte[] bytes : families) {
                columnFamilyDescriptors.add(new ColumnFamilyDescriptor(bytes, familyOptions));
                LOG.info("Load colum family of {}", new String(bytes));
            }
        }
        if (columnFamilyDescriptors.size() > 0) {
            db = RocksDB.open(dbOptions, dbPath, columnFamilyDescriptors, columnFamilyHandles);
        } else {
            db = RocksDB.open(options, dbPath);
        }
    } catch (RocksDBException e) {
        LOG.error("Failed to open db", e);
        return;
    }
    rocksDbTest(db, columnFamilyHandles);
    db.close();
}

Example 20 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project jstorm by alibaba.

the class RocksDBTest method visitorAccross.

public void visitorAccross() throws RocksDBException, InterruptedException {
    DBOptions dbOptions = null;
    TtlDB ttlDB = null;
    List<ColumnFamilyDescriptor> cfNames = new ArrayList<ColumnFamilyDescriptor>();
    List<ColumnFamilyHandle> columnFamilyHandleList = new ArrayList<ColumnFamilyHandle>();
    cfNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
    cfNames.add(new ColumnFamilyDescriptor("new_cf".getBytes()));
    List<Integer> ttlValues = new ArrayList<Integer>();
    // new column family with 1 second ttl
    ttlValues.add(1);
    // Default column family with infinite lifetime
    ttlValues.add(0);
    try {
        System.out.println("Begin to open db");
        dbOptions = new DBOptions().setCreateMissingColumnFamilies(true).setCreateIfMissing(true);
        ttlDB = TtlDB.open(dbOptions, rootDir, cfNames, columnFamilyHandleList, ttlValues, false);
        System.out.println("Successfully open db " + rootDir);
        List<String> keys = new ArrayList<String>();
        keys.add("key");
        ttlDB.put("key".getBytes(), "key".getBytes());
        for (int i = 0; i < 2; i++) {
            String key = "key" + i;
            keys.add(key);
            ttlDB.put(columnFamilyHandleList.get(i), key.getBytes(), key.getBytes());
        }
        try {
            byte[] value = ttlDB.get("others".getBytes());
            if (value != null) {
                System.out.println("Raw get :" + new String(value));
            } else {
                System.out.println("No value of other");
            }
        } catch (Exception e) {
            System.out.println("Occur exception other");
        }
        for (String key : keys) {
            try {
                byte[] value = ttlDB.get(key.getBytes());
                if (value != null) {
                    System.out.println("Raw get :" + new String(value));
                } else {
                    System.out.println("No value of " + key);
                }
            } catch (Exception e) {
                System.out.println("Occur exception " + key + ", Raw");
            }
            for (int i = 0; i < 2; i++) {
                try {
                    byte[] value = ttlDB.get(columnFamilyHandleList.get(i), key.getBytes());
                    if (value != null) {
                        System.out.println("handler index" + i + " get :" + new String(value));
                    } else {
                        System.out.println("No value of index" + i + " get :" + key);
                    }
                } catch (Exception e) {
                    System.out.println("Occur exception " + key + ", handler index:" + i);
                }
            }
        }
    } finally {
        for (ColumnFamilyHandle columnFamilyHandle : columnFamilyHandleList) {
            columnFamilyHandle.dispose();
        }
        if (ttlDB != null) {
            ttlDB.close();
        }
        if (dbOptions != null) {
            dbOptions.dispose();
        }
    }
}