Examples with ColumnFamilyDescriptor org.rocksdb.ColumnFamilyDescriptor used on opensource projects

Example 11 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project alluxio by Alluxio.

the class RocksStore method createDb.

private void createDb() throws RocksDBException {
    List<ColumnFamilyDescriptor> cfDescriptors = new ArrayList<>();
    cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
    cfDescriptors.addAll(mColumnFamilyDescriptors);
    // a list which will hold the handles for the column families once the db is opened
    List<ColumnFamilyHandle> columns = new ArrayList<>();
    final TimeoutRetry retryPolicy = new TimeoutRetry(ROCKS_OPEN_RETRY_TIMEOUT, 100);
    RocksDBException lastException = null;
    while (retryPolicy.attempt()) {
        try {
            mDb = RocksDB.open(mDbOpts, mDbPath, cfDescriptors, columns);
            break;
        } catch (RocksDBException e) {
            // sometimes the previous terminated process's lock may not have been fully cleared yet
            // retry until timeout to make sure that isn't the case
            lastException = e;
        }
    }
    if (mDb == null && lastException != null) {
        throw lastException;
    }
    mCheckpoint = Checkpoint.create(mDb);
    for (int i = 0; i < columns.size() - 1; i++) {
        // Skip the default column.
        mColumnHandles.get(i).set(columns.get(i + 1));
    }
    LOG.info("Opened rocks database under path {}", mDbPath);
}

Example 12 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project voldemort by voldemort.

the class RocksDbStorageConfiguration method getStore.

@Override
public StorageEngine<ByteArray, byte[], byte[]> getStore(StoreDefinition storeDef, RoutingStrategy strategy) {
    String storeName = storeDef.getName();
    if (!stores.containsKey(storeName)) {
        String dataDir = this.voldemortconfig.getRdbDataDirectory() + "/" + storeName;
        new File(dataDir).mkdirs();
        Properties dbProperties = parseProperties(VoldemortConfig.ROCKSDB_DB_OPTIONS);
        DBOptions dbOptions = (dbProperties.size() > 0) ? DBOptions.getDBOptionsFromProps(dbProperties) : new DBOptions();
        if (dbOptions == null) {
            throw new StorageInitializationException("Unable to parse Data Base Options.");
        }
        dbOptions.setCreateIfMissing(true);
        dbOptions.setCreateMissingColumnFamilies(true);
        dbOptions.createStatistics();
        Properties cfProperties = parseProperties(VoldemortConfig.ROCKSDB_CF_OPTIONS);
        if (this.voldemortconfig.getRocksdbPrefixKeysWithPartitionId()) {
            cfProperties.setProperty("prefix_extractor", "fixed:" + StoreBinaryFormat.PARTITIONID_PREFIX_SIZE);
        }
        ColumnFamilyOptions cfOptions = (cfProperties.size() > 0) ? ColumnFamilyOptions.getColumnFamilyOptionsFromProps(cfProperties) : new ColumnFamilyOptions();
        if (cfOptions == null) {
            throw new StorageInitializationException("Unable to parse Column Family Options.");
        }
        // Create a non default Column Family tp hold the store data.
        List<ColumnFamilyDescriptor> descriptors = new ArrayList<ColumnFamilyDescriptor>();
        descriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions));
        descriptors.add(new ColumnFamilyDescriptor(storeName.getBytes(), cfOptions));
        List<ColumnFamilyHandle> handles = new ArrayList<ColumnFamilyHandle>();
        try {
            RocksDB rdbStore = RocksDB.open(dbOptions, dataDir, descriptors, handles);
            // Dispose of the default Column Family immediately.  We don't use it and if it has not been disposed
            // by the time the DB is closed then the RocksDB code can terminate abnormally (if the RocksDB code is
            // built with assertions enabled). The handle will go out of scope on its own and the Java finalizer
            // will (eventually) do this for us, but, that is not fast enough for the unit tests.
            handles.get(0).dispose();
            ColumnFamilyHandle storeHandle = handles.get(1);
            RocksDbStorageEngine rdbStorageEngine;
            if (this.voldemortconfig.getRocksdbPrefixKeysWithPartitionId()) {
                rdbStorageEngine = new PartitionPrefixedRocksDbStorageEngine(storeName, rdbStore, storeHandle, cfOptions, lockStripes, strategy, voldemortconfig.isRocksdbEnableReadLocks());
            } else {
                rdbStorageEngine = new RocksDbStorageEngine(storeName, rdbStore, storeHandle, cfOptions, lockStripes, voldemortconfig.isRocksdbEnableReadLocks());
            }
            stores.put(storeName, rdbStorageEngine);
        } catch (Exception e) {
            throw new StorageInitializationException(e);
        }
    }
    return stores.get(storeName);
}

Example 13 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project voldemort by voldemort.

the class RocksDbStorageEngine method truncate.

@Override
public void truncate() {
    try {
        rocksDB.dropColumnFamily(storeHandle);
        storeHandle.dispose();
        storeHandle = rocksDB.createColumnFamily(new ColumnFamilyDescriptor(getName().getBytes(), storeOptions));
    } catch (RocksDBException e) {
        throw new VoldemortException("Failed to truncate DB", e);
    }
}

Example 14 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project jstorm by alibaba.

the class RocksDbUnitTest method rocksDbTest.

private static void rocksDbTest(RocksDB db, List<ColumnFamilyHandle> handlers) {
    try {
        ColumnFamilyHandle handler1 = null;
        ColumnFamilyHandle handler2 = null;
        if (handlers.size() > 0) {
            // skip default column family
            handler1 = handlers.get(1);
            handler2 = handlers.get(2);
        } else {
            handler1 = db.createColumnFamily(new ColumnFamilyDescriptor("test1".getBytes()));
            handler2 = db.createColumnFamily(new ColumnFamilyDescriptor("test2".getBytes()));
        }
        int startValue1 = getStartValue(db, handler1);
        int startValue2 = getStartValue(db, handler2);
        ;
        Checkpoint cp = Checkpoint.create(db);
        if (isCompaction) {
            db.compactRange();
            LOG.info("Compaction!");
        }
        long flushWaitTime = System.currentTimeMillis() + flushInterval;
        for (int i = 0; i < putNum || putNum == -1; i++) {
            db.put(handler1, String.valueOf(i % 1000).getBytes(), String.valueOf(startValue1 + i).getBytes());
            db.put(handler2, String.valueOf(i % 1000).getBytes(), String.valueOf(startValue2 + i).getBytes());
            if (isFlush && flushWaitTime <= System.currentTimeMillis()) {
                db.flush(new FlushOptions());
                if (isCheckpoint) {
                    cp.createCheckpoint(cpPath + "/" + i);
                }
                flushWaitTime = System.currentTimeMillis() + flushInterval;
            }
        }
    } catch (RocksDBException e) {
        LOG.error("Failed to put or flush", e);
    }
}

Example 15 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project jstorm by alibaba.

the class RocksTTLDBCache method initDb.

public void initDb(List<Integer> list) throws Exception {
    LOG.info("Begin to init rocksDB of {}", rootDir);
    DBOptions dbOptions = null;
    List<ColumnFamilyDescriptor> columnFamilyNames = new ArrayList<>();
    columnFamilyNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
    for (Integer timeout : list) {
        columnFamilyNames.add(new ColumnFamilyDescriptor(String.valueOf(timeout).getBytes()));
    }
    List<Integer> ttlValues = new ArrayList<>();
    // Default column family with infinite TTL
    // NOTE that the first must be 0, RocksDB.java API has this limitation
    ttlValues.add(0);
    // new column family with list second ttl
    ttlValues.addAll(list);
    try {
        dbOptions = new DBOptions().setCreateMissingColumnFamilies(true).setCreateIfMissing(true);
        List<ColumnFamilyHandle> columnFamilyHandleList = new ArrayList<>();
        ttlDB = TtlDB.open(dbOptions, rootDir, columnFamilyNames, columnFamilyHandleList, ttlValues, false);
        for (int i = 0; i < ttlValues.size(); i++) {
            windowHandlers.put(ttlValues.get(i), columnFamilyHandleList.get(i));
        }
        LOG.info("Successfully init rocksDB of {}", rootDir);
    } finally {
        if (dbOptions != null) {
            dbOptions.dispose();
        }
    }
}