Examples with DatabaseEntry com.sleepycat.je.DatabaseEntry used on opensource projects

Example 21 with DatabaseEntry

use of com.sleepycat.je.DatabaseEntry in project voldemort by voldemort.

the class BdbStorageEngine method get.

@Override
public List<Versioned<byte[]>> get(ByteArray key, byte[] transforms) throws PersistenceFailureException {
    StoreUtils.assertValidKey(key);
    DatabaseEntry keyEntry = new DatabaseEntry(key.get());
    DatabaseEntry valueEntry = new DatabaseEntry();
    long startTimeNs = -1;
    if (logger.isTraceEnabled())
        startTimeNs = System.nanoTime();
    try {
        // uncommitted reads are perfectly fine now, since we have no
        // je-delete() in put()
        OperationStatus status = getBdbDatabase().get(null, keyEntry, valueEntry, readLockMode);
        if (OperationStatus.SUCCESS == status) {
            return StoreBinaryFormat.fromByteArray(valueEntry.getData());
        } else {
            return Collections.emptyList();
        }
    } catch (DatabaseException e) {
        this.bdbEnvironmentStats.reportException(e);
        logger.error(e);
        throw new PersistenceFailureException(e);
    } finally {
        if (logger.isTraceEnabled()) {
            logger.trace("Completed GET (" + getName() + ") from key " + key + " (keyRef: " + System.identityHashCode(key) + ") in " + (System.nanoTime() - startTimeNs) + " ns at " + System.currentTimeMillis());
        }
    }
}

Example 22 with DatabaseEntry

use of com.sleepycat.je.DatabaseEntry in project voldemort by voldemort.

the class BdbStorageEngine method putAndUnlock.

@Override
public void putAndUnlock(ByteArray key, KeyLockHandle<byte[]> handle) {
    long startTimeNs = -1;
    if (logger.isTraceEnabled())
        startTimeNs = System.nanoTime();
    StoreUtils.assertValidKey(key);
    DatabaseEntry keyEntry = new DatabaseEntry(key.get());
    DatabaseEntry valueEntry = new DatabaseEntry();
    boolean succeeded = false;
    Transaction transaction = null;
    try {
        transaction = (Transaction) handle.getKeyLock();
        valueEntry.setData(StoreBinaryFormat.toByteArray(handle.getValues()));
        OperationStatus status = getBdbDatabase().put(transaction, keyEntry, valueEntry);
        if (status != OperationStatus.SUCCESS)
            throw new PersistenceFailureException("putAndUnlock operation failed with status: " + status);
        succeeded = true;
    } catch (DatabaseException e) {
        this.bdbEnvironmentStats.reportException(e);
        logger.error("Error in putAndUnlock for store " + this.getName(), e);
        throw new PersistenceFailureException(e);
    } finally {
        if (succeeded)
            attemptCommit(transaction);
        else
            attemptAbort(transaction);
        if (logger.isTraceEnabled()) {
            logger.trace("Completed PUTANDUNLOCK (" + getName() + ") to key " + key + " (keyRef: " + System.identityHashCode(key) + " in " + (System.nanoTime() - startTimeNs) + " ns at " + System.currentTimeMillis());
        }
        handle.close();
    }
}

Example 23 with DatabaseEntry

use of com.sleepycat.je.DatabaseEntry in project voldemort by voldemort.

the class BdbConvertNewDupToPidScan method transfer.

@Override
public void transfer() throws Exception {
    cursor = srcDB.openCursor(null, null);
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry valueEntry = new DatabaseEntry();
    HashFunction hash = new FnvHashFunction();
    int totalPartitions = cluster.getNumberOfPartitions();
    List<Versioned<byte[]>> vals;
    long startTime = System.currentTimeMillis();
    int scanCount = 0;
    int keyCount = 0;
    while (cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
        keyCount++;
        vals = StoreBinaryFormat.fromByteArray(valueEntry.getData());
        scanCount += vals.size();
        int partition = BdbConvertData.abs(hash.hash(keyEntry.getData())) % (Math.max(1, totalPartitions));
        OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(StoreBinaryFormat.makePrefixedKey(keyEntry.getData(), partition)), valueEntry);
        if (OperationStatus.SUCCESS != putStatus) {
            String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(keyEntry.getData());
            logger.error(errorStr);
            throw new Exception(errorStr);
        }
        if (scanCount % 1000000 == 0)
            logger.info("Reverted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
    }
    logger.info("Converted " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}

Example 24 with DatabaseEntry

use of com.sleepycat.je.DatabaseEntry in project voldemort by voldemort.

the class BdbGrowth method main.

public static void main(String[] args) throws Exception {
    if (args.length != 5) {
        System.err.println("USAGE: java BdbGrowth directory cache_size total_size increment threads");
        System.exit(1);
    }
    final String dir = args[0];
    final long cacheSize = Long.parseLong(args[1]);
    final int totalSize = Integer.parseInt(args[2]);
    final int increment = Integer.parseInt(args[3]);
    final int threads = Integer.parseInt(args[4]);
    Environment environment;
    EnvironmentConfig environmentConfig;
    DatabaseConfig databaseConfig;
    environmentConfig = new EnvironmentConfig();
    environmentConfig.setCacheSize(cacheSize);
    environmentConfig.setDurability(Durability.COMMIT_NO_SYNC);
    environmentConfig.setConfigParam(EnvironmentConfig.LOG_FILE_MAX, "1000000000");
    environmentConfig.setConfigParam(EnvironmentConfig.CLEANER_MAX_BATCH_FILES, "100");
    environmentConfig.setConfigParam(EnvironmentConfig.CLEANER_READ_SIZE, "52428800");
    environmentConfig.setAllowCreate(true);
    environmentConfig.setTransactional(true);
    databaseConfig = new DatabaseConfig();
    databaseConfig.setAllowCreate(true);
    // databaseConfig.setDeferredWrite(true);
    databaseConfig.setTransactional(true);
    databaseConfig.setNodeMaxEntries(1024);
    File bdbDir = new File(dir);
    if (!bdbDir.exists()) {
        bdbDir.mkdir();
    } else {
        for (File f : bdbDir.listFiles()) f.delete();
    }
    environment = new Environment(bdbDir, environmentConfig);
    final Database db = environment.openDatabase(null, "test", databaseConfig);
    final Random rand = new Random();
    int iterations = totalSize / increment;
    long[] readTimes = new long[iterations];
    long[] writeTimes = new long[iterations];
    ExecutorService service = Executors.newFixedThreadPool(threads);
    for (int i = 0; i < iterations; i++) {
        System.out.println("Starting iteration " + i);
        List<Future<Object>> results = new ArrayList<Future<Object>>(increment);
        long startTime = System.currentTimeMillis();
        final int fi = i;
        for (int j = 0; j < increment; j++) {
            final int fj = j;
            results.add(service.submit(new Callable<Object>() {

                public Object call() throws Exception {
                    db.put(null, new DatabaseEntry(Integer.toString(fi * increment + fj).getBytes()), new DatabaseEntry(Integer.toString(fi * increment + fj).getBytes()));
                    return null;
                }
            }));
        }
        for (int j = 0; j < increment; j++) results.get(j).get();
        writeTimes[i] = System.currentTimeMillis() - startTime;
        System.out.println("write: " + (writeTimes[i] / (double) increment));
        results.clear();
        startTime = System.currentTimeMillis();
        for (int j = 0; j < increment; j++) {
            results.add(service.submit(new Callable<Object>() {

                public Object call() throws Exception {
                    int value = rand.nextInt((fi + 1) * increment);
                    return db.get(null, new DatabaseEntry(Integer.toString(value).getBytes()), new DatabaseEntry(Integer.toString(value).getBytes()), null);
                }
            }));
        }
        for (int j = 0; j < increment; j++) results.get(j).get();
        readTimes[i] = (System.currentTimeMillis() - startTime);
        System.out.println("read: " + (readTimes[i] / (double) increment));
        int cleaned = 0;
        do {
            cleaned += environment.cleanLog();
        } while (cleaned > 0);
        if (cleaned > 0)
            System.out.println("Cleaned " + cleaned + " files.");
        CheckpointConfig cp = new CheckpointConfig();
        cp.setForce(true);
        environment.checkpoint(null);
        environment.compress();
        environment.sync();
        System.out.println("Cleaning, Checkpointing and compression completed.");
    }
    System.out.println();
    System.out.println("iteration read write:");
    for (int i = 0; i < iterations; i++) {
        System.out.print(i);
        System.out.print(" " + readTimes[i] / (double) increment);
        System.out.println(" " + writeTimes[i] / (double) increment);
    }
    System.out.println(environment.getStats(null));
    System.exit(0);
}

Example 25 with DatabaseEntry

use of com.sleepycat.je.DatabaseEntry in project crawler4j by yasserg.

the class DocIDServer method addUrlAndDocId.

public void addUrlAndDocId(String url, int docId) throws Exception {
    synchronized (mutex) {
        if (docId <= lastDocID) {
            throw new Exception("Requested doc id: " + docId + " is not larger than: " + lastDocID);
        }
        // Make sure that we have not already assigned a docid for this URL
        int prevDocid = getDocId(url);
        if (prevDocid > 0) {
            if (prevDocid == docId) {
                return;
            }
            throw new Exception("Doc id: " + prevDocid + " is already assigned to URL: " + url);
        }
        docIDsDB.put(null, new DatabaseEntry(url.getBytes()), new DatabaseEntry(Util.int2ByteArray(docId)));
        lastDocID = docId;
    }
}