Search in sources :

Example 26 with OperationStatus

use of com.sleepycat.je.OperationStatus in project voldemort by voldemort.

the class BdbConvertBaseToPidScan method transfer.

@Override
public void transfer() throws Exception {
    cursor = srcDB.openCursor(null, null);
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry valueEntry = new DatabaseEntry();
    byte[] prevKey = null;
    List<Versioned<byte[]>> vals = new ArrayList<Versioned<byte[]>>();
    HashFunction hash = new FnvHashFunction();
    int totalPartitions = cluster.getNumberOfPartitions();
    long startTime = System.currentTimeMillis();
    int scanCount = 0;
    int keyCount = 0;
    while (cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
        scanCount++;
        if (scanCount % 1000000 == 0)
            logger.info("Converted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
        // read the value as a versioned Object
        VectorClock clock = new VectorClock(valueEntry.getData());
        byte[] bytes = ByteUtils.copy(valueEntry.getData(), clock.sizeInBytes(), valueEntry.getData().length);
        Versioned<byte[]> value = new Versioned<byte[]>(bytes, clock);
        byte[] key = keyEntry.getData();
        if (prevKey != null && (ByteUtils.compare(prevKey, key) != 0)) {
            // there is a new key; write out the buffered values and
            // previous key
            int partition = BdbConvertData.abs(hash.hash(prevKey)) % (Math.max(1, totalPartitions));
            OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(StoreBinaryFormat.makePrefixedKey(prevKey, partition)), new DatabaseEntry(StoreBinaryFormat.toByteArray(vals)));
            if (OperationStatus.SUCCESS != putStatus) {
                String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(prevKey);
                logger.error(errorStr);
                throw new Exception(errorStr);
            }
            vals = new ArrayList<Versioned<byte[]>>();
            keyCount++;
        }
        vals.add(value);
        prevKey = key;
    }
    if (vals.size() > 0) {
        int partition = BdbConvertData.abs(hash.hash(prevKey)) % (Math.max(1, totalPartitions));
        OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(StoreBinaryFormat.makePrefixedKey(prevKey, partition)), new DatabaseEntry(StoreBinaryFormat.toByteArray(vals)));
        if (OperationStatus.SUCCESS != putStatus) {
            String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(prevKey);
            logger.error(errorStr);
            throw new Exception(errorStr);
        }
        keyCount++;
    }
    logger.info("Completed " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}
Also used : Versioned(voldemort.versioning.Versioned) VectorClock(voldemort.versioning.VectorClock) ArrayList(java.util.ArrayList) DatabaseEntry(com.sleepycat.je.DatabaseEntry) FnvHashFunction(voldemort.utils.FnvHashFunction) HashFunction(voldemort.utils.HashFunction) FnvHashFunction(voldemort.utils.FnvHashFunction) OperationStatus(com.sleepycat.je.OperationStatus)

Example 27 with OperationStatus

use of com.sleepycat.je.OperationStatus in project voldemort by voldemort.

the class BdbRevertPidScanToBase method transfer.

@Override
public void transfer() throws Exception {
    cursor = srcDB.openCursor(null, null);
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry valueEntry = new DatabaseEntry();
    VersionedSerializer<byte[]> versionedSerializer = new VersionedSerializer<byte[]>(new IdentitySerializer());
    List<Versioned<byte[]>> vals;
    long startTime = System.currentTimeMillis();
    int scanCount = 0;
    int keyCount = 0;
    while (cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
        keyCount++;
        vals = StoreBinaryFormat.fromByteArray(valueEntry.getData());
        // pull out the real key
        byte[] stripedKey = StoreBinaryFormat.extractKey(keyEntry.getData());
        for (Versioned<byte[]> val : vals) {
            OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(stripedKey), new DatabaseEntry(versionedSerializer.toBytes(val)));
            if (OperationStatus.SUCCESS != putStatus) {
                String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(stripedKey);
                logger.error(errorStr);
                throw new Exception(errorStr);
            }
            scanCount++;
        }
        if (scanCount % 1000000 == 0)
            logger.info("Reverted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
    }
    logger.info("Reverted " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}
Also used : Versioned(voldemort.versioning.Versioned) DatabaseEntry(com.sleepycat.je.DatabaseEntry) VersionedSerializer(voldemort.serialization.VersionedSerializer) OperationStatus(com.sleepycat.je.OperationStatus) IdentitySerializer(voldemort.serialization.IdentitySerializer)

Example 28 with OperationStatus

use of com.sleepycat.je.OperationStatus in project voldemort by voldemort.

the class BdbConvertBaseToNewDup method transfer.

@Override
public void transfer() throws Exception {
    cursor = srcDB.openCursor(null, null);
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry valueEntry = new DatabaseEntry();
    byte[] prevKey = null;
    List<Versioned<byte[]>> vals = new ArrayList<Versioned<byte[]>>();
    long startTime = System.currentTimeMillis();
    int scanCount = 0;
    int keyCount = 0;
    while (cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
        scanCount++;
        if (scanCount % 1000000 == 0)
            logger.info("Converted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
        // read the value as a versioned Object
        VectorClock clock = new VectorClock(valueEntry.getData());
        byte[] bytes = ByteUtils.copy(valueEntry.getData(), clock.sizeInBytes(), valueEntry.getData().length);
        Versioned<byte[]> value = new Versioned<byte[]>(bytes, clock);
        byte[] key = keyEntry.getData();
        if (prevKey != null && (ByteUtils.compare(prevKey, key) != 0)) {
            // there is a new key; write out the buffered values and
            // previous key
            OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(prevKey), new DatabaseEntry(StoreBinaryFormat.toByteArray(vals)));
            if (OperationStatus.SUCCESS != putStatus) {
                String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(prevKey);
                logger.error(errorStr);
                throw new Exception(errorStr);
            }
            vals = new ArrayList<Versioned<byte[]>>();
            keyCount++;
        }
        vals.add(value);
        prevKey = key;
    }
    if (vals.size() > 0) {
        OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(prevKey), new DatabaseEntry(StoreBinaryFormat.toByteArray(vals)));
        if (OperationStatus.SUCCESS != putStatus) {
            String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(prevKey);
            logger.error(errorStr);
            throw new Exception(errorStr);
        }
        keyCount++;
    }
    logger.info("Completed " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}
Also used : Versioned(voldemort.versioning.Versioned) VectorClock(voldemort.versioning.VectorClock) ArrayList(java.util.ArrayList) DatabaseEntry(com.sleepycat.je.DatabaseEntry) OperationStatus(com.sleepycat.je.OperationStatus)

Example 29 with OperationStatus

use of com.sleepycat.je.OperationStatus in project voldemort by voldemort.

the class BdbRevertNewDupToBase method transfer.

@Override
public void transfer() throws Exception {
    cursor = srcDB.openCursor(null, null);
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry valueEntry = new DatabaseEntry();
    VersionedSerializer<byte[]> versionedSerializer = new VersionedSerializer<byte[]>(new IdentitySerializer());
    List<Versioned<byte[]>> vals;
    long startTime = System.currentTimeMillis();
    int scanCount = 0;
    int keyCount = 0;
    while (cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
        keyCount++;
        vals = StoreBinaryFormat.fromByteArray(valueEntry.getData());
        for (Versioned<byte[]> val : vals) {
            OperationStatus putStatus = dstDB.put(null, keyEntry, new DatabaseEntry(versionedSerializer.toBytes(val)));
            if (OperationStatus.SUCCESS != putStatus) {
                String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(keyEntry.getData());
                logger.error(errorStr);
                throw new Exception(errorStr);
            }
            scanCount++;
        }
        if (scanCount % 1000000 == 0)
            logger.info("Reverted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
    }
    logger.info("Reverted " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}
Also used : Versioned(voldemort.versioning.Versioned) OperationStatus(com.sleepycat.je.OperationStatus) IdentitySerializer(voldemort.serialization.IdentitySerializer) DatabaseEntry(com.sleepycat.je.DatabaseEntry) VersionedSerializer(voldemort.serialization.VersionedSerializer)

Example 30 with OperationStatus

use of com.sleepycat.je.OperationStatus in project voldemort by voldemort.

the class BdbRevertPidScanToNewDup method transfer.

@Override
public void transfer() throws Exception {
    cursor = srcDB.openCursor(null, null);
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry valueEntry = new DatabaseEntry();
    List<Versioned<byte[]>> vals;
    long startTime = System.currentTimeMillis();
    int scanCount = 0;
    int keyCount = 0;
    while (cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
        keyCount++;
        vals = StoreBinaryFormat.fromByteArray(valueEntry.getData());
        scanCount += vals.size();
        // pull out the real key
        byte[] stripedKey = StoreBinaryFormat.extractKey(keyEntry.getData());
        OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(stripedKey), valueEntry);
        if (OperationStatus.SUCCESS != putStatus) {
            String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(keyEntry.getData());
            logger.error(errorStr);
            throw new Exception(errorStr);
        }
        if (scanCount % 1000000 == 0)
            logger.info("Reverted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
    }
    logger.info("Reverted " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}
Also used : Versioned(voldemort.versioning.Versioned) OperationStatus(com.sleepycat.je.OperationStatus) DatabaseEntry(com.sleepycat.je.DatabaseEntry)

Aggregations

OperationStatus (com.sleepycat.je.OperationStatus)76 DatabaseEntry (com.sleepycat.je.DatabaseEntry)70 Transaction (com.sleepycat.je.Transaction)23 DatabaseException (com.sleepycat.je.DatabaseException)21 Cursor (com.sleepycat.je.Cursor)15 StoreException (org.apache.qpid.server.store.StoreException)13 Versioned (voldemort.versioning.Versioned)9 Database (com.sleepycat.je.Database)8 ArrayList (java.util.ArrayList)7 LockConflictException (com.sleepycat.je.LockConflictException)6 UUIDTupleBinding (org.apache.qpid.server.store.berkeleydb.tuple.UUIDTupleBinding)6 ConnectionScopedRuntimeException (org.apache.qpid.server.util.ConnectionScopedRuntimeException)6 UUID (java.util.UUID)4 BimserverLockConflictException (org.bimserver.database.BimserverLockConflictException)4 SirixIOException (org.sirix.exception.SirixIOException)4 AsyncOperationStatus (voldemort.server.protocol.admin.AsyncOperationStatus)4 PersistenceFailureException (voldemort.store.PersistenceFailureException)4 DatabaseConfig (com.sleepycat.je.DatabaseConfig)3 LockMode (com.sleepycat.je.LockMode)3 VSystemException (io.vertigo.lang.VSystemException)3