Examples with StoreTransaction com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction used on opensource projects

Example 1 with StoreTransaction

use of com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction in project titan by thinkaurelius.

the class ConsistentKeyIDAuthority method getIDBlock.

@Override
public synchronized IDBlock getIDBlock(final int partition, final int idNamespace, Duration timeout) throws BackendException {
    Preconditions.checkArgument(partition >= 0 && partition < (1 << partitionBitWdith), "Invalid partition id [%s] for bit width [%s]", partition, partitionBitWdith);
    //can be any non-negative value
    Preconditions.checkArgument(idNamespace >= 0);
    final Timer methodTime = times.getTimer().start();
    final long blockSize = getBlockSize(idNamespace);
    final long idUpperBound = getIdUpperBound(idNamespace);
    final int maxAvailableBits = (VariableLong.unsignedBitLength(idUpperBound) - 1) - uniqueIdBitWidth;
    Preconditions.checkArgument(maxAvailableBits > 0, "Unique id bit width [%s] is too wide for id-namespace [%s] id bound [%s]", uniqueIdBitWidth, idNamespace, idUpperBound);
    final long idBlockUpperBound = (1l << maxAvailableBits);
    final List<Integer> exhaustedUniquePIDs = new ArrayList<Integer>(randomUniqueIDLimit);
    Duration backoffMS = idApplicationWaitMS;
    Preconditions.checkArgument(idBlockUpperBound > blockSize, "Block size [%s] is larger than upper bound [%s] for bit width [%s]", blockSize, idBlockUpperBound, uniqueIdBitWidth);
    while (methodTime.elapsed().compareTo(timeout) < 0) {
        final int uniquePID = getUniquePartitionID();
        final StaticBuffer partitionKey = getPartitionKey(partition, idNamespace, uniquePID);
        try {
            long nextStart = getCurrentID(partitionKey);
            if (idBlockUpperBound - blockSize <= nextStart) {
                log.info("ID overflow detected on partition({})-namespace({}) with uniqueid {}. Current id {}, block size {}, and upper bound {} for bit width {}.", partition, idNamespace, uniquePID, nextStart, blockSize, idBlockUpperBound, uniqueIdBitWidth);
                if (randomizeUniqueId) {
                    exhaustedUniquePIDs.add(uniquePID);
                    if (exhaustedUniquePIDs.size() == randomUniqueIDLimit)
                        throw new IDPoolExhaustedException(String.format("Exhausted %d uniqueid(s) on partition(%d)-namespace(%d): %s", exhaustedUniquePIDs.size(), partition, idNamespace, Joiner.on(",").join(exhaustedUniquePIDs)));
                    else
                        throw new UniqueIDExhaustedException(String.format("Exhausted ID partition(%d)-namespace(%d) with uniqueid %d (uniqueid attempt %d/%d)", partition, idNamespace, uniquePID, exhaustedUniquePIDs.size(), randomUniqueIDLimit));
                }
                throw new IDPoolExhaustedException("Exhausted id block for partition(" + partition + ")-namespace(" + idNamespace + ") with upper bound: " + idBlockUpperBound);
            }
            // calculate the start (inclusive) and end (exclusive) of the allocation we're about to attempt
            assert idBlockUpperBound - blockSize > nextStart;
            long nextEnd = nextStart + blockSize;
            StaticBuffer target = null;
            // attempt to write our claim on the next id block
            boolean success = false;
            try {
                Timer writeTimer = times.getTimer().start();
                target = getBlockApplication(nextEnd, writeTimer.getStartTime());
                // copy for the inner class
                final StaticBuffer finalTarget = target;
                BackendOperation.execute(new BackendOperation.Transactional<Boolean>() {

                    @Override
                    public Boolean call(StoreTransaction txh) throws BackendException {
                        idStore.mutate(partitionKey, Arrays.asList(StaticArrayEntry.of(finalTarget)), KeyColumnValueStore.NO_DELETIONS, txh);
                        return true;
                    }
                }, this, times);
                writeTimer.stop();
                Duration writeElapsed = writeTimer.elapsed();
                if (idApplicationWaitMS.compareTo(writeElapsed) < 0) {
                    throw new TemporaryBackendException("Wrote claim for id block [" + nextStart + ", " + nextEnd + ") in " + (writeElapsed) + " => too slow, threshold is: " + idApplicationWaitMS);
                } else {
                    assert 0 != target.length();
                    final StaticBuffer[] slice = getBlockSlice(nextEnd);
                    /* At this point we've written our claim on [nextStart, nextEnd),
                         * but we haven't yet guaranteed the absence of a contending claim on
                         * the same id block from another machine
                         */
                    sleepAndConvertInterrupts(idApplicationWaitMS.plus(waitGracePeriod));
                    // Read all id allocation claims on this partition, for the counter value we're claiming
                    List<Entry> blocks = BackendOperation.execute(new BackendOperation.Transactional<List<Entry>>() {

                        @Override
                        public List<Entry> call(StoreTransaction txh) throws BackendException {
                            return idStore.getSlice(new KeySliceQuery(partitionKey, slice[0], slice[1]), txh);
                        }
                    }, this, times);
                    if (blocks == null)
                        throw new TemporaryBackendException("Could not read from storage");
                    if (blocks.isEmpty())
                        throw new PermanentBackendException("It seems there is a race-condition in the block application. " + "If you have multiple Titan instances running on one physical machine, ensure that they have unique machine idAuthorities");
                    /* If our claim is the lexicographically first one, then our claim
                         * is the most senior one and we own this id block
                         */
                    if (target.equals(blocks.get(0).getColumnAs(StaticBuffer.STATIC_FACTORY))) {
                        ConsistentKeyIDBlock idblock = new ConsistentKeyIDBlock(nextStart, blockSize, uniqueIdBitWidth, uniquePID);
                        if (log.isDebugEnabled()) {
                            log.debug("Acquired ID block [{}] on partition({})-namespace({}) (my rid is {})", new Object[] { idblock, partition, idNamespace, new String(uid) });
                        }
                        success = true;
                        return idblock;
                    } else {
                        // Another claimant beat us to this id block -- try again.
                        log.debug("Failed to acquire ID block [{},{}) (another host claimed it first)", nextStart, nextEnd);
                    }
                }
            } finally {
                if (!success && null != target) {
                    //Delete claim to not pollute id space
                    for (int attempt = 0; attempt < rollbackAttempts; attempt++) {
                        try {
                            // copy for the inner class
                            final StaticBuffer finalTarget = target;
                            BackendOperation.execute(new BackendOperation.Transactional<Boolean>() {

                                @Override
                                public Boolean call(StoreTransaction txh) throws BackendException {
                                    idStore.mutate(partitionKey, KeyColumnValueStore.NO_ADDITIONS, Arrays.asList(finalTarget), txh);
                                    return true;
                                }
                            }, new //Use normal consistency level for these non-critical delete operations
                            BackendOperation.TransactionalProvider() {

                                @Override
                                public StoreTransaction openTx() throws BackendException {
                                    return manager.beginTransaction(storeTxConfigBuilder.build());
                                }

                                @Override
                                public void close() {
                                }
                            }, times);
                            break;
                        } catch (BackendException e) {
                            log.warn("Storage exception while deleting old block application - retrying in {}", rollbackWaitTime, e);
                            if (!rollbackWaitTime.isZero())
                                sleepAndConvertInterrupts(rollbackWaitTime);
                        }
                    }
                }
            }
        } catch (UniqueIDExhaustedException e) {
            // No need to increment the backoff wait time or to sleep
            log.warn(e.getMessage());
        } catch (TemporaryBackendException e) {
            backoffMS = Durations.min(backoffMS.multipliedBy(2), idApplicationWaitMS.multipliedBy(32));
            log.warn("Temporary storage exception while acquiring id block - retrying in {}: {}", backoffMS, e);
            sleepAndConvertInterrupts(backoffMS);
        }
    }
    throw new TemporaryLockingException(String.format("Reached timeout %d (%s elapsed) when attempting to allocate id block on partition(%d)-namespace(%d)", timeout, methodTime.toString(), partition, idNamespace));
}

Example 2 with StoreTransaction

use of com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction in project titan by thinkaurelius.

the class CassandraScanJobIT method testSimpleScan.

@Test
public void testSimpleScan() throws InterruptedException, ExecutionException, IOException, BackendException {
    int keys = 1000;
    int cols = 40;
    String[][] values = KeyValueStoreUtil.generateData(keys, cols);
    //Make it only half the number of columns for every 2nd key
    for (int i = 0; i < values.length; i++) {
        if (i % 2 == 0)
            values[i] = Arrays.copyOf(values[i], cols / 2);
    }
    log.debug("Loading values: " + keys + "x" + cols);
    KeyColumnValueStoreManager mgr = new CassandraThriftStoreManager(GraphDatabaseConfiguration.buildGraphConfiguration());
    KeyColumnValueStore store = mgr.openDatabase("edgestore");
    StoreTransaction tx = mgr.beginTransaction(StandardBaseTransactionConfig.of(TimestampProviders.MICRO));
    KeyColumnValueStoreUtil.loadValues(store, tx, values);
    // noop on Cassandra, but harmless
    tx.commit();
    SimpleScanJobRunner runner = (ScanJob job, Configuration jobConf, String rootNSName) -> {
        try {
            return new CassandraHadoopScanRunner(job).scanJobConf(jobConf).scanJobConfRoot(rootNSName).partitionerOverride("org.apache.cassandra.dht.Murmur3Partitioner").run();
        } catch (ClassNotFoundException e) {
            throw new RuntimeException(e);
        }
    };
    SimpleScanJob.runBasicTests(keys, cols, runner);
}

Example 3 with StoreTransaction

use of com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction in project titan by thinkaurelius.

the class BerkeleyJEKeyValueStore method get.

@Override
public StaticBuffer get(StaticBuffer key, StoreTransaction txh) throws BackendException {
    Transaction tx = getTransaction(txh);
    try {
        DatabaseEntry dbkey = key.as(ENTRY_FACTORY);
        DatabaseEntry data = new DatabaseEntry();
        log.trace("db={}, op=get, tx={}", name, txh);
        OperationStatus status = db.get(tx, dbkey, data, getLockMode(txh));
        if (status == OperationStatus.SUCCESS) {
            return getBuffer(data);
        } else {
            return null;
        }
    } catch (DatabaseException e) {
        throw new PermanentBackendException(e);
    }
}

Example 4 with StoreTransaction

use of com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction in project titan by thinkaurelius.

the class BerkeleyJEKeyValueStore method delete.

@Override
public void delete(StaticBuffer key, StoreTransaction txh) throws BackendException {
    log.trace("Deletion");
    Transaction tx = getTransaction(txh);
    try {
        log.trace("db={}, op=delete, tx={}", name, txh);
        OperationStatus status = db.delete(tx, key.as(ENTRY_FACTORY));
        if (status != OperationStatus.SUCCESS) {
            throw new PermanentBackendException("Could not remove: " + status);
        }
    } catch (DatabaseException e) {
        throw new PermanentBackendException(e);
    }
}

Example 5 with StoreTransaction

use of com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction in project titan by thinkaurelius.

the class BerkeleyJEKeyValueStore method getSlice.

@Override
public RecordIterator<KeyValueEntry> getSlice(KVQuery query, StoreTransaction txh) throws BackendException {
    log.trace("beginning db={}, op=getSlice, tx={}", name, txh);
    Transaction tx = getTransaction(txh);
    Cursor cursor = null;
    final StaticBuffer keyStart = query.getStart();
    final StaticBuffer keyEnd = query.getEnd();
    final KeySelector selector = query.getKeySelector();
    final List<KeyValueEntry> result = new ArrayList<KeyValueEntry>();
    try {
        DatabaseEntry foundKey = keyStart.as(ENTRY_FACTORY);
        DatabaseEntry foundData = new DatabaseEntry();
        cursor = db.openCursor(tx, null);
        OperationStatus status = cursor.getSearchKeyRange(foundKey, foundData, getLockMode(txh));
        //Iterate until given condition is satisfied or end of records
        while (status == OperationStatus.SUCCESS) {
            StaticBuffer key = getBuffer(foundKey);
            if (key.compareTo(keyEnd) >= 0)
                break;
            if (selector.include(key)) {
                result.add(new KeyValueEntry(key, getBuffer(foundData)));
            }
            if (selector.reachedLimit())
                break;
            status = cursor.getNext(foundKey, foundData, getLockMode(txh));
        }
        log.trace("db={}, op=getSlice, tx={}, resultcount={}", name, txh, result.size());
        return new RecordIterator<KeyValueEntry>() {

            private final Iterator<KeyValueEntry> entries = result.iterator();

            @Override
            public boolean hasNext() {
                return entries.hasNext();
            }

            @Override
            public KeyValueEntry next() {
                return entries.next();
            }

            @Override
            public void close() {
            }

            @Override
            public void remove() {
                throw new UnsupportedOperationException();
            }
        };
    } catch (Exception e) {
        throw new PermanentBackendException(e);
    } finally {
        try {
            if (cursor != null)
                cursor.close();
        } catch (Exception e) {
            throw new PermanentBackendException(e);
        }
    }
}