Examples with KCVMutation org.janusgraph.diskstorage.keycolumnvalue.KCVMutation used on opensource projects

Example 1 with KCVMutation

use of org.janusgraph.diskstorage.keycolumnvalue.KCVMutation in project janusgraph by JanusGraph.

the class ExpectedValueCheckingTest method testMutateManyWithLockUsesConsistentTx.

@Test
public void testMutateManyWithLockUsesConsistentTx() throws BackendException {
    final ImmutableList<Entry> adds = ImmutableList.of(StaticArrayEntry.of(DATA_COL, DATA_VAL));
    final ImmutableList<StaticBuffer> deletions = ImmutableList.of();
    Map<String, Map<StaticBuffer, KCVMutation>> mutations = ImmutableMap.of(STORE_NAME, ImmutableMap.of(DATA_KEY, new KCVMutation(adds, deletions)));
    final KeyColumn kc = new KeyColumn(LOCK_KEY, LOCK_COL);
    // Acquire a lock
    backingLocker.writeLock(kc, consistentTx);
    // 2. Run mutateMany
    // 2.1. Check locks & expected values before mutating data
    backingLocker.checkLocks(consistentTx);
    StaticBuffer nextBuf = BufferUtil.nextBiggerBuffer(kc.getColumn());
    KeySliceQuery expectedValueQuery = new KeySliceQuery(kc.getKey(), kc.getColumn(), nextBuf);
    // expected value read must use strong consistency
    expect(backingStore.getSlice(expectedValueQuery, consistentTx)).andReturn(StaticArrayEntryList.of(StaticArrayEntry.of(LOCK_COL, LOCK_VAL)));
    // 2.2. Run mutateMany on backing manager to modify data
    // writes by txs with locks must use strong consistency
    backingManager.mutateMany(mutations, consistentTx);
    ctrl.replay();
    // Lock acquisition
    expectStore.acquireLock(LOCK_KEY, LOCK_COL, LOCK_VAL, expectTx);
    // Mutate
    expectManager.mutateMany(mutations, expectTx);
}

Example 2 with KCVMutation

use of org.janusgraph.diskstorage.keycolumnvalue.KCVMutation in project janusgraph by JanusGraph.

the class HBaseStoreManager method convertToCommands.

/**
 * Convert JanusGraph internal Mutation representation into HBase native commands.
 *
 * @param mutations    Mutations to convert into HBase commands.
 * @param putTimestamp The timestamp to use for Put commands.
 * @param delTimestamp The timestamp to use for Delete commands.
 * @return Commands sorted by key converted from JanusGraph internal representation.
 * @throws org.janusgraph.diskstorage.PermanentBackendException
 */
@VisibleForTesting
Map<StaticBuffer, Pair<List<Put>, Delete>> convertToCommands(Map<String, Map<StaticBuffer, KCVMutation>> mutations, final long putTimestamp, final long delTimestamp) throws PermanentBackendException {
    // A map of rowkey to commands (list of Puts, Delete)
    final Map<StaticBuffer, Pair<List<Put>, Delete>> commandsPerKey = new HashMap<>();
    for (Map.Entry<String, Map<StaticBuffer, KCVMutation>> entry : mutations.entrySet()) {
        String cfString = getCfNameForStoreName(entry.getKey());
        byte[] cfName = Bytes.toBytes(cfString);
        for (Map.Entry<StaticBuffer, KCVMutation> m : entry.getValue().entrySet()) {
            final byte[] key = m.getKey().as(StaticBuffer.ARRAY_FACTORY);
            KCVMutation mutation = m.getValue();
            Pair<List<Put>, Delete> commands = commandsPerKey.get(m.getKey());
            // create the holder for a particular rowkey
            if (commands == null) {
                commands = new Pair<>();
                // List of all the Puts for this rowkey, including the ones without TTL and with TTL.
                final List<Put> putList = new ArrayList<>();
                commands.setFirst(putList);
                commandsPerKey.put(m.getKey(), commands);
            }
            if (mutation.hasDeletions()) {
                if (commands.getSecond() == null) {
                    Delete d = new Delete(key);
                    compat.setTimestamp(d, delTimestamp);
                    commands.setSecond(d);
                }
                for (StaticBuffer b : mutation.getDeletions()) {
                    // commands.getSecond() is a Delete for this rowkey.
                    commands.getSecond().deleteColumns(cfName, b.as(StaticBuffer.ARRAY_FACTORY), delTimestamp);
                }
            }
            if (mutation.hasAdditions()) {
                // All the entries (column cells) with the rowkey use this one Put, except the ones with TTL.
                final Put putColumnsWithoutTtl = new Put(key, putTimestamp);
                // that have TTL set.
                for (Entry e : mutation.getAdditions()) {
                    // Deal with TTL within the entry (column cell) first
                    // HBase cell level TTL is actually set at the Mutation/Put level.
                    // Therefore we need to construct a new Put for each entry (column cell) with TTL.
                    // We can not combine them because column cells within the same rowkey may:
                    // 1. have no TTL
                    // 2. have TTL
                    // 3. have different TTL
                    final Integer ttl = (Integer) e.getMetaData().get(EntryMetaData.TTL);
                    if (null != ttl && ttl > 0) {
                        // Create a new Put
                        Put putColumnWithTtl = new Put(key, putTimestamp);
                        addColumnToPut(putColumnWithTtl, cfName, putTimestamp, e);
                        // Convert ttl from second (JanusGraph TTL) to milliseconds (HBase TTL)
                        // @see JanusGraphManagement#setTTL(JanusGraphSchemaType, Duration)
                        // Cast Put to Mutation for backward compatibility with HBase 0.98.x
                        // HBase supports cell-level TTL for versions 0.98.6 and above.
                        ((Mutation) putColumnWithTtl).setTTL(ttl * 1000);
                        // commands.getFirst() is the list of Puts for this rowkey. Add this
                        // Put column with TTL to the list.
                        commands.getFirst().add(putColumnWithTtl);
                    } else {
                        addColumnToPut(putColumnsWithoutTtl, cfName, putTimestamp, e);
                    }
                }
                // If there were any mutations without TTL set, add them to commands.getFirst()
                if (!putColumnsWithoutTtl.isEmpty()) {
                    commands.getFirst().add(putColumnsWithoutTtl);
                }
            }
        }
    }
    return commandsPerKey;
}

Example 3 with KCVMutation

use of org.janusgraph.diskstorage.keycolumnvalue.KCVMutation in project janusgraph by JanusGraph.

the class AstyanaxStoreManager method mutateMany.

@Override
public void mutateMany(Map<String, Map<StaticBuffer, KCVMutation>> batch, StoreTransaction txh) throws BackendException {
    MutationBatch m = keyspaceContext.getClient().prepareMutationBatch().withAtomicBatch(atomicBatch).setConsistencyLevel(getTx(txh).getWriteConsistencyLevel().getAstyanax()).withRetryPolicy(retryPolicy.duplicate());
    final MaskedTimestamp commitTime = new MaskedTimestamp(txh);
    for (Map.Entry<String, Map<StaticBuffer, KCVMutation>> batchentry : batch.entrySet()) {
        String storeName = batchentry.getKey();
        Preconditions.checkArgument(openStores.containsKey(storeName), "Store cannot be found: " + storeName);
        ColumnFamily<ByteBuffer, ByteBuffer> columnFamily = openStores.get(storeName).getColumnFamily();
        Map<StaticBuffer, KCVMutation> mutations = batchentry.getValue();
        for (Map.Entry<StaticBuffer, KCVMutation> ent : mutations.entrySet()) {
            // The CLMs for additions and deletions are separated because
            // Astyanax's operation timestamp cannot be set on a per-delete
            // or per-addition basis.
            KCVMutation janusgraphMutation = ent.getValue();
            ByteBuffer key = ent.getKey().asByteBuffer();
            if (janusgraphMutation.hasDeletions()) {
                ColumnListMutation<ByteBuffer> deletions = m.withRow(columnFamily, key);
                deletions.setTimestamp(commitTime.getDeletionTime(times));
                for (StaticBuffer b : janusgraphMutation.getDeletions()) deletions.deleteColumn(b.as(StaticBuffer.BB_FACTORY));
            }
            if (janusgraphMutation.hasAdditions()) {
                ColumnListMutation<ByteBuffer> updates = m.withRow(columnFamily, key);
                updates.setTimestamp(commitTime.getAdditionTime(times));
                for (Entry e : janusgraphMutation.getAdditions()) {
                    Integer ttl = (Integer) e.getMetaData().get(EntryMetaData.TTL);
                    if (null != ttl && ttl > 0) {
                        updates.putColumn(e.getColumnAs(StaticBuffer.BB_FACTORY), e.getValueAs(StaticBuffer.BB_FACTORY), ttl);
                    } else {
                        updates.putColumn(e.getColumnAs(StaticBuffer.BB_FACTORY), e.getValueAs(StaticBuffer.BB_FACTORY));
                    }
                }
            }
        }
    }
    try {
        m.execute();
    } catch (ConnectionException e) {
        throw new TemporaryBackendException(e);
    }
    sleepAfterWrite(txh, commitTime);
}

Example 4 with KCVMutation

use of org.janusgraph.diskstorage.keycolumnvalue.KCVMutation in project janusgraph by JanusGraph.

the class CassandraThriftStoreManager method mutateMany.

@Override
public void mutateMany(Map<String, Map<StaticBuffer, KCVMutation>> mutations, StoreTransaction txh) throws BackendException {
    Preconditions.checkNotNull(mutations);
    final MaskedTimestamp commitTime = new MaskedTimestamp(txh);
    ConsistencyLevel consistency = getTx(txh).getWriteConsistencyLevel().getThrift();
    // Generate Thrift-compatible batch_mutate() datastructure
    // key -> cf -> cassmutation
    int size = 0;
    for (final Map<StaticBuffer, KCVMutation> mutation : mutations.values()) {
        size += mutation.size();
    }
    final Map<ByteBuffer, Map<String, List<org.apache.cassandra.thrift.Mutation>>> batch = new HashMap<>(size);
    for (final Map.Entry<String, Map<StaticBuffer, KCVMutation>> keyMutation : mutations.entrySet()) {
        final String columnFamily = keyMutation.getKey();
        for (final Map.Entry<StaticBuffer, KCVMutation> mutEntry : keyMutation.getValue().entrySet()) {
            ByteBuffer keyBB = mutEntry.getKey().asByteBuffer();
            // Get or create the single Cassandra Mutation object responsible for this key
            // Most mutations only modify the edgeStore and indexStore
            final Map<String, List<org.apache.cassandra.thrift.Mutation>> cfmutation = batch.computeIfAbsent(keyBB, k -> new HashMap<>(3));
            final KCVMutation mutation = mutEntry.getValue();
            final List<org.apache.cassandra.thrift.Mutation> thriftMutation = new ArrayList<>(mutations.size());
            if (mutation.hasDeletions()) {
                for (final StaticBuffer buf : mutation.getDeletions()) {
                    final Deletion d = new Deletion();
                    final SlicePredicate sp = new SlicePredicate();
                    sp.addToColumn_names(buf.as(StaticBuffer.BB_FACTORY));
                    d.setPredicate(sp);
                    d.setTimestamp(commitTime.getDeletionTime(times));
                    org.apache.cassandra.thrift.Mutation m = new org.apache.cassandra.thrift.Mutation();
                    m.setDeletion(d);
                    thriftMutation.add(m);
                }
            }
            if (mutation.hasAdditions()) {
                for (final Entry ent : mutation.getAdditions()) {
                    final ColumnOrSuperColumn columnOrSuperColumn = new ColumnOrSuperColumn();
                    final Column column = new Column(ent.getColumnAs(StaticBuffer.BB_FACTORY));
                    column.setValue(ent.getValueAs(StaticBuffer.BB_FACTORY));
                    column.setTimestamp(commitTime.getAdditionTime(times));
                    final Integer ttl = (Integer) ent.getMetaData().get(EntryMetaData.TTL);
                    if (null != ttl && ttl > 0) {
                        column.setTtl(ttl);
                    }
                    columnOrSuperColumn.setColumn(column);
                    org.apache.cassandra.thrift.Mutation m = new org.apache.cassandra.thrift.Mutation();
                    m.setColumn_or_supercolumn(columnOrSuperColumn);
                    thriftMutation.add(m);
                }
            }
            cfmutation.put(columnFamily, thriftMutation);
        }
    }
    CTConnection conn = null;
    try {
        conn = pool.borrowObject(keySpaceName);
        Cassandra.Client client = conn.getClient();
        if (atomicBatch) {
            client.atomic_batch_mutate(batch, consistency);
        } else {
            client.batch_mutate(batch, consistency);
        }
    } catch (Exception ex) {
        throw CassandraThriftKeyColumnValueStore.convertException(ex);
    } finally {
        pool.returnObjectUnsafe(keySpaceName, conn);
    }
    sleepAfterWrite(txh, commitTime);
}

Example 5 with KCVMutation

use of org.janusgraph.diskstorage.keycolumnvalue.KCVMutation in project janusgraph by JanusGraph.

the class CQLStoreManager method mutateManyLogged.

// Use a single logged batch
private void mutateManyLogged(final Map<String, Map<StaticBuffer, KCVMutation>> mutations, final StoreTransaction txh) throws BackendException {
    final MaskedTimestamp commitTime = new MaskedTimestamp(txh);
    final BatchStatement batchStatement = new BatchStatement(Type.LOGGED);
    batchStatement.setConsistencyLevel(getTransaction(txh).getWriteConsistencyLevel());
    batchStatement.addAll(Iterator.ofAll(mutations.entrySet()).flatMap(tableNameAndMutations -> {
        final String tableName = tableNameAndMutations.getKey();
        final Map<StaticBuffer, KCVMutation> tableMutations = tableNameAndMutations.getValue();
        final CQLKeyColumnValueStore columnValueStore = Option.of(this.openStores.get(tableName)).getOrElseThrow(() -> new IllegalStateException("Store cannot be found: " + tableName));
        return Iterator.ofAll(tableMutations.entrySet()).flatMap(keyAndMutations -> {
            final StaticBuffer key = keyAndMutations.getKey();
            final KCVMutation keyMutations = keyAndMutations.getValue();
            final Iterator<Statement> deletions = Iterator.of(commitTime.getDeletionTime(this.times)).flatMap(deleteTime -> Iterator.ofAll(keyMutations.getDeletions()).map(deletion -> columnValueStore.deleteColumn(key, deletion, deleteTime)));
            final Iterator<Statement> additions = Iterator.of(commitTime.getAdditionTime(this.times)).flatMap(addTime -> Iterator.ofAll(keyMutations.getAdditions()).map(addition -> columnValueStore.insertColumn(key, addition, addTime)));
            return Iterator.concat(deletions, additions);
        });
    }));
    final Future<ResultSet> result = Future.fromJavaFuture(this.executorService, this.session.executeAsync(batchStatement));
    result.await();
    if (result.isFailure()) {
        throw EXCEPTION_MAPPER.apply(result.getCause().get());
    }
    sleepAfterWrite(txh, commitTime);
}