Examples with Entry org.janusgraph.diskstorage.Entry used on opensource projects

Example 1 with Entry

use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.

the class HBaseStoreManager method convertToCommands.

/**
 * Convert JanusGraph internal Mutation representation into HBase native commands.
 *
 * @param mutations    Mutations to convert into HBase commands.
 * @param putTimestamp The timestamp to use for Put commands.
 * @param delTimestamp The timestamp to use for Delete commands.
 * @return Commands sorted by key converted from JanusGraph internal representation.
 * @throws org.janusgraph.diskstorage.PermanentBackendException
 */
@VisibleForTesting
Map<StaticBuffer, Pair<List<Put>, Delete>> convertToCommands(Map<String, Map<StaticBuffer, KCVMutation>> mutations, final long putTimestamp, final long delTimestamp) throws PermanentBackendException {
    // A map of rowkey to commands (list of Puts, Delete)
    final Map<StaticBuffer, Pair<List<Put>, Delete>> commandsPerKey = new HashMap<>();
    for (Map.Entry<String, Map<StaticBuffer, KCVMutation>> entry : mutations.entrySet()) {
        String cfString = getCfNameForStoreName(entry.getKey());
        byte[] cfName = Bytes.toBytes(cfString);
        for (Map.Entry<StaticBuffer, KCVMutation> m : entry.getValue().entrySet()) {
            final byte[] key = m.getKey().as(StaticBuffer.ARRAY_FACTORY);
            KCVMutation mutation = m.getValue();
            Pair<List<Put>, Delete> commands = commandsPerKey.get(m.getKey());
            // create the holder for a particular rowkey
            if (commands == null) {
                commands = new Pair<>();
                // List of all the Puts for this rowkey, including the ones without TTL and with TTL.
                final List<Put> putList = new ArrayList<>();
                commands.setFirst(putList);
                commandsPerKey.put(m.getKey(), commands);
            }
            if (mutation.hasDeletions()) {
                if (commands.getSecond() == null) {
                    Delete d = new Delete(key);
                    compat.setTimestamp(d, delTimestamp);
                    commands.setSecond(d);
                }
                for (StaticBuffer b : mutation.getDeletions()) {
                    // commands.getSecond() is a Delete for this rowkey.
                    commands.getSecond().deleteColumns(cfName, b.as(StaticBuffer.ARRAY_FACTORY), delTimestamp);
                }
            }
            if (mutation.hasAdditions()) {
                // All the entries (column cells) with the rowkey use this one Put, except the ones with TTL.
                final Put putColumnsWithoutTtl = new Put(key, putTimestamp);
                // that have TTL set.
                for (Entry e : mutation.getAdditions()) {
                    // Deal with TTL within the entry (column cell) first
                    // HBase cell level TTL is actually set at the Mutation/Put level.
                    // Therefore we need to construct a new Put for each entry (column cell) with TTL.
                    // We can not combine them because column cells within the same rowkey may:
                    // 1. have no TTL
                    // 2. have TTL
                    // 3. have different TTL
                    final Integer ttl = (Integer) e.getMetaData().get(EntryMetaData.TTL);
                    if (null != ttl && ttl > 0) {
                        // Create a new Put
                        Put putColumnWithTtl = new Put(key, putTimestamp);
                        addColumnToPut(putColumnWithTtl, cfName, putTimestamp, e);
                        // Convert ttl from second (JanusGraph TTL) to milliseconds (HBase TTL)
                        // @see JanusGraphManagement#setTTL(JanusGraphSchemaType, Duration)
                        // Cast Put to Mutation for backward compatibility with HBase 0.98.x
                        // HBase supports cell-level TTL for versions 0.98.6 and above.
                        ((Mutation) putColumnWithTtl).setTTL(ttl * 1000);
                        // commands.getFirst() is the list of Puts for this rowkey. Add this
                        // Put column with TTL to the list.
                        commands.getFirst().add(putColumnWithTtl);
                    } else {
                        addColumnToPut(putColumnsWithoutTtl, cfName, putTimestamp, e);
                    }
                }
                // If there were any mutations without TTL set, add them to commands.getFirst()
                if (!putColumnsWithoutTtl.isEmpty()) {
                    commands.getFirst().add(putColumnsWithoutTtl);
                }
            }
        }
    }
    return commandsPerKey;
}

Example 2 with Entry

use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.

the class AstyanaxStoreManager method mutateMany.

@Override
public void mutateMany(Map<String, Map<StaticBuffer, KCVMutation>> batch, StoreTransaction txh) throws BackendException {
    MutationBatch m = keyspaceContext.getClient().prepareMutationBatch().withAtomicBatch(atomicBatch).setConsistencyLevel(getTx(txh).getWriteConsistencyLevel().getAstyanax()).withRetryPolicy(retryPolicy.duplicate());
    final MaskedTimestamp commitTime = new MaskedTimestamp(txh);
    for (Map.Entry<String, Map<StaticBuffer, KCVMutation>> batchentry : batch.entrySet()) {
        String storeName = batchentry.getKey();
        Preconditions.checkArgument(openStores.containsKey(storeName), "Store cannot be found: " + storeName);
        ColumnFamily<ByteBuffer, ByteBuffer> columnFamily = openStores.get(storeName).getColumnFamily();
        Map<StaticBuffer, KCVMutation> mutations = batchentry.getValue();
        for (Map.Entry<StaticBuffer, KCVMutation> ent : mutations.entrySet()) {
            // The CLMs for additions and deletions are separated because
            // Astyanax's operation timestamp cannot be set on a per-delete
            // or per-addition basis.
            KCVMutation janusgraphMutation = ent.getValue();
            ByteBuffer key = ent.getKey().asByteBuffer();
            if (janusgraphMutation.hasDeletions()) {
                ColumnListMutation<ByteBuffer> deletions = m.withRow(columnFamily, key);
                deletions.setTimestamp(commitTime.getDeletionTime(times));
                for (StaticBuffer b : janusgraphMutation.getDeletions()) deletions.deleteColumn(b.as(StaticBuffer.BB_FACTORY));
            }
            if (janusgraphMutation.hasAdditions()) {
                ColumnListMutation<ByteBuffer> updates = m.withRow(columnFamily, key);
                updates.setTimestamp(commitTime.getAdditionTime(times));
                for (Entry e : janusgraphMutation.getAdditions()) {
                    Integer ttl = (Integer) e.getMetaData().get(EntryMetaData.TTL);
                    if (null != ttl && ttl > 0) {
                        updates.putColumn(e.getColumnAs(StaticBuffer.BB_FACTORY), e.getValueAs(StaticBuffer.BB_FACTORY), ttl);
                    } else {
                        updates.putColumn(e.getColumnAs(StaticBuffer.BB_FACTORY), e.getValueAs(StaticBuffer.BB_FACTORY));
                    }
                }
            }
        }
    }
    try {
        m.execute();
    } catch (ConnectionException e) {
        throw new TemporaryBackendException(e);
    }
    sleepAfterWrite(txh, commitTime);
}

Example 3 with Entry

use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.

the class StaticArrayEntryListBenchmark method setup.

@Setup
public void setup() {
    for (int i = 0; i < size; i++) {
        StaticArrayBuffer column = StaticArrayEntry.of(ByteBufferUtil.oneByteBuffer(20));
        StaticArrayBuffer value = StaticArrayEntry.of(ByteBufferUtil.oneByteBuffer(valueSize));
        Entry entry = StaticArrayEntry.of(column, value);
        entries.add(entry);
    }
}

Example 4 with Entry

use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.

the class IndexRepairJob method process.

@Override
public void process(JanusGraphVertex vertex, ScanMetrics metrics) {
    try {
        BackendTransaction mutator = writeTx.getTxHandle();
        if (index instanceof RelationTypeIndex) {
            RelationTypeIndexWrapper wrapper = (RelationTypeIndexWrapper) index;
            InternalRelationType wrappedType = wrapper.getWrappedType();
            EdgeSerializer edgeSerializer = writeTx.getEdgeSerializer();
            List<Entry> outAdditions = new ArrayList<>();
            Map<StaticBuffer, List<Entry>> inAdditionsMap = new HashMap<>();
            for (Object relation : vertex.query().types(indexRelationTypeName).direction(Direction.OUT).relations()) {
                InternalRelation janusgraphRelation = (InternalRelation) relation;
                for (int pos = 0; pos < janusgraphRelation.getArity(); pos++) {
                    if (!wrappedType.isUnidirected(Direction.BOTH) && !wrappedType.isUnidirected(EdgeDirection.fromPosition(pos)))
                        // Directionality is not covered
                        continue;
                    Entry entry = edgeSerializer.writeRelation(janusgraphRelation, wrappedType, pos, writeTx);
                    if (pos == 0) {
                        outAdditions.add(entry);
                    } else {
                        assert pos == 1;
                        InternalVertex otherVertex = janusgraphRelation.getVertex(1);
                        StaticBuffer otherVertexKey = writeTx.getIdInspector().getKey(otherVertex.longId());
                        inAdditionsMap.computeIfAbsent(otherVertexKey, k -> new ArrayList<>()).add(entry);
                    }
                }
            }
            // Mutating all OUT relationships for the current vertex
            StaticBuffer vertexKey = writeTx.getIdInspector().getKey(vertex.longId());
            mutator.mutateEdges(vertexKey, outAdditions, KCVSCache.NO_DELETIONS);
            // Mutating all IN relationships for the current vertex
            int totalInAdditions = 0;
            for (Map.Entry<StaticBuffer, List<Entry>> entry : inAdditionsMap.entrySet()) {
                StaticBuffer otherVertexKey = entry.getKey();
                List<Entry> inAdditions = entry.getValue();
                totalInAdditions += inAdditions.size();
                mutator.mutateEdges(otherVertexKey, inAdditions, KCVSCache.NO_DELETIONS);
            }
            metrics.incrementCustom(ADDED_RECORDS_COUNT, outAdditions.size() + totalInAdditions);
        } else if (index instanceof JanusGraphIndex) {
            IndexType indexType = managementSystem.getSchemaVertex(index).asIndexType();
            assert indexType != null;
            IndexSerializer indexSerializer = graph.getIndexSerializer();
            // Gather elements to index
            List<JanusGraphElement> elements;
            switch(indexType.getElement()) {
                case VERTEX:
                    elements = Collections.singletonList(vertex);
                    break;
                case PROPERTY:
                    elements = new ArrayList<>();
                    addIndexSchemaConstraint(vertex.query(), indexType).properties().forEach(elements::add);
                    break;
                case EDGE:
                    elements = new ArrayList<>();
                    addIndexSchemaConstraint(vertex.query().direction(Direction.OUT), indexType).edges().forEach(elements::add);
                    break;
                default:
                    throw new AssertionError("Unexpected category: " + indexType.getElement());
            }
            if (indexType.isCompositeIndex()) {
                for (JanusGraphElement element : elements) {
                    Set<IndexSerializer.IndexUpdate<StaticBuffer, Entry>> updates = indexSerializer.reindexElement(element, (CompositeIndexType) indexType);
                    for (IndexSerializer.IndexUpdate<StaticBuffer, Entry> update : updates) {
                        log.debug("Mutating index {}: {}", indexType, update.getEntry());
                        mutator.mutateIndex(update.getKey(), new ArrayList<Entry>(1) {

                            {
                                add(update.getEntry());
                            }
                        }, KCVSCache.NO_DELETIONS);
                        metrics.incrementCustom(ADDED_RECORDS_COUNT);
                    }
                }
            } else {
                assert indexType.isMixedIndex();
                for (JanusGraphElement element : elements) {
                    if (indexSerializer.reindexElement(element, (MixedIndexType) indexType, documentsPerStore)) {
                        metrics.incrementCustom(DOCUMENT_UPDATES_COUNT);
                    }
                }
            }
        } else
            throw new UnsupportedOperationException("Unsupported index found: " + index);
    } catch (final Exception e) {
        managementSystem.rollback();
        writeTx.rollback();
        metrics.incrementCustom(FAILED_TX);
        throw new JanusGraphException(e.getMessage(), e);
    }
}

Example 5 with Entry

use of org.janusgraph.diskstorage.Entry in project janusgraph by JanusGraph.

the class InMemoryColumnValueStoreTest method testMultipageSlicing.

@Test
public void testMultipageSlicing() throws TemporaryLockingException {
    int numEntries = 502;
    int windowStart = 494;
    int windowEnd = 501;
    StoreTransaction txh = mock(StoreTransaction.class);
    BaseTransactionConfig mockConfig = mock(BaseTransactionConfig.class);
    when(txh.getConfiguration()).thenReturn(mockConfig);
    when(mockConfig.getCustomOption(eq(STORAGE_TRANSACTIONAL))).thenReturn(true);
    InMemoryColumnValueStore cvs = new InMemoryColumnValueStore();
    // ColumnValueStore cvs = new DeflatedEntryColumnValueStore(false);
    List<Entry> additions = generateEntries(0, numEntries, "orig");
    cvs.mutate(additions, Collections.emptyList(), txh);
    EntryList result = cvs.getSlice(new KeySliceQuery(makeStaticBuffer("someRow"), makeStaticBuffer(VERY_START), // if we pass COL_END, it doesn't get included
    makeStaticBuffer(VERY_END)), // if we pass COL_END, it doesn't get included
    txh);
    assertEquals(additions.size(), result.size());
    // this getSlice spans two pages and doesn't retrieve either page in full
    result = cvs.getSlice(new KeySliceQuery(makeStaticBuffer("someRow"), additions.get(windowStart).getColumn(), additions.get(windowEnd).getColumn()), txh);
    assertEquals(windowEnd - windowStart, result.size());
}