Examples with ScanJobFuture org.janusgraph.diskstorage.keycolumnvalue.scan.ScanJobFuture used on opensource projects

Example 1 with ScanJobFuture

use of org.janusgraph.diskstorage.keycolumnvalue.scan.ScanJobFuture in project janusgraph by JanusGraph.

the class JanusGraphTest method testMgmtRemoveGhostVertices.

/**
 * In this test, we deliberately create ghost vertices and use ManagementSystem to purge them
 * We use two concurrent transactions and let one transaction removes a vertex and another transaction
 * updates the same vertex. When the storage backend does not have locking support, both transactions
 * will succeed, and then the vertex becomes a ghost vertex.
 *
 * @throws ExecutionException
 * @throws InterruptedException
 */
@Test
public void testMgmtRemoveGhostVertices() throws ExecutionException, InterruptedException {
    if (features.hasLocking())
        return;
    final int numOfVertices = 100;
    final int numOfGhostVertices = 80;
    final int numOfRestVertices = numOfVertices - numOfGhostVertices;
    List<Vertex> vertices = new ArrayList<>(numOfVertices);
    for (int i = 0; i < numOfVertices; i++) {
        vertices.add(tx.traversal().addV("test").next());
    }
    tx.commit();
    JanusGraphTransaction tx1 = graph.newTransaction();
    for (int i = 0; i < numOfVertices; i++) {
        tx1.traversal().V(vertices.get(i)).property("prop", "val").next();
    }
    JanusGraphTransaction tx2 = graph.newTransaction();
    for (int i = 0; i < numOfGhostVertices; i++) {
        tx2.traversal().V(vertices.get(i)).next().remove();
    }
    tx2.commit();
    tx1.commit();
    JanusGraphManagement mgmt = graph.openManagement();
    ScanJobFuture future = mgmt.removeGhostVertices();
    assertEquals(numOfGhostVertices, future.get().getCustom(GhostVertexRemover.REMOVED_VERTEX_COUNT));
    assertEquals(numOfRestVertices, graph.traversal().V().count().next());
    assertEquals(numOfRestVertices, graph.traversal().V().hasLabel("test").count().next());
    assertEquals(numOfRestVertices, graph.traversal().V().has("prop", "val").count().next());
    assertEquals(numOfRestVertices, graph.traversal().V().hasLabel("test").has("prop", "val").count().next());
    // running it again, no vertex is removed
    assertEquals(0, mgmt.removeGhostVertices().get().getCustom(GhostVertexRemover.REMOVED_VERTEX_COUNT));
}

Example 2 with ScanJobFuture

use of org.janusgraph.diskstorage.keycolumnvalue.scan.ScanJobFuture in project janusgraph by JanusGraph.

the class ManagementSystem method removeGhostVertices.

@Override
public ScanJobFuture removeGhostVertices(int numOfThreads) {
    StandardScanner.Builder builder = graph.getBackend().buildEdgeScanJob();
    builder.setJob(new GhostVertexRemover(graph));
    builder.setNumProcessingThreads(numOfThreads);
    ScanJobFuture future;
    try {
        future = builder.execute();
    } catch (BackendException e) {
        throw new JanusGraphException(e);
    }
    return future;
}

Example 3 with ScanJobFuture

use of org.janusgraph.diskstorage.keycolumnvalue.scan.ScanJobFuture in project janusgraph by JanusGraph.

the class ManagementSystem method updateIndex.

@Override
public ScanJobFuture updateIndex(Index index, SchemaAction updateAction, int numOfThreads) {
    Preconditions.checkArgument(index != null, "Need to provide an index");
    Preconditions.checkArgument(updateAction != null, "Need to provide update action");
    JanusGraphSchemaVertex schemaVertex = getSchemaVertex(index);
    Set<JanusGraphSchemaVertex> dependentTypes;
    Set<PropertyKeyVertex> keySubset = Collections.emptySet();
    if (index instanceof RelationTypeIndex) {
        dependentTypes = Collections.singleton((JanusGraphSchemaVertex) ((InternalRelationType) schemaVertex).getBaseType());
        if (!updateAction.isApplicableStatus(schemaVertex.getStatus()))
            return null;
    } else if (index instanceof JanusGraphIndex) {
        IndexType indexType = schemaVertex.asIndexType();
        dependentTypes = new HashSet<>();
        if (indexType.isCompositeIndex()) {
            if (!updateAction.isApplicableStatus(schemaVertex.getStatus()))
                return null;
            for (PropertyKey key : ((JanusGraphIndex) index).getFieldKeys()) {
                dependentTypes.add((PropertyKeyVertex) key);
            }
        } else {
            keySubset = new HashSet<>();
            MixedIndexType mixedIndexType = (MixedIndexType) indexType;
            Set<SchemaStatus> applicableStatus = updateAction.getApplicableStatus();
            for (ParameterIndexField field : mixedIndexType.getFieldKeys()) {
                if (applicableStatus.contains(field.getStatus()))
                    keySubset.add((PropertyKeyVertex) field.getFieldKey());
            }
            if (keySubset.isEmpty())
                return null;
            dependentTypes.addAll(keySubset);
        }
    } else
        throw new UnsupportedOperationException("Updates not supported for index: " + index);
    IndexIdentifier indexId = new IndexIdentifier(index);
    StandardScanner.Builder builder;
    ScanJobFuture future;
    switch(updateAction) {
        case REGISTER_INDEX:
            setStatus(schemaVertex, SchemaStatus.INSTALLED, keySubset);
            updatedTypes.add(schemaVertex);
            updatedTypes.addAll(dependentTypes);
            setUpdateTrigger(new UpdateStatusTrigger(graph, schemaVertex, SchemaStatus.REGISTERED, keySubset));
            future = new EmptyScanJobFuture();
            break;
        case REINDEX:
            builder = graph.getBackend().buildEdgeScanJob();
            builder.setFinishJob(indexId.getIndexJobFinisher(graph, SchemaAction.ENABLE_INDEX));
            builder.setJobId(indexId);
            builder.setNumProcessingThreads(numOfThreads);
            builder.setJob(VertexJobConverter.convert(graph, new IndexRepairJob(indexId.indexName, indexId.relationTypeName)));
            try {
                future = builder.execute();
            } catch (BackendException e) {
                throw new JanusGraphException(e);
            }
            break;
        case ENABLE_INDEX:
            setStatus(schemaVertex, SchemaStatus.ENABLED, keySubset);
            updatedTypes.add(schemaVertex);
            if (!keySubset.isEmpty())
                updatedTypes.addAll(dependentTypes);
            future = new EmptyScanJobFuture();
            break;
        case DISABLE_INDEX:
            setStatus(schemaVertex, SchemaStatus.INSTALLED, keySubset);
            updatedTypes.add(schemaVertex);
            if (!keySubset.isEmpty())
                updatedTypes.addAll(dependentTypes);
            setUpdateTrigger(new UpdateStatusTrigger(graph, schemaVertex, SchemaStatus.DISABLED, keySubset));
            future = new EmptyScanJobFuture();
            break;
        case REMOVE_INDEX:
            if (index instanceof RelationTypeIndex) {
                builder = graph.getBackend().buildEdgeScanJob();
            } else {
                JanusGraphIndex graphIndex = (JanusGraphIndex) index;
                if (graphIndex.isMixedIndex())
                    throw new UnsupportedOperationException("External mixed indexes must be removed in the indexing system directly.");
                builder = graph.getBackend().buildGraphIndexScanJob();
            }
            builder.setFinishJob(indexId.getIndexJobFinisher());
            builder.setJobId(indexId);
            builder.setNumProcessingThreads(numOfThreads);
            builder.setJob(new IndexRemoveJob(graph, indexId.indexName, indexId.relationTypeName));
            try {
                future = builder.execute();
            } catch (BackendException e) {
                throw new JanusGraphException(e);
            }
            break;
        default:
            throw new UnsupportedOperationException("Update action not supported: " + updateAction);
    }
    return future;
}

Example 4 with ScanJobFuture

use of org.janusgraph.diskstorage.keycolumnvalue.scan.ScanJobFuture in project janusgraph by JanusGraph.

the class JanusGraphTest method testGotGIndexRemoval.

@Test
public void testGotGIndexRemoval() throws InterruptedException, ExecutionException {
    clopen(option(LOG_SEND_DELAY, MANAGEMENT_LOG), Duration.ZERO, option(KCVSLog.LOG_READ_LAG_TIME, MANAGEMENT_LOG), Duration.ofMillis(50), option(LOG_READ_INTERVAL, MANAGEMENT_LOG), Duration.ofMillis(250));
    final String name = "name";
    // Load Graph of the Gods
    GraphOfTheGodsFactory.loadWithoutMixedIndex(graph, // True makes the index on names unique.  Test fails when this is true.
    true);
    // Change to false and test will pass.
    newTx();
    finishSchema();
    JanusGraphIndex graphIndex = mgmt.getGraphIndex(name);
    // Sanity checks on the index that we assume GraphOfTheGodsFactory created
    assertNotNull(graphIndex);
    assertEquals(1, graphIndex.getFieldKeys().length);
    assertEquals(name, graphIndex.getFieldKeys()[0].name());
    assertEquals("internalindex", graphIndex.getBackingIndex());
    assertEquals(SchemaStatus.ENABLED, graphIndex.getIndexStatus(graphIndex.getFieldKeys()[0]));
    finishSchema();
    // Disable name index
    graphIndex = mgmt.getGraphIndex(name);
    mgmt.updateIndex(graphIndex, SchemaAction.DISABLE_INDEX);
    mgmt.commit();
    tx.commit();
    ManagementUtil.awaitGraphIndexUpdate(graph, name, 5, ChronoUnit.SECONDS);
    finishSchema();
    // Remove name index
    graphIndex = mgmt.getGraphIndex(name);
    mgmt.updateIndex(graphIndex, SchemaAction.REMOVE_INDEX);
    ScanJobFuture graphMetrics = mgmt.getIndexJobStatus(graphIndex);
    finishSchema();
    // Should have deleted at least one record
    assertNotEquals(0, graphMetrics.get().getCustom(IndexRemoveJob.DELETED_RECORDS_COUNT));
}