Examples with JanusGraphIndex org.janusgraph.core.schema.JanusGraphIndex used on opensource projects

Example 36 with JanusGraphIndex

use of org.janusgraph.core.schema.JanusGraphIndex in project janusgraph by JanusGraph.

the class AbstractIndexManagementIT method prepareGraphIndex.

private void prepareGraphIndex() throws InterruptedException {
    tx.commit();
    mgmt.commit();
    // Load the "Graph of the Gods" sample data
    GraphOfTheGodsFactory.loadWithoutMixedIndex(graph, true);
    // Disable the "name" composite index
    JanusGraphManagement m = graph.openManagement();
    JanusGraphIndex nameIndex = m.getGraphIndex("name");
    m.updateIndex(nameIndex, SchemaAction.DISABLE_INDEX);
    m.commit();
    graph.tx().commit();
    // Block until the SchemaStatus transitions to DISABLED
    assertTrue(ManagementSystem.awaitGraphIndexStatus(graph, "name").status(SchemaStatus.DISABLED).call().getSucceeded());
}

Example 37 with JanusGraphIndex

use of org.janusgraph.core.schema.JanusGraphIndex in project janusgraph by JanusGraph.

the class MapReduceIndexManagement method updateIndex.

/**
 * Updates the provided index according to the given {@link SchemaAction}.
 * Only {@link SchemaAction#REINDEX} and {@link SchemaAction#REMOVE_INDEX} are supported.
 *
 * @param index the index to process
 * @param updateAction either {@code REINDEX} or {@code REMOVE_INDEX}
 * @param hadoopConf
 * @return a future that returns immediately;
 *         this method blocks until the Hadoop MapReduce job completes
 */
// TODO make this future actually async and update javadoc @return accordingly
public ScanJobFuture updateIndex(Index index, SchemaAction updateAction, Configuration hadoopConf) throws BackendException {
    Preconditions.checkNotNull(index, "Index parameter must not be null", index);
    Preconditions.checkNotNull(updateAction, "%s parameter must not be null", SchemaAction.class.getSimpleName());
    Preconditions.checkArgument(SUPPORTED_ACTIONS.contains(updateAction), "Only these %s parameters are supported: %s (was given %s)", SchemaAction.class.getSimpleName(), SUPPORTED_ACTIONS_STRING, updateAction);
    Preconditions.checkArgument(RelationTypeIndex.class.isAssignableFrom(index.getClass()) || JanusGraphIndex.class.isAssignableFrom(index.getClass()), "Index %s has class %s: must be a %s or %s (or subtype)", index.getClass(), RelationTypeIndex.class.getSimpleName(), JanusGraphIndex.class.getSimpleName());
    ModifiableHadoopConfiguration janusGraphMapReduceConfiguration = ModifiableHadoopConfiguration.of(JanusGraphHadoopConfiguration.MAPRED_NS, hadoopConf);
    // The job we'll execute to either REINDEX or REMOVE_INDEX
    final Class<? extends IndexUpdateJob> indexJobClass;
    final Class<? extends Mapper> mapperClass;
    // The class of the IndexUpdateJob and the Mapper that will be used to run it (VertexScanJob vs ScanJob)
    if (updateAction.equals(SchemaAction.REINDEX)) {
        indexJobClass = IndexRepairJob.class;
        mapperClass = HadoopVertexScanMapper.class;
    } else {
        assert updateAction.equals(SchemaAction.REMOVE_INDEX);
        indexJobClass = IndexRemoveJob.class;
        mapperClass = HadoopScanMapper.class;
    }
    // The column family that serves as input to the IndexUpdateJob
    final String readCF;
    if (RelationTypeIndex.class.isAssignableFrom(index.getClass())) {
        readCF = Backend.EDGESTORE_NAME;
    } else {
        JanusGraphIndex graphIndex = (JanusGraphIndex) index;
        if (graphIndex.isMixedIndex() && !updateAction.equals(SchemaAction.REINDEX))
            throw new UnsupportedOperationException("External mixed indexes must be removed in the indexing system directly.");
        Preconditions.checkState(JanusGraphIndex.class.isAssignableFrom(index.getClass()));
        if (updateAction.equals(SchemaAction.REMOVE_INDEX))
            readCF = Backend.INDEXSTORE_NAME;
        else
            readCF = Backend.EDGESTORE_NAME;
    }
    janusGraphMapReduceConfiguration.set(JanusGraphHadoopConfiguration.COLUMN_FAMILY_NAME, readCF);
    // The MapReduce InputFormat class based on the open graph's store manager
    HadoopStoreManager storeManager = (HadoopStoreManager) graph.getBackend().getStoreManager().getHadoopManager();
    if (storeManager == null) {
        throw new IllegalArgumentException("Store manager class " + graph.getBackend().getStoreManagerClass() + "is not supported");
    }
    final Class<? extends InputFormat> inputFormat = storeManager.getInputFormat(hadoopConf);
    // The index name and relation type name (if the latter is applicable)
    final String indexName = index.name();
    final RelationType relationType = RelationTypeIndex.class.isAssignableFrom(index.getClass()) ? ((RelationTypeIndex) index).getType() : null;
    final String relationTypeName = relationType == null ? StringUtils.EMPTY : relationType.name();
    Preconditions.checkNotNull(indexName);
    // Set the class of the IndexUpdateJob
    janusGraphMapReduceConfiguration.set(JanusGraphHadoopConfiguration.SCAN_JOB_CLASS, indexJobClass.getName());
    // Set the configuration of the IndexUpdateJob
    copyIndexJobKeys(hadoopConf, indexName, relationTypeName);
    janusGraphMapReduceConfiguration.set(JanusGraphHadoopConfiguration.SCAN_JOB_CONFIG_ROOT, GraphDatabaseConfiguration.class.getName() + "#JOB_NS");
    // Copy the StandardJanusGraph configuration under JanusGraphHadoopConfiguration.GRAPH_CONFIG_KEYS
    org.apache.commons.configuration2.Configuration localConfiguration = graph.getConfiguration().getConfigurationAtOpen();
    localConfiguration.clearProperty(Graph.GRAPH);
    copyInputKeys(hadoopConf, localConfiguration);
    String jobName = HadoopScanMapper.class.getSimpleName() + "[" + indexJobClass.getSimpleName() + "]";
    try {
        return new CompletedJobFuture(HadoopScanRunner.runJob(hadoopConf, inputFormat, jobName, mapperClass));
    } catch (Exception e) {
        return new FailedJobFuture(e);
    }
}

Example 38 with JanusGraphIndex

use of org.janusgraph.core.schema.JanusGraphIndex in project janusgraph by JanusGraph.

the class ConfigurationManagementGraphTest method shouldReindexIfPropertyKeyExists.

@Test
public void shouldReindexIfPropertyKeyExists() {
    final Map<String, Object> map = new HashMap<>();
    map.put(STORAGE_BACKEND.toStringWithoutRoot(), "inmemory");
    final MapConfiguration config = ConfigurationUtil.loadMapConfiguration(map);
    final StandardJanusGraph graph = new StandardJanusGraph(new GraphDatabaseConfigurationBuilder().build(new CommonsConfiguration(config)));
    final String propertyKeyName = "Created_Using_Template";
    final Class dataType = Boolean.class;
    JanusGraphManagement management = graph.openManagement();
    management.makePropertyKey(propertyKeyName).dataType(dataType).make();
    management.commit();
    // Instantiate the ConfigurationManagementGraph Singleton
    // This is purposefully done after a property key is created to ensure that a REDINDEX is initiated
    new ConfigurationManagementGraph(graph);
    management = graph.openManagement();
    final JanusGraphIndex index = management.getGraphIndex("Created_Using_Template_Index");
    final PropertyKey propertyKey = management.getPropertyKey("Created_Using_Template");
    assertNotNull(index);
    assertNotNull(propertyKey);
    assertEquals(ENABLED, index.getIndexStatus(propertyKey));
    management.commit();
}

Example 39 with JanusGraphIndex

use of org.janusgraph.core.schema.JanusGraphIndex in project janusgraph by JanusGraph.

the class QueryTest method testIndexQueryCache.

@Test
public void testIndexQueryCache() throws Exception {
    JanusGraphManagement mgmt = graph.openManagement();
    final PropertyKey prop = mgmt.makePropertyKey("prop").dataType(String.class).cardinality(Cardinality.SINGLE).make();
    final JanusGraphIndex index = mgmt.buildIndex("index", Vertex.class).addKey(prop).buildCompositeIndex();
    mgmt.commit();
    // cache is used when there is no result for given query
    assertBackendHit(graph.traversal().V().has("prop", "value").profile().next());
    assertNoBackendHit(graph.traversal().V().has("prop", "value").profile().next());
    assertEquals(0, graph.traversal().V().has("prop", "value").toList().size());
    graph.tx().rollback();
    for (int i = 0; i < 100; i++) {
        tx.addVertex("prop", "value");
    }
    tx.commit();
    // cache is used when there are results for given query
    assertBackendHit(graph.traversal().V().has("prop", "value").profile().next());
    assertNoBackendHit(graph.traversal().V().has("prop", "value").profile().next());
    assertEquals(100, graph.traversal().V().has("prop", "value").toList().size());
    graph.tx().rollback();
    // cache is used with limit
    assertBackendHit(graph.traversal().V().has("prop", "value").limit(10).profile().next());
    assertNoBackendHit(graph.traversal().V().has("prop", "value").limit(10).profile().next());
    assertEquals(10, graph.traversal().V().has("prop", "value").limit(10).toList().size());
    graph.tx().rollback();
    // result is cached and cache is used with limit larger than number of possible results
    assertBackendHit(graph.traversal().V().has("prop", "value").limit(1000).profile().next());
    assertNoBackendHit(graph.traversal().V().has("prop", "value").limit(1000).profile().next());
    assertEquals(100, graph.traversal().V().has("prop", "value").limit(1000).toList().size());
    graph.tx().rollback();
    // cache is not used when second query has higher limit
    assertBackendHit(graph.traversal().V().has("prop", "value").limit(10).profile().next());
    assertBackendHit(graph.traversal().V().has("prop", "value").limit(11).profile().next());
    assertEquals(11, graph.traversal().V().has("prop", "value").limit(11).toList().size());
    graph.tx().rollback();
    // cache is used when first query exhausts all results
    assertBackendHit(graph.traversal().V().has("prop", "value").limit(200).profile().next());
    assertNoBackendHit(graph.traversal().V().has("prop", "value").limit(1000).profile().next());
    assertEquals(100, graph.traversal().V().has("prop", "value").limit(1000).toList().size());
    graph.tx().rollback();
    assertBackendHit(graph.traversal().V().has("prop", "value").profile().next());
    assertNoBackendHit(graph.traversal().V().has("prop", "value").limit(1000).profile().next());
    assertEquals(100, graph.traversal().V().has("prop", "value").limit(1000).toList().size());
    graph.tx().rollback();
    // cache is used when second query has lower limit
    assertBackendHit(graph.traversal().V().has("prop", "value").limit(10).profile().next());
    assertNoBackendHit(graph.traversal().V().has("prop", "value").limit(9).profile().next());
    assertEquals(9, graph.traversal().V().has("prop", "value").limit(9).toList().size());
    graph.tx().rollback();
    // incomplete results are not put in cache if iterator is not exhausted
    GraphTraversal<Vertex, Vertex> iter = graph.traversal().V().has("prop", "value");
    for (int i = 0; i < 10; i++) {
        iter.next();
    }
    iter.close();
    assertBackendHit(graph.traversal().V().has("prop", "value").limit(1).profile().next());
    graph.tx().rollback();
    try (GraphTraversal<Vertex, Vertex> it = graph.traversal().V().has("prop", "value")) {
        for (int i = 0; i < 10; i++) {
            it.next();
        }
    }
    assertBackendHit(graph.traversal().V().has("prop", "value").limit(1).profile().next());
    graph.tx().rollback();
    // complete results are put in cache if iterator is exhausted
    iter = graph.traversal().V().has("prop", "value");
    while (iter.hasNext()) {
        iter.next();
    }
    assertNoBackendHit(graph.traversal().V().has("prop", "value").limit(1).profile().next());
    graph.tx().rollback();
    iter = graph.traversal().V().has("prop", "value");
    for (int i = 0; i < 100; i++) {
        iter.next();
    }
    assertNoBackendHit(graph.traversal().V().has("prop", "value").limit(1).profile().next());
    graph.tx().rollback();
}

Example 40 with JanusGraphIndex

use of org.janusgraph.core.schema.JanusGraphIndex in project janusgraph by JanusGraph.

the class IndexRepairJob method validateIndexStatus.

/**
 * Check that our target index is in either the ENABLED or REGISTERED state.
 */
@Override
protected void validateIndexStatus() {
    JanusGraphSchemaVertex schemaVertex = managementSystem.getSchemaVertex(index);
    Set<SchemaStatus> acceptableStatuses = SchemaAction.REINDEX.getApplicableStatus();
    boolean isValidIndex = true;
    String invalidIndexHint;
    if (index instanceof RelationTypeIndex || (index instanceof JanusGraphIndex && ((JanusGraphIndex) index).isCompositeIndex())) {
        SchemaStatus actualStatus = schemaVertex.getStatus();
        isValidIndex = acceptableStatuses.contains(actualStatus);
        invalidIndexHint = String.format("The index has status %s, but one of %s is required", actualStatus, acceptableStatuses);
    } else {
        Preconditions.checkArgument(index instanceof JanusGraphIndex, "Unexpected index: %s", index);
        JanusGraphIndex graphIndex = (JanusGraphIndex) index;
        Preconditions.checkArgument(graphIndex.isMixedIndex());
        Map<String, SchemaStatus> invalidKeyStatuses = new HashMap<>();
        int acceptableFields = 0;
        for (PropertyKey key : graphIndex.getFieldKeys()) {
            SchemaStatus status = graphIndex.getIndexStatus(key);
            if (status != SchemaStatus.DISABLED && !acceptableStatuses.contains(status)) {
                isValidIndex = false;
                invalidKeyStatuses.put(key.name(), status);
                log.warn("Index {} has key {} in an invalid status {}", index, key, status);
            }
            if (acceptableStatuses.contains(status))
                acceptableFields++;
        }
        invalidIndexHint = String.format("The following index keys have invalid status: %s (status must be one of %s)", StringUtils.join(invalidKeyStatuses, " has status ", ","), acceptableStatuses);
        if (isValidIndex && acceptableFields == 0) {
            isValidIndex = false;
            invalidIndexHint = "The index does not contain any valid keys";
        }
    }
    Preconditions.checkArgument(isValidIndex, "The index %s is in an invalid state and cannot be indexed. %s", indexName, invalidIndexHint);
    // TODO consider retrieving the current Job object and calling killJob() if !isValidIndex -- would be more efficient than throwing an exception on the first pair processed by each mapper
    if (log.isDebugEnabled()) {
        log.debug("Index " + index.name() + " is valid for re-indexing");
    }
}