Search in sources :

Example 1 with NODE_CREATE

use of org.neo4j.kernel.api.impl.fulltext.FulltextIndexProceduresUtil.NODE_CREATE in project neo4j by neo4j.

the class FulltextProceduresTest method concurrentPopulationAndUpdatesToAnEventuallyConsistentIndexMustEventuallyResultInCorrectIndexState.

@Test
void concurrentPopulationAndUpdatesToAnEventuallyConsistentIndexMustEventuallyResultInCorrectIndexState() throws Exception {
    String oldValue = "red";
    String newValue = "green";
    // First we create the nodes and relationships with the property value "red".
    LongHashSet nodeIds = new LongHashSet();
    LongHashSet relIds = new LongHashSet();
    generateNodesAndRelationshipsWithProperty(200, nodeIds, relIds, oldValue);
    // Then, in two concurrent transactions, we create our indexes AND change all the property values to "green".
    CountDownLatch readyLatch = new CountDownLatch(2);
    BinaryLatch startLatch = new BinaryLatch();
    Runnable createIndexes = () -> {
        readyLatch.countDown();
        startLatch.await();
        try (Transaction tx = db.beginTx()) {
            tx.execute(format(NODE_CREATE, DEFAULT_NODE_IDX_NAME, asStrList(LABEL.name()), asStrList(PROP) + EVENTUALLY_CONSISTENT));
            tx.execute(format(RELATIONSHIP_CREATE, DEFAULT_REL_IDX_NAME, asStrList(REL.name()), asStrList(PROP) + EVENTUALLY_CONSISTENT));
            tx.commit();
        }
    };
    Runnable makeAllEntitiesGreen = () -> {
        try (Transaction tx = db.beginTx()) {
            // Prepare our transaction state first.
            nodeIds.forEach(nodeId -> tx.getNodeById(nodeId).setProperty(PROP, newValue));
            relIds.forEach(relId -> tx.getRelationshipById(relId).setProperty(PROP, newValue));
            // Okay, NOW we're ready to race!
            readyLatch.countDown();
            startLatch.await();
            tx.commit();
        }
    };
    ExecutorService executor = Executors.newFixedThreadPool(2);
    Future<?> future1 = executor.submit(createIndexes);
    Future<?> future2 = executor.submit(makeAllEntitiesGreen);
    readyLatch.await();
    startLatch.release();
    // Finally, when everything has settled down, we should see that all of the nodes and relationships are indexed with the value "green".
    try {
        future1.get();
        future2.get();
        awaitIndexesOnline();
        try (Transaction tx = db.beginTx()) {
            tx.execute(AWAIT_REFRESH).close();
        }
        assertQueryFindsIds(db, true, DEFAULT_NODE_IDX_NAME, newValue, nodeIds);
        assertQueryFindsIds(db, false, DEFAULT_REL_IDX_NAME, newValue, relIds);
    } finally {
        IOUtils.closeAllSilently(executor::shutdown);
    }
}
Also used : Arrays(java.util.Arrays) ResourceIterator(org.neo4j.graphdb.ResourceIterator) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) Config(org.neo4j.configuration.Config) Value(org.neo4j.values.storable.Value) FulltextIndexProviderFactory(org.neo4j.kernel.impl.index.schema.FulltextIndexProviderFactory) NullLogProvider(org.neo4j.logging.NullLogProvider) DatabaseLayout(org.neo4j.io.layout.DatabaseLayout) Future(java.util.concurrent.Future) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) LongHashSet(org.eclipse.collections.impl.set.mutable.primitive.LongHashSet) FulltextProcedures(org.neo4j.procedure.builtin.FulltextProcedures) Arrays.asList(java.util.Arrays.asList) Map(java.util.Map) IndexSettingImpl(org.neo4j.graphdb.schema.IndexSettingImpl) ThreadTestUtils(org.neo4j.test.ThreadTestUtils) Transaction(org.neo4j.graphdb.Transaction) BinaryLatch(org.neo4j.util.concurrent.BinaryLatch) MethodSource(org.junit.jupiter.params.provider.MethodSource) QueryExecutionException(org.neo4j.graphdb.QueryExecutionException) Result(org.neo4j.graphdb.Result) RepeatedPropertyInSchemaException(org.neo4j.kernel.api.exceptions.schema.RepeatedPropertyInSchemaException) Set(java.util.Set) DROP(org.neo4j.kernel.api.impl.fulltext.FulltextIndexProceduresUtil.DROP) Arguments(org.junit.jupiter.params.provider.Arguments) Collectors(java.util.stream.Collectors) StandardCharsets(java.nio.charset.StandardCharsets) Executors(java.util.concurrent.Executors) String.format(java.lang.String.format) Test(org.junit.jupiter.api.Test) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) Stream(java.util.stream.Stream) RepeatedLabelInSchemaException(org.neo4j.kernel.api.exceptions.schema.RepeatedLabelInSchemaException) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) RelationshipType(org.neo4j.graphdb.RelationshipType) LongHashSet.newSetWith(org.eclipse.collections.impl.set.mutable.primitive.LongHashSet.newSetWith) ConsistencyCheckService(org.neo4j.consistency.ConsistencyCheckService) SHOW_FULLTEXT_INDEXES(org.neo4j.kernel.api.impl.fulltext.FulltextIndexProceduresUtil.SHOW_FULLTEXT_INDEXES) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) RELATIONSHIP_CREATE(org.neo4j.kernel.api.impl.fulltext.FulltextIndexProceduresUtil.RELATIONSHIP_CREATE) CsvSource(org.junit.jupiter.params.provider.CsvSource) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) Label(org.neo4j.graphdb.Label) AWAIT_REFRESH(org.neo4j.kernel.api.impl.fulltext.FulltextIndexProceduresUtil.AWAIT_REFRESH) DB_AWAIT_INDEX(org.neo4j.kernel.api.impl.fulltext.FulltextIndexProceduresUtil.DB_AWAIT_INDEX) IOUtils(org.neo4j.io.IOUtils) ArrayUtils(org.apache.commons.lang3.ArrayUtils) EnumSource(org.junit.jupiter.params.provider.EnumSource) HashMap(java.util.HashMap) NODE_CREATE(org.neo4j.kernel.api.impl.fulltext.FulltextIndexProceduresUtil.NODE_CREATE) TreeSet(java.util.TreeSet) Node(org.neo4j.graphdb.Node) IndexType(org.neo4j.graphdb.schema.IndexType) HashSet(java.util.HashSet) Iterables(org.neo4j.internal.helpers.collection.Iterables) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) NON_ASCII_LETTERS(org.neo4j.kernel.api.impl.fulltext.analyzer.StandardFoldingAnalyzer.NON_ASCII_LETTERS) ExecutorService(java.util.concurrent.ExecutorService) LIST_AVAILABLE_ANALYZERS(org.neo4j.kernel.api.impl.fulltext.FulltextIndexProceduresUtil.LIST_AVAILABLE_ANALYZERS) Files(java.nio.file.Files) Iterables.stream(org.neo4j.internal.helpers.collection.Iterables.stream) ProgressMonitorFactory(org.neo4j.internal.helpers.progress.ProgressMonitorFactory) InputStreamReader(java.io.InputStreamReader) MutableLongSet(org.eclipse.collections.api.set.primitive.MutableLongSet) TimeUnit(java.util.concurrent.TimeUnit) Assertions.assertArrayEquals(org.junit.jupiter.api.Assertions.assertArrayEquals) RepeatedRelationshipTypeInSchemaException(org.neo4j.kernel.api.exceptions.schema.RepeatedRelationshipTypeInSchemaException) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Collectors.toList(java.util.stream.Collectors.toList) ASCIIFoldingFilter(org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter) Relationship(org.neo4j.graphdb.Relationship) FulltextIndexProceduresUtil.asStrList(org.neo4j.kernel.api.impl.fulltext.FulltextIndexProceduresUtil.asStrList) BufferedReader(java.io.BufferedReader) IndexDefinition(org.neo4j.graphdb.schema.IndexDefinition) IndexSetting(org.neo4j.graphdb.schema.IndexSetting) ConsistencyFlags(org.neo4j.consistency.checking.full.ConsistencyFlags) LongHashSet(org.eclipse.collections.impl.set.mutable.primitive.LongHashSet) Transaction(org.neo4j.graphdb.Transaction) ExecutorService(java.util.concurrent.ExecutorService) CountDownLatch(java.util.concurrent.CountDownLatch) BinaryLatch(org.neo4j.util.concurrent.BinaryLatch) Test(org.junit.jupiter.api.Test) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 2 with NODE_CREATE

use of org.neo4j.kernel.api.impl.fulltext.FulltextIndexProceduresUtil.NODE_CREATE in project neo4j by neo4j.

the class FulltextIndexConsistencyCheckIT method consistencyCheckerMustBeAbleToRunOnStoreWithFulltextIndexes.

@Test
void consistencyCheckerMustBeAbleToRunOnStoreWithFulltextIndexes() throws Exception {
    GraphDatabaseService db = createDatabase();
    Label[] labels = IntStream.range(1, 7).mapToObj(i -> Label.label("LABEL" + i)).toArray(Label[]::new);
    RelationshipType[] relTypes = IntStream.range(1, 5).mapToObj(i -> RelationshipType.withName("REL" + i)).toArray(RelationshipType[]::new);
    String[] propertyKeys = IntStream.range(1, 7).mapToObj(i -> "PROP" + i).toArray(String[]::new);
    RandomValues randomValues = random.randomValues();
    try (Transaction tx = db.beginTx()) {
        int nodeCount = 1000;
        List<Node> nodes = new ArrayList<>(nodeCount);
        for (int i = 0; i < nodeCount; i++) {
            Label[] nodeLabels = random.ints(random.nextInt(labels.length), 0, labels.length).distinct().mapToObj(x -> labels[x]).toArray(Label[]::new);
            Node node = tx.createNode(nodeLabels);
            Stream.of(propertyKeys).forEach(p -> node.setProperty(p, random.nextBoolean() ? p : randomValues.nextValue().asObject()));
            nodes.add(node);
            int localRelCount = Math.min(nodes.size(), 5);
            random.ints(localRelCount, 0, localRelCount).distinct().mapToObj(x -> node.createRelationshipTo(nodes.get(x), relTypes[random.nextInt(relTypes.length)])).forEach(r -> Stream.of(propertyKeys).forEach(p -> r.setProperty(p, random.nextBoolean() ? p : randomValues.nextValue().asObject())));
        }
        tx.commit();
    }
    try (Transaction tx = db.beginTx()) {
        for (int i = 1; i < labels.length; i++) {
            tx.execute(format(NODE_CREATE, "nodes" + i, asStrList(Arrays.stream(labels).limit(i).map(Label::name).toArray(String[]::new)), asStrList(Arrays.copyOf(propertyKeys, i)))).close();
        }
        for (int i = 1; i < relTypes.length; i++) {
            tx.execute(format(RELATIONSHIP_CREATE, "rels" + i, asStrList(Arrays.stream(relTypes).limit(i).map(RelationshipType::name).toArray(String[]::new)), asStrList(Arrays.copyOf(propertyKeys, i)))).close();
        }
        tx.commit();
    }
    try (Transaction tx = db.beginTx()) {
        tx.schema().awaitIndexesOnline(2, TimeUnit.MINUTES);
        tx.commit();
    }
    managementService.shutdown();
    assertIsConsistent(checkConsistency());
}
Also used : BeforeEach(org.junit.jupiter.api.BeforeEach) Arrays(java.util.Arrays) RandomExtension(org.neo4j.test.extension.RandomExtension) Date(java.util.Date) Config(org.neo4j.configuration.Config) Disabled(org.junit.jupiter.api.Disabled) FailingGenericNativeIndexProviderFactory(org.neo4j.kernel.impl.index.schema.FailingGenericNativeIndexProviderFactory) NullLogProvider(org.neo4j.logging.NullLogProvider) DatabaseLayout(org.neo4j.io.layout.DatabaseLayout) DEFAULT_DATABASE_NAME(org.neo4j.configuration.GraphDatabaseSettings.DEFAULT_DATABASE_NAME) RandomValues(org.neo4j.values.storable.RandomValues) ExtendWith(org.junit.jupiter.api.extension.ExtendWith) IndexUpdater(org.neo4j.kernel.api.index.IndexUpdater) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) RandomRule(org.neo4j.test.rule.RandomRule) AbstractBaseRecord(org.neo4j.kernel.impl.store.record.AbstractBaseRecord) NULL(org.neo4j.io.pagecache.context.CursorContext.NULL) SKIP_ONLINE_UPDATES(org.neo4j.kernel.impl.index.schema.FailingGenericNativeIndexProviderFactory.FailureType.SKIP_ONLINE_UPDATES) IndexUpdateMode(org.neo4j.kernel.impl.api.index.IndexUpdateMode) IndexDefinitionImpl(org.neo4j.kernel.impl.coreapi.schema.IndexDefinitionImpl) Transaction(org.neo4j.graphdb.Transaction) TokenIndexProviderFactory(org.neo4j.kernel.impl.index.schema.TokenIndexProviderFactory) RecordStorageEngine(org.neo4j.internal.recordstorage.RecordStorageEngine) IndexEntryUpdate(org.neo4j.storageengine.api.IndexEntryUpdate) Neo4jLayoutExtension(org.neo4j.test.extension.Neo4jLayoutExtension) String.format(java.lang.String.format) GraphDatabaseAPI(org.neo4j.kernel.internal.GraphDatabaseAPI) Test(org.junit.jupiter.api.Test) List(java.util.List) Stream(java.util.stream.Stream) RelationshipType(org.neo4j.graphdb.RelationshipType) IndexDescriptor(org.neo4j.internal.schema.IndexDescriptor) DatabaseManagementService(org.neo4j.dbms.api.DatabaseManagementService) ConsistencyCheckService(org.neo4j.consistency.ConsistencyCheckService) INDEX_PROVIDERS_FILTER(org.neo4j.test.TestDatabaseManagementServiceBuilder.INDEX_PROVIDERS_FILTER) IntStream(java.util.stream.IntStream) IndexingService(org.neo4j.kernel.impl.api.index.IndexingService) Assertions.fail(org.junit.jupiter.api.Assertions.fail) RELATIONSHIP_CREATE(org.neo4j.kernel.api.impl.fulltext.FulltextIndexProceduresUtil.RELATIONSHIP_CREATE) Label(org.neo4j.graphdb.Label) IndexProxy(org.neo4j.kernel.impl.api.index.IndexProxy) GraphDatabaseSettings(org.neo4j.configuration.GraphDatabaseSettings) NODE_CREATE(org.neo4j.kernel.api.impl.fulltext.FulltextIndexProceduresUtil.NODE_CREATE) Node(org.neo4j.graphdb.Node) ArrayList(java.util.ArrayList) IndexType(org.neo4j.graphdb.schema.IndexType) Values(org.neo4j.values.storable.Values) TestDatabaseManagementServiceBuilder(org.neo4j.test.TestDatabaseManagementServiceBuilder) GraphDatabaseService(org.neo4j.graphdb.GraphDatabaseService) NeoStores(org.neo4j.kernel.impl.store.NeoStores) Inject(org.neo4j.test.extension.Inject) DependencyResolver(org.neo4j.common.DependencyResolver) Files(java.nio.file.Files) PropertyRecord(org.neo4j.kernel.impl.store.record.PropertyRecord) DatabaseManagementServiceBuilder(org.neo4j.dbms.api.DatabaseManagementServiceBuilder) IOException(java.io.IOException) ProgressMonitorFactory(org.neo4j.internal.helpers.progress.ProgressMonitorFactory) TimeUnit(java.util.concurrent.TimeUnit) ConsistencyCheckIncompleteException(org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException) AfterEach(org.junit.jupiter.api.AfterEach) Relationship(org.neo4j.graphdb.Relationship) FulltextIndexProceduresUtil.asStrList(org.neo4j.kernel.api.impl.fulltext.FulltextIndexProceduresUtil.asStrList) IndexDefinition(org.neo4j.graphdb.schema.IndexDefinition) ConsistencyFlags(org.neo4j.consistency.checking.full.ConsistencyFlags) RecordLoad(org.neo4j.kernel.impl.store.record.RecordLoad) RelationshipRecord(org.neo4j.kernel.impl.store.record.RelationshipRecord) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) GraphDatabaseService(org.neo4j.graphdb.GraphDatabaseService) Node(org.neo4j.graphdb.Node) Label(org.neo4j.graphdb.Label) RelationshipType(org.neo4j.graphdb.RelationshipType) ArrayList(java.util.ArrayList) RandomValues(org.neo4j.values.storable.RandomValues) Transaction(org.neo4j.graphdb.Transaction) Test(org.junit.jupiter.api.Test)

Aggregations

String.format (java.lang.String.format)2 Files (java.nio.file.Files)2 Arrays (java.util.Arrays)2 List (java.util.List)2 TimeUnit (java.util.concurrent.TimeUnit)2 Stream (java.util.stream.Stream)2 Assertions.assertFalse (org.junit.jupiter.api.Assertions.assertFalse)2 Test (org.junit.jupiter.api.Test)2 Config (org.neo4j.configuration.Config)2 ConsistencyCheckService (org.neo4j.consistency.ConsistencyCheckService)2 ConsistencyFlags (org.neo4j.consistency.checking.full.ConsistencyFlags)2 Label (org.neo4j.graphdb.Label)2 Node (org.neo4j.graphdb.Node)2 Relationship (org.neo4j.graphdb.Relationship)2 RelationshipType (org.neo4j.graphdb.RelationshipType)2 Transaction (org.neo4j.graphdb.Transaction)2 IndexDefinition (org.neo4j.graphdb.schema.IndexDefinition)2 IndexType (org.neo4j.graphdb.schema.IndexType)2 ProgressMonitorFactory (org.neo4j.internal.helpers.progress.ProgressMonitorFactory)2 DatabaseLayout (org.neo4j.io.layout.DatabaseLayout)2