Search in sources :

Example 1 with Randoms

use of org.neo4j.test.Randoms in project neo4j by neo4j.

the class ParallelBatchImporterTest method relationships.

private InputIterable<InputRelationship> relationships(final long randomSeed, final long count, final InputIdGenerator idGenerator, final IdGroupDistribution groups) {
    return new InputIterable<InputRelationship>() {

        private int calls;

        @Override
        public InputIterator<InputRelationship> iterator() {
            calls++;
            assertTrue("Unexpected use of input iterator " + multiPassIterators + ", " + calls, multiPassIterators || (!multiPassIterators && calls == 1));
            // since we use it to compare the imported data against after the import has been completed.
            return new SimpleInputIterator<InputRelationship>("test relationships") {

                private final Random random = new Random(randomSeed);

                private final Randoms randoms = new Randoms(random, Randoms.DEFAULT);

                private int cursor;

                private final MutableLong nodeIndex = new MutableLong(-1);

                @Override
                protected InputRelationship fetchNextOrNull() {
                    if (cursor < count) {
                        Object[] properties = randomProperties(randoms, "Name " + cursor);
                        try {
                            Object startNode = idGenerator.randomExisting(random, nodeIndex);
                            Group startNodeGroup = groups.groupOf(nodeIndex.longValue());
                            Object endNode = idGenerator.randomExisting(random, nodeIndex);
                            Group endNodeGroup = groups.groupOf(nodeIndex.longValue());
                            // miss some
                            startNode = idGenerator.miss(random, startNode, 0.001f);
                            endNode = idGenerator.miss(random, endNode, 0.001f);
                            String type = idGenerator.randomType(random);
                            if (random.nextFloat() < 0.00005) {
                                // Let there be a small chance of introducing a one-off relationship
                                // with a type that no, or at least very few, other relationships have.
                                type += "_odd";
                            }
                            return new InputRelationship(sourceDescription, itemNumber, itemNumber, properties, null, startNodeGroup, startNode, endNodeGroup, endNode, type, null);
                        } finally {
                            cursor++;
                        }
                    }
                    return null;
                }
            };
        }

        @Override
        public boolean supportsMultiplePasses() {
            return multiPassIterators;
        }
    };
}
Also used : Group(org.neo4j.unsafe.impl.batchimport.input.Group) Randoms(org.neo4j.test.Randoms) MutableLong(org.apache.commons.lang3.mutable.MutableLong) SimpleInputIterator(org.neo4j.unsafe.impl.batchimport.input.SimpleInputIterator) Random(java.util.Random) InputRelationship(org.neo4j.unsafe.impl.batchimport.input.InputRelationship)

Example 2 with Randoms

use of org.neo4j.test.Randoms in project neo4j by neo4j.

the class ParallelBatchImporterTest method nodes.

private InputIterable<InputNode> nodes(final long randomSeed, final long count, final InputIdGenerator inputIdGenerator, final IdGroupDistribution groups) {
    return new InputIterable<InputNode>() {

        private int calls;

        @Override
        public InputIterator<InputNode> iterator() {
            calls++;
            assertTrue("Unexpected use of input iterator " + multiPassIterators + ", " + calls, multiPassIterators || (!multiPassIterators && calls == 1));
            return new SimpleInputIterator<InputNode>("test nodes") {

                private final Random random = new Random(randomSeed);

                private final Randoms randoms = new Randoms(random, Randoms.DEFAULT);

                private int cursor;

                @Override
                protected InputNode fetchNextOrNull() {
                    if (cursor < count) {
                        Object nodeId = inputIdGenerator.nextNodeId(random);
                        Object[] properties = randomProperties(randoms, nodeId);
                        String[] labels = randoms.selection(TOKENS, 0, TOKENS.length, true);
                        try {
                            Group group = groups.groupOf(cursor);
                            return new InputNode(sourceDescription, itemNumber, itemNumber, group, nodeId, properties, null, labels, null);
                        } finally {
                            cursor++;
                        }
                    }
                    return null;
                }
            };
        }

        @Override
        public boolean supportsMultiplePasses() {
            return multiPassIterators;
        }
    };
}
Also used : InputNode(org.neo4j.unsafe.impl.batchimport.input.InputNode) Group(org.neo4j.unsafe.impl.batchimport.input.Group) Randoms(org.neo4j.test.Randoms) SimpleInputIterator(org.neo4j.unsafe.impl.batchimport.input.SimpleInputIterator) Random(java.util.Random)

Example 3 with Randoms

use of org.neo4j.test.Randoms in project neo4j by neo4j.

the class RandomRule method reset.

// ============================
// Other utility methods
// ============================
public void reset() {
    random = new Random(seed);
    randoms = new Randoms(random, config);
}
Also used : Randoms(org.neo4j.test.Randoms) Random(java.util.Random)

Example 4 with Randoms

use of org.neo4j.test.Randoms in project neo4j by neo4j.

the class InputCacheTest method shouldCacheAndRetrieveRelationships.

@Test
public void shouldCacheAndRetrieveRelationships() throws Exception {
    // GIVEN
    try (InputCache cache = new InputCache(fileSystemRule.get(), dir.directory(), Standard.LATEST_RECORD_FORMATS, withMaxProcessors(50), (int) ByteUnit.kibiBytes(8), BATCH_SIZE)) {
        List<InputRelationship> relationships = new ArrayList<>();
        Randoms random = getRandoms();
        try (Receiver<InputRelationship[], IOException> cacher = cache.cacheRelationships(MAIN)) {
            InputRelationship[] batch = new InputRelationship[BATCH_SIZE];
            for (int b = 0; b < BATCHES; b++) {
                for (int i = 0; i < BATCH_SIZE; i++) {
                    InputRelationship relationship = randomRelationship(random);
                    batch[i] = relationship;
                    relationships.add(relationship);
                }
                cacher.receive(batch);
            }
        }
        // WHEN/THEN
        try (InputIterator<InputRelationship> reader = cache.relationships(MAIN, true).iterator()) {
            reader.processors(50 - reader.processors(0));
            Iterator<InputRelationship> expected = relationships.iterator();
            while (expected.hasNext()) {
                assertTrue(reader.hasNext());
                InputRelationship expectedRelationship = expected.next();
                InputRelationship relationship = reader.next();
                assertRelationshipsEquals(expectedRelationship, relationship);
            }
            assertFalse(reader.hasNext());
        }
    }
    assertNoFilesLeftBehind();
}
Also used : Randoms(org.neo4j.test.Randoms) ArrayList(java.util.ArrayList) IOException(java.io.IOException) Test(org.junit.Test)

Example 5 with Randoms

use of org.neo4j.test.Randoms in project neo4j by neo4j.

the class MultipleIndexPopulationStressIT method populateDbAndIndexes.

private void populateDbAndIndexes(int nodeCount, boolean multiThreaded) throws InterruptedException {
    final GraphDatabaseService db = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder(directory.graphDbDir()).setConfig(GraphDatabaseSettings.multi_threaded_schema_index_population_enabled, multiThreaded + "").newGraphDatabase();
    try {
        createIndexes(db);
        final AtomicBoolean end = new AtomicBoolean();
        ExecutorService executor = cleanup.add(Executors.newCachedThreadPool());
        for (int i = 0; i < 10; i++) {
            executor.submit(() -> {
                Randoms random = new Randoms();
                while (!end.get()) {
                    changeRandomNode(db, nodeCount, random);
                }
            });
        }
        while (!indexesAreOnline(db)) {
            Thread.sleep(100);
        }
        end.set(true);
        executor.shutdown();
        executor.awaitTermination(10, SECONDS);
    } finally {
        db.shutdown();
    }
}
Also used : GraphDatabaseService(org.neo4j.graphdb.GraphDatabaseService) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Randoms(org.neo4j.test.Randoms) TestGraphDatabaseFactory(org.neo4j.test.TestGraphDatabaseFactory) ExecutorService(java.util.concurrent.ExecutorService)

Aggregations

Randoms (org.neo4j.test.Randoms)6 Random (java.util.Random)3 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 Test (org.junit.Test)2 Group (org.neo4j.unsafe.impl.batchimport.input.Group)2 SimpleInputIterator (org.neo4j.unsafe.impl.batchimport.input.SimpleInputIterator)2 ExecutorService (java.util.concurrent.ExecutorService)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1 MutableLong (org.apache.commons.lang3.mutable.MutableLong)1 GraphDatabaseService (org.neo4j.graphdb.GraphDatabaseService)1 TestGraphDatabaseFactory (org.neo4j.test.TestGraphDatabaseFactory)1 InputNode (org.neo4j.unsafe.impl.batchimport.input.InputNode)1 InputRelationship (org.neo4j.unsafe.impl.batchimport.input.InputRelationship)1