Examples with SimpleInputIterator org.neo4j.unsafe.impl.batchimport.input.SimpleInputIterator used on opensource projects

Example 1 with SimpleInputIterator

use of org.neo4j.unsafe.impl.batchimport.input.SimpleInputIterator in project neo4j by neo4j.

the class ParallelBatchImporterTest method relationships.

private InputIterable<InputRelationship> relationships(final long randomSeed, final long count, final InputIdGenerator idGenerator, final IdGroupDistribution groups) {
    return new InputIterable<InputRelationship>() {

        private int calls;

        @Override
        public InputIterator<InputRelationship> iterator() {
            calls++;
            assertTrue("Unexpected use of input iterator " + multiPassIterators + ", " + calls, multiPassIterators || (!multiPassIterators && calls == 1));
            // since we use it to compare the imported data against after the import has been completed.
            return new SimpleInputIterator<InputRelationship>("test relationships") {

                private final Random random = new Random(randomSeed);

                private final Randoms randoms = new Randoms(random, Randoms.DEFAULT);

                private int cursor;

                private final MutableLong nodeIndex = new MutableLong(-1);

                @Override
                protected InputRelationship fetchNextOrNull() {
                    if (cursor < count) {
                        Object[] properties = randomProperties(randoms, "Name " + cursor);
                        try {
                            Object startNode = idGenerator.randomExisting(random, nodeIndex);
                            Group startNodeGroup = groups.groupOf(nodeIndex.longValue());
                            Object endNode = idGenerator.randomExisting(random, nodeIndex);
                            Group endNodeGroup = groups.groupOf(nodeIndex.longValue());
                            // miss some
                            startNode = idGenerator.miss(random, startNode, 0.001f);
                            endNode = idGenerator.miss(random, endNode, 0.001f);
                            String type = idGenerator.randomType(random);
                            if (random.nextFloat() < 0.00005) {
                                // Let there be a small chance of introducing a one-off relationship
                                // with a type that no, or at least very few, other relationships have.
                                type += "_odd";
                            }
                            return new InputRelationship(sourceDescription, itemNumber, itemNumber, properties, null, startNodeGroup, startNode, endNodeGroup, endNode, type, null);
                        } finally {
                            cursor++;
                        }
                    }
                    return null;
                }
            };
        }

        @Override
        public boolean supportsMultiplePasses() {
            return multiPassIterators;
        }
    };
}

Example 2 with SimpleInputIterator

use of org.neo4j.unsafe.impl.batchimport.input.SimpleInputIterator in project neo4j by neo4j.

the class ParallelBatchImporterTest method nodes.

private InputIterable<InputNode> nodes(final long randomSeed, final long count, final InputIdGenerator inputIdGenerator, final IdGroupDistribution groups) {
    return new InputIterable<InputNode>() {

        private int calls;

        @Override
        public InputIterator<InputNode> iterator() {
            calls++;
            assertTrue("Unexpected use of input iterator " + multiPassIterators + ", " + calls, multiPassIterators || (!multiPassIterators && calls == 1));
            return new SimpleInputIterator<InputNode>("test nodes") {

                private final Random random = new Random(randomSeed);

                private final Randoms randoms = new Randoms(random, Randoms.DEFAULT);

                private int cursor;

                @Override
                protected InputNode fetchNextOrNull() {
                    if (cursor < count) {
                        Object nodeId = inputIdGenerator.nextNodeId(random);
                        Object[] properties = randomProperties(randoms, nodeId);
                        String[] labels = randoms.selection(TOKENS, 0, TOKENS.length, true);
                        try {
                            Group group = groups.groupOf(cursor);
                            return new InputNode(sourceDescription, itemNumber, itemNumber, group, nodeId, properties, null, labels, null);
                        } finally {
                            cursor++;
                        }
                    }
                    return null;
                }
            };
        }

        @Override
        public boolean supportsMultiplePasses() {
            return multiPassIterators;
        }
    };
}