Examples with Groups org.neo4j.internal.batchimport.input.Groups used on opensource projects

Example 6 with Groups

use of org.neo4j.internal.batchimport.input.Groups in project neo4j by neo4j.

the class ParallelBatchImporterTest method verifyData.

private void verifyData(int nodeCount, int relationshipCount, GraphDatabaseService db, Transaction tx, IdGroupDistribution groups, long nodeRandomSeed, long relationshipRandomSeed) throws IOException {
    // Read all nodes, relationships and properties ad verify against the input data.
    LongAdder propertyCount = new LongAdder();
    try (InputIterator nodes = nodes(nodeRandomSeed, nodeCount, config.batchSize(), inputIdGenerator, groups, propertyCount).iterator();
        InputIterator relationships = relationships(relationshipRandomSeed, relationshipCount, config.batchSize(), inputIdGenerator, groups, propertyCount, new LongAdder()).iterator();
        ResourceIterator<Node> dbNodes = tx.getAllNodes().iterator()) {
        // Nodes
        Map<String, Node> nodeByInputId = new HashMap<>(nodeCount);
        while (dbNodes.hasNext()) {
            Node node = dbNodes.next();
            String id = (String) node.getProperty("id");
            assertNull(nodeByInputId.put(id, node));
        }
        int verifiedNodes = 0;
        long allNodesScanLabelCount = 0;
        InputChunk chunk = nodes.newChunk();
        InputEntity input = new InputEntity();
        while (nodes.next(chunk)) {
            while (chunk.next(input)) {
                String iid = uniqueId(input.idGroup, input.objectId);
                Node node = nodeByInputId.get(iid);
                assertNodeEquals(input, node);
                verifiedNodes++;
                assertDegrees(node);
                allNodesScanLabelCount += Iterables.count(node.getLabels());
            }
        }
        assertEquals(nodeCount, verifiedNodes);
        // Labels
        long labelScanStoreEntryCount = stream(tx.getAllLabels()).flatMap(l -> tx.findNodes(l).stream()).count();
        assertEquals(allNodesScanLabelCount, labelScanStoreEntryCount, format("Expected label scan store and node store to have same number labels. But %n" + "#labelsInNodeStore=%d%n" + "#labelsInLabelScanStore=%d%n", allNodesScanLabelCount, labelScanStoreEntryCount));
        // Relationships
        chunk = relationships.newChunk();
        Map<String, Relationship> relationshipByName = new HashMap<>();
        for (Relationship relationship : tx.getAllRelationships()) {
            relationshipByName.put((String) relationship.getProperty("id"), relationship);
        }
        int verifiedRelationships = 0;
        while (relationships.next(chunk)) {
            while (chunk.next(input)) {
                if (!inputIdGenerator.isMiss(input.objectStartId) && !inputIdGenerator.isMiss(input.objectEndId)) {
                    // A relationship referring to missing nodes. The InputIdGenerator is expected to generate
                    // some (very few) of those. Skip it.
                    String name = (String) propertyOf(input, "id");
                    Relationship relationship = relationshipByName.get(name);
                    assertNotNull(relationship, "Expected there to be a relationship with name '" + name + "'");
                    assertEquals(nodeByInputId.get(uniqueId(input.startIdGroup, input.objectStartId)), relationship.getStartNode());
                    assertEquals(nodeByInputId.get(uniqueId(input.endIdGroup, input.objectEndId)), relationship.getEndNode());
                    assertRelationshipEquals(input, relationship);
                }
                verifiedRelationships++;
            }
        }
        assertEquals(relationshipCount, verifiedRelationships);
    }
}

Example 7 with Groups

use of org.neo4j.internal.batchimport.input.Groups in project neo4j by neo4j.

the class ParallelBatchImporterTest method shouldImportCsvData.

@ParameterizedTest
@MethodSource("params")
void shouldImportCsvData(InputIdGenerator inputIdGenerator, IdType idType) throws Exception {
    this.inputIdGenerator = inputIdGenerator;
    // GIVEN
    ExecutionMonitor processorAssigner = ProcessorAssignmentStrategies.eagerRandomSaturation(config.maxNumberOfProcessors());
    CapturingMonitor monitor = new CapturingMonitor(processorAssigner);
    boolean successful = false;
    Groups groups = new Groups();
    IdGroupDistribution groupDistribution = new IdGroupDistribution(NODE_COUNT, NUMBER_OF_ID_GROUPS, random.random(), groups);
    long nodeRandomSeed = random.nextLong();
    long relationshipRandomSeed = random.nextLong();
    var pageCacheTracer = new DefaultPageCacheTracer();
    JobScheduler jobScheduler = new ThreadPoolJobScheduler();
    // This will have statistically half the nodes be considered dense
    Config dbConfig = Config.defaults(GraphDatabaseSettings.dense_node_threshold, RELATIONSHIPS_PER_NODE * 2);
    IndexImporterFactoryImpl indexImporterFactory = new IndexImporterFactoryImpl(dbConfig);
    final BatchImporter inserter = new ParallelBatchImporter(databaseLayout, fs, pageCacheTracer, config, NullLogService.getInstance(), monitor, EMPTY, dbConfig, getFormat(), ImportLogic.NO_MONITOR, jobScheduler, Collector.EMPTY, TransactionLogInitializer.getLogFilesInitializer(), indexImporterFactory, INSTANCE);
    LongAdder propertyCount = new LongAdder();
    LongAdder relationshipCount = new LongAdder();
    try {
        // WHEN
        inserter.doImport(Input.input(nodes(nodeRandomSeed, NODE_COUNT, config.batchSize(), inputIdGenerator, groupDistribution, propertyCount), relationships(relationshipRandomSeed, RELATIONSHIP_COUNT, config.batchSize(), inputIdGenerator, groupDistribution, propertyCount, relationshipCount), idType, knownEstimates(NODE_COUNT, RELATIONSHIP_COUNT, NODE_COUNT * TOKENS.length / 2, RELATIONSHIP_COUNT * TOKENS.length / 2, NODE_COUNT * TOKENS.length / 2 * Long.BYTES, RELATIONSHIP_COUNT * TOKENS.length / 2 * Long.BYTES, NODE_COUNT * TOKENS.length / 2), groups));
        assertThat(pageCacheTracer.pins()).isGreaterThan(0);
        assertThat(pageCacheTracer.pins()).isEqualTo(pageCacheTracer.unpins());
        assertThat(pageCacheTracer.pins()).isEqualTo(Math.addExact(pageCacheTracer.faults(), pageCacheTracer.hits()));
        // THEN
        DatabaseManagementService managementService = getDBMSBuilder(databaseLayout).build();
        GraphDatabaseService db = managementService.database(DEFAULT_DATABASE_NAME);
        try (Transaction tx = db.beginTx()) {
            inputIdGenerator.reset();
            verifyData(NODE_COUNT, RELATIONSHIP_COUNT, db, tx, groupDistribution, nodeRandomSeed, relationshipRandomSeed);
            tx.commit();
        } finally {
            managementService.shutdown();
        }
        assertConsistent(databaseLayout);
        successful = true;
    } finally {
        jobScheduler.close();
        if (!successful) {
            Path failureFile = databaseLayout.databaseDirectory().resolve("input");
            try (PrintStream out = new PrintStream(Files.newOutputStream(failureFile))) {
                out.println("Seed used in this failing run: " + random.seed());
                out.println(inputIdGenerator);
                inputIdGenerator.reset();
                out.println();
                out.println("Processor assignments");
                out.println(processorAssigner.toString());
            }
            System.err.println("Additional debug information stored in " + failureFile);
        }
    }
}