Examples with PerTypeRelationshipSplitter org.neo4j.unsafe.impl.batchimport.input.PerTypeRelationshipSplitter used on opensource projects

Example 1 with PerTypeRelationshipSplitter

use of org.neo4j.unsafe.impl.batchimport.input.PerTypeRelationshipSplitter in project neo4j by neo4j.

the class ParallelBatchImporter method importRelationships.

private void importRelationships(NodeRelationshipCache nodeRelationshipCache, CountingStoreUpdateMonitor storeUpdateMonitor, BatchingNeoStores neoStore, IoMonitor writeMonitor, IdMapper idMapper, InputIterable<InputRelationship> relationships, InputCache inputCache, Object[] allRelationshipTypes, Object[] minorityRelationshipTypes) {
    // Imports the relationships from the Input. This isn't a straight forward as importing nodes,
    // since keeping track of and updating heads of relationship chains in scenarios where most nodes
    // are dense and there are many relationship types scales poorly w/ regards to cache memory usage
    // also as a side-effect time required to update this cache.
    //
    // The approach is instead to do multiple iterations where each iteration imports relationships
    // of a single type. For each iteration Node --> Relationship and Relationship --> Relationship
    // stages _for dense nodes only_ are run so that the cache can be reused to hold relationship chain heads
    // of the next type in the next iteration. All relationships will be imported this way and then
    // finally there will be one Node --> Relationship and Relationship --> Relationship stage linking
    // all sparse relationship chains together.
    Set<Object> minorityRelationshipTypeSet = asSet(minorityRelationshipTypes);
    PerTypeRelationshipSplitter perTypeIterator = new PerTypeRelationshipSplitter(relationships.iterator(), allRelationshipTypes, minorityRelationshipTypeSet::contains, neoStore.getRelationshipTypeRepository(), inputCache);
    long nextRelationshipId = 0;
    Configuration relationshipConfig = withBatchSize(config, neoStore.getRelationshipStore().getRecordsPerPage());
    Configuration nodeConfig = withBatchSize(config, neoStore.getNodeStore().getRecordsPerPage());
    for (int i = 0; perTypeIterator.hasNext(); i++) {
        // Stage 3a -- relationships, properties
        nodeRelationshipCache.setForwardScan(true);
        Object currentType = perTypeIterator.currentType();
        int currentTypeId = neoStore.getRelationshipTypeRepository().getOrCreateId(currentType);
        InputIterator<InputRelationship> perType = perTypeIterator.next();
        String topic = " [:" + currentType + "] (" + (i + 1) + "/" + allRelationshipTypes.length + ")";
        final RelationshipStage relationshipStage = new RelationshipStage(topic, config, writeMonitor, perType, idMapper, neoStore, nodeRelationshipCache, storeUpdateMonitor, nextRelationshipId);
        executeStage(relationshipStage);
        // Stage 4a -- set node nextRel fields for dense nodes
        executeStage(new NodeFirstRelationshipStage(topic, nodeConfig, neoStore.getNodeStore(), neoStore.getTemporaryRelationshipGroupStore(), nodeRelationshipCache, true, /*dense*/
        currentTypeId));
        // Stage 5a -- link relationship chains together for dense nodes
        nodeRelationshipCache.setForwardScan(false);
        executeStage(new RelationshipLinkbackStage(topic, relationshipConfig, neoStore.getRelationshipStore(), nodeRelationshipCache, nextRelationshipId, relationshipStage.getNextRelationshipId(), true));
        nextRelationshipId = relationshipStage.getNextRelationshipId();
        // cheap higher level clearing
        nodeRelationshipCache.clearChangedChunks(true);
    }
    String topic = " Sparse";
    nodeRelationshipCache.setForwardScan(true);
    // Stage 4b -- set node nextRe fields for sparse nodes
    executeStage(new NodeFirstRelationshipStage(topic, nodeConfig, neoStore.getNodeStore(), neoStore.getTemporaryRelationshipGroupStore(), nodeRelationshipCache, false, /*sparse*/
    -1));
    // Stage 5b -- link relationship chains together for sparse nodes
    nodeRelationshipCache.setForwardScan(false);
    executeStage(new RelationshipLinkbackStage(topic, relationshipConfig, neoStore.getRelationshipStore(), nodeRelationshipCache, 0, nextRelationshipId, false));
    if (minorityRelationshipTypes.length > 0) {
        // Do some batch insertion style random-access insertions for super small minority types
        executeStage(new BatchInsertRelationshipsStage(config, idMapper, perTypeIterator.getMinorityRelationships(), neoStore, nextRelationshipId));
    }
}