Examples with Workers org.neo4j.unsafe.impl.batchimport.cache.idmapping.string.Workers used on opensource projects

Example 1 with Workers

use of org.neo4j.unsafe.impl.batchimport.cache.idmapping.string.Workers in project neo4j by neo4j.

the class IndexWorkSyncTransactionApplicationStressIT method shouldApplyIndexUpdatesInWorkSyncedBatches.

@Test
public void shouldApplyIndexUpdatesInWorkSyncedBatches() throws Exception {
    // GIVEN
    long duration = parseTimeMillis.apply(System.getProperty(getClass().getName() + ".duration", "2s"));
    int numThreads = Integer.getInteger(getClass().getName() + ".numThreads", Runtime.getRuntime().availableProcessors());
    RecordStorageEngine storageEngine = storageEngineRule.getWith(fileSystemRule.get(), pageCacheRule.getPageCache(fileSystemRule.get())).storeDirectory(directory.directory()).indexProvider(new InMemoryIndexProvider()).build();
    storageEngine.apply(tx(asList(createIndexRule(InMemoryIndexProviderFactory.PROVIDER_DESCRIPTOR, 1, SchemaBoundary.map(descriptor)))), TransactionApplicationMode.EXTERNAL);
    Dependencies dependencies = new Dependencies();
    storageEngine.satisfyDependencies(dependencies);
    IndexProxy index = dependencies.resolveDependency(IndexingService.class).getIndexProxy(descriptor);
    awaitOnline(index);
    // WHEN
    Workers<Worker> workers = new Workers<>(getClass().getSimpleName());
    final AtomicBoolean end = new AtomicBoolean();
    for (int i = 0; i < numThreads; i++) {
        workers.start(new Worker(i, end, storageEngine, 10, index));
    }
    // let the threads hammer the storage engine for some time
    Thread.sleep(duration);
    end.set(true);
    // THEN (assertions as part of the workers applying transactions)
    workers.awaitAndThrowOnError(RuntimeException.class);
}

Example 2 with Workers

use of org.neo4j.unsafe.impl.batchimport.cache.idmapping.string.Workers in project neo4j by neo4j.

the class TransactionRepresentationCommitProcessIT method commitDuringContinuousCheckpointing.

@Test(timeout = 15000)
public void commitDuringContinuousCheckpointing() throws Exception {
    final Index<Node> index;
    try (Transaction tx = db.beginTx()) {
        index = db.index().forNodes(INDEX_NAME, stringMap(IndexManager.PROVIDER, DummyIndexExtensionFactory.IDENTIFIER));
        tx.success();
    }
    final AtomicBoolean done = new AtomicBoolean();
    Workers<Runnable> workers = new Workers<>(getClass().getSimpleName());
    for (int i = 0; i < TOTAL_ACTIVE_THREADS; i++) {
        workers.start(new Runnable() {

            private final ThreadLocalRandom random = ThreadLocalRandom.current();

            @Override
            public void run() {
                while (!done.get()) {
                    try (Transaction tx = db.beginTx()) {
                        Node node = db.createNode();
                        index.add(node, "key", node.getId());
                        tx.success();
                    }
                    randomSleep();
                }
            }

            private void randomSleep() {
                try {
                    Thread.sleep(random.nextInt(50));
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }
            }
        });
    }
    Thread.sleep(SECONDS.toMillis(2));
    done.set(true);
    workers.awaitAndThrowOnError(RuntimeException.class);
    NeoStores neoStores = getDependency(RecordStorageEngine.class).testAccessNeoStores();
    assertThat("Count store should be rotated once at least", neoStores.getCounts().txId(), greaterThan(0L));
    long lastRotationTx = getDependency(CheckPointer.class).forceCheckPoint(new SimpleTriggerInfo("test"));
    assertEquals("NeoStore last closed transaction id should be equal last count store rotation transaction id.", neoStores.getMetaDataStore().getLastClosedTransactionId(), lastRotationTx);
    assertEquals("Last closed transaction should be last rotated tx in count store", neoStores.getMetaDataStore().getLastClosedTransactionId(), neoStores.getCounts().txId());
}

Example 3 with Workers

use of org.neo4j.unsafe.impl.batchimport.cache.idmapping.string.Workers in project neo4j by neo4j.

the class TransactionThroughMasterSwitchStressIT method oneRound.

private void oneRound() throws Throwable {
    // GIVEN a cluster and a node
    final String key = "key";
    ManagedCluster cluster = clusterRule.startCluster();
    final GraphDatabaseService master = cluster.getMaster();
    final long nodeId = createNode(master);
    cluster.sync();
    // and a bunch of workers contending on that node, each changing it
    Workers<Runnable> transactors = new Workers<>("Transactors");
    final AtomicInteger successes = new AtomicInteger();
    final AtomicBoolean end = new AtomicBoolean();
    for (int i = 0; i < 10; i++) {
        transactors.start(new Runnable() {

            @Override
            public void run() {
                Random random = ThreadLocalRandom.current();
                while (!end.get()) {
                    boolean committed = true;
                    try (Transaction tx = master.beginTx()) {
                        Node node = master.getNodeById(nodeId);
                        // Acquiring lock, read int property value, increment, set incremented int property
                        // should not break under any circumstances.
                        tx.acquireWriteLock(node);
                        node.setProperty(key, (Integer) node.getProperty(key, 0) + 1);
                        // Throw in relationship for good measure
                        node.createRelationshipTo(master.createNode(), TEST);
                        Thread.sleep(random.nextInt(1_000));
                        tx.success();
                    } catch (Throwable e) {
                        // It's OK
                        committed = false;
                    }
                    if (committed) {
                        successes.incrementAndGet();
                    }
                }
            }
        });
    }
    // WHEN entering a period of induced cluster instabilities
    reelectTheSameMasterMakingItGoToPendingAndBack(cluster);
    // ... letting transactions run a bit after the role switch as well.
    long targetSuccesses = successes.get() + 20;
    while (successes.get() < targetSuccesses) {
        Thread.sleep(100);
    }
    end.set(true);
    transactors.awaitAndThrowOnError(RuntimeException.class);
    // THEN verify that the count is equal to the number of successful transactions
    assertEquals(successes.get(), getNodePropertyValue(master, nodeId, key));
}

Example 4 with Workers

use of org.neo4j.unsafe.impl.batchimport.cache.idmapping.string.Workers in project neo4j by neo4j.

the class RecordDistributor method distributeRecords.

public static <RECORD> void distributeRecords(int numberOfThreads, String workerNames, int queueSize, Iterator<RECORD> records, final ProgressListener progress, RecordProcessor<RECORD> processor, QueueDistributor<RECORD> idDistributor) {
    if (!records.hasNext()) {
        return;
    }
    @SuppressWarnings("unchecked") final ArrayBlockingQueue<RECORD>[] recordQ = new ArrayBlockingQueue[numberOfThreads];
    final Workers<RecordCheckWorker<RECORD>> workers = new Workers<>(workerNames);
    final AtomicInteger idGroup = new AtomicInteger(-1);
    for (int threadId = 0; threadId < numberOfThreads; threadId++) {
        recordQ[threadId] = new ArrayBlockingQueue<>(queueSize);
        workers.start(new RecordCheckWorker<>(threadId, idGroup, recordQ[threadId], processor));
    }
    final int[] recsProcessed = new int[numberOfThreads];
    RecordConsumer<RECORD> recordConsumer = (record, qIndex) -> {
        recordQ[qIndex].put(record);
        recsProcessed[qIndex]++;
    };
    try {
        while (records.hasNext()) {
            try {
                // Put records into the queues using the queue distributor. Each Worker will pull and process.
                RECORD record = records.next();
                idDistributor.distribute(record, recordConsumer);
                progress.add(1);
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                break;
            }
        }
        // No more records to distribute, mark as done so that the workers will exit when no more records in queue.
        for (RecordCheckWorker<RECORD> worker : workers) {
            worker.done();
        }
        workers.awaitAndThrowOnError(RuntimeException.class);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new RuntimeException("Was interrupted while awaiting completion");
    }
}