Examples with MutableLong org.apache.commons.lang3.mutable.MutableLong used on opensource projects

Example 36 with MutableLong

use of org.apache.commons.lang3.mutable.MutableLong in project apex-malhar by apache.

the class Application method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration configuration) {
    WordGenerator inputOperator = new WordGenerator();
    KeyedWindowedOperatorImpl<String, Long, MutableLong, Long> windowedOperator = new KeyedWindowedOperatorImpl<>();
    Accumulation<Long, MutableLong, Long> sum = new SumAccumulation();
    windowedOperator.setAccumulation(sum);
    windowedOperator.setDataStorage(new InMemoryWindowedKeyedStorage<String, MutableLong>());
    windowedOperator.setRetractionStorage(new InMemoryWindowedKeyedStorage<String, Long>());
    windowedOperator.setWindowStateStorage(new InMemoryWindowedStorage<WindowState>());
    windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.standardMinutes(1)));
    windowedOperator.setTriggerOption(TriggerOption.AtWatermark().withEarlyFiringsAtEvery(Duration.millis(1000)).accumulatingAndRetractingFiredPanes());
    // windowedOperator.setAllowedLateness(Duration.millis(14000));
    ConsoleOutputOperator outputOperator = new ConsoleOutputOperator();
    dag.addOperator("inputOperator", inputOperator);
    dag.addOperator("windowedOperator", windowedOperator);
    dag.addOperator("outputOperator", outputOperator);
    dag.addStream("input_windowed", inputOperator.output, windowedOperator.input);
    dag.addStream("windowed_output", windowedOperator.output, outputOperator.input);
}

Example 37 with MutableLong

use of org.apache.commons.lang3.mutable.MutableLong in project neo4j by neo4j.

the class FormatCompatibilityTest method shouldDetectFormatChange.

@Test
public void shouldDetectFormatChange() throws Throwable {
    // GIVEN stored tree
    File storeFile = directory.file(STORE);
    try {
        unzipTo(storeFile);
    } catch (FileNotFoundException e) {
        // First time this test is run, eh?
        createAndZipTree(storeFile);
        tellDeveloperToCommitThisFormatVersion();
    }
    assertTrue(CURRENT_FORMAT_ZIP + " seems to be missing from resources directory", fsRule.get().fileExists(storeFile));
    // WHEN reading from the tree
    // THEN everything should work, otherwise there has likely been a format change
    PageCache pageCache = pageCacheRule.getPageCache(fsRule.get());
    try (GBPTree<MutableLong, MutableLong> tree = new GBPTree<>(pageCache, storeFile, new SimpleLongLayout(), 0, NO_MONITOR, NO_HEADER)) {
        try {
            tree.consistencyCheck();
            try (RawCursor<Hit<MutableLong, MutableLong>, IOException> cursor = tree.seek(new MutableLong(0), new MutableLong(KEY_COUNT))) {
                for (long expectedKey = 0; cursor.next(); expectedKey++) {
                    Hit<MutableLong, MutableLong> hit = cursor.get();
                    assertEquals(expectedKey, hit.key().longValue());
                    assertEquals(value(expectedKey), hit.value().longValue());
                }
                assertFalse(cursor.next());
            }
        } catch (Throwable t) {
            throw new AssertionError(format("If this is the single failing test for %s this failure is a strong indication that format " + "has changed without also incrementing %s.FORMAT_VERSION. " + "Please go ahead and increment the format version", TREE_CLASS_NAME, TREE_CLASS_NAME), t);
        }
    } catch (MetadataMismatchException e) {
        // Good actually, or?
        assertThat(e.getMessage(), containsString("format version"));
        fsRule.get().deleteFile(storeFile);
        createAndZipTree(storeFile);
        tellDeveloperToCommitThisFormatVersion();
    }
}

Example 38 with MutableLong

use of org.apache.commons.lang3.mutable.MutableLong in project neo4j by neo4j.

the class FormatCompatibilityTest method createAndZipTree.

private void createAndZipTree(File storeFile) throws IOException {
    PageCache pageCache = pageCacheRule.getPageCache(fsRule.get());
    try (GBPTree<MutableLong, MutableLong> tree = new GBPTree<>(pageCache, storeFile, new SimpleLongLayout(), 0, NO_MONITOR, NO_HEADER)) {
        MutableLong insertKey = new MutableLong();
        MutableLong insertValue = new MutableLong();
        int batchSize = KEY_COUNT / 10;
        for (int i = 0, key = 0; i < 10; i++) {
            try (Writer<MutableLong, MutableLong> writer = tree.writer()) {
                for (int j = 0; j < batchSize; j++, key++) {
                    insertKey.setValue(key);
                    insertValue.setValue(value(key));
                    writer.put(insertKey, insertValue);
                }
            }
            tree.checkpoint(IOLimiter.unlimited());
        }
    }
    zip(storeFile);
}

Example 39 with MutableLong

use of org.apache.commons.lang3.mutable.MutableLong in project neo4j by neo4j.

the class GBPTreeRecoveryTest method doShouldRecoverFromAnything.

private void doShouldRecoverFromAnything(boolean replayRecoveryExactlyFromCheckpoint) throws Exception {
    assertInitialized();
    // GIVEN
    // a tree which has had random updates and checkpoints in it, load generated with specific seed
    File file = directory.file("index");
    List<Action> load = generateLoad();
    int lastCheckPointIndex = indexOfLastCheckpoint(load);
    {
        // _,_,_,_,_,_,_,c,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,c,_,_,_,_,_,_,_,_,_,_,_
        //                                                 ^             ^
        //                                                 |             |------------ crash flush index
        //                                                 |-------------------------- last checkpoint index
        //
        PageCache pageCache = createPageCache();
        GBPTree<MutableLong, MutableLong> index = createIndex(pageCache, file);
        // Execute all actions up to and including last checkpoint ...
        execute(load.subList(0, lastCheckPointIndex + 1), index);
        // ... a random amount of the remaining "unsafe" actions ...
        int numberOfRemainingActions = load.size() - lastCheckPointIndex - 1;
        int crashFlushIndex = lastCheckPointIndex + random.nextInt(numberOfRemainingActions) + 1;
        execute(load.subList(lastCheckPointIndex + 1, crashFlushIndex), index);
        // ... flush ...
        pageCache.flushAndForce();
        // ... execute the remaining actions
        execute(load.subList(crashFlushIndex, load.size()), index);
        // ... and finally crash
        fs.snapshot(throwing(() -> {
            index.close();
            pageCache.close();
        }));
    }
    // WHEN doing recovery
    List<Action> recoveryActions;
    if (replayRecoveryExactlyFromCheckpoint) {
        recoveryActions = recoveryActions(load, lastCheckPointIndex + 1);
    } else {
        recoveryActions = recoveryActions(load, random.nextInt(lastCheckPointIndex + 1));
    }
    // first crashing during recovery
    int numberOfCrashesDuringRecovery = random.intBetween(0, 3);
    for (int i = 0; i < numberOfCrashesDuringRecovery; i++) {
        PageCache pageCache = createPageCache();
        GBPTree<MutableLong, MutableLong> index = createIndex(pageCache, file);
        int numberOfActionsToRecoverBeforeCrashing = random.intBetween(1, recoveryActions.size());
        recover(recoveryActions.subList(0, numberOfActionsToRecoverBeforeCrashing), index);
        pageCache.flushAndForce();
        fs.snapshot(throwing(() -> {
            index.close();
            pageCache.close();
        }));
    }
    // to finally apply all actions after last checkpoint and verify tree
    try (PageCache pageCache = createPageCache();
        GBPTree<MutableLong, MutableLong> index = createIndex(pageCache, file)) {
        recover(recoveryActions, index);
        index.finishRecovery();
        // THEN
        // we should end up with a consistent index containing all the stuff load says
        index.consistencyCheck();
        long[] /*key,value,key,value...*/
        aggregate = expectedSortedAggregatedDataFromGeneratedLoad(load);
        try (RawCursor<Hit<MutableLong, MutableLong>, IOException> cursor = index.seek(new MutableLong(Long.MIN_VALUE), new MutableLong(Long.MAX_VALUE))) {
            for (int i = 0; i < aggregate.length; ) {
                assertTrue(cursor.next());
                Hit<MutableLong, MutableLong> hit = cursor.get();
                assertEquals(aggregate[i++], hit.key().longValue());
                assertEquals(aggregate[i++], hit.value().longValue());
            }
            assertFalse(cursor.next());
        }
    }
}

Example 40 with MutableLong

use of org.apache.commons.lang3.mutable.MutableLong in project neo4j by neo4j.

the class GBPTreeRecoveryTest method shouldRecoverFromCrashBeforeFirstCheckpoint.

@Test
public void shouldRecoverFromCrashBeforeFirstCheckpoint() throws Exception {
    // GIVEN
    // a tree with only small amount of data that has not yet seen checkpoint from outside
    File file = directory.file("index");
    {
        PageCache pageCache = createPageCache();
        GBPTree<MutableLong, MutableLong> index = createIndex(pageCache, file);
        Writer<MutableLong, MutableLong> writer = index.writer();
        key.setValue(1L);
        value.setValue(10L);
        writer.put(key, value);
        pageCache.flushAndForce();
        fs.snapshot(throwing(() -> {
            writer.close();
            index.close();
            pageCache.close();
        }));
    }
    // WHEN
    try (PageCache pageCache = createPageCache();
        GBPTree<MutableLong, MutableLong> index = createIndex(pageCache, file)) {
        // this is the mimic:ed recovery
        index.prepareForRecovery();
        index.finishRecovery();
        try (Writer<MutableLong, MutableLong> writer = index.writer()) {
            writer.put(key, value);
        }
        // THEN
        // we should end up with a consistent index
        index.consistencyCheck();
        // ... containing all the stuff load says
        try (RawCursor<Hit<MutableLong, MutableLong>, IOException> cursor = index.seek(new MutableLong(Long.MIN_VALUE), new MutableLong(Long.MAX_VALUE))) {
            assertTrue(cursor.next());
            Hit<MutableLong, MutableLong> hit = cursor.get();
            assertEquals(key.getValue(), hit.key().getValue());
            assertEquals(value.getValue(), hit.value().getValue());
        }
    }
}