Examples with HashMapStateBackend org.apache.flink.runtime.state.hashmap.HashMapStateBackend used on opensource projects

Example 11 with HashMapStateBackend

use of org.apache.flink.runtime.state.hashmap.HashMapStateBackend in project flink by apache.

the class ChangelogStateBackendLoadingTest method testApplicationDisableChangelogStateBackend.

@Test
public void testApplicationDisableChangelogStateBackend() throws Exception {
    final StateBackend backend = StateBackendLoader.fromApplicationOrConfigOrDefault(null, TernaryBoolean.FALSE, config(true), cl, null);
    assertTrue(backend instanceof HashMapStateBackend);
}

Example 12 with HashMapStateBackend

use of org.apache.flink.runtime.state.hashmap.HashMapStateBackend in project flink by apache.

the class LatencyTrackingStateTestBase method createKeyedBackend.

protected AbstractKeyedStateBackend<K> createKeyedBackend(TypeSerializer<K> keySerializer) throws Exception {
    Environment env = new DummyEnvironment();
    KeyGroupRange keyGroupRange = new KeyGroupRange(0, 127);
    int numberOfKeyGroups = keyGroupRange.getNumberOfKeyGroups();
    Configuration configuration = new Configuration();
    configuration.setBoolean(StateBackendOptions.LATENCY_TRACK_ENABLED, true);
    configuration.setInteger(StateBackendOptions.LATENCY_TRACK_SAMPLE_INTERVAL, SAMPLE_INTERVAL);
    // use a very large value to not let metrics data overridden.
    int historySize = 1000_000;
    configuration.setInteger(StateBackendOptions.LATENCY_TRACK_HISTORY_SIZE, historySize);
    HashMapStateBackend stateBackend = new HashMapStateBackend().configure(configuration, Thread.currentThread().getContextClassLoader());
    return stateBackend.createKeyedStateBackend(env, new JobID(), "test_op", keySerializer, numberOfKeyGroups, keyGroupRange, env.getTaskKvStateRegistry(), TtlTimeProvider.DEFAULT, new UnregisteredMetricsGroup(), Collections.emptyList(), new CloseableRegistry());
}

Example 13 with HashMapStateBackend

use of org.apache.flink.runtime.state.hashmap.HashMapStateBackend in project flink by apache.

the class StateBackendLoadingTest method testInstantiateHashMapStateBackendBackendByDefault.

@Test
public void testInstantiateHashMapStateBackendBackendByDefault() throws Exception {
    StateBackend backend = StateBackendLoader.fromApplicationOrConfigOrDefault(null, TernaryBoolean.UNDEFINED, new Configuration(), cl, null);
    assertTrue(backend instanceof HashMapStateBackend);
}

Example 14 with HashMapStateBackend

use of org.apache.flink.runtime.state.hashmap.HashMapStateBackend in project flink by apache.

the class StateBackendLoadingTest method testMemoryBackendHighAvailabilityDefault.

private void testMemoryBackendHighAvailabilityDefault(String haPersistenceDir, Path checkpointPath) throws Exception {
    final Configuration config1 = new Configuration();
    config1.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
    config1.setString(HighAvailabilityOptions.HA_CLUSTER_ID, "myCluster");
    config1.setString(HighAvailabilityOptions.HA_STORAGE_PATH, haPersistenceDir);
    final Configuration config2 = new Configuration();
    config2.setString(backendKey, "jobmanager");
    config2.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
    config2.setString(HighAvailabilityOptions.HA_CLUSTER_ID, "myCluster");
    config2.setString(HighAvailabilityOptions.HA_STORAGE_PATH, haPersistenceDir);
    if (checkpointPath != null) {
        config1.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointPath.toUri().toString());
        config2.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointPath.toUri().toString());
    }
    final MemoryStateBackend appBackend = new MemoryStateBackend();
    final StateBackend loaded1 = StateBackendLoader.fromApplicationOrConfigOrDefault(appBackend, TernaryBoolean.UNDEFINED, config1, cl, null);
    final StateBackend loaded2 = StateBackendLoader.fromApplicationOrConfigOrDefault(null, TernaryBoolean.UNDEFINED, config1, cl, null);
    final StateBackend loaded3 = StateBackendLoader.fromApplicationOrConfigOrDefault(null, TernaryBoolean.UNDEFINED, config2, cl, null);
    assertTrue(loaded1 instanceof MemoryStateBackend);
    assertTrue(loaded2 instanceof HashMapStateBackend);
    assertTrue(loaded3 instanceof MemoryStateBackend);
    final MemoryStateBackend memBackend1 = (MemoryStateBackend) loaded1;
    final MemoryStateBackend memBackend2 = (MemoryStateBackend) loaded3;
    assertNull(memBackend1.getSavepointPath());
    if (checkpointPath != null) {
        assertNotNull(memBackend1.getCheckpointPath());
        assertNotNull(memBackend2.getCheckpointPath());
        assertEquals(checkpointPath, memBackend1.getCheckpointPath());
        assertEquals(checkpointPath, memBackend2.getCheckpointPath());
    } else {
        assertNull(memBackend1.getCheckpointPath());
        assertNull(memBackend2.getCheckpointPath());
    }
}

Example 15 with HashMapStateBackend

use of org.apache.flink.runtime.state.hashmap.HashMapStateBackend in project flink by apache.

the class TypeSerializerSnapshotMigrationITCase method testSnapshot.

@Test
public void testSnapshot() throws Exception {
    final int parallelism = 1;
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setRestartStrategy(RestartStrategies.noRestart());
    switch(snapshotSpec.getStateBackendType()) {
        case StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME:
            env.setStateBackend(new EmbeddedRocksDBStateBackend());
            break;
        case StateBackendLoader.MEMORY_STATE_BACKEND_NAME:
            env.setStateBackend(new MemoryStateBackend());
            break;
        case StateBackendLoader.HASHMAP_STATE_BACKEND_NAME:
            env.setStateBackend(new HashMapStateBackend());
            break;
        default:
            throw new UnsupportedOperationException();
    }
    env.enableChangelogStateBackend(false);
    env.enableCheckpointing(500);
    env.setParallelism(parallelism);
    env.setMaxParallelism(parallelism);
    SourceFunction<Tuple2<Long, Long>> nonParallelSource = new MigrationTestUtils.CheckpointingNonParallelSourceWithListState(NUM_SOURCE_ELEMENTS);
    env.addSource(nonParallelSource).keyBy(0).map(new TestMapFunction()).addSink(new MigrationTestUtils.AccumulatorCountingSink<>());
    final String snapshotPath = getSnapshotPath(snapshotSpec);
    if (executionMode == ExecutionMode.CREATE_SNAPSHOT) {
        executeAndSnapshot(env, "src/test/resources/" + snapshotPath, snapshotSpec.getSnapshotType(), Tuple2.of(MigrationTestUtils.AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
    } else if (executionMode == ExecutionMode.VERIFY_SNAPSHOT) {
        restoreAndExecute(env, getResourceFilename(snapshotPath), Tuple2.of(MigrationTestUtils.AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
    } else {
        throw new IllegalStateException("Unknown ExecutionMode " + executionMode);
    }
}