Examples with RegisteredBackendStateMetaInfo org.apache.flink.runtime.state.RegisteredBackendStateMetaInfo used on opensource projects

Example 1 with RegisteredBackendStateMetaInfo

use of org.apache.flink.runtime.state.RegisteredBackendStateMetaInfo in project flink by apache.

the class StateTableSnapshotCompatibilityTest method checkCompatibleSerializationFormats.

/**
	 * This test ensures that different implementations of {@link StateTable} are compatible in their serialization
	 * format.
	 */
@Test
public void checkCompatibleSerializationFormats() throws IOException {
    final Random r = new Random(42);
    RegisteredBackendStateMetaInfo<Integer, ArrayList<Integer>> metaInfo = new RegisteredBackendStateMetaInfo<>(StateDescriptor.Type.UNKNOWN, "test", IntSerializer.INSTANCE, new ArrayListSerializer<>(IntSerializer.INSTANCE));
    final CopyOnWriteStateTableTest.MockInternalKeyContext<Integer> keyContext = new CopyOnWriteStateTableTest.MockInternalKeyContext<>(IntSerializer.INSTANCE);
    CopyOnWriteStateTable<Integer, Integer, ArrayList<Integer>> cowStateTable = new CopyOnWriteStateTable<>(keyContext, metaInfo);
    for (int i = 0; i < 100; ++i) {
        ArrayList<Integer> list = new ArrayList<>(5);
        int end = r.nextInt(5);
        for (int j = 0; j < end; ++j) {
            list.add(r.nextInt(100));
        }
        cowStateTable.put(r.nextInt(10), r.nextInt(2), list);
    }
    StateTableSnapshot snapshot = cowStateTable.createSnapshot();
    final NestedMapsStateTable<Integer, Integer, ArrayList<Integer>> nestedMapsStateTable = new NestedMapsStateTable<>(keyContext, metaInfo);
    restoreStateTableFromSnapshot(nestedMapsStateTable, snapshot, keyContext.getKeyGroupRange());
    snapshot.release();
    Assert.assertEquals(cowStateTable.size(), nestedMapsStateTable.size());
    for (StateEntry<Integer, Integer, ArrayList<Integer>> entry : cowStateTable) {
        Assert.assertEquals(entry.getState(), nestedMapsStateTable.get(entry.getKey(), entry.getNamespace()));
    }
    snapshot = nestedMapsStateTable.createSnapshot();
    cowStateTable = new CopyOnWriteStateTable<>(keyContext, metaInfo);
    restoreStateTableFromSnapshot(cowStateTable, snapshot, keyContext.getKeyGroupRange());
    snapshot.release();
    Assert.assertEquals(nestedMapsStateTable.size(), cowStateTable.size());
    for (StateEntry<Integer, Integer, ArrayList<Integer>> entry : cowStateTable) {
        Assert.assertEquals(nestedMapsStateTable.get(entry.getKey(), entry.getNamespace()), entry.getState());
    }
}

Example 2 with RegisteredBackendStateMetaInfo

use of org.apache.flink.runtime.state.RegisteredBackendStateMetaInfo in project flink by apache.

the class CopyOnWriteStateTableTest method testRandomModificationsAndCopyOnWriteIsolation.

/**
	 * This test does some random modifications to a state table and a reference (hash map). Then draws snapshots,
	 * performs more modifications and checks snapshot integrity.
	 */
@Test
public void testRandomModificationsAndCopyOnWriteIsolation() throws Exception {
    final RegisteredBackendStateMetaInfo<Integer, ArrayList<Integer>> metaInfo = new RegisteredBackendStateMetaInfo<>(StateDescriptor.Type.UNKNOWN, "test", IntSerializer.INSTANCE, // we use mutable state objects.
    new ArrayListSerializer<>(IntSerializer.INSTANCE));
    final MockInternalKeyContext<Integer> keyContext = new MockInternalKeyContext<>(IntSerializer.INSTANCE);
    final CopyOnWriteStateTable<Integer, Integer, ArrayList<Integer>> stateTable = new CopyOnWriteStateTable<>(keyContext, metaInfo);
    final HashMap<Tuple2<Integer, Integer>, ArrayList<Integer>> referenceMap = new HashMap<>();
    final Random random = new Random(42);
    // holds snapshots from the map under test
    CopyOnWriteStateTable.StateTableEntry<Integer, Integer, ArrayList<Integer>>[] snapshot = null;
    int snapshotSize = 0;
    // holds a reference snapshot from our reference map that we compare against
    Tuple3<Integer, Integer, ArrayList<Integer>>[] reference = null;
    int val = 0;
    int snapshotCounter = 0;
    int referencedSnapshotId = 0;
    final StateTransformationFunction<ArrayList<Integer>, Integer> transformationFunction = new StateTransformationFunction<ArrayList<Integer>, Integer>() {

        @Override
        public ArrayList<Integer> apply(ArrayList<Integer> previousState, Integer value) throws Exception {
            if (previousState == null) {
                previousState = new ArrayList<>();
            }
            previousState.add(value);
            // we give back the original, attempting to spot errors in to copy-on-write
            return previousState;
        }
    };
    // the main loop for modifications
    for (int i = 0; i < 10_000_000; ++i) {
        int key = random.nextInt(20);
        int namespace = random.nextInt(4);
        Tuple2<Integer, Integer> compositeKey = new Tuple2<>(key, namespace);
        int op = random.nextInt(7);
        ArrayList<Integer> state = null;
        ArrayList<Integer> referenceState = null;
        switch(op) {
            case 0:
            case 1:
                {
                    state = stateTable.get(key, namespace);
                    referenceState = referenceMap.get(compositeKey);
                    if (null == state) {
                        state = new ArrayList<>();
                        stateTable.put(key, namespace, state);
                        referenceState = new ArrayList<>();
                        referenceMap.put(compositeKey, referenceState);
                    }
                    break;
                }
            case 2:
                {
                    stateTable.put(key, namespace, new ArrayList<Integer>());
                    referenceMap.put(compositeKey, new ArrayList<Integer>());
                    break;
                }
            case 3:
                {
                    state = stateTable.putAndGetOld(key, namespace, new ArrayList<Integer>());
                    referenceState = referenceMap.put(compositeKey, new ArrayList<Integer>());
                    break;
                }
            case 4:
                {
                    stateTable.remove(key, namespace);
                    referenceMap.remove(compositeKey);
                    break;
                }
            case 5:
                {
                    state = stateTable.removeAndGetOld(key, namespace);
                    referenceState = referenceMap.remove(compositeKey);
                    break;
                }
            case 6:
                {
                    final int updateValue = random.nextInt(1000);
                    stateTable.transform(key, namespace, updateValue, transformationFunction);
                    referenceMap.put(compositeKey, transformationFunction.apply(referenceMap.remove(compositeKey), updateValue));
                    break;
                }
            default:
                {
                    Assert.fail("Unknown op-code " + op);
                }
        }
        Assert.assertEquals(referenceMap.size(), stateTable.size());
        if (state != null) {
            // mutate the states a bit...
            if (random.nextBoolean() && !state.isEmpty()) {
                state.remove(state.size() - 1);
                referenceState.remove(referenceState.size() - 1);
            } else {
                state.add(val);
                referenceState.add(val);
                ++val;
            }
        }
        Assert.assertEquals(referenceState, state);
        // snapshot triggering / comparison / release
        if (i > 0 && i % 500 == 0) {
            if (snapshot != null) {
                // check our referenced snapshot
                deepCheck(reference, convert(snapshot, snapshotSize));
                if (i % 1_000 == 0) {
                    // draw and release some other snapshot while holding on the old snapshot
                    ++snapshotCounter;
                    stateTable.snapshotTableArrays();
                    stateTable.releaseSnapshot(snapshotCounter);
                }
                //release the snapshot after some time
                if (i % 5_000 == 0) {
                    snapshot = null;
                    reference = null;
                    snapshotSize = 0;
                    stateTable.releaseSnapshot(referencedSnapshotId);
                }
            } else {
                // if there is no more referenced snapshot, we create one
                ++snapshotCounter;
                referencedSnapshotId = snapshotCounter;
                snapshot = stateTable.snapshotTableArrays();
                snapshotSize = stateTable.size();
                reference = manualDeepDump(referenceMap);
            }
        }
    }
}

Example 3 with RegisteredBackendStateMetaInfo

use of org.apache.flink.runtime.state.RegisteredBackendStateMetaInfo in project flink by apache.

the class AbstractMigrationRestoreStrategy method deserialize.

@Override
public StateTable<K, N, S> deserialize(String stateName, HeapKeyedStateBackend<K> stateBackend) throws IOException {
    Preconditions.checkNotNull(stateName, "State name is null. Cannot deserialize snapshot.");
    Preconditions.checkNotNull(stateBackend, "State backend is null. Cannot deserialize snapshot.");
    final KeyGroupRange keyGroupRange = stateBackend.getKeyGroupRange();
    Preconditions.checkState(1 == keyGroupRange.getNumberOfKeyGroups(), "Unexpected number of key-groups for restoring from Flink 1.1");
    TypeSerializer<N> patchedNamespaceSerializer = this.namespaceSerializer;
    if (patchedNamespaceSerializer instanceof VoidSerializer) {
        patchedNamespaceSerializer = (TypeSerializer<N>) VoidNamespaceSerializer.INSTANCE;
    }
    RegisteredBackendStateMetaInfo<N, S> registeredBackendStateMetaInfo = new RegisteredBackendStateMetaInfo<>(StateDescriptor.Type.UNKNOWN, stateName, patchedNamespaceSerializer, stateSerializer);
    final StateTable<K, N, S> stateTable = stateBackend.newStateTable(registeredBackendStateMetaInfo);
    final DataInputView inView = openDataInputView();
    final int keyGroup = keyGroupRange.getStartKeyGroup();
    final int numNamespaces = inView.readInt();
    for (int i = 0; i < numNamespaces; i++) {
        N namespace = namespaceSerializer.deserialize(inView);
        if (null == namespace) {
            namespace = (N) VoidNamespace.INSTANCE;
        }
        final int numKV = inView.readInt();
        for (int j = 0; j < numKV; j++) {
            K key = keySerializer.deserialize(inView);
            S value = stateSerializer.deserialize(inView);
            stateTable.put(key, keyGroup, namespace, value);
        }
    }
    return stateTable;
}

Example 4 with RegisteredBackendStateMetaInfo

use of org.apache.flink.runtime.state.RegisteredBackendStateMetaInfo in project flink by apache.

the class QueryableStateClientTest method testIntegrationWithKvStateServer.

/**
	 * Tests queries against multiple servers.
	 *
	 * <p>The servers are populated with different keys and the client queries
	 * all available keys from all servers.
	 */
@Test
public void testIntegrationWithKvStateServer() throws Exception {
    // Config
    int numServers = 2;
    int numKeys = 1024;
    int numKeyGroups = 1;
    JobID jobId = new JobID();
    JobVertexID jobVertexId = new JobVertexID();
    KvStateServer[] servers = new KvStateServer[numServers];
    AtomicKvStateRequestStats[] serverStats = new AtomicKvStateRequestStats[numServers];
    QueryableStateClient client = null;
    KvStateClient networkClient = null;
    AtomicKvStateRequestStats networkClientStats = new AtomicKvStateRequestStats();
    MemoryStateBackend backend = new MemoryStateBackend();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    AbstractKeyedStateBackend<Integer> keyedStateBackend = backend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()));
    try {
        KvStateRegistry[] registries = new KvStateRegistry[numServers];
        KvStateID[] kvStateIds = new KvStateID[numServers];
        List<HeapValueState<Integer, VoidNamespace, Integer>> kvStates = new ArrayList<>();
        // Start the servers
        for (int i = 0; i < numServers; i++) {
            registries[i] = new KvStateRegistry();
            serverStats[i] = new AtomicKvStateRequestStats();
            servers[i] = new KvStateServer(InetAddress.getLocalHost(), 0, 1, 1, registries[i], serverStats[i]);
            servers[i].start();
            ValueStateDescriptor<Integer> descriptor = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
            RegisteredBackendStateMetaInfo<VoidNamespace, Integer> registeredBackendStateMetaInfo = new RegisteredBackendStateMetaInfo<>(descriptor.getType(), descriptor.getName(), VoidNamespaceSerializer.INSTANCE, IntSerializer.INSTANCE);
            // Register state
            HeapValueState<Integer, VoidNamespace, Integer> kvState = new HeapValueState<>(descriptor, new NestedMapsStateTable<Integer, VoidNamespace, Integer>(keyedStateBackend, registeredBackendStateMetaInfo), IntSerializer.INSTANCE, VoidNamespaceSerializer.INSTANCE);
            kvStates.add(kvState);
            kvStateIds[i] = registries[i].registerKvState(jobId, new JobVertexID(), new KeyGroupRange(i, i), "choco", kvState);
        }
        int[] expectedRequests = new int[numServers];
        for (int key = 0; key < numKeys; key++) {
            int targetKeyGroupIndex = MathUtils.murmurHash(key) % numServers;
            expectedRequests[targetKeyGroupIndex]++;
            HeapValueState<Integer, VoidNamespace, Integer> kvState = kvStates.get(targetKeyGroupIndex);
            keyedStateBackend.setCurrentKey(key);
            kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
            kvState.update(1337 + key);
        }
        // Location lookup service
        KvStateLocation location = new KvStateLocation(jobId, jobVertexId, numServers, "choco");
        for (int keyGroupIndex = 0; keyGroupIndex < numServers; keyGroupIndex++) {
            location.registerKvState(new KeyGroupRange(keyGroupIndex, keyGroupIndex), kvStateIds[keyGroupIndex], servers[keyGroupIndex].getAddress());
        }
        KvStateLocationLookupService lookupService = mock(KvStateLocationLookupService.class);
        when(lookupService.getKvStateLookupInfo(eq(jobId), eq("choco"))).thenReturn(Futures.successful(location));
        // The client
        networkClient = new KvStateClient(1, networkClientStats);
        client = new QueryableStateClient(lookupService, networkClient, testActorSystem.dispatcher());
        // Send all queries
        List<Future<byte[]>> futures = new ArrayList<>(numKeys);
        for (int key = 0; key < numKeys; key++) {
            byte[] serializedKeyAndNamespace = KvStateRequestSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
            futures.add(client.getKvState(jobId, "choco", key, serializedKeyAndNamespace));
        }
        // Verify results
        Future<Iterable<byte[]>> future = Futures.sequence(futures, testActorSystem.dispatcher());
        Iterable<byte[]> results = Await.result(future, timeout);
        int index = 0;
        for (byte[] buffer : results) {
            int deserializedValue = KvStateRequestSerializer.deserializeValue(buffer, IntSerializer.INSTANCE);
            assertEquals(1337 + index, deserializedValue);
            index++;
        }
        // Verify requests
        for (int i = 0; i < numServers; i++) {
            int numRetries = 10;
            for (int retry = 0; retry < numRetries; retry++) {
                try {
                    assertEquals("Unexpected number of requests", expectedRequests[i], serverStats[i].getNumRequests());
                    assertEquals("Unexpected success requests", expectedRequests[i], serverStats[i].getNumSuccessful());
                    assertEquals("Unexpected failed requests", 0, serverStats[i].getNumFailed());
                    break;
                } catch (Throwable t) {
                    // Retry
                    if (retry == numRetries - 1) {
                        throw t;
                    } else {
                        Thread.sleep(100);
                    }
                }
            }
        }
    } finally {
        if (client != null) {
            client.shutDown();
        }
        if (networkClient != null) {
            networkClient.shutDown();
        }
        for (KvStateServer server : servers) {
            if (server != null) {
                server.shutDown();
            }
        }
    }
}

Example 5 with RegisteredBackendStateMetaInfo

use of org.apache.flink.runtime.state.RegisteredBackendStateMetaInfo in project flink by apache.

the class RocksDBKeyedStateBackend method getColumnFamily.

// ------------------------------------------------------------------------
//  State factories
// ------------------------------------------------------------------------
/**
	 * Creates a column family handle for use with a k/v state. When restoring from a snapshot
	 * we don't restore the individual k/v states, just the global RocksDB data base and the
	 * list of column families. When a k/v state is first requested we check here whether we
	 * already have a column family for that and return it or create a new one if it doesn't exist.
	 *
	 * <p>This also checks whether the {@link StateDescriptor} for a state matches the one
	 * that we checkpointed, i.e. is already in the map of column families.
	 */
@SuppressWarnings("rawtypes, unchecked")
protected <N, S> ColumnFamilyHandle getColumnFamily(StateDescriptor<?, S> descriptor, TypeSerializer<N> namespaceSerializer) throws IOException {
    Tuple2<ColumnFamilyHandle, RegisteredBackendStateMetaInfo<?, ?>> stateInfo = kvStateInformation.get(descriptor.getName());
    RegisteredBackendStateMetaInfo<N, S> newMetaInfo = new RegisteredBackendStateMetaInfo<>(descriptor.getType(), descriptor.getName(), namespaceSerializer, descriptor.getSerializer());
    if (stateInfo != null) {
        if (newMetaInfo.isCompatibleWith(stateInfo.f1)) {
            stateInfo.f1 = newMetaInfo;
            return stateInfo.f0;
        } else {
            throw new IOException("Trying to access state using wrong meta info, was " + stateInfo.f1 + " trying access with " + newMetaInfo);
        }
    }
    ColumnFamilyDescriptor columnDescriptor = new ColumnFamilyDescriptor(descriptor.getName().getBytes(ConfigConstants.DEFAULT_CHARSET), columnOptions);
    try {
        ColumnFamilyHandle columnFamily = db.createColumnFamily(columnDescriptor);
        Tuple2<ColumnFamilyHandle, RegisteredBackendStateMetaInfo<N, S>> tuple = new Tuple2<>(columnFamily, newMetaInfo);
        Map rawAccess = kvStateInformation;
        rawAccess.put(descriptor.getName(), tuple);
        return columnFamily;
    } catch (RocksDBException e) {
        throw new IOException("Error creating ColumnFamilyHandle.", e);
    }
}