Examples with GenericTypeInfo org.apache.flink.api.java.typeutils.GenericTypeInfo used on opensource projects

Example 21 with GenericTypeInfo

use of org.apache.flink.api.java.typeutils.GenericTypeInfo in project flink by apache.

the class StateBackendTestBase method testKryoRestoreResilienceWithDifferentRegistrationOrder.

@Test
@SuppressWarnings("unchecked")
public void testKryoRestoreResilienceWithDifferentRegistrationOrder() throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    // register A first then B
    env.getExecutionConfig().registerKryoType(TestNestedPojoClassA.class);
    env.getExecutionConfig().registerKryoType(TestNestedPojoClassB.class);
    CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);
    try {
        TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
        // make sure that we are in fact using the KryoSerializer
        assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
        ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id", pojoType);
        ValueState<TestPojo> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        // access the internal state representation to retrieve the original Kryo registration
        // ids;
        // these will be later used to check that on restore, the new Kryo serializer has
        // reconfigured itself to
        // have identical mappings
        InternalKvState internalKvState = (InternalKvState) state;
        KryoSerializer<TestPojo> kryoSerializer = (KryoSerializer<TestPojo>) internalKvState.getValueSerializer();
        int mainPojoClassRegistrationId = kryoSerializer.getKryo().getRegistration(TestPojo.class).getId();
        int nestedPojoClassARegistrationId = kryoSerializer.getKryo().getRegistration(TestNestedPojoClassA.class).getId();
        int nestedPojoClassBRegistrationId = kryoSerializer.getKryo().getRegistration(TestNestedPojoClassB.class).getId();
        // ============== create snapshot of current configuration ==============
        // make some more modifications
        backend.setCurrentKey(1);
        state.update(new TestPojo("u1", 1, new TestNestedPojoClassA(1.0, 2), new TestNestedPojoClassB(2.3, "foo")));
        backend.setCurrentKey(2);
        state.update(new TestPojo("u2", 2, new TestNestedPojoClassA(2.0, 5), new TestNestedPojoClassB(3.1, "bar")));
        KeyedStateHandle snapshot = runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        backend.dispose();
        // ========== restore snapshot, with a different registration order in the configuration
        // ==========
        env.close();
        env = buildMockEnv();
        env.getExecutionConfig().registerKryoType(// this time register B first
        TestNestedPojoClassB.class);
        env.getExecutionConfig().registerKryoType(TestNestedPojoClassA.class);
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot, env);
        // re-initialize to ensure that we create the KryoSerializer from scratch, otherwise
        // initializeSerializerUnlessSet would not pick up our new config
        kvId = new ValueStateDescriptor<>("id", pojoType);
        state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        // verify that on restore, the serializer that the state handle uses has reconfigured
        // itself to have
        // identical Kryo registration ids compared to the previous execution
        internalKvState = (InternalKvState) state;
        kryoSerializer = (KryoSerializer<TestPojo>) internalKvState.getValueSerializer();
        assertEquals(mainPojoClassRegistrationId, kryoSerializer.getKryo().getRegistration(TestPojo.class).getId());
        assertEquals(nestedPojoClassARegistrationId, kryoSerializer.getKryo().getRegistration(TestNestedPojoClassA.class).getId());
        assertEquals(nestedPojoClassBRegistrationId, kryoSerializer.getKryo().getRegistration(TestNestedPojoClassB.class).getId());
        backend.setCurrentKey(1);
        // update to test state backends that eagerly serialize, such as RocksDB
        state.update(new TestPojo("u1", 11, new TestNestedPojoClassA(22.1, 12), new TestNestedPojoClassB(1.23, "foobar")));
        // this tests backends that lazily serialize, such as memory state backend
        runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        snapshot.discardState();
    } finally {
        backend.dispose();
    }
}

Example 22 with GenericTypeInfo

use of org.apache.flink.api.java.typeutils.GenericTypeInfo in project flink by apache.

the class ChangelogStateBackendTestUtils method testMaterializedRestore.

public static void testMaterializedRestore(StateBackend stateBackend, StateTtlConfig stateTtlConfig, Environment env, CheckpointStreamFactory streamFactory) throws Exception {
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    TypeInformation<StateBackendTestBase.TestPojo> pojoType = new GenericTypeInfo<>(StateBackendTestBase.TestPojo.class);
    ValueStateDescriptor<StateBackendTestBase.TestPojo> kvId = new ValueStateDescriptor<>("id", pojoType);
    if (stateTtlConfig.isEnabled()) {
        kvId.enableTimeToLive(stateTtlConfig);
    }
    ChangelogKeyedStateBackend<Integer> keyedBackend = (ChangelogKeyedStateBackend<Integer>) createKeyedBackend(stateBackend, env);
    CompletableFuture<Void> asyncComplete = new CompletableFuture<>();
    PeriodicMaterializationManager periodicMaterializationManager = periodicMaterializationManager(keyedBackend, asyncComplete);
    try {
        ValueState<StateBackendTestBase.TestPojo> state = keyedBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        keyedBackend.setCurrentKey(1);
        state.update(new StateBackendTestBase.TestPojo("u1", 1));
        keyedBackend.setCurrentKey(2);
        state.update(new StateBackendTestBase.TestPojo("u2", 2));
        materialize(keyedBackend, periodicMaterializationManager);
        keyedBackend.setCurrentKey(2);
        state.update(new StateBackendTestBase.TestPojo("u2", 22));
        keyedBackend.setCurrentKey(3);
        state.update(new StateBackendTestBase.TestPojo("u3", 3));
        materialize(keyedBackend, periodicMaterializationManager);
        keyedBackend.setCurrentKey(4);
        state.update(new StateBackendTestBase.TestPojo("u4", 4));
        keyedBackend.setCurrentKey(2);
        state.update(new StateBackendTestBase.TestPojo("u2", 222));
        KeyedStateHandle snapshot = runSnapshot(keyedBackend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        IOUtils.closeQuietly(keyedBackend);
        keyedBackend.dispose();
        // make sure the asycn phase completes successfully
        if (asyncComplete.isCompletedExceptionally()) {
            asyncComplete.get();
        }
        // ============================ restore snapshot ===============================
        env.getExecutionConfig().registerKryoType(StateBackendTestBase.TestPojo.class);
        keyedBackend = (ChangelogKeyedStateBackend<Integer>) restoreKeyedBackend(stateBackend, snapshot, env);
        snapshot.discardState();
        state = keyedBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        keyedBackend.setCurrentKey(1);
        assertEquals(new StateBackendTestBase.TestPojo("u1", 1), state.value());
        keyedBackend.setCurrentKey(2);
        assertEquals(new StateBackendTestBase.TestPojo("u2", 222), state.value());
        keyedBackend.setCurrentKey(3);
        assertEquals(new StateBackendTestBase.TestPojo("u3", 3), state.value());
    } finally {
        IOUtils.closeQuietly(keyedBackend);
        keyedBackend.dispose();
    }
}

Example 23 with GenericTypeInfo

use of org.apache.flink.api.java.typeutils.GenericTypeInfo in project flink by apache.

the class TypeStringUtils method writeTypeInfo.

public static String writeTypeInfo(TypeInformation<?> typeInfo) {
    if (typeInfo.equals(Types.STRING)) {
        return VARCHAR;
    } else if (typeInfo.equals(Types.BOOLEAN)) {
        return BOOLEAN;
    } else if (typeInfo.equals(Types.BYTE)) {
        return TINYINT;
    } else if (typeInfo.equals(Types.SHORT)) {
        return SMALLINT;
    } else if (typeInfo.equals(Types.INT)) {
        return INT;
    } else if (typeInfo.equals(Types.LONG)) {
        return BIGINT;
    } else if (typeInfo.equals(Types.FLOAT)) {
        return FLOAT;
    } else if (typeInfo.equals(Types.DOUBLE)) {
        return DOUBLE;
    } else if (typeInfo.equals(Types.BIG_DEC)) {
        return DECIMAL;
    } else if (typeInfo.equals(Types.SQL_DATE) || typeInfo.equals(Types.LOCAL_DATE)) {
        // write LOCAL_DATE as "DATE" to keep compatible when using new types
        return DATE;
    } else if (typeInfo.equals(Types.SQL_TIME) || typeInfo.equals(Types.LOCAL_TIME)) {
        // write LOCAL_TIME as "TIME" to keep compatible when using new types
        return TIME;
    } else if (typeInfo.equals(Types.SQL_TIMESTAMP) || typeInfo.equals(Types.LOCAL_DATE_TIME)) {
        // write LOCAL_DATE_TIME as "TIMESTAMP" to keep compatible when using new types
        return TIMESTAMP;
    } else if (typeInfo instanceof RowTypeInfo) {
        final RowTypeInfo rt = (RowTypeInfo) typeInfo;
        final String[] fieldNames = rt.getFieldNames();
        final TypeInformation<?>[] fieldTypes = rt.getFieldTypes();
        final StringBuilder result = new StringBuilder();
        result.append(ROW);
        result.append('<');
        for (int i = 0; i < fieldNames.length; i++) {
            // escape field name if it contains delimiters
            if (containsDelimiter(fieldNames[i])) {
                result.append('`');
                result.append(fieldNames[i].replace("`", "``"));
                result.append('`');
            } else {
                result.append(fieldNames[i]);
            }
            result.append(' ');
            result.append(writeTypeInfo(fieldTypes[i]));
            if (i < fieldNames.length - 1) {
                result.append(", ");
            }
        }
        result.append('>');
        return result.toString();
    } else if (typeInfo instanceof GenericTypeInfo) {
        return ANY + '<' + typeInfo.getTypeClass().getName() + '>';
    } else if (typeInfo instanceof PojoTypeInfo) {
        // we only support very simple POJOs that only contain extracted fields
        // (not manually specified)
        TypeInformation<?> extractedPojo;
        try {
            extractedPojo = TypeExtractor.createTypeInfo(typeInfo.getTypeClass());
        } catch (InvalidTypesException e) {
            extractedPojo = null;
        }
        if (extractedPojo == null || !typeInfo.equals(extractedPojo)) {
            throw new TableException("A string representation for custom POJO types is not supported yet.");
        }
        return POJO + '<' + typeInfo.getTypeClass().getName() + '>';
    } else if (typeInfo instanceof PrimitiveArrayTypeInfo) {
        final PrimitiveArrayTypeInfo arrayTypeInfo = (PrimitiveArrayTypeInfo) typeInfo;
        return PRIMITIVE_ARRAY + '<' + writeTypeInfo(arrayTypeInfo.getComponentType()) + '>';
    } else if (typeInfo instanceof ObjectArrayTypeInfo) {
        final ObjectArrayTypeInfo arrayTypeInfo = (ObjectArrayTypeInfo) typeInfo;
        return OBJECT_ARRAY + '<' + writeTypeInfo(arrayTypeInfo.getComponentInfo()) + '>';
    } else if (typeInfo instanceof MultisetTypeInfo) {
        final MultisetTypeInfo multisetTypeInfo = (MultisetTypeInfo) typeInfo;
        return MULTISET + '<' + writeTypeInfo(multisetTypeInfo.getElementTypeInfo()) + '>';
    } else if (typeInfo instanceof MapTypeInfo) {
        final MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
        final String keyTypeInfo = writeTypeInfo(mapTypeInfo.getKeyTypeInfo());
        final String valueTypeInfo = writeTypeInfo(mapTypeInfo.getValueTypeInfo());
        return MAP + '<' + keyTypeInfo + ", " + valueTypeInfo + '>';
    } else {
        return ANY + '<' + typeInfo.getTypeClass().getName() + ", " + EncodingUtils.encodeObjectToString(typeInfo) + '>';
    }
}

Example 24 with GenericTypeInfo

use of org.apache.flink.api.java.typeutils.GenericTypeInfo in project flink by apache.

the class BinaryRowDataTest method testZeroOutPaddingGeneric.

@Test
public void testZeroOutPaddingGeneric() {
    GenericTypeInfo<MyObj> info = new GenericTypeInfo<>(MyObj.class);
    TypeSerializer<MyObj> genericSerializer = info.createSerializer(new ExecutionConfig());
    Random random = new Random();
    byte[] bytes = new byte[1024];
    BinaryRowData row = new BinaryRowData(1);
    BinaryRowWriter writer = new BinaryRowWriter(row);
    // let's random the bytes
    writer.reset();
    random.nextBytes(bytes);
    writer.writeBinary(0, bytes);
    writer.reset();
    writer.writeRawValue(0, RawValueData.fromObject(new MyObj(0, 1)), new RawValueDataSerializer<>(genericSerializer));
    writer.complete();
    int hash1 = row.hashCode();
    writer.reset();
    random.nextBytes(bytes);
    writer.writeBinary(0, bytes);
    writer.reset();
    writer.writeRawValue(0, RawValueData.fromObject(new MyObj(0, 1)), new RawValueDataSerializer<>(genericSerializer));
    writer.complete();
    int hash2 = row.hashCode();
    assertEquals(hash1, hash2);
}

Example 25 with GenericTypeInfo

use of org.apache.flink.api.java.typeutils.GenericTypeInfo in project flink by apache.

the class NestedRowDataTest method testNestedRowDataWithMultipleSegments.

@Test
public void testNestedRowDataWithMultipleSegments() {
    BinaryRowData row = getBinaryRowData();
    GenericTypeInfo<MyObj> info = new GenericTypeInfo<>(MyObj.class);
    TypeSerializer<MyObj> genericSerializer = info.createSerializer(new ExecutionConfig());
    MemorySegment[] segments = splitBytes(row.getSegments()[0].getHeapMemory(), 3);
    row.pointTo(segments, 3, row.getSizeInBytes());
    {
        RowData nestedRow = row.getRow(0, 5);
        assertEquals(nestedRow.getInt(0), 1);
        assertEquals(nestedRow.getLong(1), 5L);
        assertEquals(nestedRow.getString(2), StringData.fromString("12345678"));
        assertTrue(nestedRow.isNullAt(3));
        assertEquals(new MyObj(15, 5), nestedRow.<MyObj>getRawValue(4).toObject(genericSerializer));
    }
}