Examples with RowTypeInfo org.apache.flink.api.java.typeutils.RowTypeInfo used on opensource projects

Example 46 with RowTypeInfo

use of org.apache.flink.api.java.typeutils.RowTypeInfo in project zeppelin by apache.

the class AbstractStreamSqlJob method run.

public String run(Table table, String tableName) throws IOException {
    try {
        this.table = table;
        int parallelism = Integer.parseInt(context.getLocalProperties().getOrDefault("parallelism", defaultParallelism + ""));
        this.schema = removeTimeAttributes(flinkShims, table.getSchema());
        checkTableSchema(schema);
        LOGGER.info("ResultTable Schema: " + this.schema);
        final RowTypeInfo outputType = new RowTypeInfo(schema.getFieldTypes(), schema.getFieldNames());
        // create socket stream iterator
        TypeInformation<Tuple2<Boolean, Row>> socketType = Types.TUPLE(Types.BOOLEAN, outputType);
        TypeSerializer<Tuple2<Boolean, Row>> serializer = socketType.createSerializer(senv.getConfig());
        // pass gateway port and address such that iterator knows where to bind to
        iterator = new SocketStreamIterator<>(0, InetAddress.getByName(RemoteInterpreterUtils.findAvailableHostAddress()), serializer);
        // create table sink
        // pass binding address and port such that sink knows where to send to
        LOGGER.debug("Collecting data at address: " + iterator.getBindAddress() + ":" + iterator.getPort());
        RetractStreamTableSink collectTableSink = (RetractStreamTableSink) flinkShims.getCollectStreamTableSink(iterator.getBindAddress(), iterator.getPort(), serializer);
        // new CollectStreamTableSink(iterator.getBindAddress(), iterator.getPort(), serializer);
        collectTableSink = (RetractStreamTableSink) collectTableSink.configure(outputType.getFieldNames(), outputType.getFieldTypes());
        // workaround, otherwise it won't find the sink properly
        String originalCatalog = stenv.getCurrentCatalog();
        String originalDatabase = stenv.getCurrentDatabase();
        try {
            stenv.useCatalog("default_catalog");
            stenv.useDatabase("default_database");
            flinkShims.registerTableSink(stenv, tableName, collectTableSink);
            table.insertInto(tableName);
        } finally {
            stenv.useCatalog(originalCatalog);
            stenv.useDatabase(originalDatabase);
        }
        long delay = 1000L;
        long period = Long.parseLong(context.getLocalProperties().getOrDefault("refreshInterval", "3000"));
        refreshScheduler.scheduleAtFixedRate(new RefreshTask(context), delay, period, MILLISECONDS);
        ResultRetrievalThread retrievalThread = new ResultRetrievalThread(refreshScheduler);
        retrievalThread.start();
        LOGGER.info("Run job: " + tableName + ", parallelism: " + parallelism);
        String jobName = context.getStringLocalProperty("jobName", tableName);
        stenv.execute(jobName);
        LOGGER.info("Flink Job is finished, jobName: " + jobName);
        // wait for retrieve thread consume all data
        LOGGER.info("Waiting for retrieve thread to be done");
        retrievalThread.join();
        refresh(context);
        String finalResult = buildResult();
        LOGGER.info("Final Result: " + finalResult);
        return finalResult;
    } catch (Exception e) {
        LOGGER.error("Fail to run stream sql job", e);
        throw new IOException("Fail to run stream sql job", e);
    } finally {
        refreshScheduler.shutdownNow();
    }
}

Example 47 with RowTypeInfo

use of org.apache.flink.api.java.typeutils.RowTypeInfo in project flink by apache.

the class FieldAccessorTest method testRowTypeInfo.

/**
 * Validates that no ClassCastException happens should not fail e.g. like in FLINK-8255.
 */
@Test(expected = CompositeType.InvalidFieldReferenceException.class)
public void testRowTypeInfo() {
    TypeInformation<?>[] typeList = new TypeInformation<?>[] { new RowTypeInfo(BasicTypeInfo.SHORT_TYPE_INFO, BasicTypeInfo.BIG_DEC_TYPE_INFO) };
    String[] fieldNames = new String[] { "row" };
    RowTypeInfo rowTypeInfo = new RowTypeInfo(typeList, fieldNames);
    FieldAccessor f = FieldAccessorFactory.getAccessor(rowTypeInfo, "row.0", null);
}

Example 48 with RowTypeInfo

use of org.apache.flink.api.java.typeutils.RowTypeInfo in project flink by apache.

the class TypeStringUtils method writeTypeInfo.

public static String writeTypeInfo(TypeInformation<?> typeInfo) {
    if (typeInfo.equals(Types.STRING)) {
        return VARCHAR;
    } else if (typeInfo.equals(Types.BOOLEAN)) {
        return BOOLEAN;
    } else if (typeInfo.equals(Types.BYTE)) {
        return TINYINT;
    } else if (typeInfo.equals(Types.SHORT)) {
        return SMALLINT;
    } else if (typeInfo.equals(Types.INT)) {
        return INT;
    } else if (typeInfo.equals(Types.LONG)) {
        return BIGINT;
    } else if (typeInfo.equals(Types.FLOAT)) {
        return FLOAT;
    } else if (typeInfo.equals(Types.DOUBLE)) {
        return DOUBLE;
    } else if (typeInfo.equals(Types.BIG_DEC)) {
        return DECIMAL;
    } else if (typeInfo.equals(Types.SQL_DATE) || typeInfo.equals(Types.LOCAL_DATE)) {
        // write LOCAL_DATE as "DATE" to keep compatible when using new types
        return DATE;
    } else if (typeInfo.equals(Types.SQL_TIME) || typeInfo.equals(Types.LOCAL_TIME)) {
        // write LOCAL_TIME as "TIME" to keep compatible when using new types
        return TIME;
    } else if (typeInfo.equals(Types.SQL_TIMESTAMP) || typeInfo.equals(Types.LOCAL_DATE_TIME)) {
        // write LOCAL_DATE_TIME as "TIMESTAMP" to keep compatible when using new types
        return TIMESTAMP;
    } else if (typeInfo instanceof RowTypeInfo) {
        final RowTypeInfo rt = (RowTypeInfo) typeInfo;
        final String[] fieldNames = rt.getFieldNames();
        final TypeInformation<?>[] fieldTypes = rt.getFieldTypes();
        final StringBuilder result = new StringBuilder();
        result.append(ROW);
        result.append('<');
        for (int i = 0; i < fieldNames.length; i++) {
            // escape field name if it contains delimiters
            if (containsDelimiter(fieldNames[i])) {
                result.append('`');
                result.append(fieldNames[i].replace("`", "``"));
                result.append('`');
            } else {
                result.append(fieldNames[i]);
            }
            result.append(' ');
            result.append(writeTypeInfo(fieldTypes[i]));
            if (i < fieldNames.length - 1) {
                result.append(", ");
            }
        }
        result.append('>');
        return result.toString();
    } else if (typeInfo instanceof GenericTypeInfo) {
        return ANY + '<' + typeInfo.getTypeClass().getName() + '>';
    } else if (typeInfo instanceof PojoTypeInfo) {
        // we only support very simple POJOs that only contain extracted fields
        // (not manually specified)
        TypeInformation<?> extractedPojo;
        try {
            extractedPojo = TypeExtractor.createTypeInfo(typeInfo.getTypeClass());
        } catch (InvalidTypesException e) {
            extractedPojo = null;
        }
        if (extractedPojo == null || !typeInfo.equals(extractedPojo)) {
            throw new TableException("A string representation for custom POJO types is not supported yet.");
        }
        return POJO + '<' + typeInfo.getTypeClass().getName() + '>';
    } else if (typeInfo instanceof PrimitiveArrayTypeInfo) {
        final PrimitiveArrayTypeInfo arrayTypeInfo = (PrimitiveArrayTypeInfo) typeInfo;
        return PRIMITIVE_ARRAY + '<' + writeTypeInfo(arrayTypeInfo.getComponentType()) + '>';
    } else if (typeInfo instanceof ObjectArrayTypeInfo) {
        final ObjectArrayTypeInfo arrayTypeInfo = (ObjectArrayTypeInfo) typeInfo;
        return OBJECT_ARRAY + '<' + writeTypeInfo(arrayTypeInfo.getComponentInfo()) + '>';
    } else if (typeInfo instanceof MultisetTypeInfo) {
        final MultisetTypeInfo multisetTypeInfo = (MultisetTypeInfo) typeInfo;
        return MULTISET + '<' + writeTypeInfo(multisetTypeInfo.getElementTypeInfo()) + '>';
    } else if (typeInfo instanceof MapTypeInfo) {
        final MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
        final String keyTypeInfo = writeTypeInfo(mapTypeInfo.getKeyTypeInfo());
        final String valueTypeInfo = writeTypeInfo(mapTypeInfo.getValueTypeInfo());
        return MAP + '<' + keyTypeInfo + ", " + valueTypeInfo + '>';
    } else {
        return ANY + '<' + typeInfo.getTypeClass().getName() + ", " + EncodingUtils.encodeObjectToString(typeInfo) + '>';
    }
}

Example 49 with RowTypeInfo

use of org.apache.flink.api.java.typeutils.RowTypeInfo in project flink by apache.

the class LogicalTypeChecksTest method testIsCompositeTypeLegacyCompositeType.

@Test
public void testIsCompositeTypeLegacyCompositeType() {
    DataType dataType = TypeConversions.fromLegacyInfoToDataType(new RowTypeInfo(Types.STRING, Types.INT));
    assertThat(LogicalTypeChecks.isCompositeType(dataType.getLogicalType())).isTrue();
}

Example 50 with RowTypeInfo

use of org.apache.flink.api.java.typeutils.RowTypeInfo in project flink by apache.

the class TypeInfoDataTypeConverter method fromDataTypeToTypeInfo.

public static TypeInformation<?> fromDataTypeToTypeInfo(DataType dataType) {
    Class<?> clazz = dataType.getConversionClass();
    if (clazz.isPrimitive()) {
        final TypeInformation<?> foundTypeInfo = primitiveDataTypeTypeInfoMap.get(clazz.getName());
        if (foundTypeInfo != null) {
            return foundTypeInfo;
        }
    }
    LogicalType logicalType = fromDataTypeToLogicalType(dataType);
    switch(logicalType.getTypeRoot()) {
        case TIMESTAMP_WITHOUT_TIME_ZONE:
            TimestampType timestampType = (TimestampType) logicalType;
            int precision = timestampType.getPrecision();
            if (timestampType.getKind() == TimestampKind.REGULAR) {
                return clazz == TimestampData.class ? new TimestampDataTypeInfo(precision) : (clazz == LocalDateTime.class ? ((3 == precision) ? Types.LOCAL_DATE_TIME : new LegacyLocalDateTimeTypeInfo(precision)) : ((3 == precision) ? Types.SQL_TIMESTAMP : new LegacyTimestampTypeInfo(precision)));
            } else {
                return TypeConversions.fromDataTypeToLegacyInfo(dataType);
            }
        case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
            LocalZonedTimestampType lzTs = (LocalZonedTimestampType) logicalType;
            int precisionLzTs = lzTs.getPrecision();
            if (lzTs.getKind() == TimestampKind.REGULAR) {
                return clazz == TimestampData.class ? new TimestampDataTypeInfo(precisionLzTs) : (clazz == Instant.class ? ((3 == precisionLzTs) ? Types.INSTANT : new LegacyInstantTypeInfo(precisionLzTs)) : TypeConversions.fromDataTypeToLegacyInfo(dataType));
            } else {
                return TypeConversions.fromDataTypeToLegacyInfo(dataType);
            }
        case DECIMAL:
            DecimalType decimalType = (DecimalType) logicalType;
            return clazz == DecimalData.class ? new DecimalDataTypeInfo(decimalType.getPrecision(), decimalType.getScale()) : new BigDecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale());
        case CHAR:
        case // ignore precision
        VARCHAR:
            return clazz == StringData.class ? StringDataTypeInfo.INSTANCE : BasicTypeInfo.STRING_TYPE_INFO;
        case BINARY:
        case // ignore precision
        VARBINARY:
            return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
        case INTERVAL_YEAR_MONTH:
            return TimeIntervalTypeInfo.INTERVAL_MONTHS;
        case INTERVAL_DAY_TIME:
            return TimeIntervalTypeInfo.INTERVAL_MILLIS;
        case ARRAY:
            if (dataType instanceof CollectionDataType && !isPrimitive(((CollectionDataType) dataType).getElementDataType().getLogicalType())) {
                return ObjectArrayTypeInfo.getInfoFor(fromDataTypeToTypeInfo(((CollectionDataType) dataType).getElementDataType()));
            } else {
                return TypeConversions.fromDataTypeToLegacyInfo(dataType);
            }
        case MAP:
            KeyValueDataType mapType = (KeyValueDataType) dataType;
            return new MapTypeInfo(fromDataTypeToTypeInfo(mapType.getKeyDataType()), fromDataTypeToTypeInfo(mapType.getValueDataType()));
        case MULTISET:
            return MultisetTypeInfo.getInfoFor(fromDataTypeToTypeInfo(((CollectionDataType) dataType).getElementDataType()));
        case ROW:
            if (RowData.class.isAssignableFrom(dataType.getConversionClass())) {
                return InternalTypeInfo.of((RowType) fromDataTypeToLogicalType(dataType));
            } else if (Row.class == dataType.getConversionClass()) {
                RowType logicalRowType = (RowType) logicalType;
                return new RowTypeInfo(dataType.getChildren().stream().map(TypeInfoDataTypeConverter::fromDataTypeToTypeInfo).toArray(TypeInformation[]::new), logicalRowType.getFieldNames().toArray(new String[0]));
            } else {
                return TypeConversions.fromDataTypeToLegacyInfo(dataType);
            }
        case RAW:
            if (logicalType instanceof RawType) {
                return ExternalTypeInfo.of(dataType);
            }
            return TypeConversions.fromDataTypeToLegacyInfo(dataType);
        default:
            return TypeConversions.fromDataTypeToLegacyInfo(dataType);
    }
}