Examples with DataGenerator org.apache.flink.streaming.api.functions.source.datagen.DataGenerator used on opensource projects

Example 1 with DataGenerator

use of org.apache.flink.streaming.api.functions.source.datagen.DataGenerator in project flink by apache.

the class DataGenTableSourceFactory method createDynamicTableSource.

@Override
public DynamicTableSource createDynamicTableSource(Context context) {
    Configuration options = new Configuration();
    context.getCatalogTable().getOptions().forEach(options::setString);
    DataType rowDataType = context.getPhysicalRowDataType();
    DataGenerator<?>[] fieldGenerators = new DataGenerator[DataType.getFieldCount(rowDataType)];
    Set<ConfigOption<?>> optionalOptions = new HashSet<>();
    List<String> fieldNames = DataType.getFieldNames(rowDataType);
    List<DataType> fieldDataTypes = DataType.getFieldDataTypes(rowDataType);
    for (int i = 0; i < fieldGenerators.length; i++) {
        String name = fieldNames.get(i);
        DataType type = fieldDataTypes.get(i);
        ConfigOption<String> kind = key(DataGenConnectorOptionsUtil.FIELDS + "." + name + "." + DataGenConnectorOptionsUtil.KIND).stringType().defaultValue(DataGenConnectorOptionsUtil.RANDOM);
        DataGeneratorContainer container = createContainer(name, type, options.get(kind), options);
        fieldGenerators[i] = container.getGenerator();
        optionalOptions.add(kind);
        optionalOptions.addAll(container.getOptions());
    }
    FactoryUtil.validateFactoryOptions(requiredOptions(), optionalOptions, options);
    Set<String> consumedOptionKeys = new HashSet<>();
    consumedOptionKeys.add(CONNECTOR.key());
    consumedOptionKeys.add(DataGenConnectorOptions.ROWS_PER_SECOND.key());
    consumedOptionKeys.add(DataGenConnectorOptions.NUMBER_OF_ROWS.key());
    optionalOptions.stream().map(ConfigOption::key).forEach(consumedOptionKeys::add);
    FactoryUtil.validateUnconsumedKeys(factoryIdentifier(), options.keySet(), consumedOptionKeys);
    String name = context.getObjectIdentifier().toString();
    return new DataGenTableSource(fieldGenerators, name, rowDataType, options.get(DataGenConnectorOptions.ROWS_PER_SECOND), options.get(DataGenConnectorOptions.NUMBER_OF_ROWS));
}

Example 2 with DataGenerator

use of org.apache.flink.streaming.api.functions.source.datagen.DataGenerator in project flink by apache.

the class RandomGeneratorVisitor method visit.

@Override
public DataGeneratorContainer visit(RowType rowType) {
    List<DataGeneratorContainer> fieldContainers = rowType.getFields().stream().map(field -> {
        String fieldName = name + "." + field.getName();
        return field.getType().accept(new RandomGeneratorVisitor(fieldName, config));
    }).collect(Collectors.toList());
    ConfigOption<?>[] options = fieldContainers.stream().flatMap(container -> container.getOptions().stream()).toArray(ConfigOption[]::new);
    DataGenerator[] generators = fieldContainers.stream().map(DataGeneratorContainer::getGenerator).toArray(DataGenerator[]::new);
    return DataGeneratorContainer.of(new RowDataGenerator(generators, rowType.getFieldNames()), options);
}