Examples with FieldSpec com.linkedin.pinot.common.data.FieldSpec used on opensource projects

Example 1 with FieldSpec

use of com.linkedin.pinot.common.data.FieldSpec in project pinot by linkedin.

the class GenerateDataCommand method buildDataGeneratorSpec.

private DataGeneratorSpec buildDataGeneratorSpec(Schema schema, List<String> columns, HashMap<String, DataType> dataTypes, HashMap<String, FieldType> fieldTypes, HashMap<String, TimeUnit> timeUnits, HashMap<String, Integer> cardinality, HashMap<String, IntRange> range) {
    for (final FieldSpec fs : schema.getAllFieldSpecs()) {
        String col = fs.getName();
        columns.add(col);
        dataTypes.put(col, fs.getDataType());
        fieldTypes.put(col, fs.getFieldType());
        switch(fs.getFieldType()) {
            case DIMENSION:
                if (cardinality.get(col) == null) {
                    cardinality.put(col, 1000);
                }
                break;
            case METRIC:
                if (!range.containsKey(col)) {
                    range.put(col, new IntRange(1, 1000));
                }
                break;
            case TIME:
                if (!range.containsKey(col)) {
                    range.put(col, new IntRange(1, 1000));
                }
                TimeFieldSpec tfs = (TimeFieldSpec) fs;
                timeUnits.put(col, tfs.getIncomingGranularitySpec().getTimeType());
                break;
            default:
                throw new RuntimeException("Invalid field type.");
        }
    }
    return new DataGeneratorSpec(columns, cardinality, range, dataTypes, fieldTypes, timeUnits, FileFormat.AVRO, _outDir, _overwrite);
}

Example 2 with FieldSpec

use of com.linkedin.pinot.common.data.FieldSpec in project pinot by linkedin.

the class DataGenerator method fetchSchema.

public Schema fetchSchema() {
    final Schema schema = new Schema();
    for (final String column : genSpec.getColumns()) {
        final FieldSpec spec = buildSpec(genSpec, column);
        schema.addField(spec);
    }
    return schema;
}

Example 3 with FieldSpec

use of com.linkedin.pinot.common.data.FieldSpec in project pinot by linkedin.

the class DataGenerator method buildSpec.

private FieldSpec buildSpec(DataGeneratorSpec genSpec, String column) {
    DataType dataType = genSpec.getDataTypesMap().get(column);
    FieldType fieldType = genSpec.getFieldTypesMap().get(column);
    FieldSpec spec;
    switch(fieldType) {
        case DIMENSION:
            spec = new DimensionFieldSpec();
            break;
        case METRIC:
            spec = new MetricFieldSpec();
            break;
        case TIME:
            spec = new TimeFieldSpec(column, dataType, genSpec.getTimeUnitMap().get(column));
            break;
        default:
            throw new RuntimeException("Invalid Field type.");
    }
    spec.setName(column);
    spec.setDataType(dataType);
    spec.setSingleValueField(true);
    return spec;
}

Example 4 with FieldSpec

use of com.linkedin.pinot.common.data.FieldSpec in project pinot by linkedin.

the class AvroWriter method getJSONSchema.

public JSONObject getJSONSchema() throws JSONException {
    final JSONObject ret = new JSONObject();
    ret.put("name", "data_gen_record");
    ret.put("type", "record");
    final JSONArray fields = new JSONArray();
    for (final FieldSpec spec : schema.getAllFieldSpecs()) {
        fields.put(spec.getDataType().toJSONSchemaFor(spec.getName()));
    }
    ret.put("fields", fields);
    return ret;
}

Example 5 with FieldSpec

use of com.linkedin.pinot.common.data.FieldSpec in project pinot by linkedin.

the class ThirdEyeUtils method createSchema.

public static Schema createSchema(CollectionSchema collectionSchema) {
    Schema schema = new Schema();
    for (DimensionSpec dimensionSpec : collectionSchema.getDimensions()) {
        FieldSpec fieldSpec = new DimensionFieldSpec();
        String dimensionName = dimensionSpec.getName();
        fieldSpec.setName(dimensionName);
        fieldSpec.setDataType(DataType.STRING);
        fieldSpec.setSingleValueField(true);
        schema.addField(dimensionName, fieldSpec);
    }
    for (MetricSpec metricSpec : collectionSchema.getMetrics()) {
        FieldSpec fieldSpec = new MetricFieldSpec();
        String metricName = metricSpec.getName();
        fieldSpec.setName(metricName);
        fieldSpec.setDataType(DataType.valueOf(metricSpec.getType().toString()));
        fieldSpec.setSingleValueField(true);
        schema.addField(metricName, fieldSpec);
    }
    TimeSpec timeSpec = collectionSchema.getTime();
    String timeFormat = timeSpec.getFormat().equals("sinceEpoch") ? TimeFormat.EPOCH.toString() : TimeFormat.SIMPLE_DATE_FORMAT.toString() + ":" + timeSpec.getFormat();
    TimeGranularitySpec incoming = new TimeGranularitySpec(DataType.LONG, timeSpec.getDataGranularity().getSize(), timeSpec.getDataGranularity().getUnit(), timeFormat, timeSpec.getColumnName());
    TimeGranularitySpec outgoing = new TimeGranularitySpec(DataType.LONG, timeSpec.getDataGranularity().getSize(), timeSpec.getDataGranularity().getUnit(), timeFormat, timeSpec.getColumnName());
    schema.addField(timeSpec.getColumnName(), new TimeFieldSpec(incoming, outgoing));
    schema.setSchemaName(collectionSchema.getCollection());
    return schema;
}