Examples with Field org.apache.arrow.vector.types.pojo.Field used on opensource projects

Example 6 with Field

use of org.apache.arrow.vector.types.pojo.Field in project twister2 by DSC-SPIDAL.

the class RowSchema method fromArrow.

public static RowSchema fromArrow(org.apache.arrow.vector.types.pojo.Schema schema) {
    List<Field> fields = schema.getFields();
    List<TField> tFields = new ArrayList<>();
    for (Field f : fields) {
        TField tField;
        if (f.getFieldType().equals(ArrowTypes.INT_FIELD_TYPE)) {
            tField = new TField(f.getName(), MessageTypes.INTEGER);
        } else if (f.getFieldType().equals(ArrowTypes.LONG_FIELD_TYPE)) {
            tField = new TField(f.getName(), MessageTypes.LONG);
        } else if (f.getFieldType().equals(ArrowTypes.SHORT_FIELD_TYPE)) {
            tField = new TField(f.getName(), MessageTypes.SHORT);
        } else if (f.getFieldType().equals(ArrowTypes.FLOAT_FIELD_TYPE)) {
            tField = new TField(f.getName(), MessageTypes.FLOAT);
        } else if (f.getFieldType().equals(ArrowTypes.DOUBLE_FIELD_TYPE)) {
            tField = new TField(f.getName(), MessageTypes.DOUBLE);
        } else if (f.getFieldType().equals(ArrowTypes.STRING_FIELD_TYPE)) {
            tField = new TField(f.getName(), MessageTypes.STRING);
        } else if (f.getFieldType().equals(ArrowTypes.BINARY_FILED_TYPE)) {
            tField = new TField(f.getName(), MessageTypes.BYTE);
        } else {
            throw new Twister2RuntimeException("Unknown type");
        }
        tFields.add(tField);
    }
    return new RowSchema(tFields);
}

Example 7 with Field

use of org.apache.arrow.vector.types.pojo.Field in project twister2 by DSC-SPIDAL.

the class BTAllToAll method execute.

@Override
public void execute(Config config, JobAPI.Job job, IWorkerController workerController, IPersistentVolume persistentVolume, IVolatileVolume volatileVolume) {
    this.jobParameters = JobParameters.build(config);
    // create a worker environment
    this.wEnv = WorkerEnvironment.init(config, job, workerController, persistentVolume, volatileVolume);
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), wEnv).withFairDistribution();
    RootAllocator rootAllocator = new RootAllocator();
    IntVector intVector = new IntVector("fist", rootAllocator);
    Float8Vector float8Vector = new Float8Vector("second", rootAllocator);
    for (int i = 0; i < 1000; i++) {
        intVector.setSafe(i, i);
        float8Vector.setSafe(i, i);
    }
    intVector.setValueCount(1000);
    float8Vector.setValueCount(1000);
    List<Field> fieldList = Arrays.asList(intVector.getField(), float8Vector.getField());
    Schema schema = new Schema(fieldList);
    Table t = new ArrowTable(schema, Arrays.asList(new FieldVector[] { intVector, float8Vector }));
    allToAll = new ArrowAllToAll(wEnv.getConfig(), wEnv.getWorkerController(), logicalPlanBuilder.getSources(), logicalPlanBuilder.getTargets(), logicalPlanBuilder.build(), wEnv.getCommunicator().nextEdge(), new ArrowReceiver(), schema, rootAllocator);
    for (int i : logicalPlanBuilder.getTargets()) {
        allToAll.insert(t, i);
    }
    for (int s : logicalPlanBuilder.getSourcesOnThisWorker()) {
        allToAll.finish(s);
    }
    while (!allToAll.isComplete()) {
    // wait
    }
}

Example 8 with Field

use of org.apache.arrow.vector.types.pojo.Field in project twister2 by DSC-SPIDAL.

the class ArrowTSetSourceExample method makeSchema.

private Schema makeSchema() {
    ImmutableList.Builder<Field> builder = ImmutableList.builder();
    builder.add(new Field("int", FieldType.nullable(new ArrowType.Int(32, true)), null));
    // builder.add(new Field("long", FieldType.nullable(new ArrowType.Int(64, true)), null));
    return new Schema(builder.build(), null);
}

Example 9 with Field

use of org.apache.arrow.vector.types.pojo.Field in project parquet-mr by apache.

the class SchemaConverter method fromParquet.

/**
 * @param type parquet type
 * @param name overrides parquet.getName)
 * @param repetition overrides parquet.getRepetition()
 * @return a type mapping from the Parquet type to an Arrow type
 */
private TypeMapping fromParquet(Type type, String name, Repetition repetition) {
    if (repetition == REPEATED) {
        // case where we have a repeated field that is not in a List/Map
        TypeMapping child = fromParquet(type, null, REQUIRED);
        Field arrowField = new Field(name, false, new ArrowType.List(), asList(child.getArrowField()));
        return new RepeatedTypeMapping(arrowField, type, child);
    }
    if (type.isPrimitive()) {
        return fromParquetPrimitive(type.asPrimitiveType(), name);
    } else {
        return fromParquetGroup(type.asGroupType(), name);
    }
}

Example 10 with Field

use of org.apache.arrow.vector.types.pojo.Field in project parquet-mr by apache.

the class SchemaConverter method fromArrow.

/**
 * Creates a Parquet Schema from an Arrow one and returns the mapping
 * @param arrowSchema the provided Arrow Schema
 * @return the mapping between the 2
 */
public SchemaMapping fromArrow(Schema arrowSchema) {
    List<Field> fields = arrowSchema.getFields();
    List<TypeMapping> parquetFields = fromArrow(fields);
    MessageType parquetType = addToBuilder(parquetFields, Types.buildMessage()).named("root");
    return new SchemaMapping(arrowSchema, parquetType, parquetFields);
}