Examples with ColumnMetadata org.apache.drill.exec.record.metadata.ColumnMetadata used on opensource projects

Example 56 with ColumnMetadata

use of org.apache.drill.exec.record.metadata.ColumnMetadata in project drill by apache.

the class ShpBatchReader method writeTimeColumn.

private void writeTimeColumn(TupleWriter rowWriter, String name, long value) {
    int index = rowWriter.tupleSchema().index(name);
    Instant instant = Instant.ofEpochMilli(value);
    if (index == -1) {
        ColumnMetadata colSchema = MetadataUtils.newScalar(name, TypeProtos.MinorType.INT, TypeProtos.DataMode.OPTIONAL);
        index = rowWriter.addColumn(colSchema);
    }
    ScalarWriter colWriter = rowWriter.scalar(index);
    colWriter.setTimestamp(instant);
}

Example 57 with ColumnMetadata

use of org.apache.drill.exec.record.metadata.ColumnMetadata in project drill by apache.

the class WriterSpec method makeWriter.

public ValueWriter makeWriter(String name, MinorType type, DataMode mode) {
    ColumnMetadata providedCol = providedSchema == null ? null : providedSchema.metadata(name);
    if (providedCol == null) {
        ColumnMetadata colSchema = MetadataUtils.newScalar(name, type, mode);
        int index = tupleWriter.addColumn(colSchema);
        return tupleWriter.scalar(index);
    } else {
        int index = tupleWriter.addColumn(providedCol);
        return conversions.converterFor(tupleWriter.scalar(index), type);
    }
}

Example 58 with ColumnMetadata

use of org.apache.drill.exec.record.metadata.ColumnMetadata in project drill by apache.

the class CompliantTextBatchReader method mergeSchemas.

private TupleMetadata mergeSchemas(TupleMetadata providedSchema, String[] fieldNames) {
    final TupleMetadata readerSchema = new TupleSchema();
    for (String fieldName : fieldNames) {
        final ColumnMetadata providedCol = providedSchema.metadata(fieldName);
        readerSchema.addColumn(providedCol == null ? textColumn(fieldName) : providedCol);
    }
    return readerSchema;
}

Example 59 with ColumnMetadata

use of org.apache.drill.exec.record.metadata.ColumnMetadata in project drill by apache.

the class LogFormatPlugin method defineOutputSchemaFromConfig.

/**
 * Define the output schema: the schema after type conversions.
 * Does not include the special columns as those are added only when
 * requested, and are always VARCHAR.
 *
 * @param capturingGroups the number of capturing groups in the regex
 * (the number that return fields)
 * @return a schema for the plugin config with names and types filled
 * in. This schema drives type conversions if no schema is provided
 * for the table
 */
private TupleMetadata defineOutputSchemaFromConfig(int capturingGroups) {
    List<String> fields = formatConfig.getFieldNames();
    for (int i = fields.size(); i < capturingGroups; i++) {
        fields.add("field_" + i);
    }
    SchemaBuilder builder = new SchemaBuilder();
    for (int i = 0; i < capturingGroups; i++) {
        makeColumn(builder, fields.get(i), i);
    }
    TupleMetadata schema = builder.buildSchema();
    // Populate the date formats, if provided.
    if (formatConfig.getSchema() == null) {
        return schema;
    }
    for (int i = 0; i < formatConfig.getSchema().size(); i++) {
        ColumnMetadata col = schema.metadata(i);
        switch(col.type()) {
            case DATE:
            case TIMESTAMP:
            case TIME:
                break;
            default:
                continue;
        }
        String format = formatConfig.getDateFormat(i);
        if (format == null) {
            continue;
        }
        col.setProperty(ColumnMetadata.FORMAT_PROP, format);
    }
    addImplicitCols(schema);
    return schema;
}

Example 60 with ColumnMetadata

use of org.apache.drill.exec.record.metadata.ColumnMetadata in project drill by apache.

the class TestSchemaProvider method testPathProviderRead.

@Test
public void testPathProviderRead() throws Exception {
    Path schemaPath = folder.newFile("schema").toPath();
    String schema = "{\n" + "  \"table\" : \"tbl\",\n" + "  \"schema\" : {\n" + "    \"columns\" : [\n" + "      {\n" + "        \"name\" : \"i\",\n" + "        \"type\" : \"INT\",\n" + "        \"mode\" : \"REQUIRED\",\n" + "        \"properties\" : {\n" + "          \"drill.default\" : \"10\"\n" + "        }\n" + "      },\n" + "      {\n" + "        \"name\" : \"a\",\n" + "        \"type\" : \"ARRAY<VARCHAR(10)>\",\n" + "        \"mode\" : \"REPEATED\",\n" + "        \"properties\" : {\n" + "          \"ck1\" : \"cv1\",\n" + "          \"ck2\" : \"cv2\"\n" + "        }\n" + "      },\n" + "      {\n" + "        \"name\" : \"t\",\n" + "        \"type\" : \"DATE\",\n" + "        \"mode\" : \"OPTIONAL\",\n" + "        \"properties\" : {\n" + "          \"drill.format\" : \"yyyy-mm-dd\"\n" + "        }\n" + "      }\n" + "    ],\n" + "    \"properties\" : {\n" + "      \"sk1\" : \"sv1\",\n" + "      \"sk2\" : \"sv2\"\n" + "    }\n" + "  }\n" + "}";
    Files.write(schemaPath, Collections.singletonList(schema));
    SchemaProvider provider = new PathSchemaProvider(new org.apache.hadoop.fs.Path(schemaPath.toUri().getPath()));
    assertTrue(provider.exists());
    SchemaContainer schemaContainer = provider.read();
    assertNotNull(schemaContainer);
    assertEquals("tbl", schemaContainer.getTable());
    TupleMetadata metadata = schemaContainer.getSchema();
    assertNotNull(metadata);
    Map<String, String> schemaProperties = new LinkedHashMap<>();
    schemaProperties.put("sk1", "sv1");
    schemaProperties.put("sk2", "sv2");
    assertEquals(schemaProperties, metadata.properties());
    assertEquals(3, metadata.size());
    ColumnMetadata i = metadata.metadata("i");
    assertEquals(TypeProtos.MinorType.INT, i.type());
    assertEquals(TypeProtos.DataMode.REQUIRED, i.mode());
    assertEquals(10, i.decodeDefaultValue());
    ColumnMetadata a = metadata.metadata("a");
    assertEquals(TypeProtos.MinorType.VARCHAR, a.type());
    assertEquals(TypeProtos.DataMode.REPEATED, a.mode());
    Map<String, String> columnProperties = new LinkedHashMap<>();
    columnProperties.put("ck1", "cv1");
    columnProperties.put("ck2", "cv2");
    assertEquals(columnProperties, a.properties());
    ColumnMetadata t = metadata.metadata("t");
    assertEquals(TypeProtos.MinorType.DATE, t.type());
    assertEquals(TypeProtos.DataMode.OPTIONAL, t.mode());
    assertEquals("yyyy-mm-dd", t.format());
    assertTrue(schemaContainer.getVersion().isUndefined());
}