Examples with Text org.apache.drill.exec.util.Text used on opensource projects

Example 6 with Text

use of org.apache.drill.exec.util.Text in project drill by axbaretto.

the class DrillTestWrapper method addToCombinedVectorResults.

/**
 * Add to result vectors and compare batch schema against expected schema while iterating batches.
 * @param batches
 * @param  expectedSchema: the expected schema the batches should contain. Through SchemaChangeException
 *                       if encounter different batch schema.
 * @param combinedVectors: the vectors to hold the values when iterate the batches.
 *
 * @return number of batches
 * @throws SchemaChangeException
 * @throws UnsupportedEncodingException
 */
public static int addToCombinedVectorResults(Iterable<VectorAccessible> batches, BatchSchema expectedSchema, Long expectedBatchSize, Integer expectedNumBatches, Map<String, List<Object>> combinedVectors) throws SchemaChangeException, UnsupportedEncodingException {
    // TODO - this does not handle schema changes
    int numBatch = 0;
    long totalRecords = 0;
    BatchSchema schema = null;
    for (VectorAccessible loader : batches) {
        numBatch++;
        if (expectedSchema != null) {
            if (!expectedSchema.isEquivalent(loader.getSchema())) {
                throw new SchemaChangeException(String.format("Batch schema does not match expected schema\n" + "Actual schema: %s.  Expected schema : %s", loader.getSchema(), expectedSchema));
            }
        }
        if (expectedBatchSize != null) {
            RecordBatchSizer sizer = new RecordBatchSizer(loader);
            // Not checking actualSize as accounting is not correct when we do
            // split and transfer ownership across operators.
            Assert.assertTrue(sizer.netSize() <= expectedBatchSize);
        }
        // SchemaChangeException, so check/clean throws clause above.
        if (schema == null) {
            schema = loader.getSchema();
            for (MaterializedField mf : schema) {
                combinedVectors.put(SchemaPath.getSimplePath(mf.getName()).toExpr(), new ArrayList<>());
            }
        } else {
            // TODO - actually handle schema changes, this is just to get access to the SelectionVectorMode
            // of the current batch, the check for a null schema is used to only mutate the schema once
            // need to add new vectors and null fill for previous batches? distinction between null and non-existence important?
            schema = loader.getSchema();
        }
        logger.debug("reading batch with " + loader.getRecordCount() + " rows, total read so far " + totalRecords);
        totalRecords += loader.getRecordCount();
        for (VectorWrapper<?> w : loader) {
            String field = SchemaPath.getSimplePath(w.getField().getName()).toExpr();
            ValueVector[] vectors;
            if (w.isHyper()) {
                vectors = w.getValueVectors();
            } else {
                vectors = new ValueVector[] { w.getValueVector() };
            }
            SelectionVector2 sv2 = null;
            SelectionVector4 sv4 = null;
            switch(schema.getSelectionVectorMode()) {
                case TWO_BYTE:
                    sv2 = loader.getSelectionVector2();
                    break;
                case FOUR_BYTE:
                    sv4 = loader.getSelectionVector4();
                    break;
            }
            if (sv4 != null) {
                for (int j = 0; j < sv4.getCount(); j++) {
                    int complexIndex = sv4.get(j);
                    int batchIndex = complexIndex >> 16;
                    int recordIndexInBatch = complexIndex & 65535;
                    Object obj = vectors[batchIndex].getAccessor().getObject(recordIndexInBatch);
                    if (obj != null) {
                        if (obj instanceof Text) {
                            obj = obj.toString();
                        }
                    }
                    combinedVectors.get(field).add(obj);
                }
            } else {
                for (ValueVector vv : vectors) {
                    for (int j = 0; j < loader.getRecordCount(); j++) {
                        int index;
                        if (sv2 != null) {
                            index = sv2.getIndex(j);
                        } else {
                            index = j;
                        }
                        Object obj = vv.getAccessor().getObject(index);
                        if (obj != null) {
                            if (obj instanceof Text) {
                                obj = obj.toString();
                            }
                        }
                        combinedVectors.get(field).add(obj);
                    }
                }
            }
        }
    }
    if (expectedNumBatches != null) {
        // Based on how much memory is actually taken by value vectors (because of doubling stuff),
        // we have to do complex math for predicting exact number of batches.
        // Instead, check that number of batches is at least the minimum that is expected
        // and no more than twice of that.
        Assert.assertTrue(numBatch >= expectedNumBatches);
        Assert.assertTrue(numBatch <= (2 * expectedNumBatches));
    }
    return numBatch;
}

Example 7 with Text

use of org.apache.drill.exec.util.Text in project drill by axbaretto.

the class TestBuilder method mapOf.

/**
 * Convenience method to create a {@link JsonStringHashMap<String, Object> map} instance with the given key value sequence.
 *
 * Key value sequence consists of key - value pairs such that a key precedes its value. For instance:
 *
 * mapOf("name", "Adam", "age", 41) corresponds to {"name": "Adam", "age": 41} in JSON.
 */
public static JsonStringHashMap<String, Object> mapOf(Object... keyValueSequence) {
    Preconditions.checkArgument(keyValueSequence.length % 2 == 0, "Length of key value sequence must be even");
    final JsonStringHashMap<String, Object> map = new JsonStringHashMap<>();
    for (int i = 0; i < keyValueSequence.length; i += 2) {
        Object value = keyValueSequence[i + 1];
        if (value instanceof CharSequence) {
            value = new Text(value.toString());
        }
        map.put(String.class.cast(keyValueSequence[i]), value);
    }
    return map;
}

Example 8 with Text

use of org.apache.drill.exec.util.Text in project drill by apache.

the class TestDrillParquetReader method testTimeMicros.

@Test
public void testTimeMicros() throws Exception {
    testBuilder().unOrdered().sqlQuery("select * from cp.`parquet2/allTypes.parquet`").baselineColumns("int_field", "long_field", "float_field", "double_field", "string_field", "boolean_field", "time_field", "timestamp_field", "date_field", "decimal_field", "uuid_field", "fixed_field", "binary_field", "list_field", "map_field", "struct_field", "repeated_struct_field", "repeated_list_field", "repeated_map_field").baselineValues(1, 100L, 0.5F, 1.5D, "abc", true, LocalTime.of(2, 42, 42), LocalDateTime.of(1994, 4, 18, 11, 0, 0), LocalDate.of(1994, 4, 18), new BigDecimal("12.34"), new byte[16], new byte[10], "hello".getBytes(StandardCharsets.UTF_8), listOf("a", "b", "c"), mapOfObject(new Text("a"), 0.1F, new Text("b"), 0.2F), mapOf("struct_int_field", 123, "struct_string_field", "abc"), listOf(mapOf("struct_int_field", 123, "struct_string_field", "abc"), mapOf("struct_int_field", 123, "struct_string_field", "abc")), listOf(listOf("a", "b", "c"), listOf("a", "b", "c")), listOf(mapOfObject(new Text("a"), 0.1F, new Text("b"), 0.2F), mapOfObject(new Text("a"), 0.1F, new Text("b"), 0.2F))).baselineValues(null, null, null, null, null, null, null, null, null, null, null, null, null, listOf(), mapOfObject(), mapOf(), listOf(), listOf(), listOf()).go();
    testRunAndPrint(UserBitShared.QueryType.SQL, "select * from cp.`parquet2/allTypes.parquet`");
}

Example 9 with Text

use of org.apache.drill.exec.util.Text in project drill by apache.

the class TestAggregateFunctions method getBaselineRecords.

private Map<String, Object> getBaselineRecords(String tableName) throws Exception {
    QueryDataBatch result = queryBuilder().sql("select * from %s limit 1", tableName).results().get(0);
    Map<String, Object> resultingValues = new HashMap<>();
    RecordBatchLoader loader = new RecordBatchLoader(cluster.allocator());
    loader.load(result.getHeader().getDef(), result.getData());
    for (VectorWrapper<?> vectorWrapper : loader.getContainer()) {
        String fieldName = vectorWrapper.getField().getName();
        Object object = vectorWrapper.getValueVector().getAccessor().getObject(0);
        // VarCharVector returns Text instance, but baseline values should contain String value
        if (object instanceof Text) {
            object = object.toString();
        }
        resultingValues.put(fieldName, object);
    }
    loader.clear();
    result.release();
    return resultingValues;
}

Example 10 with Text

use of org.apache.drill.exec.util.Text in project drill by apache.

the class TestStringFunctions method testSplitWithNullInput.

@Test
public void testSplitWithNullInput() throws Exception {
    /*
      {"a": "aaaaaa.bbb.cc.ddddd"}
      {"a": null}
      {"a": "aa"}
     */
    try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dirTestWatcher.getRootDir(), "nullable_strings.json")))) {
        String[] fieldValue = { "\"aaaaaa.bbb.cc.ddddd\"", null, "\"aa\"" };
        for (String value : fieldValue) {
            String entry = String.format("{ \"a\": %s}\n", value);
            writer.write(entry);
        }
    }
    testBuilder().sqlQuery("select split(a, '.') wordsCount from dfs.`nullable_strings.json` t").unOrdered().baselineColumns("wordsCount").baselineValues(ImmutableList.of(new Text("aaaaaa"), new Text("bbb"), new Text("cc"), new Text("ddddd"))).baselineValues(ImmutableList.of()).baselineValues(ImmutableList.of(new Text("aa"))).go();
}