use of io.prestosql.testing.MaterializedRow in project boostkit-bigdata by kunpengcompute.
the class AbstractTestHive method testGetPartialRecords.
@Test
public void testGetPartialRecords() throws Exception {
try (Transaction transaction = newTransaction()) {
ConnectorMetadata metadata = transaction.getMetadata();
ConnectorSession session = newSession();
metadata.beginQuery(session);
ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat);
List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values());
Map<String, Integer> columnIndex = indexColumns(columnHandles);
List<ConnectorSplit> splits = getAllSplits(tableHandle, transaction, session);
assertEquals(splits.size(), tablePartitionFormatPartitions.size());
for (ConnectorSplit split : splits) {
HiveSplit hiveSplit = HiveSplitWrapper.getOnlyHiveSplit(split);
List<HivePartitionKey> partitionKeys = hiveSplit.getPartitionKeys();
String ds = partitionKeys.get(0).getValue();
String fileFormat = partitionKeys.get(1).getValue();
HiveStorageFormat fileType = HiveStorageFormat.valueOf(fileFormat.toUpperCase(ENGLISH));
int dummyPartition = Integer.parseInt(partitionKeys.get(2).getValue());
long rowNumber = 0;
try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, tableHandle, columnHandles)) {
assertPageSourceType(pageSource, fileType);
MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles));
for (MaterializedRow row : result) {
rowNumber++;
assertEquals(row.getField(columnIndex.get("t_double")), 6.2 + rowNumber);
assertEquals(row.getField(columnIndex.get("ds")), ds);
assertEquals(row.getField(columnIndex.get("file_format")), fileFormat);
assertEquals(row.getField(columnIndex.get("dummy")), dummyPartition);
}
}
assertEquals(rowNumber, 100);
}
}
}
use of io.prestosql.testing.MaterializedRow in project boostkit-bigdata by kunpengcompute.
the class AbstractTestHive method readTable.
private MaterializedResult readTable(Transaction transaction, ConnectorTableHandle connectorTableHandle, List<ColumnHandle> columnHandles, ConnectorSession session, TupleDomain<ColumnHandle> tupleDomain, OptionalInt expectedSplitCount, Optional<HiveStorageFormat> expectedStorageFormat) throws Exception {
ConnectorTableHandle tableHandle = connectorTableHandle;
tableHandle = applyFilter(transaction.getMetadata(), tableHandle, new Constraint(tupleDomain));
List<ConnectorSplit> splits = getAllSplits(splitManager.getSplits(transaction.getTransactionHandle(), session, tableHandle, UNGROUPED_SCHEDULING));
if (expectedSplitCount.isPresent()) {
assertEquals(splits.size(), expectedSplitCount.getAsInt());
}
ImmutableList.Builder<MaterializedRow> allRows = ImmutableList.builder();
for (ConnectorSplit split : splits) {
try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, tableHandle, columnHandles)) {
expectedStorageFormat.ifPresent(format -> assertPageSourceType(pageSource, format));
MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles));
allRows.addAll(result.getMaterializedRows());
}
}
return new MaterializedResult(allRows.build(), getTypes(columnHandles));
}
use of io.prestosql.testing.MaterializedRow in project boostkit-bigdata by kunpengcompute.
the class AbstractTestHive method assertGetRecords.
protected void assertGetRecords(HiveStorageFormat hiveStorageFormat, ConnectorTableMetadata tableMetadata, HiveSplit hiveSplit, ConnectorPageSource pageSource, List<? extends ColumnHandle> columnHandles) throws IOException {
try {
MaterializedResult result = materializeSourceDataStream(newSession(), pageSource, getTypes(columnHandles));
assertPageSourceType(pageSource, hiveStorageFormat);
ImmutableMap<String, Integer> columnIndex = indexColumns(tableMetadata);
long rowNumber = 0;
long completedBytes = 0;
for (MaterializedRow row : result) {
try {
assertValueTypes(row, tableMetadata.getColumns());
} catch (RuntimeException e) {
throw new RuntimeException("row " + rowNumber, e);
}
rowNumber++;
Integer index;
Object value;
// STRING
index = columnIndex.get("t_string");
value = row.getField(index);
if (rowNumber % 19 == 0) {
assertNull(value);
} else if (rowNumber % 19 == 1) {
assertEquals(value, "");
} else {
assertEquals(value, "test");
}
// NUMBERS
assertEquals(row.getField(columnIndex.get("t_tinyint")), (byte) (1 + rowNumber));
assertEquals(row.getField(columnIndex.get("t_smallint")), (short) (2 + rowNumber));
assertEquals(row.getField(columnIndex.get("t_int")), (int) (3 + rowNumber));
index = columnIndex.get("t_bigint");
if ((rowNumber % 13) == 0) {
assertNull(row.getField(index));
} else {
assertEquals(row.getField(index), 4 + rowNumber);
}
assertEquals((Float) row.getField(columnIndex.get("t_float")), 5.1f + rowNumber, 0.001);
assertEquals(row.getField(columnIndex.get("t_double")), 6.2 + rowNumber);
// BOOLEAN
index = columnIndex.get("t_boolean");
if ((rowNumber % 3) == 2) {
assertNull(row.getField(index));
} else {
assertEquals(row.getField(index), (rowNumber % 3) != 0);
}
// TIMESTAMP
index = columnIndex.get("t_timestamp");
if (index != null) {
if ((rowNumber % 17) == 0) {
assertNull(row.getField(index));
} else {
SqlTimestamp expected = sqlTimestampOf(2011, 5, 6, 7, 8, 9, 123);
assertEquals(row.getField(index), expected);
}
}
// BINARY
index = columnIndex.get("t_binary");
if (index != null) {
if ((rowNumber % 23) == 0) {
assertNull(row.getField(index));
} else {
assertEquals(row.getField(index), new SqlVarbinary("test binary".getBytes(UTF_8)));
}
}
// DATE
index = columnIndex.get("t_date");
if (index != null) {
if ((rowNumber % 37) == 0) {
assertNull(row.getField(index));
} else {
SqlDate expected = new SqlDate(toIntExact(MILLISECONDS.toDays(new DateTime(2013, 8, 9, 0, 0, 0, UTC).getMillis())));
assertEquals(row.getField(index), expected);
}
}
// VARCHAR(50)
index = columnIndex.get("t_varchar");
if (index != null) {
value = row.getField(index);
if (rowNumber % 39 == 0) {
assertNull(value);
} else if (rowNumber % 39 == 1) {
// RCBINARY reads empty VARCHAR as null
if (hiveStorageFormat == RCBINARY) {
assertNull(value);
} else {
assertEquals(value, "");
}
} else {
assertEquals(value, "test varchar");
}
}
// CHAR(25)
index = columnIndex.get("t_char");
if (index != null) {
value = row.getField(index);
if ((rowNumber % 41) == 0) {
assertNull(value);
} else {
assertEquals(value, (rowNumber % 41) == 1 ? " " : "test char ");
}
}
// MAP<STRING, STRING>
index = columnIndex.get("t_map");
if (index != null) {
if ((rowNumber % 27) == 0) {
assertNull(row.getField(index));
} else {
assertEquals(row.getField(index), ImmutableMap.of("test key", "test value"));
}
}
// ARRAY<STRING>
index = columnIndex.get("t_array_string");
if (index != null) {
if ((rowNumber % 29) == 0) {
assertNull(row.getField(index));
} else {
assertEquals(row.getField(index), ImmutableList.of("abc", "xyz", "data"));
}
}
// ARRAY<TIMESTAMP>
index = columnIndex.get("t_array_timestamp");
if (index != null) {
if ((rowNumber % 43) == 0) {
assertNull(row.getField(index));
} else {
SqlTimestamp expected = sqlTimestampOf(LocalDateTime.of(2011, 5, 6, 7, 8, 9, 123_000_000));
assertEquals(row.getField(index), ImmutableList.of(expected));
}
}
// ARRAY<STRUCT<s_string: STRING, s_double:DOUBLE>>
index = columnIndex.get("t_array_struct");
if (index != null) {
if ((rowNumber % 31) == 0) {
assertNull(row.getField(index));
} else {
List<Object> expected1 = ImmutableList.of("test abc", 0.1);
List<Object> expected2 = ImmutableList.of("test xyz", 0.2);
assertEquals(row.getField(index), ImmutableList.of(expected1, expected2));
}
}
// STRUCT<s_string: STRING, s_double:DOUBLE>
index = columnIndex.get("t_struct");
if (index != null) {
if ((rowNumber % 31) == 0) {
assertNull(row.getField(index));
} else {
assertTrue(row.getField(index) instanceof List);
List<?> values = (List<?>) row.getField(index);
assertEquals(values.size(), 2);
assertEquals(values.get(0), "test abc");
assertEquals(values.get(1), 0.1);
}
}
// MAP<INT, ARRAY<STRUCT<s_string: STRING, s_double:DOUBLE>>>
index = columnIndex.get("t_complex");
if (index != null) {
if ((rowNumber % 33) == 0) {
assertNull(row.getField(index));
} else {
List<Object> expected1 = ImmutableList.of("test abc", 0.1);
List<Object> expected2 = ImmutableList.of("test xyz", 0.2);
assertEquals(row.getField(index), ImmutableMap.of(1, ImmutableList.of(expected1, expected2)));
}
}
// NEW COLUMN
assertNull(row.getField(columnIndex.get("new_column")));
long newCompletedBytes = pageSource.getCompletedBytes();
assertTrue(newCompletedBytes >= completedBytes);
assertTrue(newCompletedBytes <= hiveSplit.getLength());
completedBytes = newCompletedBytes;
}
assertTrue(completedBytes <= hiveSplit.getLength());
assertEquals(rowNumber, 100);
} finally {
pageSource.close();
}
}
use of io.prestosql.testing.MaterializedRow in project boostkit-bigdata by kunpengcompute.
the class AbstractTestHive method testBucketedTableBigintBoolean.
@SuppressWarnings("ConstantConditions")
@Test
public void testBucketedTableBigintBoolean() throws Exception {
try (Transaction transaction = newTransaction()) {
ConnectorMetadata metadata = transaction.getMetadata();
ConnectorSession session = newSession();
metadata.beginQuery(session);
ConnectorTableHandle tableHandle = getTableHandle(metadata, tableBucketedBigintBoolean);
List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values());
Map<String, Integer> columnIndex = indexColumns(columnHandles);
assertTableIsBucketed(tableHandle, transaction, session);
String testString = "test";
Long testBigint = 89L;
Boolean testBoolean = true;
ImmutableMap<ColumnHandle, NullableValue> bindings = ImmutableMap.<ColumnHandle, NullableValue>builder().put(columnHandles.get(columnIndex.get("t_string")), NullableValue.of(createUnboundedVarcharType(), utf8Slice(testString))).put(columnHandles.get(columnIndex.get("t_bigint")), NullableValue.of(BIGINT, testBigint)).put(columnHandles.get(columnIndex.get("t_boolean")), NullableValue.of(BOOLEAN, testBoolean)).build();
MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.fromFixedValues(bindings), OptionalInt.of(1), Optional.empty());
boolean rowFound = false;
for (MaterializedRow row : result) {
if (testString.equals(row.getField(columnIndex.get("t_string"))) && testBigint.equals(row.getField(columnIndex.get("t_bigint"))) && testBoolean.equals(row.getField(columnIndex.get("t_boolean")))) {
rowFound = true;
break;
}
}
assertTrue(rowFound);
}
}
use of io.prestosql.testing.MaterializedRow in project boostkit-bigdata by kunpengcompute.
the class TestHiveIntegrationSmokeTest method testCreateAndInsert.
@Test
public void testCreateAndInsert() {
Session session = getSession();
List<MaterializedRow> expected = MaterializedResult.resultBuilder(session, BIGINT, BIGINT).row(101L, 1L).row(201L, 2L).row(202L, 2L).row(301L, 3L).row(302L, 3L).build().getMaterializedRows();
transaction(getQueryRunner().getTransactionManager(), getQueryRunner().getAccessControl()).execute(session, transactionSession -> {
assertUpdate(transactionSession, "CREATE TABLE tmp_create_insert WITH (partitioned_by=array ['z']) AS " + "SELECT * from (VALUES (CAST (101 AS BIGINT), CAST (1 AS BIGINT)), (201, 2), (202, 2)) t(a, z)", 3);
assertUpdate(transactionSession, "INSERT INTO tmp_create_insert VALUES (301, 3), (302, 3)", 2);
MaterializedResult actualFromCurrentTransaction = computeActual(transactionSession, "SELECT * FROM tmp_create_insert");
assertEqualsIgnoreOrder(actualFromCurrentTransaction, expected);
});
MaterializedResult actualAfterTransaction = computeActual(session, "SELECT * FROM tmp_create_insert");
assertEqualsIgnoreOrder(actualAfterTransaction, expected);
}
Aggregations