Search in sources :

Example 1 with SESSION

use of io.trino.testing.TestingConnectorSession.SESSION in project trino by trinodb.

the class TestTransactionLogAccess method testSnapshotsAreConsistent.

@Test
public void testSnapshotsAreConsistent() throws Exception {
    String tableName = "person";
    File tempDir = Files.createTempDir();
    File tableDir = new File(tempDir, tableName);
    File transactionLogDir = new File(tableDir, TRANSACTION_LOG_DIRECTORY);
    transactionLogDir.mkdirs();
    File resourceDir = new File(getClass().getClassLoader().getResource("databricks/person/_delta_log").toURI());
    copyTransactionLogEntry(0, 12, resourceDir, transactionLogDir);
    Files.copy(new File(resourceDir, LAST_CHECKPOINT_FILENAME), new File(transactionLogDir, LAST_CHECKPOINT_FILENAME));
    setupTransactionLogAccess(tableName, new Path(tableDir.toURI()));
    List<AddFileEntry> expectedDataFiles = transactionLogAccess.getActiveFiles(tableSnapshot, SESSION);
    copyTransactionLogEntry(12, 14, resourceDir, transactionLogDir);
    Set<String> newDataFiles = ImmutableSet.of("age=28/part-00000-40dd1707-1d42-4328-a59a-21f5c945fe60.c000.snappy.parquet", "age=29/part-00000-3794c463-cb0c-4beb-8d07-7cc1e3b5920f.c000.snappy.parquet");
    TableSnapshot updatedTableSnapshot = transactionLogAccess.loadSnapshot(new SchemaTableName("schema", tableName), new Path(tableDir.toURI()), SESSION);
    List<AddFileEntry> allDataFiles = transactionLogAccess.getActiveFiles(updatedTableSnapshot, SESSION);
    List<AddFileEntry> dataFilesWithFixedVersion = transactionLogAccess.getActiveFiles(tableSnapshot, SESSION);
    for (String newFilePath : newDataFiles) {
        assertTrue(allDataFiles.stream().anyMatch(entry -> entry.getPath().equals(newFilePath)));
        assertTrue(dataFilesWithFixedVersion.stream().noneMatch(entry -> entry.getPath().equals(newFilePath)));
    }
    assertEquals(expectedDataFiles.size(), dataFilesWithFixedVersion.size());
    List<ColumnMetadata> columns = extractSchema(transactionLogAccess.getMetadataEntry(tableSnapshot, SESSION).get(), TESTING_TYPE_MANAGER);
    for (int i = 0; i < expectedDataFiles.size(); i++) {
        AddFileEntry expected = expectedDataFiles.get(i);
        AddFileEntry actual = dataFilesWithFixedVersion.get(i);
        assertEquals(expected.getPath(), actual.getPath());
        assertEquals(expected.getPartitionValues(), actual.getPartitionValues());
        assertEquals(expected.getSize(), actual.getSize());
        assertEquals(expected.getModificationTime(), actual.getModificationTime());
        assertEquals(expected.isDataChange(), actual.isDataChange());
        assertEquals(expected.getTags(), actual.getTags());
        assertTrue(expected.getStats().isPresent());
        assertTrue(actual.getStats().isPresent());
        for (ColumnMetadata column : columns) {
            DeltaLakeColumnHandle columnHandle = new DeltaLakeColumnHandle(column.getName(), column.getType(), REGULAR);
            assertEquals(expected.getStats().get().getMinColumnValue(columnHandle), actual.getStats().get().getMinColumnValue(columnHandle));
            assertEquals(expected.getStats().get().getMaxColumnValue(columnHandle), actual.getStats().get().getMaxColumnValue(columnHandle));
            assertEquals(expected.getStats().get().getNullCount(columnHandle.getName()), actual.getStats().get().getNullCount(columnHandle.getName()));
            assertEquals(expected.getStats().get().getNumRecords(), actual.getStats().get().getNumRecords());
        }
    }
}
Also used : Path(org.apache.hadoop.fs.Path) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) TableSnapshot(io.trino.plugin.deltalake.transactionlog.TableSnapshot) Test(org.testng.annotations.Test) DeltaLakeSchemaSupport.extractSchema(io.trino.plugin.deltalake.transactionlog.DeltaLakeSchemaSupport.extractSchema) AddFileEntry(io.trino.plugin.deltalake.transactionlog.AddFileEntry) RemoveFileEntry(io.trino.plugin.deltalake.transactionlog.RemoveFileEntry) NoHdfsAuthentication(io.trino.plugin.hive.authentication.NoHdfsAuthentication) BigDecimal(java.math.BigDecimal) CheckpointSchemaManager(io.trino.plugin.deltalake.transactionlog.checkpoint.CheckpointSchemaManager) Map(java.util.Map) Sets.union(com.google.common.collect.Sets.union) Path(org.apache.hadoop.fs.Path) HiveHdfsConfiguration(io.trino.plugin.hive.HiveHdfsConfiguration) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) LAST_CHECKPOINT_FILENAME(io.trino.plugin.deltalake.transactionlog.TransactionLogParser.LAST_CHECKPOINT_FILENAME) Assert.assertFalse(org.testng.Assert.assertFalse) Assert.assertEquals(io.trino.testing.assertions.Assert.assertEquals) TRANSACTION_LOG_DIRECTORY(io.trino.plugin.deltalake.transactionlog.TransactionLogUtil.TRANSACTION_LOG_DIRECTORY) ImmutableSet(com.google.common.collect.ImmutableSet) FileFormatDataSourceStats(io.trino.plugin.hive.FileFormatDataSourceStats) HdfsEnvironment(io.trino.plugin.hive.HdfsEnvironment) ImmutableMap(com.google.common.collect.ImmutableMap) UTC_KEY(io.trino.spi.type.TimeZoneKey.UTC_KEY) SESSION(io.trino.testing.TestingConnectorSession.SESSION) MetadataEntry(io.trino.plugin.deltalake.transactionlog.MetadataEntry) Set(java.util.Set) Collectors(java.util.stream.Collectors) SchemaTableName(io.trino.spi.connector.SchemaTableName) String.format(java.lang.String.format) List(java.util.List) Stream(java.util.stream.Stream) HdfsConfig(io.trino.plugin.hive.HdfsConfig) LocalDate(java.time.LocalDate) HdfsConfigurationInitializer(io.trino.plugin.hive.HdfsConfigurationInitializer) Decimals(io.trino.spi.type.Decimals) UTC(java.time.ZoneOffset.UTC) Optional(java.util.Optional) ParquetReaderConfig(io.trino.plugin.hive.parquet.ParquetReaderConfig) REGULAR(io.trino.plugin.deltalake.DeltaLakeColumnType.REGULAR) DeltaLakeFileStatistics(io.trino.plugin.deltalake.transactionlog.statistics.DeltaLakeFileStatistics) DataProvider(org.testng.annotations.DataProvider) ColumnMetadata(io.trino.spi.connector.ColumnMetadata) LocalDateTime(java.time.LocalDateTime) DateTimeEncoding(io.trino.spi.type.DateTimeEncoding) ImmutableList(com.google.common.collect.ImmutableList) Files(com.google.common.io.Files) IntegerType(io.trino.spi.type.IntegerType) TESTING_TYPE_MANAGER(io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER) IOException(java.io.IOException) HdfsConfiguration(io.trino.plugin.hive.HdfsConfiguration) TupleDomain(io.trino.spi.predicate.TupleDomain) TestingConnectorContext(io.trino.testing.TestingConnectorContext) File(java.io.File) Assertions.assertEqualsIgnoreOrder(io.airlift.testing.Assertions.assertEqualsIgnoreOrder) ProtocolEntry(io.trino.plugin.deltalake.transactionlog.ProtocolEntry) Assert.assertTrue(org.testng.Assert.assertTrue) CommitInfoEntry(io.trino.plugin.deltalake.transactionlog.CommitInfoEntry) TypeManager(io.trino.spi.type.TypeManager) ColumnMetadata(io.trino.spi.connector.ColumnMetadata) SchemaTableName(io.trino.spi.connector.SchemaTableName) AddFileEntry(io.trino.plugin.deltalake.transactionlog.AddFileEntry) TableSnapshot(io.trino.plugin.deltalake.transactionlog.TableSnapshot) File(java.io.File) Test(org.testng.annotations.Test)

Example 2 with SESSION

use of io.trino.testing.TestingConnectorSession.SESSION in project trino by trinodb.

the class TestDefaultJdbcMetadata method testAggregationPushdownForTableHandle.

@Test
public void testAggregationPushdownForTableHandle() {
    ConnectorSession session = TestingConnectorSession.builder().setPropertyMetadata(new JdbcMetadataSessionProperties(new JdbcMetadataConfig().setAggregationPushdownEnabled(true), Optional.empty()).getSessionProperties()).build();
    ColumnHandle groupByColumn = metadata.getColumnHandles(session, tableHandle).get("text");
    Function<ConnectorTableHandle, Optional<AggregationApplicationResult<ConnectorTableHandle>>> applyAggregation = handle -> metadata.applyAggregation(session, handle, ImmutableList.of(new AggregateFunction("count", BIGINT, List.of(), List.of(), false, Optional.empty())), ImmutableMap.of(), ImmutableList.of(ImmutableList.of(groupByColumn)));
    ConnectorTableHandle baseTableHandle = metadata.getTableHandle(session, new SchemaTableName("example", "numbers"));
    Optional<AggregationApplicationResult<ConnectorTableHandle>> aggregationResult = applyAggregation.apply(baseTableHandle);
    assertThat(aggregationResult).isPresent();
    SchemaTableName noAggregationPushdownTable = new SchemaTableName("example", "no_aggregation_pushdown");
    metadata.createTable(SESSION, new ConnectorTableMetadata(noAggregationPushdownTable, ImmutableList.of(new ColumnMetadata("text", VARCHAR))), false);
    ConnectorTableHandle noAggregationPushdownTableHandle = metadata.getTableHandle(session, noAggregationPushdownTable);
    aggregationResult = applyAggregation.apply(noAggregationPushdownTableHandle);
    assertThat(aggregationResult).isEmpty();
}
Also used : Constraint(io.trino.spi.connector.Constraint) Assert.assertNull(org.testng.Assert.assertNull) ColumnMetadata(io.trino.spi.connector.ColumnMetadata) AggregateFunction(io.trino.spi.connector.AggregateFunction) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) Assert.assertEquals(org.testng.Assert.assertEquals) ConnectorTableMetadata(io.trino.spi.connector.ConnectorTableMetadata) Test(org.testng.annotations.Test) AfterMethod(org.testng.annotations.AfterMethod) TrinoExceptionAssert.assertTrinoExceptionThrownBy(io.trino.testing.assertions.TrinoExceptionAssert.assertTrinoExceptionThrownBy) Function(java.util.function.Function) VARCHAR(io.trino.spi.type.VarcharType.VARCHAR) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) ImmutableList(com.google.common.collect.ImmutableList) Assertions.assertThatThrownBy(org.assertj.core.api.Assertions.assertThatThrownBy) ConnectorTableHandle(io.trino.spi.connector.ConnectorTableHandle) Map(java.util.Map) ColumnHandle(io.trino.spi.connector.ColumnHandle) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) NOT_FOUND(io.trino.spi.StandardErrorCode.NOT_FOUND) JDBC_VARCHAR(io.trino.plugin.jdbc.TestingJdbcTypeHandle.JDBC_VARCHAR) Collections.emptyMap(java.util.Collections.emptyMap) ImmutableSet(com.google.common.collect.ImmutableSet) ConstraintApplicationResult(io.trino.spi.connector.ConstraintApplicationResult) ImmutableMap(com.google.common.collect.ImmutableMap) Domain(io.trino.spi.predicate.Domain) BeforeMethod(org.testng.annotations.BeforeMethod) JDBC_BIGINT(io.trino.plugin.jdbc.TestingJdbcTypeHandle.JDBC_BIGINT) SESSION(io.trino.testing.TestingConnectorSession.SESSION) ConnectorSession(io.trino.spi.connector.ConnectorSession) TupleDomain(io.trino.spi.predicate.TupleDomain) SchemaTableName(io.trino.spi.connector.SchemaTableName) AggregationApplicationResult(io.trino.spi.connector.AggregationApplicationResult) TestingConnectorSession(io.trino.testing.TestingConnectorSession) List(java.util.List) BIGINT(io.trino.spi.type.BigintType.BIGINT) Optional(java.util.Optional) Assert.assertTrue(org.testng.Assert.assertTrue) VarcharType.createVarcharType(io.trino.spi.type.VarcharType.createVarcharType) ColumnHandle(io.trino.spi.connector.ColumnHandle) ColumnMetadata(io.trino.spi.connector.ColumnMetadata) Optional(java.util.Optional) SchemaTableName(io.trino.spi.connector.SchemaTableName) ConnectorTableHandle(io.trino.spi.connector.ConnectorTableHandle) AggregateFunction(io.trino.spi.connector.AggregateFunction) ConnectorSession(io.trino.spi.connector.ConnectorSession) TestingConnectorSession(io.trino.testing.TestingConnectorSession) AggregationApplicationResult(io.trino.spi.connector.AggregationApplicationResult) ConnectorTableMetadata(io.trino.spi.connector.ConnectorTableMetadata) Test(org.testng.annotations.Test)

Example 3 with SESSION

use of io.trino.testing.TestingConnectorSession.SESSION in project trino by trinodb.

the class TestLocalFileRecordSet method assertData.

private static void assertData(LocalFileTables localFileTables, LocalFileMetadata metadata) {
    LocalFileTableHandle tableHandle = new LocalFileTableHandle(getSchemaTableName(), OptionalInt.of(0), OptionalInt.of(-1));
    List<LocalFileColumnHandle> columnHandles = metadata.getColumnHandles(SESSION, tableHandle).values().stream().map(column -> (LocalFileColumnHandle) column).collect(Collectors.toList());
    LocalFileSplit split = new LocalFileSplit(address);
    RecordSet recordSet = new LocalFileRecordSet(localFileTables, split, tableHandle, columnHandles);
    RecordCursor cursor = recordSet.cursor();
    for (int i = 0; i < columnHandles.size(); i++) {
        assertEquals(cursor.getType(i), columnHandles.get(i).getColumnType());
    }
    // test one row
    assertTrue(cursor.advanceNextPosition());
    assertEquals(cursor.getSlice(0).toStringUtf8(), address.toString());
    assertEquals(cursor.getSlice(2).toStringUtf8(), "127.0.0.1");
    assertEquals(cursor.getSlice(3).toStringUtf8(), "POST");
    assertEquals(cursor.getSlice(4).toStringUtf8(), "/v1/memory");
    assertTrue(cursor.isNull(5));
    assertTrue(cursor.isNull(6));
    assertEquals(cursor.getLong(7), 200);
    assertEquals(cursor.getLong(8), 0);
    assertEquals(cursor.getLong(9), 1000);
    assertEquals(cursor.getLong(10), 10);
    assertTrue(cursor.isNull(11));
    assertTrue(cursor.advanceNextPosition());
    assertEquals(cursor.getSlice(0).toStringUtf8(), address.toString());
    assertEquals(cursor.getSlice(2).toStringUtf8(), "127.0.0.1");
    assertEquals(cursor.getSlice(3).toStringUtf8(), "GET");
    assertEquals(cursor.getSlice(4).toStringUtf8(), "/v1/service/presto/general");
    assertEquals(cursor.getSlice(5).toStringUtf8(), "foo");
    assertEquals(cursor.getSlice(6).toStringUtf8(), "ffffffff-ffff-ffff-ffff-ffffffffffff");
    assertEquals(cursor.getLong(7), 200);
    assertEquals(cursor.getLong(8), 0);
    assertEquals(cursor.getLong(9), 37);
    assertEquals(cursor.getLong(10), 1094);
    assertEquals(cursor.getSlice(11).toStringUtf8(), "a7229d56-5cbd-4e23-81ff-312ba6be0f12");
}
Also used : List(java.util.List) RecordCursor(io.trino.spi.connector.RecordCursor) HttpRequestLogTable.getSchemaTableName(io.trino.plugin.localfile.LocalFileTables.HttpRequestLogTable.getSchemaTableName) SESSION(io.trino.testing.TestingConnectorSession.SESSION) RecordSet(io.trino.spi.connector.RecordSet) Assert.assertEquals(org.testng.Assert.assertEquals) Assert.assertTrue(org.testng.Assert.assertTrue) Test(org.testng.annotations.Test) OptionalInt(java.util.OptionalInt) Collectors(java.util.stream.Collectors) HostAddress(io.trino.spi.HostAddress) RecordCursor(io.trino.spi.connector.RecordCursor) RecordSet(io.trino.spi.connector.RecordSet)

Aggregations

SESSION (io.trino.testing.TestingConnectorSession.SESSION)3 List (java.util.List)3 Assert.assertTrue (org.testng.Assert.assertTrue)3 Test (org.testng.annotations.Test)3 ImmutableList (com.google.common.collect.ImmutableList)2 ImmutableMap (com.google.common.collect.ImmutableMap)2 ImmutableSet (com.google.common.collect.ImmutableSet)2 Slices.utf8Slice (io.airlift.slice.Slices.utf8Slice)2 ColumnMetadata (io.trino.spi.connector.ColumnMetadata)2 SchemaTableName (io.trino.spi.connector.SchemaTableName)2 TupleDomain (io.trino.spi.predicate.TupleDomain)2 Map (java.util.Map)2 Optional (java.util.Optional)2 Assertions.assertThat (org.assertj.core.api.Assertions.assertThat)2 Sets.union (com.google.common.collect.Sets.union)1 Files (com.google.common.io.Files)1 Assertions.assertEqualsIgnoreOrder (io.airlift.testing.Assertions.assertEqualsIgnoreOrder)1 REGULAR (io.trino.plugin.deltalake.DeltaLakeColumnType.REGULAR)1 AddFileEntry (io.trino.plugin.deltalake.transactionlog.AddFileEntry)1 CommitInfoEntry (io.trino.plugin.deltalake.transactionlog.CommitInfoEntry)1