Examples with NodeVersion io.trino.plugin.hive.NodeVersion used on opensource projects

Example 1 with NodeVersion

use of io.trino.plugin.hive.NodeVersion in project trino by trinodb.

the class TestDeltaLakeMetastoreStatistics method setupMetastore.

@BeforeClass
public void setupMetastore() {
    TestingConnectorContext context = new TestingConnectorContext();
    TypeManager typeManager = context.getTypeManager();
    CheckpointSchemaManager checkpointSchemaManager = new CheckpointSchemaManager(typeManager);
    HdfsConfig hdfsConfig = new HdfsConfig();
    HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hdfsConfig), ImmutableSet.of());
    HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hdfsConfig, new NoHdfsAuthentication());
    FileFormatDataSourceStats fileFormatDataSourceStats = new FileFormatDataSourceStats();
    TransactionLogAccess transactionLogAccess = new TransactionLogAccess(typeManager, checkpointSchemaManager, new DeltaLakeConfig(), fileFormatDataSourceStats, hdfsEnvironment, new ParquetReaderConfig(), new DeltaLakeConfig());
    File tmpDir = Files.createTempDir();
    File metastoreDir = new File(tmpDir, "metastore");
    hiveMetastore = new FileHiveMetastore(new NodeVersion("test_version"), hdfsEnvironment, new MetastoreConfig(), new FileHiveMetastoreConfig().setCatalogDirectory(metastoreDir.toURI().toString()).setMetastoreUser("test"));
    hiveMetastore.createDatabase(new Database("db_name", Optional.empty(), Optional.of("test"), Optional.of(PrincipalType.USER), Optional.empty(), ImmutableMap.of()));
    CachingDeltaLakeStatisticsAccess statistics = new CachingDeltaLakeStatisticsAccess(new MetaDirStatisticsAccess(hdfsEnvironment, new JsonCodecFactory().jsonCodec(DeltaLakeStatistics.class)));
    deltaLakeMetastore = new HiveMetastoreBackedDeltaLakeMetastore(hiveMetastore, transactionLogAccess, typeManager, statistics);
}

Example 2 with NodeVersion

use of io.trino.plugin.hive.NodeVersion in project trino by trinodb.

the class TestOrcPredicates method testOrcPredicates.

private void testOrcPredicates(ConnectorSession session) throws Exception {
    List<TestColumn> columnsToWrite = ImmutableList.of(columnPrimitiveInteger, columnStruct, columnPrimitiveBigInt);
    File file = File.createTempFile("test", "orc_predicate");
    file.delete();
    try {
        // Write data
        OrcFileWriterFactory writerFactory = new OrcFileWriterFactory(HDFS_ENVIRONMENT, TESTING_TYPE_MANAGER, new NodeVersion("test"), STATS, new OrcWriterOptions());
        FileSplit split = createTestFileTrino(file.getAbsolutePath(), ORC, HiveCompressionCodec.NONE, columnsToWrite, session, NUM_ROWS, writerFactory);
        TupleDomain<TestColumn> testingPredicate;
        // Verify predicates on base column
        List<TestColumn> columnsToRead = columnsToWrite;
        // All rows returned for a satisfying predicate
        testingPredicate = TupleDomain.withColumnDomains(ImmutableMap.of(columnPrimitiveBigInt, Domain.singleValue(BIGINT, 6L)));
        assertFilteredRows(testingPredicate, columnsToRead, session, split, NUM_ROWS);
        // No rows returned for a mismatched predicate
        testingPredicate = TupleDomain.withColumnDomains(ImmutableMap.of(columnPrimitiveBigInt, Domain.singleValue(BIGINT, 1L)));
        assertFilteredRows(testingPredicate, columnsToRead, session, split, 0);
        // Verify predicates on projected column
        TestColumn projectedColumn = new TestColumn(columnStruct.getBaseName(), columnStruct.getBaseObjectInspector(), ImmutableList.of("field1"), ImmutableList.of(1), javaLongObjectInspector, 5L, 5L, false);
        columnsToRead = ImmutableList.of(columnPrimitiveBigInt, projectedColumn);
        // All rows returned for a satisfying predicate
        testingPredicate = TupleDomain.withColumnDomains(ImmutableMap.of(projectedColumn, Domain.singleValue(BIGINT, 5L)));
        assertFilteredRows(testingPredicate, columnsToRead, session, split, NUM_ROWS);
        // No rows returned for a mismatched predicate
        testingPredicate = TupleDomain.withColumnDomains(ImmutableMap.of(projectedColumn, Domain.singleValue(BIGINT, 6L)));
        assertFilteredRows(testingPredicate, columnsToRead, session, split, 0);
    } finally {
        file.delete();
    }
}

Example 3 with NodeVersion

use of io.trino.plugin.hive.NodeVersion in project trino by trinodb.

the class TestHivePlans method createLocalQueryRunner.

@Override
protected LocalQueryRunner createLocalQueryRunner() {
    baseDir = Files.createTempDir();
    HdfsConfig config = new HdfsConfig();
    HdfsConfiguration configuration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(config), ImmutableSet.of());
    HdfsEnvironment environment = new HdfsEnvironment(configuration, config, new NoHdfsAuthentication());
    HiveMetastore metastore = new FileHiveMetastore(new NodeVersion("test_version"), environment, new MetastoreConfig(), new FileHiveMetastoreConfig().setCatalogDirectory(baseDir.toURI().toString()).setMetastoreUser("test"));
    Database database = Database.builder().setDatabaseName(SCHEMA_NAME).setOwnerName(Optional.of("public")).setOwnerType(Optional.of(PrincipalType.ROLE)).build();
    metastore.createDatabase(database);
    return createQueryRunner(HIVE_SESSION, metastore);
}

Example 4 with NodeVersion

use of io.trino.plugin.hive.NodeVersion in project trino by trinodb.

the class TestHiveProjectionPushdownIntoTableScan method createLocalQueryRunner.

@Override
protected LocalQueryRunner createLocalQueryRunner() {
    baseDir = Files.createTempDir();
    HdfsConfig config = new HdfsConfig();
    HdfsConfiguration configuration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(config), ImmutableSet.of());
    HdfsEnvironment environment = new HdfsEnvironment(configuration, config, new NoHdfsAuthentication());
    HiveMetastore metastore = new FileHiveMetastore(new NodeVersion("test_version"), environment, new MetastoreConfig(), new FileHiveMetastoreConfig().setCatalogDirectory(baseDir.toURI().toString()).setMetastoreUser("test"));
    Database database = Database.builder().setDatabaseName(SCHEMA_NAME).setOwnerName(Optional.of("public")).setOwnerType(Optional.of(PrincipalType.ROLE)).build();
    metastore.createDatabase(database);
    LocalQueryRunner queryRunner = LocalQueryRunner.create(HIVE_SESSION);
    queryRunner.createCatalog(HIVE_CATALOG_NAME, new TestingHiveConnectorFactory(metastore), ImmutableMap.of());
    return queryRunner;
}

Example 5 with NodeVersion

use of io.trino.plugin.hive.NodeVersion in project trino by trinodb.

the class InternalDeltaLakeConnectorFactory method createConnector.

@VisibleForTesting
public static Connector createConnector(String catalogName, Map<String, String> config, ConnectorContext context, Module extraModule) {
    ClassLoader classLoader = InternalDeltaLakeConnectorFactory.class.getClassLoader();
    try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) {
        Bootstrap app = new Bootstrap(new EventModule(), new MBeanModule(), new JsonModule(), new MBeanServerModule(), new HiveHdfsModule(), new HiveS3Module(), new HiveAzureModule(), new HdfsAuthenticationModule(), new CatalogNameModule(catalogName), new DeltaLakeMetastoreModule(), new DeltaLakeModule(), binder -> {
            binder.bind(NodeVersion.class).toInstance(new NodeVersion(context.getNodeManager().getCurrentNode().getVersion()));
            binder.bind(NodeManager.class).toInstance(context.getNodeManager());
            binder.bind(TypeManager.class).toInstance(context.getTypeManager());
            binder.bind(PageIndexerFactory.class).toInstance(context.getPageIndexerFactory());
            binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName));
            newSetBinder(binder, EventListener.class);
        }, binder -> bindSessionPropertiesProvider(binder, DeltaLakeSessionProperties.class), extraModule);
        Injector injector = app.doNotInitializeLogging().setRequiredConfigurationProperties(config).initialize();
        LifeCycleManager lifeCycleManager = injector.getInstance(LifeCycleManager.class);
        ConnectorSplitManager splitManager = injector.getInstance(ConnectorSplitManager.class);
        ConnectorPageSourceProvider connectorPageSource = injector.getInstance(ConnectorPageSourceProvider.class);
        ConnectorPageSinkProvider connectorPageSink = injector.getInstance(ConnectorPageSinkProvider.class);
        ConnectorNodePartitioningProvider connectorDistributionProvider = injector.getInstance(ConnectorNodePartitioningProvider.class);
        Set<SessionPropertiesProvider> sessionPropertiesProviders = injector.getInstance(Key.get(new TypeLiteral<Set<SessionPropertiesProvider>>() {
        }));
        DeltaLakeTableProperties deltaLakeTableProperties = injector.getInstance(DeltaLakeTableProperties.class);
        DeltaLakeAnalyzeProperties deltaLakeAnalyzeProperties = injector.getInstance(DeltaLakeAnalyzeProperties.class);
        DeltaLakeTransactionManager transactionManager = injector.getInstance(DeltaLakeTransactionManager.class);
        Set<EventListener> eventListeners = injector.getInstance(Key.get(new TypeLiteral<Set<EventListener>>() {
        })).stream().map(listener -> new ClassLoaderSafeEventListener(listener, classLoader)).collect(toImmutableSet());
        Set<Procedure> procedures = injector.getInstance(Key.get(new TypeLiteral<Set<Procedure>>() {
        }));
        Set<TableProcedureMetadata> tableProcedures = injector.getInstance(Key.get(new TypeLiteral<Set<TableProcedureMetadata>>() {
        }));
        return new DeltaLakeConnector(lifeCycleManager, new ClassLoaderSafeConnectorSplitManager(splitManager, classLoader), new ClassLoaderSafeConnectorPageSourceProvider(connectorPageSource, classLoader), new ClassLoaderSafeConnectorPageSinkProvider(connectorPageSink, classLoader), new ClassLoaderSafeNodePartitioningProvider(connectorDistributionProvider, classLoader), ImmutableSet.of(), procedures, tableProcedures, sessionPropertiesProviders, DeltaLakeSchemaProperties.SCHEMA_PROPERTIES, deltaLakeTableProperties.getTableProperties(), deltaLakeAnalyzeProperties.getAnalyzeProperties(), eventListeners, transactionManager);
    }
}