Examples with CarbondataTableCacheModel io.hetu.core.plugin.carbondata.impl.CarbondataTableCacheModel used on opensource projects

Example 1 with CarbondataTableCacheModel

use of io.hetu.core.plugin.carbondata.impl.CarbondataTableCacheModel in project hetu-core by openlookeng.

the class CarbondataPageSourceProvider method getCarbonTable.

/**
 * @param carbonSplit
 * @return
 */
private CarbonTable getCarbonTable(HiveSplit carbonSplit, Configuration configuration) {
    CarbondataTableCacheModel tableCacheModel = carbonTableReader.getCarbonCache(new SchemaTableName(carbonSplit.getDatabase(), carbonSplit.getTable()), carbonSplit.getSchema().getProperty("tablePath"), configuration);
    requireNonNull(tableCacheModel, "tableCacheModel should not be null");
    requireNonNull(tableCacheModel.getCarbonTable(), "tableCacheModel.carbonTable should not be null");
    requireNonNull(tableCacheModel.getCarbonTable().getTableInfo(), "tableCacheModel.carbonTable.tableInfo should not be null");
    return tableCacheModel.getCarbonTable();
}

Example 2 with CarbondataTableCacheModel

use of io.hetu.core.plugin.carbondata.impl.CarbondataTableCacheModel in project hetu-core by openlookeng.

the class CarbondataSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableHandle tableHandle, SplitSchedulingStrategy splitSchedulingStrategy, Supplier<List<Set<DynamicFilter>>> dynamicFilterSupplier, Optional<QueryType> queryType, Map<String, Object> queryProperties, Set<TupleDomain<ColumnMetadata>> userDefinedCachePredicates, boolean partOfReuse) {
    HiveTableHandle hiveTable = (HiveTableHandle) tableHandle;
    SchemaTableName schemaTableName = hiveTable.getSchemaTableName();
    // get table metadata
    HiveIdentity identity = new HiveIdentity(session);
    SemiTransactionalHiveMetastore metastore = metastoreProvider.apply((HiveTransactionHandle) transactionHandle);
    Table table = metastore.getTable(identity, schemaTableName.getSchemaName(), schemaTableName.getTableName()).orElseThrow(() -> new TableNotFoundException(schemaTableName));
    if (!table.getStorage().getStorageFormat().getInputFormat().contains("carbon")) {
        throw new PrestoException(NOT_SUPPORTED, "Carbondata connector can only read carbondata tables");
    }
    return hdfsEnvironment.doAs(session.getUser(), () -> {
        String location = table.getStorage().getLocation();
        String queryId = System.nanoTime() + "";
        QueryStatistic statistic = new QueryStatistic();
        QueryStatisticsRecorder statisticRecorder = CarbonTimeStatisticsFactory.createDriverRecorder();
        statistic.addStatistics(QueryStatisticsConstants.BLOCK_ALLOCATION, System.currentTimeMillis());
        statisticRecorder.recordStatisticsForDriver(statistic, queryId);
        statistic = new QueryStatistic();
        carbonTableReader.setQueryId(queryId);
        TupleDomain<HiveColumnHandle> predicate = (TupleDomain<HiveColumnHandle>) hiveTable.getCompactEffectivePredicate();
        Configuration configuration = this.hdfsEnvironment.getConfiguration(new HdfsEnvironment.HdfsContext(session, schemaTableName.getSchemaName(), schemaTableName.getTableName()), new Path(location));
        // set the hadoop configuration to thread local, so that FileFactory can use it.
        ThreadLocalSessionInfo.setConfigurationToCurrentThread(configuration);
        CarbondataTableCacheModel cache = carbonTableReader.getCarbonCache(schemaTableName, location, configuration);
        Expression filters = CarbondataHetuFilterUtil.parseFilterExpression(predicate);
        try {
            List<CarbondataLocalMultiBlockSplit> splits = carbonTableReader.getInputSplits(cache, filters, predicate, configuration);
            ImmutableList.Builder<ConnectorSplit> cSplits = ImmutableList.builder();
            long index = 0;
            for (CarbondataLocalMultiBlockSplit split : splits) {
                index++;
                Properties properties = new Properties();
                for (Map.Entry<String, String> entry : table.getStorage().getSerdeParameters().entrySet()) {
                    properties.setProperty(entry.getKey(), entry.getValue());
                }
                properties.setProperty("tablePath", cache.getCarbonTable().getTablePath());
                properties.setProperty("carbonSplit", split.getJsonString());
                properties.setProperty("queryId", queryId);
                properties.setProperty("index", String.valueOf(index));
                cSplits.add(HiveSplitWrapper.wrap(new HiveSplit(schemaTableName.getSchemaName(), schemaTableName.getTableName(), schemaTableName.getTableName(), cache.getCarbonTable().getTablePath(), 0, 0, 0, 0, properties, new ArrayList(), getHostAddresses(split.getLocations()), OptionalInt.empty(), false, new HashMap<>(), Optional.empty(), false, Optional.empty(), Optional.empty(), false, ImmutableMap.of())));
            /* Todo: Make this part aligned with rest of the HiveSlipt loading flow...
                     *   and figure out how to pass valid transaction Ids to CarbonData? */
            }
            statisticRecorder.logStatisticsAsTableDriver();
            statistic.addStatistics(QueryStatisticsConstants.BLOCK_IDENTIFICATION, System.currentTimeMillis());
            statisticRecorder.recordStatisticsForDriver(statistic, queryId);
            statisticRecorder.logStatisticsAsTableDriver();
            if (queryType != null && queryType.isPresent() && queryType.get().equals(QueryType.VACUUM)) {
                // Get Splits for compaction
                return getSplitsForCompaction(identity, transactionHandle, tableHandle, cache.getCarbonTable().getTablePath(), queryProperties, queryId, cSplits, configuration);
            }
            return new FixedSplitSource(cSplits.build());
        } catch (IOException ex) {
            throw new PrestoException(GENERIC_INTERNAL_ERROR, "Failed while trying to get splits ", ex);
        }
    });
}

Example 3 with CarbondataTableCacheModel

use of io.hetu.core.plugin.carbondata.impl.CarbondataTableCacheModel in project hetu-core by openlookeng.

the class CarbondataMetadata method getCarbonTable.

static CarbonTable getCarbonTable(String dbName, String tableName, Properties schema, Configuration configuration, CarbondataTableReader carbondataTableReader) {
    CarbondataTableCacheModel tableCacheModel = carbondataTableReader.getCarbonCache(new SchemaTableName(dbName, tableName), schema.getProperty("tablePath"), configuration);
    requireNonNull(tableCacheModel, "tableCacheModel should not be null");
    requireNonNull(tableCacheModel.getCarbonTable(), "tableCacheModel.carbonTable should not be null");
    requireNonNull(tableCacheModel.getCarbonTable().getTableInfo(), "tableCacheModel.carbonTable.tableInfo should not be null");
    return tableCacheModel.getCarbonTable();
}