Examples with FileGroupScan org.apache.drill.exec.physical.base.FileGroupScan used on opensource projects

Example 1 with FileGroupScan

use of org.apache.drill.exec.physical.base.FileGroupScan in project drill by apache.

the class FileSystemPartitionDescriptor method createTableScan.

@Override
public TableScan createTableScan(List<PartitionLocation> newPartitionLocation, String cacheFileRoot, boolean wasAllPartitionsPruned, MetadataContext metaContext) throws Exception {
    List<String> newFiles = Lists.newArrayList();
    for (final PartitionLocation location : newPartitionLocation) {
        if (!location.isCompositePartition()) {
            newFiles.add(location.getEntirePartitionLocation());
        } else {
            final Collection<SimplePartitionLocation> subPartitions = location.getPartitionLocationRecursive();
            for (final PartitionLocation subPart : subPartitions) {
                newFiles.add(subPart.getEntirePartitionLocation());
            }
        }
    }
    if (scanRel instanceof DrillScanRel) {
        final FormatSelection formatSelection = (FormatSelection) table.getSelection();
        final FileSelection newFileSelection = new FileSelection(null, newFiles, getBaseTableLocation(), cacheFileRoot, wasAllPartitionsPruned, formatSelection.getSelection().getDirStatus());
        newFileSelection.setMetaContext(metaContext);
        final FileGroupScan newGroupScan = ((FileGroupScan) ((DrillScanRel) scanRel).getGroupScan()).clone(newFileSelection);
        return new DrillScanRel(scanRel.getCluster(), scanRel.getTraitSet().plus(DrillRel.DRILL_LOGICAL), scanRel.getTable(), newGroupScan, scanRel.getRowType(), ((DrillScanRel) scanRel).getColumns(), true);
    } else if (scanRel instanceof EnumerableTableScan) {
        return createNewTableScanFromSelection((EnumerableTableScan) scanRel, newFiles, cacheFileRoot, wasAllPartitionsPruned, metaContext);
    } else {
        throw new UnsupportedOperationException("Only DrillScanRel and EnumerableTableScan is allowed!");
    }
}

Example 2 with FileGroupScan

use of org.apache.drill.exec.physical.base.FileGroupScan in project drill by apache.

the class ParquetPartitionDescriptor method createNewGroupScan.

private GroupScan createNewGroupScan(List<String> newFiles, String cacheFileRoot, boolean wasAllPartitionsPruned, MetadataContext metaContext) throws IOException {
    final FileSelection newSelection = FileSelection.create(null, newFiles, getBaseTableLocation(), cacheFileRoot, wasAllPartitionsPruned);
    newSelection.setMetaContext(metaContext);
    final FileGroupScan newScan = ((FileGroupScan) scanRel.getGroupScan()).clone(newSelection);
    return newScan;
}

Example 3 with FileGroupScan

use of org.apache.drill.exec.physical.base.FileGroupScan in project drill by axbaretto.

the class FileSystemPartitionDescriptor method createTableScan.

@Override
public TableScan createTableScan(List<PartitionLocation> newPartitionLocation, String cacheFileRoot, boolean wasAllPartitionsPruned, MetadataContext metaContext) throws Exception {
    List<String> newFiles = Lists.newArrayList();
    for (final PartitionLocation location : newPartitionLocation) {
        if (!location.isCompositePartition()) {
            newFiles.add(location.getEntirePartitionLocation());
        } else {
            final Collection<SimplePartitionLocation> subPartitions = location.getPartitionLocationRecursive();
            for (final PartitionLocation subPart : subPartitions) {
                newFiles.add(subPart.getEntirePartitionLocation());
            }
        }
    }
    if (scanRel instanceof DrillScanRel) {
        final FormatSelection formatSelection = (FormatSelection) table.getSelection();
        final FileSelection newFileSelection = new FileSelection(null, newFiles, getBaseTableLocation(), cacheFileRoot, wasAllPartitionsPruned, formatSelection.getSelection().getDirStatus());
        newFileSelection.setMetaContext(metaContext);
        final FileGroupScan newGroupScan = ((FileGroupScan) ((DrillScanRel) scanRel).getGroupScan()).clone(newFileSelection);
        return new DrillScanRel(scanRel.getCluster(), scanRel.getTraitSet().plus(DrillRel.DRILL_LOGICAL), scanRel.getTable(), newGroupScan, scanRel.getRowType(), ((DrillScanRel) scanRel).getColumns(), true);
    } else if (scanRel instanceof EnumerableTableScan) {
        return createNewTableScanFromSelection((EnumerableTableScan) scanRel, newFiles, cacheFileRoot, wasAllPartitionsPruned, metaContext);
    } else {
        throw new UnsupportedOperationException("Only DrillScanRel and EnumerableTableScan is allowed!");
    }
}

Example 4 with FileGroupScan

use of org.apache.drill.exec.physical.base.FileGroupScan in project drill by axbaretto.

the class ParquetPartitionDescriptor method createNewGroupScan.

private GroupScan createNewGroupScan(List<String> newFiles, String cacheFileRoot, boolean wasAllPartitionsPruned, MetadataContext metaContext) throws IOException {
    final FileSelection newSelection = FileSelection.create(null, newFiles, getBaseTableLocation(), cacheFileRoot, wasAllPartitionsPruned);
    newSelection.setMetaContext(metaContext);
    final FileGroupScan newScan = ((FileGroupScan) scanRel.getGroupScan()).clone(newSelection);
    return newScan;
}

Example 5 with FileGroupScan

use of org.apache.drill.exec.physical.base.FileGroupScan in project drill by apache.

the class FileSystemPartitionDescriptor method createTableScan.

@Override
public TableScan createTableScan(List<PartitionLocation> newPartitionLocation, Path cacheFileRoot, boolean wasAllPartitionsPruned, MetadataContext metaContext) throws Exception {
    List<Path> newFiles = new ArrayList<>();
    for (PartitionLocation location : newPartitionLocation) {
        if (!location.isCompositePartition()) {
            newFiles.add(location.getEntirePartitionLocation());
        } else {
            final Collection<SimplePartitionLocation> subPartitions = location.getPartitionLocationRecursive();
            for (PartitionLocation subPart : subPartitions) {
                newFiles.add(subPart.getEntirePartitionLocation());
            }
        }
    }
    FormatSelection formatSelection = (FormatSelection) table.getSelection();
    FileSelection newFileSelection = new FileSelection(null, newFiles, getBaseTableLocation(), cacheFileRoot, wasAllPartitionsPruned, formatSelection.getSelection().getDirStatus());
    newFileSelection.setMetaContext(metaContext);
    RelOptTable relOptTable = scanRel.getTable();
    if (scanRel instanceof DrillScanRel) {
        FileGroupScan newGroupScan = ((FileGroupScan) ((DrillScanRel) scanRel).getGroupScan()).clone(newFileSelection);
        return new DrillScanRel(scanRel.getCluster(), scanRel.getTraitSet().plus(DrillRel.DRILL_LOGICAL), relOptTable, newGroupScan, scanRel.getRowType(), ((DrillScanRel) scanRel).getColumns(), true);
    } else if (scanRel instanceof EnumerableTableScan) {
        FormatSelection newFormatSelection = new FormatSelection(formatSelection.getFormat(), newFileSelection);
        DynamicDrillTable dynamicDrillTable = new DynamicDrillTable(table.getPlugin(), table.getStorageEngineName(), table.getUserName(), newFormatSelection, table.getMetadataProviderManager());
        /* Copy statistics from the original relOptTable */
        DrillTranslatableTable newTable = new DrillTranslatableTable(dynamicDrillTable);
        RelOptTableImpl newOptTableImpl = RelOptTableImpl.create(relOptTable.getRelOptSchema(), relOptTable.getRowType(), newTable, GuavaUtils.convertToUnshadedImmutableList(ImmutableList.of()));
        // return an EnumerableTableScan with fileSelection being part of digest of TableScan node.
        return DirPrunedEnumerableTableScan.create(scanRel.getCluster(), newOptTableImpl, newFileSelection.toString());
    } else {
        throw new UnsupportedOperationException("Only DrillScanRel and EnumerableTableScan is allowed!");
    }
}