Examples with CarbonTable org.apache.carbondata.core.metadata.schema.table.CarbonTable used on opensource projects

Example 41 with CarbonTable

use of org.apache.carbondata.core.metadata.schema.table.CarbonTable in project carbondata by apache.

the class BlockletIndexUtil method getBlockMetaInfoMap.

public static Map<String, BlockMetaInfo> getBlockMetaInfoMap(TableBlockIndexUniqueIdentifierWrapper identifierWrapper, SegmentIndexFileStore indexFileStore, Set<String> filesRead, Map<String, BlockMetaInfo> fileNameToMetaInfoMapping, List<DataFileFooter> indexInfos) throws IOException {
    boolean isTransactionalTable = true;
    TableBlockIndexUniqueIdentifier identifier = identifierWrapper.getTableBlockIndexUniqueIdentifier();
    List<ColumnSchema> tableColumnList = null;
    if (identifier.getMergeIndexFileName() != null && indexFileStore.getFileData(identifier.getIndexFileName()) == null) {
        CarbonFile indexMergeFile = FileFactory.getCarbonFile(identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier.getMergeIndexFileName(), identifierWrapper.getConfiguration());
        if (indexMergeFile.exists() && !filesRead.contains(indexMergeFile.getPath())) {
            indexFileStore.readAllIIndexOfSegment(new CarbonFile[] { indexMergeFile });
            filesRead.add(indexMergeFile.getPath());
        }
    }
    if (indexFileStore.getFileData(identifier.getIndexFileName()) == null) {
        indexFileStore.readAllIIndexOfSegment(new CarbonFile[] { FileFactory.getCarbonFile(identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier.getIndexFileName(), identifierWrapper.getConfiguration()) });
    }
    Map<String, BlockMetaInfo> blockMetaInfoMap = new HashMap<>();
    CarbonTable carbonTable = identifierWrapper.getCarbonTable();
    if (carbonTable != null) {
        isTransactionalTable = carbonTable.getTableInfo().isTransactionalTable();
        tableColumnList = carbonTable.getTableInfo().getFactTable().getListOfColumns();
    }
    DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter(identifierWrapper.getConfiguration());
    List<DataFileFooter> indexInfo = fileFooterConverter.getIndexInfo(identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier.getIndexFileName(), indexFileStore.getFileData(identifier.getIndexFileName()), isTransactionalTable);
    indexInfos.addAll(indexInfo);
    for (DataFileFooter footer : indexInfo) {
        if ((!isTransactionalTable) && (tableColumnList.size() != 0) && !isSameColumnAndDifferentDatatypeInSchema(footer.getColumnInTable(), tableColumnList)) {
            LOG.error("Datatype of the common columns present in " + identifier.getIndexFileName() + " doesn't match with the column's datatype in table schema");
            throw new IOException("All common columns present in the files doesn't have same datatype. " + "Unsupported operation on nonTransactional table. Check logs.");
        }
        if ((tableColumnList != null) && (tableColumnList.size() == 0)) {
            // Carbon reader have used dummy columnSchema. Update it with inferred schema now
            carbonTable.getTableInfo().getFactTable().setListOfColumns(footer.getColumnInTable());
            CarbonTable.updateTableByTableInfo(carbonTable, carbonTable.getTableInfo());
        }
        String blockPath = footer.getBlockInfo().getFilePath();
        if (null == blockMetaInfoMap.get(blockPath)) {
            BlockMetaInfo blockMetaInfo = createBlockMetaInfo(fileNameToMetaInfoMapping, footer.getBlockInfo());
            // complete block is deleted but the entry still exists in index or merge index file
            if (null != blockMetaInfo) {
                blockMetaInfoMap.put(blockPath, blockMetaInfo);
            }
        }
    }
    return blockMetaInfoMap;
}

Example 42 with CarbonTable

use of org.apache.carbondata.core.metadata.schema.table.CarbonTable in project carbondata by apache.

the class StageInputCollector method collectStageFiles.

/**
 * Collect all stage files and matched success files.
 * A stage file without success file will not be collected
 */
public static void collectStageFiles(CarbonTable table, Configuration hadoopConf, List<CarbonFile> stageInputList, List<CarbonFile> successFileList) {
    Objects.requireNonNull(table);
    Objects.requireNonNull(hadoopConf);
    Objects.requireNonNull(stageInputList);
    Objects.requireNonNull(successFileList);
    CarbonFile dir = FileFactory.getCarbonFile(table.getStagePath(), hadoopConf);
    if (dir.exists()) {
        // list the stage folder and collect all stage files who has corresponding success file,
        // which means the file is committed
        CarbonFile[] allFiles = dir.listFiles();
        Map<String, CarbonFile> map = new HashMap<>();
        Arrays.stream(allFiles).filter(file -> file.getName().endsWith(SUCCESS_FILE_SUFFIX)).forEach(file -> map.put(file.getName().substring(0, file.getName().indexOf(".")), file));
        Arrays.stream(allFiles).filter(file -> !file.getName().endsWith(SUCCESS_FILE_SUFFIX)).filter(file -> map.containsKey(file.getName())).forEach(carbonFile -> {
            stageInputList.add(carbonFile);
            successFileList.add(map.get(carbonFile.getName()));
        });
    }
}

Example 43 with CarbonTable

use of org.apache.carbondata.core.metadata.schema.table.CarbonTable in project carbondata by apache.

the class CarbonMetadata method loadTableMetadata.

/**
 * method load the table
 *
 * @param tableInfo
 */
public void loadTableMetadata(TableInfo tableInfo) {
    CarbonTable carbonTable = tableInfoMap.get(convertToLowerCase(tableInfo.getTableUniqueName()));
    if (null == carbonTable || carbonTable.getTableLastUpdatedTime() < tableInfo.getLastUpdatedTime()) {
        carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
        tableInfoMap.put(convertToLowerCase(tableInfo.getTableUniqueName()), carbonTable);
    }
}

Example 44 with CarbonTable

use of org.apache.carbondata.core.metadata.schema.table.CarbonTable in project carbondata by apache.

the class CarbonMetadataTest method testGetCarbonDimensionBasedOnColIdentifierNullCase.

@Test
public void testGetCarbonDimensionBasedOnColIdentifierNullCase() {
    CarbonTable carbonTable = CarbonTable.buildFromTableInfo(getTableInfo(1000L));
    String columnIdentifier = "3";
    final List<CarbonDimension> carbonDimensions = new ArrayList();
    ColumnSchema colSchema1 = new ColumnSchema();
    colSchema1.setColumnUniqueId("1");
    colSchema1.setNumberOfChild(1);
    CarbonDimension carbonDimension = new CarbonDimension(colSchema1, 1, 1, 1);
    carbonDimensions.add(carbonDimension);
    final List<CarbonDimension> carbonChildDimensions = new ArrayList();
    ColumnSchema colSchema2 = new ColumnSchema();
    colSchema2.setColumnUniqueId("9");
    colSchema2.setNumberOfChild(0);
    carbonChildDimensions.add(new CarbonDimension(colSchema2, 1, 1, 1));
    new MockUp<CarbonTable>() {

        @Mock
        public String getTableName() {
            return "carbonTestTable";
        }

        @Mock
        public List<CarbonDimension> getVisibleDimensions() {
            return carbonDimensions;
        }
    };
    new MockUp<CarbonDimension>() {

        @Mock
        public List<CarbonDimension> getListOfChildDimensions() {
            return carbonChildDimensions;
        }
    };
    CarbonDimension result = carbonMetadata.getCarbonDimensionBasedOnColIdentifier(carbonTable, columnIdentifier);
    assertNull(result);
}

Example 45 with CarbonTable

use of org.apache.carbondata.core.metadata.schema.table.CarbonTable in project carbondata by apache.

the class RestructureUtilTest method testToGetAggregatorInfos.

@Test
public void testToGetAggregatorInfos() {
    ColumnSchema columnSchema1 = new ColumnSchema();
    columnSchema1.setColumnName("Id");
    columnSchema1.setDataType(DataTypes.STRING);
    columnSchema1.setColumnUniqueId(UUID.randomUUID().toString());
    ColumnSchema columnSchema2 = new ColumnSchema();
    columnSchema2.setColumnName("Name");
    columnSchema2.setDataType(DataTypes.STRING);
    columnSchema2.setColumnUniqueId(UUID.randomUUID().toString());
    ColumnSchema columnSchema3 = new ColumnSchema();
    columnSchema3.setColumnName("Age");
    columnSchema3.setDataType(DataTypes.STRING);
    columnSchema3.setColumnUniqueId(UUID.randomUUID().toString());
    CarbonMeasure carbonMeasure1 = new CarbonMeasure(columnSchema1, 1);
    CarbonMeasure carbonMeasure2 = new CarbonMeasure(columnSchema2, 2);
    CarbonMeasure carbonMeasure3 = new CarbonMeasure(columnSchema3, 3);
    carbonMeasure3.getColumnSchema().setDefaultValue("3".getBytes());
    List<CarbonMeasure> currentBlockMeasures = Arrays.asList(carbonMeasure1, carbonMeasure2);
    ProjectionMeasure queryMeasure1 = new ProjectionMeasure(carbonMeasure1);
    ProjectionMeasure queryMeasure2 = new ProjectionMeasure(carbonMeasure2);
    ProjectionMeasure queryMeasure3 = new ProjectionMeasure(carbonMeasure3);
    ProjectionMeasure[] queryMeasures = new ProjectionMeasure[] { queryMeasure1, queryMeasure2, queryMeasure3 };
    BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
    RestructureUtil.createMeasureInfoAndGetCurrentBlockQueryMeasures(blockExecutionInfo, queryMeasures, currentBlockMeasures, true, QueryModel.newInstance(new CarbonTable()));
    MeasureInfo measureInfo = blockExecutionInfo.getMeasureInfo();
    boolean[] measuresExist = { true, true, false };
    assertThat(measureInfo.getMeasureExists(), is(equalTo(measuresExist)));
    Object[] defaultValues = { null, null, 3.0 };
    assertThat(measureInfo.getDefaultValues(), is(equalTo(defaultValues)));
}