Search in sources :

Example 1 with ThriftReader

use of org.apache.carbondata.core.reader.ThriftReader in project carbondata by apache.

the class SchemaReader method readCarbonTableFromStore.

public static CarbonTable readCarbonTableFromStore(AbsoluteTableIdentifier identifier) throws IOException {
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(identifier);
    String schemaFilePath = carbonTablePath.getSchemaFilePath();
    if (FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.LOCAL) || FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.HDFS) || FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.VIEWFS)) {
        String tableName = identifier.getCarbonTableIdentifier().getTableName();
        ThriftReader.TBaseCreator createTBase = new ThriftReader.TBaseCreator() {

            public TBase create() {
                return new org.apache.carbondata.format.TableInfo();
            }
        };
        ThriftReader thriftReader = new ThriftReader(carbonTablePath.getSchemaFilePath(), createTBase);
        thriftReader.open();
        org.apache.carbondata.format.TableInfo tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
        thriftReader.close();
        SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
        TableInfo wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(tableInfo, identifier.getCarbonTableIdentifier().getDatabaseName(), tableName, identifier.getStorePath());
        wrapperTableInfo.setMetaDataFilepath(CarbonTablePath.getFolderContainingFile(schemaFilePath));
        CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo);
        return CarbonMetadata.getInstance().getCarbonTable(identifier.getCarbonTableIdentifier().getTableUniqueName());
    } else {
        throw new IOException("File does not exist: " + schemaFilePath);
    }
}
Also used : IOException(java.io.IOException) ThriftReader(org.apache.carbondata.core.reader.ThriftReader) CarbonTablePath(org.apache.carbondata.core.util.path.CarbonTablePath) SchemaConverter(org.apache.carbondata.core.metadata.converter.SchemaConverter) TableInfo(org.apache.carbondata.core.metadata.schema.table.TableInfo) ThriftWrapperSchemaConverterImpl(org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl)

Example 2 with ThriftReader

use of org.apache.carbondata.core.reader.ThriftReader in project carbondata by apache.

the class CarbonUtil method readSchemaFile.

/**
 * This method will read the schema file from a given path
 *
 * @param schemaFilePath
 * @return
 */
public static org.apache.carbondata.format.TableInfo readSchemaFile(String schemaFilePath) throws IOException {
    TBaseCreator createTBase = new ThriftReader.TBaseCreator() {

        public org.apache.thrift.TBase<org.apache.carbondata.format.TableInfo, org.apache.carbondata.format.TableInfo._Fields> create() {
            return new org.apache.carbondata.format.TableInfo();
        }
    };
    ThriftReader thriftReader = new ThriftReader(schemaFilePath, createTBase);
    thriftReader.open();
    org.apache.carbondata.format.TableInfo tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
    thriftReader.close();
    return tableInfo;
}
Also used : ThriftReader(org.apache.carbondata.core.reader.ThriftReader) TBaseCreator(org.apache.carbondata.core.reader.ThriftReader.TBaseCreator) TableInfo(org.apache.carbondata.core.metadata.schema.table.TableInfo)

Example 3 with ThriftReader

use of org.apache.carbondata.core.reader.ThriftReader in project carbondata by apache.

the class CarbonUtil method inferSchema.

/**
 * This method will read the schema file from a given path
 *
 * @param schemaFilePath
 * @return
 */
public static org.apache.carbondata.format.TableInfo inferSchema(String carbonDataFilePath, AbsoluteTableIdentifier absoluteTableIdentifier, boolean schemaExists) throws IOException {
    TBaseCreator createTBase = new ThriftReader.TBaseCreator() {

        public org.apache.thrift.TBase<org.apache.carbondata.format.TableInfo, org.apache.carbondata.format.TableInfo._Fields> create() {
            return new org.apache.carbondata.format.TableInfo();
        }
    };
    if (schemaExists == false) {
        List<String> filePaths = getFilePathExternalFilePath(carbonDataFilePath + "/Fact/Part0/Segment_null");
        String fistFilePath = null;
        try {
            fistFilePath = filePaths.get(0);
        } catch (Exception e) {
            LOGGER.error("CarbonData file is not present in the table location");
        }
        CarbonHeaderReader carbonHeaderReader = new CarbonHeaderReader(fistFilePath);
        FileHeader fileHeader = carbonHeaderReader.readHeader();
        List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
        List<org.apache.carbondata.format.ColumnSchema> table_columns = fileHeader.getColumn_schema();
        for (int i = 0; i < table_columns.size(); i++) {
            ColumnSchema col = thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i));
            col.setColumnReferenceId(col.getColumnUniqueId());
            columnSchemaList.add(col);
        }
        TableSchema tableSchema = new TableSchema();
        tableSchema.setTableName(absoluteTableIdentifier.getTableName());
        tableSchema.setBucketingInfo(null);
        tableSchema.setSchemaEvalution(null);
        tableSchema.setTableId(UUID.randomUUID().toString());
        tableSchema.setListOfColumns(columnSchemaList);
        ThriftWrapperSchemaConverterImpl thriftWrapperSchemaConverter = new ThriftWrapperSchemaConverterImpl();
        SchemaEvolutionEntry schemaEvolutionEntry = new SchemaEvolutionEntry();
        schemaEvolutionEntry.setTimeStamp(System.currentTimeMillis());
        SchemaEvolution schemaEvol = new SchemaEvolution();
        List<SchemaEvolutionEntry> schEntryList = new ArrayList<>();
        schEntryList.add(schemaEvolutionEntry);
        schemaEvol.setSchemaEvolutionEntryList(schEntryList);
        tableSchema.setSchemaEvalution(schemaEvol);
        org.apache.carbondata.format.TableSchema thriftFactTable = thriftWrapperSchemaConverter.fromWrapperToExternalTableSchema(tableSchema);
        org.apache.carbondata.format.TableInfo tableInfo = new org.apache.carbondata.format.TableInfo(thriftFactTable, new ArrayList<org.apache.carbondata.format.TableSchema>());
        tableInfo.setDataMapSchemas(null);
        return tableInfo;
    } else {
        ThriftReader thriftReader = new ThriftReader(carbonDataFilePath, createTBase);
        thriftReader.open();
        org.apache.carbondata.format.TableInfo tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
        thriftReader.close();
        return tableInfo;
    }
}
Also used : TableSchema(org.apache.carbondata.core.metadata.schema.table.TableSchema) TBaseCreator(org.apache.carbondata.core.reader.ThriftReader.TBaseCreator) ColumnSchema(org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema) SchemaEvolution(org.apache.carbondata.core.metadata.schema.SchemaEvolution) SchemaEvolutionEntry(org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry) ThriftReader(org.apache.carbondata.core.reader.ThriftReader) TableInfo(org.apache.carbondata.core.metadata.schema.table.TableInfo) FileHeader(org.apache.carbondata.format.FileHeader) CarbonHeaderReader(org.apache.carbondata.core.reader.CarbonHeaderReader) InvalidConfigurationException(org.apache.carbondata.core.exception.InvalidConfigurationException) CarbonDataWriterException(org.apache.carbondata.core.datastore.exception.CarbonDataWriterException) TException(org.apache.thrift.TException) ThriftWrapperSchemaConverterImpl(org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl)

Example 4 with ThriftReader

use of org.apache.carbondata.core.reader.ThriftReader in project carbondata by apache.

the class SegmentIndexFileStore method readMergeFile.

/**
 * Read carbonindexmerge file and update the map
 *
 * @param mergeFilePath
 * @throws IOException
 */
private void readMergeFile(String mergeFilePath) throws IOException {
    ThriftReader thriftReader = new ThriftReader(mergeFilePath);
    try {
        thriftReader.open();
        MergedBlockIndexHeader indexHeader = readMergeBlockIndexHeader(thriftReader);
        MergedBlockIndex mergedBlockIndex = readMergeBlockIndex(thriftReader);
        List<String> file_names = indexHeader.getFile_names();
        List<ByteBuffer> fileData = mergedBlockIndex.getFileData();
        CarbonFile mergeFile = FileFactory.getCarbonFile(mergeFilePath);
        assert (file_names.size() == fileData.size());
        for (int i = 0; i < file_names.size(); i++) {
            carbonIndexMap.put(file_names.get(i), fileData.get(i).array());
            carbonIndexMapWithFullPath.put(mergeFile.getParentFile().getAbsolutePath() + CarbonCommonConstants.FILE_SEPARATOR + file_names.get(i), fileData.get(i).array());
        }
    } finally {
        thriftReader.close();
    }
}
Also used : ThriftReader(org.apache.carbondata.core.reader.ThriftReader) MergedBlockIndex(org.apache.carbondata.format.MergedBlockIndex) CarbonFile(org.apache.carbondata.core.datastore.filesystem.CarbonFile) MergedBlockIndexHeader(org.apache.carbondata.format.MergedBlockIndexHeader) ByteBuffer(java.nio.ByteBuffer)

Example 5 with ThriftReader

use of org.apache.carbondata.core.reader.ThriftReader in project carbondata by apache.

the class ThriftReaderTest method testReadForException.

@Test(expected = java.io.IOException.class)
public void testReadForException() throws IOException {
    ThriftReader.TBaseCreator tBaseCreator = new ThriftReader.TBaseCreator() {

        @Override
        public TBase create() {
            return new ColumnDictionaryChunkMeta();
        }
    };
    new MockUp<ColumnDictionaryChunkMeta>() {

        @Mock
        public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
            throw new TException("TException Occur");
        }
    };
    thriftReader = new ThriftReader("TestFile.carbon", tBaseCreator);
    thriftReader.read();
}
Also used : TException(org.apache.thrift.TException) ThriftReader(org.apache.carbondata.core.reader.ThriftReader) ColumnDictionaryChunkMeta(org.apache.carbondata.format.ColumnDictionaryChunkMeta) MockUp(mockit.MockUp) Test(org.junit.Test)

Aggregations

ThriftReader (org.apache.carbondata.core.reader.ThriftReader)9 TableInfo (org.apache.carbondata.core.metadata.schema.table.TableInfo)4 ThriftWrapperSchemaConverterImpl (org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl)3 IOException (java.io.IOException)2 MockUp (mockit.MockUp)2 SchemaConverter (org.apache.carbondata.core.metadata.converter.SchemaConverter)2 TBaseCreator (org.apache.carbondata.core.reader.ThriftReader.TBaseCreator)2 MergedBlockIndexHeader (org.apache.carbondata.format.MergedBlockIndexHeader)2 TException (org.apache.thrift.TException)2 Test (org.junit.Test)2 TableNotFoundException (com.facebook.presto.spi.TableNotFoundException)1 ByteArrayInputStream (java.io.ByteArrayInputStream)1 DataInputStream (java.io.DataInputStream)1 ByteBuffer (java.nio.ByteBuffer)1 ArrayList (java.util.ArrayList)1 TableBlockInfo (org.apache.carbondata.core.datastore.block.TableBlockInfo)1 CarbonDataWriterException (org.apache.carbondata.core.datastore.exception.CarbonDataWriterException)1 CarbonFile (org.apache.carbondata.core.datastore.filesystem.CarbonFile)1 InvalidConfigurationException (org.apache.carbondata.core.exception.InvalidConfigurationException)1 CarbonTableIdentifier (org.apache.carbondata.core.metadata.CarbonTableIdentifier)1