use of org.apache.carbondata.core.reader.ThriftReader in project carbondata by apache.
the class SchemaReader method readCarbonTableFromStore.
public static CarbonTable readCarbonTableFromStore(AbsoluteTableIdentifier identifier) throws IOException {
CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(identifier);
String schemaFilePath = carbonTablePath.getSchemaFilePath();
if (FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.LOCAL) || FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.HDFS) || FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.VIEWFS)) {
String tableName = identifier.getCarbonTableIdentifier().getTableName();
ThriftReader.TBaseCreator createTBase = new ThriftReader.TBaseCreator() {
public TBase create() {
return new org.apache.carbondata.format.TableInfo();
}
};
ThriftReader thriftReader = new ThriftReader(carbonTablePath.getSchemaFilePath(), createTBase);
thriftReader.open();
org.apache.carbondata.format.TableInfo tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
thriftReader.close();
SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
TableInfo wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(tableInfo, identifier.getCarbonTableIdentifier().getDatabaseName(), tableName, identifier.getStorePath());
wrapperTableInfo.setMetaDataFilepath(CarbonTablePath.getFolderContainingFile(schemaFilePath));
CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo);
return CarbonMetadata.getInstance().getCarbonTable(identifier.getCarbonTableIdentifier().getTableUniqueName());
} else {
throw new IOException("File does not exist: " + schemaFilePath);
}
}
use of org.apache.carbondata.core.reader.ThriftReader in project carbondata by apache.
the class CarbonUtil method readSchemaFile.
/**
* This method will read the schema file from a given path
*
* @param schemaFilePath
* @return
*/
public static org.apache.carbondata.format.TableInfo readSchemaFile(String schemaFilePath) throws IOException {
TBaseCreator createTBase = new ThriftReader.TBaseCreator() {
public org.apache.thrift.TBase<org.apache.carbondata.format.TableInfo, org.apache.carbondata.format.TableInfo._Fields> create() {
return new org.apache.carbondata.format.TableInfo();
}
};
ThriftReader thriftReader = new ThriftReader(schemaFilePath, createTBase);
thriftReader.open();
org.apache.carbondata.format.TableInfo tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
thriftReader.close();
return tableInfo;
}
use of org.apache.carbondata.core.reader.ThriftReader in project carbondata by apache.
the class CarbonUtil method inferSchema.
/**
* This method will read the schema file from a given path
*
* @param schemaFilePath
* @return
*/
public static org.apache.carbondata.format.TableInfo inferSchema(String carbonDataFilePath, AbsoluteTableIdentifier absoluteTableIdentifier, boolean schemaExists) throws IOException {
TBaseCreator createTBase = new ThriftReader.TBaseCreator() {
public org.apache.thrift.TBase<org.apache.carbondata.format.TableInfo, org.apache.carbondata.format.TableInfo._Fields> create() {
return new org.apache.carbondata.format.TableInfo();
}
};
if (schemaExists == false) {
List<String> filePaths = getFilePathExternalFilePath(carbonDataFilePath + "/Fact/Part0/Segment_null");
String fistFilePath = null;
try {
fistFilePath = filePaths.get(0);
} catch (Exception e) {
LOGGER.error("CarbonData file is not present in the table location");
}
CarbonHeaderReader carbonHeaderReader = new CarbonHeaderReader(fistFilePath);
FileHeader fileHeader = carbonHeaderReader.readHeader();
List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
List<org.apache.carbondata.format.ColumnSchema> table_columns = fileHeader.getColumn_schema();
for (int i = 0; i < table_columns.size(); i++) {
ColumnSchema col = thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i));
col.setColumnReferenceId(col.getColumnUniqueId());
columnSchemaList.add(col);
}
TableSchema tableSchema = new TableSchema();
tableSchema.setTableName(absoluteTableIdentifier.getTableName());
tableSchema.setBucketingInfo(null);
tableSchema.setSchemaEvalution(null);
tableSchema.setTableId(UUID.randomUUID().toString());
tableSchema.setListOfColumns(columnSchemaList);
ThriftWrapperSchemaConverterImpl thriftWrapperSchemaConverter = new ThriftWrapperSchemaConverterImpl();
SchemaEvolutionEntry schemaEvolutionEntry = new SchemaEvolutionEntry();
schemaEvolutionEntry.setTimeStamp(System.currentTimeMillis());
SchemaEvolution schemaEvol = new SchemaEvolution();
List<SchemaEvolutionEntry> schEntryList = new ArrayList<>();
schEntryList.add(schemaEvolutionEntry);
schemaEvol.setSchemaEvolutionEntryList(schEntryList);
tableSchema.setSchemaEvalution(schemaEvol);
org.apache.carbondata.format.TableSchema thriftFactTable = thriftWrapperSchemaConverter.fromWrapperToExternalTableSchema(tableSchema);
org.apache.carbondata.format.TableInfo tableInfo = new org.apache.carbondata.format.TableInfo(thriftFactTable, new ArrayList<org.apache.carbondata.format.TableSchema>());
tableInfo.setDataMapSchemas(null);
return tableInfo;
} else {
ThriftReader thriftReader = new ThriftReader(carbonDataFilePath, createTBase);
thriftReader.open();
org.apache.carbondata.format.TableInfo tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
thriftReader.close();
return tableInfo;
}
}
use of org.apache.carbondata.core.reader.ThriftReader in project carbondata by apache.
the class SegmentIndexFileStore method readMergeFile.
/**
* Read carbonindexmerge file and update the map
*
* @param mergeFilePath
* @throws IOException
*/
private void readMergeFile(String mergeFilePath) throws IOException {
ThriftReader thriftReader = new ThriftReader(mergeFilePath);
try {
thriftReader.open();
MergedBlockIndexHeader indexHeader = readMergeBlockIndexHeader(thriftReader);
MergedBlockIndex mergedBlockIndex = readMergeBlockIndex(thriftReader);
List<String> file_names = indexHeader.getFile_names();
List<ByteBuffer> fileData = mergedBlockIndex.getFileData();
CarbonFile mergeFile = FileFactory.getCarbonFile(mergeFilePath);
assert (file_names.size() == fileData.size());
for (int i = 0; i < file_names.size(); i++) {
carbonIndexMap.put(file_names.get(i), fileData.get(i).array());
carbonIndexMapWithFullPath.put(mergeFile.getParentFile().getAbsolutePath() + CarbonCommonConstants.FILE_SEPARATOR + file_names.get(i), fileData.get(i).array());
}
} finally {
thriftReader.close();
}
}
use of org.apache.carbondata.core.reader.ThriftReader in project carbondata by apache.
the class ThriftReaderTest method testReadForException.
@Test(expected = java.io.IOException.class)
public void testReadForException() throws IOException {
ThriftReader.TBaseCreator tBaseCreator = new ThriftReader.TBaseCreator() {
@Override
public TBase create() {
return new ColumnDictionaryChunkMeta();
}
};
new MockUp<ColumnDictionaryChunkMeta>() {
@Mock
public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
throw new TException("TException Occur");
}
};
thriftReader = new ThriftReader("TestFile.carbon", tBaseCreator);
thriftReader.read();
}
Aggregations