use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.
the class AbstractDictionaryCache method getDictionaryMetaCarbonFile.
/**
* This method will return the carbon file objetc based on its type (local, HDFS)
*
* @param dictionaryColumnUniqueIdentifier
* @return
*/
private CarbonFile getDictionaryMetaCarbonFile(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException {
PathService pathService = CarbonCommonFactory.getPathService();
CarbonTablePath carbonTablePath = pathService.getCarbonTablePath(carbonStorePath, dictionaryColumnUniqueIdentifier.getCarbonTableIdentifier());
String dictionaryFilePath = carbonTablePath.getDictionaryMetaFilePath(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId());
FileFactory.FileType fileType = FileFactory.getFileType(dictionaryFilePath);
CarbonFile dictFile = FileFactory.getCarbonFile(dictionaryFilePath, fileType);
// When rename table triggered parallely with select query, dictionary files may not exist
if (!dictFile.exists()) {
throw new IOException("Dictionary file does not exist: " + dictionaryFilePath);
}
return dictFile;
}
use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.
the class CarbonTableReader method getFileStatusOfSegments.
private FileSystem getFileStatusOfSegments(String[] segmentsToConsider, CarbonTablePath tablePath, List<FileStatus> result) throws IOException {
String[] partitionsToConsider = getValidPartitions();
if (partitionsToConsider.length == 0) {
throw new IOException("No partitions/data found");
}
FileSystem fs = null;
//get all data files of valid partitions and segments
for (int i = 0; i < partitionsToConsider.length; ++i) {
String partition = partitionsToConsider[i];
for (int j = 0; j < segmentsToConsider.length; ++j) {
String segmentId = segmentsToConsider[j];
Path segmentPath = new Path(tablePath.getCarbonDataDirectoryPath(partition, segmentId));
try {
Configuration conf = new Configuration();
fs = segmentPath.getFileSystem(conf);
RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(segmentPath);
while (iter.hasNext()) {
LocatedFileStatus stat = iter.next();
if (DefaultFilter.accept(stat.getPath())) {
if (stat.isDirectory()) {
addInputPathRecursively(result, fs, stat.getPath(), DefaultFilter);
} else {
result.add(stat);
}
}
}
} catch (Exception ex) {
System.out.println(ex.toString());
}
}
}
return fs;
}
use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.
the class CarbonLoaderUtil method checkAndCreateCarbonDataLocation.
/**
* This method will get the store location for the given path, segment id and partition id
*
* @param carbonStorePath
* @param segmentId
*/
public static void checkAndCreateCarbonDataLocation(String carbonStorePath, String segmentId, CarbonTable carbonTable) {
CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier();
CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonStorePath, carbonTableIdentifier);
String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
CarbonUtil.checkAndCreateFolder(carbonDataDirectoryPath);
}
use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.
the class CarbonDataMergerUtil method updateStatusFile.
public static Boolean updateStatusFile(List<CarbonDataMergerUtilResult> updateDataMergerDetailsList, CarbonTable table, String timestamp, SegmentUpdateStatusManager segmentUpdateStatusManager) {
List<SegmentUpdateDetails> segmentUpdateDetails = new ArrayList<SegmentUpdateDetails>(updateDataMergerDetailsList.size());
// Check the list output.
for (CarbonDataMergerUtilResult carbonDataMergerUtilResult : updateDataMergerDetailsList) {
if (carbonDataMergerUtilResult.getCompactionStatus()) {
SegmentUpdateDetails tempSegmentUpdateDetails = new SegmentUpdateDetails();
tempSegmentUpdateDetails.setSegmentName(carbonDataMergerUtilResult.getSegmentName());
tempSegmentUpdateDetails.setBlockName(carbonDataMergerUtilResult.getBlockName());
for (SegmentUpdateDetails origDetails : segmentUpdateStatusManager.getUpdateStatusDetails()) {
if (origDetails.getBlockName().equalsIgnoreCase(carbonDataMergerUtilResult.getBlockName()) && origDetails.getSegmentName().equalsIgnoreCase(carbonDataMergerUtilResult.getSegmentName())) {
tempSegmentUpdateDetails.setDeletedRowsInBlock(origDetails.getDeletedRowsInBlock());
tempSegmentUpdateDetails.setStatus(origDetails.getStatus());
break;
}
}
tempSegmentUpdateDetails.setDeleteDeltaStartTimestamp(carbonDataMergerUtilResult.getDeleteDeltaStartTimestamp());
tempSegmentUpdateDetails.setDeleteDeltaEndTimestamp(carbonDataMergerUtilResult.getDeleteDeltaEndTimestamp());
segmentUpdateDetails.add(tempSegmentUpdateDetails);
} else
return false;
}
CarbonUpdateUtil.updateSegmentStatus(segmentUpdateDetails, table, timestamp, true);
// Update the Table Status.
String metaDataFilepath = table.getMetaDataFilepath();
AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
String tableStatusPath = carbonTablePath.getTableStatusFilePath();
SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
boolean lockStatus = false;
try {
lockStatus = carbonLock.lockWithRetries();
if (lockStatus) {
LOGGER.info("Acquired lock for table" + table.getDatabaseName() + "." + table.getFactTableName() + " for table status updation");
LoadMetadataDetails[] listOfLoadFolderDetailsArray = segmentStatusManager.readLoadMetadata(metaDataFilepath);
for (LoadMetadataDetails loadMetadata : listOfLoadFolderDetailsArray) {
if (loadMetadata.getLoadName().equalsIgnoreCase("0")) {
loadMetadata.setUpdateStatusFileName(CarbonUpdateUtil.getUpdateStatusFileName(timestamp));
}
}
try {
segmentStatusManager.writeLoadDetailsIntoFile(tableStatusPath, listOfLoadFolderDetailsArray);
} catch (IOException e) {
return false;
}
} else {
LOGGER.error("Not able to acquire the lock for Table status updation for table " + table.getDatabaseName() + "." + table.getFactTableName());
}
} finally {
if (lockStatus) {
if (carbonLock.unlock()) {
LOGGER.info("Table unlocked successfully after table status updation" + table.getDatabaseName() + "." + table.getFactTableName());
} else {
LOGGER.error("Unable to unlock Table lock for table" + table.getDatabaseName() + "." + table.getFactTableName() + " during table status updation");
}
}
}
return true;
}
use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.
the class CarbonUpdateUtil method updateSegmentStatus.
/**
* @param updateDetailsList
* @param table
* @param updateStatusFileIdentifier
* @return
*/
public static boolean updateSegmentStatus(List<SegmentUpdateDetails> updateDetailsList, CarbonTable table, String updateStatusFileIdentifier, boolean isCompaction) {
boolean status = false;
SegmentUpdateStatusManager segmentUpdateStatusManager = new SegmentUpdateStatusManager(table.getAbsoluteTableIdentifier());
ICarbonLock updateLock = segmentUpdateStatusManager.getTableUpdateStatusLock();
boolean lockStatus = false;
try {
lockStatus = updateLock.lockWithRetries();
if (lockStatus) {
AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
// read the existing file if present and update the same.
SegmentUpdateDetails[] oldDetails = segmentUpdateStatusManager.getUpdateStatusDetails();
List<SegmentUpdateDetails> oldList = new ArrayList(Arrays.asList(oldDetails));
for (SegmentUpdateDetails newBlockEntry : updateDetailsList) {
int index = oldList.indexOf(newBlockEntry);
if (index != -1) {
// update the element in existing list.
SegmentUpdateDetails blockDetail = oldList.get(index);
if (blockDetail.getDeleteDeltaStartTimestamp().isEmpty() || (isCompaction == true)) {
blockDetail.setDeleteDeltaStartTimestamp(newBlockEntry.getDeleteDeltaStartTimestamp());
}
blockDetail.setDeleteDeltaEndTimestamp(newBlockEntry.getDeleteDeltaEndTimestamp());
blockDetail.setStatus(newBlockEntry.getStatus());
blockDetail.setDeletedRowsInBlock(newBlockEntry.getDeletedRowsInBlock());
} else {
// add the new details to the list.
oldList.add(newBlockEntry);
}
}
segmentUpdateStatusManager.writeLoadDetailsIntoFile(oldList, updateStatusFileIdentifier);
status = true;
} else {
LOGGER.error("Not able to acquire the segment update lock.");
status = false;
}
} catch (IOException e) {
status = false;
} finally {
if (lockStatus) {
if (updateLock.unlock()) {
LOGGER.info("Unlock the segment update lock successfull.");
} else {
LOGGER.error("Not able to unlock the segment update lock.");
}
}
}
return status;
}
Aggregations