Examples with CarbonTablePath org.apache.carbondata.core.util.path.CarbonTablePath used on opensource projects

Example 41 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class AbstractDictionaryCache method getDictionaryMetaCarbonFile.

/**
   * This method will return the carbon file objetc based on its type (local, HDFS)
   *
   * @param dictionaryColumnUniqueIdentifier
   * @return
   */
private CarbonFile getDictionaryMetaCarbonFile(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException {
    PathService pathService = CarbonCommonFactory.getPathService();
    CarbonTablePath carbonTablePath = pathService.getCarbonTablePath(carbonStorePath, dictionaryColumnUniqueIdentifier.getCarbonTableIdentifier());
    String dictionaryFilePath = carbonTablePath.getDictionaryMetaFilePath(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId());
    FileFactory.FileType fileType = FileFactory.getFileType(dictionaryFilePath);
    CarbonFile dictFile = FileFactory.getCarbonFile(dictionaryFilePath, fileType);
    // When rename table triggered parallely with select query, dictionary files may not exist
    if (!dictFile.exists()) {
        throw new IOException("Dictionary file does not exist: " + dictionaryFilePath);
    }
    return dictFile;
}

Example 42 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonTableReader method getFileStatusOfSegments.

private FileSystem getFileStatusOfSegments(String[] segmentsToConsider, CarbonTablePath tablePath, List<FileStatus> result) throws IOException {
    String[] partitionsToConsider = getValidPartitions();
    if (partitionsToConsider.length == 0) {
        throw new IOException("No partitions/data found");
    }
    FileSystem fs = null;
    //get all data files of valid partitions and segments
    for (int i = 0; i < partitionsToConsider.length; ++i) {
        String partition = partitionsToConsider[i];
        for (int j = 0; j < segmentsToConsider.length; ++j) {
            String segmentId = segmentsToConsider[j];
            Path segmentPath = new Path(tablePath.getCarbonDataDirectoryPath(partition, segmentId));
            try {
                Configuration conf = new Configuration();
                fs = segmentPath.getFileSystem(conf);
                RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(segmentPath);
                while (iter.hasNext()) {
                    LocatedFileStatus stat = iter.next();
                    if (DefaultFilter.accept(stat.getPath())) {
                        if (stat.isDirectory()) {
                            addInputPathRecursively(result, fs, stat.getPath(), DefaultFilter);
                        } else {
                            result.add(stat);
                        }
                    }
                }
            } catch (Exception ex) {
                System.out.println(ex.toString());
            }
        }
    }
    return fs;
}

Example 43 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonLoaderUtil method checkAndCreateCarbonDataLocation.

/**
   * This method will get the store location for the given path, segment id and partition id
   *
   * @param carbonStorePath
   * @param segmentId
   */
public static void checkAndCreateCarbonDataLocation(String carbonStorePath, String segmentId, CarbonTable carbonTable) {
    CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonStorePath, carbonTableIdentifier);
    String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
    CarbonUtil.checkAndCreateFolder(carbonDataDirectoryPath);
}

Example 44 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonDataMergerUtil method updateStatusFile.

public static Boolean updateStatusFile(List<CarbonDataMergerUtilResult> updateDataMergerDetailsList, CarbonTable table, String timestamp, SegmentUpdateStatusManager segmentUpdateStatusManager) {
    List<SegmentUpdateDetails> segmentUpdateDetails = new ArrayList<SegmentUpdateDetails>(updateDataMergerDetailsList.size());
    // Check the list output.
    for (CarbonDataMergerUtilResult carbonDataMergerUtilResult : updateDataMergerDetailsList) {
        if (carbonDataMergerUtilResult.getCompactionStatus()) {
            SegmentUpdateDetails tempSegmentUpdateDetails = new SegmentUpdateDetails();
            tempSegmentUpdateDetails.setSegmentName(carbonDataMergerUtilResult.getSegmentName());
            tempSegmentUpdateDetails.setBlockName(carbonDataMergerUtilResult.getBlockName());
            for (SegmentUpdateDetails origDetails : segmentUpdateStatusManager.getUpdateStatusDetails()) {
                if (origDetails.getBlockName().equalsIgnoreCase(carbonDataMergerUtilResult.getBlockName()) && origDetails.getSegmentName().equalsIgnoreCase(carbonDataMergerUtilResult.getSegmentName())) {
                    tempSegmentUpdateDetails.setDeletedRowsInBlock(origDetails.getDeletedRowsInBlock());
                    tempSegmentUpdateDetails.setStatus(origDetails.getStatus());
                    break;
                }
            }
            tempSegmentUpdateDetails.setDeleteDeltaStartTimestamp(carbonDataMergerUtilResult.getDeleteDeltaStartTimestamp());
            tempSegmentUpdateDetails.setDeleteDeltaEndTimestamp(carbonDataMergerUtilResult.getDeleteDeltaEndTimestamp());
            segmentUpdateDetails.add(tempSegmentUpdateDetails);
        } else
            return false;
    }
    CarbonUpdateUtil.updateSegmentStatus(segmentUpdateDetails, table, timestamp, true);
    // Update the Table Status.
    String metaDataFilepath = table.getMetaDataFilepath();
    AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String tableStatusPath = carbonTablePath.getTableStatusFilePath();
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
    boolean lockStatus = false;
    try {
        lockStatus = carbonLock.lockWithRetries();
        if (lockStatus) {
            LOGGER.info("Acquired lock for table" + table.getDatabaseName() + "." + table.getFactTableName() + " for table status updation");
            LoadMetadataDetails[] listOfLoadFolderDetailsArray = segmentStatusManager.readLoadMetadata(metaDataFilepath);
            for (LoadMetadataDetails loadMetadata : listOfLoadFolderDetailsArray) {
                if (loadMetadata.getLoadName().equalsIgnoreCase("0")) {
                    loadMetadata.setUpdateStatusFileName(CarbonUpdateUtil.getUpdateStatusFileName(timestamp));
                }
            }
            try {
                segmentStatusManager.writeLoadDetailsIntoFile(tableStatusPath, listOfLoadFolderDetailsArray);
            } catch (IOException e) {
                return false;
            }
        } else {
            LOGGER.error("Not able to acquire the lock for Table status updation for table " + table.getDatabaseName() + "." + table.getFactTableName());
        }
    } finally {
        if (lockStatus) {
            if (carbonLock.unlock()) {
                LOGGER.info("Table unlocked successfully after table status updation" + table.getDatabaseName() + "." + table.getFactTableName());
            } else {
                LOGGER.error("Unable to unlock Table lock for table" + table.getDatabaseName() + "." + table.getFactTableName() + " during table status updation");
            }
        }
    }
    return true;
}

Example 45 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonUpdateUtil method updateSegmentStatus.

/**
   * @param updateDetailsList
   * @param table
   * @param updateStatusFileIdentifier
   * @return
   */
public static boolean updateSegmentStatus(List<SegmentUpdateDetails> updateDetailsList, CarbonTable table, String updateStatusFileIdentifier, boolean isCompaction) {
    boolean status = false;
    SegmentUpdateStatusManager segmentUpdateStatusManager = new SegmentUpdateStatusManager(table.getAbsoluteTableIdentifier());
    ICarbonLock updateLock = segmentUpdateStatusManager.getTableUpdateStatusLock();
    boolean lockStatus = false;
    try {
        lockStatus = updateLock.lockWithRetries();
        if (lockStatus) {
            AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
            CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
            // read the existing file if present and update the same.
            SegmentUpdateDetails[] oldDetails = segmentUpdateStatusManager.getUpdateStatusDetails();
            List<SegmentUpdateDetails> oldList = new ArrayList(Arrays.asList(oldDetails));
            for (SegmentUpdateDetails newBlockEntry : updateDetailsList) {
                int index = oldList.indexOf(newBlockEntry);
                if (index != -1) {
                    // update the element in existing list.
                    SegmentUpdateDetails blockDetail = oldList.get(index);
                    if (blockDetail.getDeleteDeltaStartTimestamp().isEmpty() || (isCompaction == true)) {
                        blockDetail.setDeleteDeltaStartTimestamp(newBlockEntry.getDeleteDeltaStartTimestamp());
                    }
                    blockDetail.setDeleteDeltaEndTimestamp(newBlockEntry.getDeleteDeltaEndTimestamp());
                    blockDetail.setStatus(newBlockEntry.getStatus());
                    blockDetail.setDeletedRowsInBlock(newBlockEntry.getDeletedRowsInBlock());
                } else {
                    // add the new details to the list.
                    oldList.add(newBlockEntry);
                }
            }
            segmentUpdateStatusManager.writeLoadDetailsIntoFile(oldList, updateStatusFileIdentifier);
            status = true;
        } else {
            LOGGER.error("Not able to acquire the segment update lock.");
            status = false;
        }
    } catch (IOException e) {
        status = false;
    } finally {
        if (lockStatus) {
            if (updateLock.unlock()) {
                LOGGER.info("Unlock the segment update lock successfull.");
            } else {
                LOGGER.error("Not able to unlock the segment update lock.");
            }
        }
    }
    return status;
}