Examples with CarbonTablePath org.apache.carbondata.core.util.path.CarbonTablePath used on opensource projects

Example 1 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonUtil method calculateDriverBTreeSize.

/**
   * The method returns the B-Tree for a particular taskId
   *
   * @param taskId
   * @param tableBlockInfoList
   * @param absoluteTableIdentifier
   */
public static long calculateDriverBTreeSize(String taskId, String bucketNumber, List<TableBlockInfo> tableBlockInfoList, AbsoluteTableIdentifier absoluteTableIdentifier) {
    // need to sort the  block info list based for task in ascending  order so
    // it will be sinkup with block index read from file
    Collections.sort(tableBlockInfoList);
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    // geting the index file path
    //TODO need to pass proper partition number when partiton will be supported
    String carbonIndexFilePath = carbonTablePath.getCarbonIndexFilePath(taskId, "0", tableBlockInfoList.get(0).getSegmentId(), bucketNumber, CarbonTablePath.DataFileUtil.getTimeStampFromFileName(tableBlockInfoList.get(0).getFilePath()), tableBlockInfoList.get(0).getVersion());
    CarbonFile carbonFile = FileFactory.getCarbonFile(carbonIndexFilePath, FileFactory.getFileType(carbonIndexFilePath));
    // in case of carbonIndex file whole file is meta only so reading complete file.
    return carbonFile.getSize();
}

Example 2 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class SegmentStatusManager method updateDeletionStatus.

/**
   * updates deletion status
   *
   * @param loadDate
   * @param tableFolderPath
   * @return
   */
public static List<String> updateDeletionStatus(AbsoluteTableIdentifier identifier, String loadDate, String tableFolderPath, Long loadStartTime) throws Exception {
    CarbonTableIdentifier carbonTableIdentifier = identifier.getCarbonTableIdentifier();
    ICarbonLock carbonDeleteSegmentLock = CarbonLockFactory.getCarbonLockObj(carbonTableIdentifier, LockUsage.DELETE_SEGMENT_LOCK);
    ICarbonLock carbonTableStatusLock = CarbonLockFactory.getCarbonLockObj(carbonTableIdentifier, LockUsage.TABLE_STATUS_LOCK);
    String tableDetails = carbonTableIdentifier.getDatabaseName() + "." + carbonTableIdentifier.getTableName();
    List<String> invalidLoadTimestamps = new ArrayList<String>(0);
    try {
        if (carbonDeleteSegmentLock.lockWithRetries()) {
            LOG.info("Delete segment lock has been successfully acquired");
            CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(identifier.getStorePath(), identifier.getCarbonTableIdentifier());
            String dataLoadLocation = carbonTablePath.getTableStatusFilePath();
            LoadMetadataDetails[] listOfLoadFolderDetailsArray = null;
            if (!FileFactory.isFileExist(dataLoadLocation, FileFactory.getFileType(dataLoadLocation))) {
                // log error.
                LOG.error("Error message: " + "Load metadata file is not present.");
                invalidLoadTimestamps.add(loadDate);
                return invalidLoadTimestamps;
            }
            // read existing metadata details in load metadata.
            listOfLoadFolderDetailsArray = readLoadMetadata(tableFolderPath);
            if (listOfLoadFolderDetailsArray != null && listOfLoadFolderDetailsArray.length != 0) {
                updateDeletionStatus(loadDate, listOfLoadFolderDetailsArray, invalidLoadTimestamps, loadStartTime);
                if (invalidLoadTimestamps.isEmpty()) {
                    if (carbonTableStatusLock.lockWithRetries()) {
                        LOG.info("Table status lock has been successfully acquired.");
                        // To handle concurrency scenarios, always take latest metadata before writing
                        // into status file.
                        LoadMetadataDetails[] latestLoadMetadataDetails = readLoadMetadata(tableFolderPath);
                        updateLatestTableStatusDetails(listOfLoadFolderDetailsArray, latestLoadMetadataDetails);
                        writeLoadDetailsIntoFile(dataLoadLocation, listOfLoadFolderDetailsArray);
                    } else {
                        String errorMsg = "Delete segment by date is failed for " + tableDetails + ". Not able to acquire the table status lock due to other operation running " + "in the background.";
                        LOG.audit(errorMsg);
                        LOG.error(errorMsg);
                        throw new Exception(errorMsg + " Please try after some time.");
                    }
                } else {
                    return invalidLoadTimestamps;
                }
            } else {
                LOG.audit("Delete segment by date is failed. No matching segment found.");
                invalidLoadTimestamps.add(loadDate);
                return invalidLoadTimestamps;
            }
        } else {
            String errorMsg = "Delete segment by date is failed for " + tableDetails + ". Not able to acquire the delete segment lock due to another delete " + "operation is running in the background.";
            LOG.audit(errorMsg);
            LOG.error(errorMsg);
            throw new Exception(errorMsg + " Please try after some time.");
        }
    } catch (IOException e) {
        LOG.error("Error message: " + "IOException" + e.getMessage());
        throw e;
    } finally {
        CarbonLockUtil.fileUnlock(carbonTableStatusLock, LockUsage.TABLE_STATUS_LOCK);
        CarbonLockUtil.fileUnlock(carbonDeleteSegmentLock, LockUsage.DELETE_SEGMENT_LOCK);
    }
    return invalidLoadTimestamps;
}

Example 3 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class SegmentUpdateStatusManager method getTimestampForRefreshCache.

/**
   * compares passed time stamp with status file delete timestamp and
   * returns latest timestamp from status file if both are not equal
   * returns null otherwise
   *
   * @param completeBlockName
   * @param timestamp
   * @return
   */
public String getTimestampForRefreshCache(String completeBlockName, String timestamp) {
    long cacheTimestamp = 0;
    if (null != timestamp) {
        cacheTimestamp = CarbonUpdateUtil.getTimeStampAsLong(timestamp);
    }
    String blockName = CarbonTablePath.addDataPartPrefix(CarbonUpdateUtil.getBlockName(CarbonUpdateUtil.getRequiredFieldFromTID(completeBlockName, TupleIdEnum.BLOCK_ID)));
    String segmentId = CarbonUpdateUtil.getRequiredFieldFromTID(completeBlockName, TupleIdEnum.SEGMENT_ID);
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    SegmentUpdateDetails[] listOfSegmentUpdateDetailsArray = readLoadMetadata();
    for (SegmentUpdateDetails block : listOfSegmentUpdateDetailsArray) {
        if (segmentId.equalsIgnoreCase(block.getSegmentName()) && block.getBlockName().equalsIgnoreCase(blockName) && !CarbonUpdateUtil.isBlockInvalid(block.getStatus())) {
            long deleteTimestampFromStatusFile = block.getDeleteDeltaEndTimeAsLong();
            if (Long.compare(deleteTimestampFromStatusFile, cacheTimestamp) == 0) {
                return null;
            } else {
                return block.getDeleteDeltaEndTimestamp();
            }
        }
    }
    return null;
}

Example 4 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class SegmentUpdateStatusManager method writeLoadDetailsIntoFile.

/**
   * writes segment update details into a given file at @param dataLoadLocation
   *
   * @param listOfSegmentUpdateDetailsArray
   * @throws IOException
   */
public void writeLoadDetailsIntoFile(List<SegmentUpdateDetails> listOfSegmentUpdateDetailsArray, String updateStatusFileIdentifier) throws IOException {
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String fileLocation = carbonTablePath.getMetadataDirectoryPath() + CarbonCommonConstants.FILE_SEPARATOR + CarbonUpdateUtil.getUpdateStatusFileName(updateStatusFileIdentifier);
    AtomicFileOperations fileWrite = new AtomicFileOperationsImpl(fileLocation, FileFactory.getFileType(fileLocation));
    BufferedWriter brWriter = null;
    DataOutputStream dataOutputStream = null;
    Gson gsonObjectToWrite = new Gson();
    try {
        dataOutputStream = fileWrite.openForWrite(FileWriteOperation.OVERWRITE);
        brWriter = new BufferedWriter(new OutputStreamWriter(dataOutputStream, CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT));
        String metadataInstance = gsonObjectToWrite.toJson(listOfSegmentUpdateDetailsArray);
        brWriter.write(metadataInstance);
    } catch (IOException ioe) {
        LOG.error("Error message: " + ioe.getLocalizedMessage());
    } finally {
        if (null != brWriter) {
            brWriter.flush();
        }
        CarbonUtil.closeStreams(brWriter);
        fileWrite.close();
    }
}

Example 5 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class SegmentUpdateStatusManager method getUpdatedStatusIdentifier.

/**
   * @return updateStatusFileName
   */
private String getUpdatedStatusIdentifier() {
    SegmentStatusManager ssm = new SegmentStatusManager(absoluteTableIdentifier);
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    LoadMetadataDetails[] loadDetails = ssm.readLoadMetadata(carbonTablePath.getMetadataDirectoryPath());
    if (loadDetails.length == 0) {
        return null;
    }
    return loadDetails[0].getUpdateStatusFileName();
}