Examples with CarbonTablePath org.apache.carbondata.core.util.path.CarbonTablePath used on opensource projects

Example 36 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonDataProcessorUtil method getLocalDataFolderLocation.

/**
   * This method will form the local data folder store location
   *
   * @param databaseName
   * @param tableName
   * @param taskId
   * @param partitionId
   * @param segmentId
   * @return
   */
public static String getLocalDataFolderLocation(String databaseName, String tableName, String taskId, String partitionId, String segmentId, boolean isCompactionFlow) {
    String tempLocationKey = getTempStoreLocationKey(databaseName, tableName, taskId, isCompactionFlow);
    String baseStorePath = CarbonProperties.getInstance().getProperty(tempLocationKey, CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL);
    CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(databaseName + CarbonCommonConstants.UNDERSCORE + tableName);
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(baseStorePath, carbonTable.getCarbonTableIdentifier());
    String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath(partitionId, segmentId + "");
    return carbonDataDirectoryPath + File.separator + taskId;
}

Example 37 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class SegmentStatusManager method getValidAndInvalidSegments.

/**
   * get valid segment for given table
   *
   * @return
   * @throws IOException
   */
public ValidAndInvalidSegmentsInfo getValidAndInvalidSegments() throws IOException {
    // @TODO: move reading LoadStatus file to separate class
    List<String> listOfValidSegments = new ArrayList<String>(10);
    List<String> listOfValidUpdatedSegments = new ArrayList<String>(10);
    List<String> listOfInvalidSegments = new ArrayList<String>(10);
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String dataPath = carbonTablePath.getTableStatusFilePath();
    DataInputStream dataInputStream = null;
    Gson gsonObjectToRead = new Gson();
    AtomicFileOperations fileOperation = new AtomicFileOperationsImpl(dataPath, FileFactory.getFileType(dataPath));
    LoadMetadataDetails[] loadFolderDetailsArray;
    try {
        if (FileFactory.isFileExist(dataPath, FileFactory.getFileType(dataPath))) {
            dataInputStream = fileOperation.openForRead();
            BufferedReader buffReader = new BufferedReader(new InputStreamReader(dataInputStream, "UTF-8"));
            loadFolderDetailsArray = gsonObjectToRead.fromJson(buffReader, LoadMetadataDetails[].class);
            //just directly iterate Array
            for (LoadMetadataDetails loadMetadataDetails : loadFolderDetailsArray) {
                if (CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS.equalsIgnoreCase(loadMetadataDetails.getLoadStatus()) || CarbonCommonConstants.MARKED_FOR_UPDATE.equalsIgnoreCase(loadMetadataDetails.getLoadStatus()) || CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS.equalsIgnoreCase(loadMetadataDetails.getLoadStatus())) {
                    // check for merged loads.
                    if (null != loadMetadataDetails.getMergedLoadName()) {
                        if (!listOfValidSegments.contains(loadMetadataDetails.getMergedLoadName())) {
                            listOfValidSegments.add(loadMetadataDetails.getMergedLoadName());
                        }
                        // if merged load is updated then put it in updated list
                        if (CarbonCommonConstants.MARKED_FOR_UPDATE.equalsIgnoreCase(loadMetadataDetails.getLoadStatus())) {
                            listOfValidUpdatedSegments.add(loadMetadataDetails.getMergedLoadName());
                        }
                        continue;
                    }
                    if (CarbonCommonConstants.MARKED_FOR_UPDATE.equalsIgnoreCase(loadMetadataDetails.getLoadStatus())) {
                        listOfValidUpdatedSegments.add(loadMetadataDetails.getLoadName());
                    }
                    listOfValidSegments.add(loadMetadataDetails.getLoadName());
                } else if ((CarbonCommonConstants.STORE_LOADSTATUS_FAILURE.equalsIgnoreCase(loadMetadataDetails.getLoadStatus()) || CarbonCommonConstants.COMPACTED.equalsIgnoreCase(loadMetadataDetails.getLoadStatus()) || CarbonCommonConstants.MARKED_FOR_DELETE.equalsIgnoreCase(loadMetadataDetails.getLoadStatus()))) {
                    listOfInvalidSegments.add(loadMetadataDetails.getLoadName());
                }
            }
        }
    } catch (IOException e) {
        LOG.error(e);
        throw e;
    } finally {
        try {
            if (null != dataInputStream) {
                dataInputStream.close();
            }
        } catch (Exception e) {
            LOG.error(e);
            throw e;
        }
    }
    return new ValidAndInvalidSegmentsInfo(listOfValidSegments, listOfValidUpdatedSegments, listOfInvalidSegments);
}

Example 38 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class SegmentStatusManager method updateDeletionStatus.

/**
   * updates deletion status
   *
   * @param loadIds
   * @param tableFolderPath
   * @return
   */
public static List<String> updateDeletionStatus(AbsoluteTableIdentifier identifier, List<String> loadIds, String tableFolderPath) throws Exception {
    CarbonTableIdentifier carbonTableIdentifier = identifier.getCarbonTableIdentifier();
    ICarbonLock carbonDeleteSegmentLock = CarbonLockFactory.getCarbonLockObj(carbonTableIdentifier, LockUsage.DELETE_SEGMENT_LOCK);
    ICarbonLock carbonTableStatusLock = CarbonLockFactory.getCarbonLockObj(carbonTableIdentifier, LockUsage.TABLE_STATUS_LOCK);
    String tableDetails = carbonTableIdentifier.getDatabaseName() + "." + carbonTableIdentifier.getTableName();
    List<String> invalidLoadIds = new ArrayList<String>(0);
    try {
        if (carbonDeleteSegmentLock.lockWithRetries()) {
            LOG.info("Delete segment lock has been successfully acquired");
            CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(identifier.getStorePath(), identifier.getCarbonTableIdentifier());
            String dataLoadLocation = carbonTablePath.getTableStatusFilePath();
            LoadMetadataDetails[] listOfLoadFolderDetailsArray = null;
            if (!FileFactory.isFileExist(dataLoadLocation, FileFactory.getFileType(dataLoadLocation))) {
                // log error.
                LOG.error("Load metadata file is not present.");
                return loadIds;
            }
            // read existing metadata details in load metadata.
            listOfLoadFolderDetailsArray = readLoadMetadata(tableFolderPath);
            if (listOfLoadFolderDetailsArray != null && listOfLoadFolderDetailsArray.length != 0) {
                updateDeletionStatus(loadIds, listOfLoadFolderDetailsArray, invalidLoadIds);
                if (invalidLoadIds.isEmpty()) {
                    // All or None , if anything fails then dont write
                    if (carbonTableStatusLock.lockWithRetries()) {
                        LOG.info("Table status lock has been successfully acquired");
                        // To handle concurrency scenarios, always take latest metadata before writing
                        // into status file.
                        LoadMetadataDetails[] latestLoadMetadataDetails = readLoadMetadata(tableFolderPath);
                        updateLatestTableStatusDetails(listOfLoadFolderDetailsArray, latestLoadMetadataDetails);
                        writeLoadDetailsIntoFile(dataLoadLocation, listOfLoadFolderDetailsArray);
                    } else {
                        String errorMsg = "Delete segment by id is failed for " + tableDetails + ". Not able to acquire the table status lock due to other operation running " + "in the background.";
                        LOG.audit(errorMsg);
                        LOG.error(errorMsg);
                        throw new Exception(errorMsg + " Please try after some time.");
                    }
                } else {
                    return invalidLoadIds;
                }
            } else {
                LOG.audit("Delete segment by Id is failed. No matching segment id found.");
                return loadIds;
            }
        } else {
            String errorMsg = "Delete segment by id is failed for " + tableDetails + ". Not able to acquire the delete segment lock due to another delete " + "operation is running in the background.";
            LOG.audit(errorMsg);
            LOG.error(errorMsg);
            throw new Exception(errorMsg + " Please try after some time.");
        }
    } catch (IOException e) {
        LOG.error("IOException" + e.getMessage());
        throw e;
    } finally {
        CarbonLockUtil.fileUnlock(carbonTableStatusLock, LockUsage.TABLE_STATUS_LOCK);
        CarbonLockUtil.fileUnlock(carbonDeleteSegmentLock, LockUsage.DELETE_SEGMENT_LOCK);
    }
    return invalidLoadIds;
}

Example 39 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class SegmentUpdateStatusManager method getDeleteDeltaFilesList.

/**
   * Return all delta file for a block.
   * @param segmentId
   * @param blockName
   * @return
   */
public CarbonFile[] getDeleteDeltaFilesList(final String segmentId, final String blockName) {
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
    CarbonFile segDir = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
    for (SegmentUpdateDetails block : updateDetails) {
        if ((block.getBlockName().equalsIgnoreCase(blockName)) && (block.getSegmentName().equalsIgnoreCase(segmentId)) && !CarbonUpdateUtil.isBlockInvalid((block.getStatus()))) {
            final long deltaStartTimestamp = getStartTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
            final long deltaEndTimeStamp = getEndTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
            return segDir.listFiles(new CarbonFileFilter() {

                @Override
                public boolean accept(CarbonFile pathName) {
                    String fileName = pathName.getName();
                    if (fileName.endsWith(CarbonCommonConstants.DELETE_DELTA_FILE_EXT)) {
                        String firstPart = fileName.substring(0, fileName.indexOf('.'));
                        String blkName = firstPart.substring(0, firstPart.lastIndexOf("-"));
                        long timestamp = Long.parseLong(firstPart.substring(firstPart.lastIndexOf("-") + 1, firstPart.length()));
                        if (blockName.equals(blkName) && (Long.compare(timestamp, deltaEndTimeStamp) <= 0) && (Long.compare(timestamp, deltaStartTimestamp) >= 0)) {
                            return true;
                        }
                    }
                    return false;
                }
            });
        }
    }
    return null;
}

Example 40 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class SegmentUpdateStatusManager method getInvalidBlockList.

/**
   * Get the invalid tasks in that segment.
   * @param segmentId
   * @return
   */
public List<String> getInvalidBlockList(String segmentId) {
    // get the original fact file timestamp from the table status file.
    List<String> listOfInvalidBlocks = new ArrayList<String>();
    SegmentStatusManager ssm = new SegmentStatusManager(absoluteTableIdentifier);
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    LoadMetadataDetails[] segmentDetails = ssm.readLoadMetadata(carbonTablePath.getMetadataDirectoryPath());
    long timestampOfOriginalFacts = 0;
    String startTimestampOfUpdate = "";
    String endTimestampOfUpdate = "";
    for (LoadMetadataDetails segment : segmentDetails) {
        // find matching segment and return timestamp.
        if (segment.getLoadName().equalsIgnoreCase(segmentId)) {
            timestampOfOriginalFacts = segment.getLoadStartTime();
            startTimestampOfUpdate = segment.getUpdateDeltaStartTimestamp();
            endTimestampOfUpdate = segment.getUpdateDeltaEndTimestamp();
        }
    }
    if (startTimestampOfUpdate.isEmpty()) {
        return listOfInvalidBlocks;
    }
    // now after getting the original fact timestamp, what ever is remaining
    // files need to cross check it with table status file.
    // filter out the fact files.
    String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
    CarbonFile segDir = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
    final Long endTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(endTimestampOfUpdate);
    final Long startTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(startTimestampOfUpdate);
    final Long timeStampOriginalFactFinal = timestampOfOriginalFacts;
    CarbonFile[] files = segDir.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile pathName) {
            String fileName = pathName.getName();
            if (fileName.endsWith(CarbonCommonConstants.UPDATE_DELTA_FILE_EXT)) {
                String firstPart = fileName.substring(0, fileName.indexOf('.'));
                long timestamp = Long.parseLong(firstPart.substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1, firstPart.length()));
                if (Long.compare(timestamp, endTimeStampFinal) <= 0 && Long.compare(timestamp, startTimeStampFinal) >= 0) {
                    return false;
                }
                if (Long.compare(timestamp, timeStampOriginalFactFinal) == 0) {
                    return false;
                }
                // take the rest of files as they are invalid.
                return true;
            }
            return false;
        }
    });
    // gather the task numbers.
    for (CarbonFile updateFiles : files) {
        listOfInvalidBlocks.add(updateFiles.getName());
    }
    return listOfInvalidBlocks;
}