Examples with CarbonTablePath org.apache.carbondata.core.util.path.CarbonTablePath used on opensource projects

Example 6 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class SegmentUpdateStatusManager method readLoadMetadata.

/**
   * This method loads segment update details
   *
   * @return
   */
public SegmentUpdateDetails[] readLoadMetadata() {
    Gson gsonObjectToRead = new Gson();
    DataInputStream dataInputStream = null;
    BufferedReader buffReader = null;
    InputStreamReader inStream = null;
    SegmentUpdateDetails[] listOfSegmentUpdateDetailsArray;
    // get the updated status file identifier from the table status.
    String tableUpdateStatusIdentifier = getUpdatedStatusIdentifier();
    if (null == tableUpdateStatusIdentifier) {
        return new SegmentUpdateDetails[0];
    }
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String tableUpdateStatusPath = carbonTablePath.getMetadataDirectoryPath() + CarbonCommonConstants.FILE_SEPARATOR + tableUpdateStatusIdentifier;
    AtomicFileOperations fileOperation = new AtomicFileOperationsImpl(tableUpdateStatusPath, FileFactory.getFileType(tableUpdateStatusPath));
    try {
        if (!FileFactory.isFileExist(tableUpdateStatusPath, FileFactory.getFileType(tableUpdateStatusPath))) {
            return new SegmentUpdateDetails[0];
        }
        dataInputStream = fileOperation.openForRead();
        inStream = new InputStreamReader(dataInputStream, CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT);
        buffReader = new BufferedReader(inStream);
        listOfSegmentUpdateDetailsArray = gsonObjectToRead.fromJson(buffReader, SegmentUpdateDetails[].class);
    } catch (IOException e) {
        return new SegmentUpdateDetails[0];
    } finally {
        closeStreams(buffReader, inStream, dataInputStream);
    }
    return listOfSegmentUpdateDetailsArray;
}

Example 7 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class SegmentUpdateStatusManager method getUpdateDeltaFilesList.

/**
   * Returns all update delta files of specified Segment.
   *
   * @param segmentId
   * @param validUpdateFiles if true then only the valid range files will be returned.
   * @return
   */
public CarbonFile[] getUpdateDeltaFilesList(String segmentId, final boolean validUpdateFiles, final String fileExtension, final boolean excludeOriginalFact, CarbonFile[] allFilesOfSegment) {
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    String endTimeStamp = "";
    String startTimeStamp = "";
    long factTimeStamp = 0;
    LoadMetadataDetails[] segmentDetails = segmentStatusManager.readLoadMetadata(carbonTablePath.getMetadataDirectoryPath());
    for (LoadMetadataDetails eachSeg : segmentDetails) {
        if (eachSeg.getLoadName().equalsIgnoreCase(segmentId)) {
            // if the segment is found then take the start and end time stamp.
            startTimeStamp = eachSeg.getUpdateDeltaStartTimestamp();
            endTimeStamp = eachSeg.getUpdateDeltaEndTimestamp();
            factTimeStamp = eachSeg.getLoadStartTime();
        }
    }
    // if start timestamp is empty then no update delta is found. so return empty list.
    if (startTimeStamp.isEmpty()) {
        return new CarbonFile[0];
    }
    final Long endTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(endTimeStamp);
    final Long startTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(startTimeStamp);
    final long factTimeStampFinal = factTimeStamp;
    List<CarbonFile> listOfCarbonFiles = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    for (CarbonFile eachFile : allFilesOfSegment) {
        String fileName = eachFile.getName();
        if (fileName.endsWith(fileExtension)) {
            String firstPart = fileName.substring(0, fileName.indexOf('.'));
            long timestamp = Long.parseLong(firstPart.substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1, firstPart.length()));
            if (excludeOriginalFact) {
                if (Long.compare(factTimeStampFinal, timestamp) == 0) {
                    continue;
                }
            }
            if (validUpdateFiles) {
                if (Long.compare(timestamp, endTimeStampFinal) <= 0 && Long.compare(timestamp, startTimeStampFinal) >= 0) {
                    listOfCarbonFiles.add(eachFile);
                }
            } else {
                // invalid cases.
                if (Long.compare(timestamp, startTimeStampFinal) < 0) {
                    listOfCarbonFiles.add(eachFile);
                }
            }
        }
    }
    return listOfCarbonFiles.toArray(new CarbonFile[listOfCarbonFiles.size()]);
}

Example 8 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonUpdateUtil method cleanUpDeltaFiles.

/**
   * Handling of the clean up of old carbondata files, index files , delte delta,
   * update status files.
   * @param table clean up will be handled on this table.
   * @param forceDelete if true then max query execution timeout will not be considered.
   */
public static void cleanUpDeltaFiles(CarbonTable table, boolean forceDelete) {
    SegmentStatusManager ssm = new SegmentStatusManager(table.getAbsoluteTableIdentifier());
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(table.getAbsoluteTableIdentifier().getStorePath(), table.getAbsoluteTableIdentifier().getCarbonTableIdentifier());
    LoadMetadataDetails[] details = ssm.readLoadMetadata(table.getMetaDataFilepath());
    String validUpdateStatusFile = "";
    for (LoadMetadataDetails segment : details) {
        // take the update status file name from 0th segment.
        validUpdateStatusFile = ssm.getUpdateStatusFileName(details);
        if (segment.getLoadStatus().equalsIgnoreCase(CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS) || segment.getLoadStatus().equalsIgnoreCase(CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS)) {
            // take the list of files from this segment.
            String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segment.getLoadName());
            CarbonFile segDir = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
            CarbonFile[] allSegmentFiles = segDir.listFiles();
            // scan through the segment and find the carbondatafiles and index files.
            SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(table.getAbsoluteTableIdentifier());
            // get Invalid update  delta files.
            CarbonFile[] invalidUpdateDeltaFiles = updateStatusManager.getUpdateDeltaFilesList(segment.getLoadName(), false, CarbonCommonConstants.UPDATE_DELTA_FILE_EXT, true, allSegmentFiles);
            for (CarbonFile invalidFile : invalidUpdateDeltaFiles) {
                compareTimestampsAndDelete(invalidFile, forceDelete, false);
            }
            // do the same for the index files.
            CarbonFile[] invalidIndexFiles = updateStatusManager.getUpdateDeltaFilesList(segment.getLoadName(), false, CarbonCommonConstants.UPDATE_INDEX_FILE_EXT, true, allSegmentFiles);
            for (CarbonFile invalidFile : invalidIndexFiles) {
                compareTimestampsAndDelete(invalidFile, forceDelete, false);
            }
            // now handle all the delete delta files which needs to be deleted.
            // there are 2 cases here .
            // 1. if the block is marked as compacted then the corresponding delta files
            //    can be deleted if query exec timeout is done.
            // 2. if the block is in success state then also there can be delete
            //    delta compaction happened and old files can be deleted.
            SegmentUpdateDetails[] updateDetails = updateStatusManager.readLoadMetadata();
            for (SegmentUpdateDetails block : updateDetails) {
                CarbonFile[] completeListOfDeleteDeltaFiles;
                CarbonFile[] invalidDeleteDeltaFiles;
                if (!block.getSegmentName().equalsIgnoreCase(segment.getLoadName())) {
                    continue;
                }
                // case 1
                if (CarbonUpdateUtil.isBlockInvalid(block.getStatus())) {
                    completeListOfDeleteDeltaFiles = updateStatusManager.getDeleteDeltaInvalidFilesList(segment.getLoadName(), block, true, allSegmentFiles);
                    for (CarbonFile invalidFile : completeListOfDeleteDeltaFiles) {
                        compareTimestampsAndDelete(invalidFile, forceDelete, false);
                    }
                    CarbonFile[] blockRelatedFiles = updateStatusManager.getAllBlockRelatedFiles(block.getBlockName(), allSegmentFiles, block.getActualBlockName());
                    for (CarbonFile invalidFile : blockRelatedFiles) {
                        compareTimestampsAndDelete(invalidFile, forceDelete, false);
                    }
                } else {
                    invalidDeleteDeltaFiles = updateStatusManager.getDeleteDeltaInvalidFilesList(segment.getLoadName(), block, false, allSegmentFiles);
                    for (CarbonFile invalidFile : invalidDeleteDeltaFiles) {
                        compareTimestampsAndDelete(invalidFile, forceDelete, false);
                    }
                }
            }
        }
    }
    // delete the update table status files which are old.
    if (null != validUpdateStatusFile && !validUpdateStatusFile.isEmpty()) {
        final String updateStatusTimestamp = validUpdateStatusFile.substring(validUpdateStatusFile.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1);
        CarbonFile metaFolder = FileFactory.getCarbonFile(carbonTablePath.getMetadataDirectoryPath(), FileFactory.getFileType(carbonTablePath.getMetadataDirectoryPath()));
        CarbonFile[] invalidUpdateStatusFiles = metaFolder.listFiles(new CarbonFileFilter() {

            @Override
            public boolean accept(CarbonFile file) {
                if (file.getName().startsWith(CarbonCommonConstants.TABLEUPDATESTATUS_FILENAME)) {
                    // we only send invalid ones to delete.
                    if (!file.getName().endsWith(updateStatusTimestamp)) {
                        return true;
                    }
                }
                return false;
            }
        });
        for (CarbonFile invalidFile : invalidUpdateStatusFiles) {
            compareTimestampsAndDelete(invalidFile, forceDelete, true);
        }
    }
}

Example 9 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonDictionarySortIndexReaderImpl method initPath.

protected void initPath() {
    PathService pathService = CarbonCommonFactory.getPathService();
    CarbonTablePath carbonTablePath = pathService.getCarbonTablePath(carbonStorePath, carbonTableIdentifier);
    try {
        CarbonDictionaryColumnMetaChunk chunkMetaObjectForLastSegmentEntry = getChunkMetaObjectForLastSegmentEntry();
        long dictOffset = chunkMetaObjectForLastSegmentEntry.getEnd_offset();
        this.sortIndexFilePath = carbonTablePath.getSortIndexFilePath(columnIdentifier.getColumnId(), dictOffset);
        if (!FileFactory.isFileExist(this.sortIndexFilePath, FileFactory.getFileType(this.sortIndexFilePath))) {
            this.sortIndexFilePath = carbonTablePath.getSortIndexFilePath(columnIdentifier.getColumnId());
        }
    } catch (IOException e) {
        this.sortIndexFilePath = carbonTablePath.getSortIndexFilePath(columnIdentifier.getColumnId());
    }
}

Example 10 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonDictionaryReaderImpl method initFileLocation.

/**
   * This method will form the path for dictionary file for a given column
   */
protected void initFileLocation() {
    PathService pathService = CarbonCommonFactory.getPathService();
    CarbonTablePath carbonTablePath = pathService.getCarbonTablePath(this.storePath, carbonTableIdentifier);
    this.columnDictionaryFilePath = carbonTablePath.getDictionaryFilePath(columnIdentifier.getColumnId());
}