Examples with CarbonFileFilter org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter used on opensource projects

Example 11 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class FileFooterValidator method setUp.

@Before
public void setUp() throws Exception {
    if (setUpIsDone) {
        return;
    }
    CarbonHiveContext.sql("CREATE CUBE validatefooter DIMENSIONS (empno Integer, empname String," + " designation String," + " doj Timestamp, workgroupcategory Integer, workgroupcategoryname String, " + "deptno Integer, deptname String, projectcode Integer, projectjoindate Timestamp," + " projectenddate Timestamp) MEASURES (attendance Integer,utilization Integer," + "salary Integer) OPTIONS (PARTITIONER [PARTITION_COUNT=1])");
    CarbonHiveContext.sql("LOAD DATA fact from './src/test/resources/data.csv' INTO CUBE validatefooter " + "PARTITIONDATA(DELIMITER ',', QUOTECHAR '\"')");
    String storePath = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION);
    CarbonTableIdentifier tableIdentifier = new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "validatefooter", "1");
    String segmentPath = CarbonStorePath.getCarbonTablePath(storePath, tableIdentifier).getCarbonDataDirectoryPath("0", "0");
    CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
    CarbonFile[] list = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            if (file.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT)) {
                return true;
            }
            return false;
        }
    });
    for (CarbonFile file : list) {
        String fileLocation = file.getAbsolutePath();
        CarbonFile factFile = FileFactory.getCarbonFile(fileLocation, FileFactory.getFileType(fileLocation));
        long offset = factFile.getSize() - CarbonCommonConstants.LONG_SIZE_IN_BYTE;
        FileHolder fileHolder = FileFactory.getFileHolder(FileFactory.getFileType(fileLocation));
        offset = fileHolder.readLong(fileLocation, offset);
        CarbonFooterReader metaDataReader = new CarbonFooterReader(fileLocation, offset);
        fileFooter = metaDataReader.readFooter();
    }
    setUpIsDone = true;
}

Example 12 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class CarbonLoaderUtil method deletePartialLoadDataIfExist.

public static void deletePartialLoadDataIfExist(CarbonLoadModel loadModel, final boolean isCompactionFlow) throws IOException {
    CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
    String metaDataLocation = carbonTable.getMetaDataFilepath();
    final LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(metaDataLocation);
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(loadModel.getStorePath(), carbonTable.getCarbonTableIdentifier());
    //delete folder which metadata no exist in tablestatus
    for (int i = 0; i < carbonTable.getPartitionCount(); i++) {
        final String partitionCount = i + "";
        String partitionPath = carbonTablePath.getPartitionDir(partitionCount);
        FileType fileType = FileFactory.getFileType(partitionPath);
        if (FileFactory.isFileExist(partitionPath, fileType)) {
            CarbonFile carbonFile = FileFactory.getCarbonFile(partitionPath, fileType);
            CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {

                @Override
                public boolean accept(CarbonFile path) {
                    String segmentId = CarbonTablePath.DataPathUtil.getSegmentId(path.getAbsolutePath() + "/dummy");
                    boolean found = false;
                    for (int j = 0; j < details.length; j++) {
                        if (details[j].getLoadName().equals(segmentId) && details[j].getPartitionCount().equals(partitionCount)) {
                            found = true;
                            break;
                        }
                    }
                    return !found;
                }
            });
            for (int k = 0; k < listFiles.length; k++) {
                String segmentId = CarbonTablePath.DataPathUtil.getSegmentId(listFiles[k].getAbsolutePath() + "/dummy");
                if (isCompactionFlow) {
                    if (segmentId.contains(".")) {
                        deleteStorePath(listFiles[k].getAbsolutePath());
                    }
                } else {
                    if (!segmentId.contains(".")) {
                        deleteStorePath(listFiles[k].getAbsolutePath());
                    }
                }
            }
        }
    }
}

Example 13 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class CarbonLoaderUtil method isValidSegment.

/**
   * the method returns true if the segment has carbondata file else returns false.
   *
   * @param loadModel
   * @param currentLoad
   * @return
   */
public static boolean isValidSegment(CarbonLoadModel loadModel, int currentLoad) {
    CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(loadModel.getStorePath(), carbonTable.getCarbonTableIdentifier());
    int fileCount = 0;
    int partitionCount = carbonTable.getPartitionCount();
    for (int i = 0; i < partitionCount; i++) {
        String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(i + "", currentLoad + "");
        CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
        CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {

            @Override
            public boolean accept(CarbonFile file) {
                return file.getName().endsWith(CarbonTablePath.getCarbonIndexExtension()) || file.getName().endsWith(CarbonTablePath.getCarbonDataExtension());
            }
        });
        fileCount += files.length;
        if (files.length > 0) {
            return true;
        }
    }
    if (fileCount == 0) {
        return false;
    }
    return true;
}

Example 14 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class SegmentUpdateStatusManager method getDeleteDeltaFilesList.

/**
   * Return all delta file for a block.
   * @param segmentId
   * @param blockName
   * @return
   */
public CarbonFile[] getDeleteDeltaFilesList(final String segmentId, final String blockName) {
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
    CarbonFile segDir = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
    for (SegmentUpdateDetails block : updateDetails) {
        if ((block.getBlockName().equalsIgnoreCase(blockName)) && (block.getSegmentName().equalsIgnoreCase(segmentId)) && !CarbonUpdateUtil.isBlockInvalid((block.getStatus()))) {
            final long deltaStartTimestamp = getStartTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
            final long deltaEndTimeStamp = getEndTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
            return segDir.listFiles(new CarbonFileFilter() {

                @Override
                public boolean accept(CarbonFile pathName) {
                    String fileName = pathName.getName();
                    if (fileName.endsWith(CarbonCommonConstants.DELETE_DELTA_FILE_EXT)) {
                        String firstPart = fileName.substring(0, fileName.indexOf('.'));
                        String blkName = firstPart.substring(0, firstPart.lastIndexOf("-"));
                        long timestamp = Long.parseLong(firstPart.substring(firstPart.lastIndexOf("-") + 1, firstPart.length()));
                        if (blockName.equals(blkName) && (Long.compare(timestamp, deltaEndTimeStamp) <= 0) && (Long.compare(timestamp, deltaStartTimestamp) >= 0)) {
                            return true;
                        }
                    }
                    return false;
                }
            });
        }
    }
    return null;
}

Example 15 with CarbonFileFilter

use of org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter in project carbondata by apache.

the class SegmentUpdateStatusManager method getInvalidBlockList.

/**
   * Get the invalid tasks in that segment.
   * @param segmentId
   * @return
   */
public List<String> getInvalidBlockList(String segmentId) {
    // get the original fact file timestamp from the table status file.
    List<String> listOfInvalidBlocks = new ArrayList<String>();
    SegmentStatusManager ssm = new SegmentStatusManager(absoluteTableIdentifier);
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    LoadMetadataDetails[] segmentDetails = ssm.readLoadMetadata(carbonTablePath.getMetadataDirectoryPath());
    long timestampOfOriginalFacts = 0;
    String startTimestampOfUpdate = "";
    String endTimestampOfUpdate = "";
    for (LoadMetadataDetails segment : segmentDetails) {
        // find matching segment and return timestamp.
        if (segment.getLoadName().equalsIgnoreCase(segmentId)) {
            timestampOfOriginalFacts = segment.getLoadStartTime();
            startTimestampOfUpdate = segment.getUpdateDeltaStartTimestamp();
            endTimestampOfUpdate = segment.getUpdateDeltaEndTimestamp();
        }
    }
    if (startTimestampOfUpdate.isEmpty()) {
        return listOfInvalidBlocks;
    }
    // now after getting the original fact timestamp, what ever is remaining
    // files need to cross check it with table status file.
    // filter out the fact files.
    String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
    CarbonFile segDir = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
    final Long endTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(endTimestampOfUpdate);
    final Long startTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(startTimestampOfUpdate);
    final Long timeStampOriginalFactFinal = timestampOfOriginalFacts;
    CarbonFile[] files = segDir.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile pathName) {
            String fileName = pathName.getName();
            if (fileName.endsWith(CarbonCommonConstants.UPDATE_DELTA_FILE_EXT)) {
                String firstPart = fileName.substring(0, fileName.indexOf('.'));
                long timestamp = Long.parseLong(firstPart.substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1, firstPart.length()));
                if (Long.compare(timestamp, endTimeStampFinal) <= 0 && Long.compare(timestamp, startTimeStampFinal) >= 0) {
                    return false;
                }
                if (Long.compare(timestamp, timeStampOriginalFactFinal) == 0) {
                    return false;
                }
                // take the rest of files as they are invalid.
                return true;
            }
            return false;
        }
    });
    // gather the task numbers.
    for (CarbonFile updateFiles : files) {
        listOfInvalidBlocks.add(updateFiles.getName());
    }
    return listOfInvalidBlocks;
}