Examples with FileType org.apache.carbondata.core.datastore.impl.FileFactory.FileType used on opensource projects

Example 1 with FileType

use of org.apache.carbondata.core.datastore.impl.FileFactory.FileType in project carbondata by apache.

the class BadRecordsLogger method writeBadRecordsToFile.

/**
   *
   */
private synchronized void writeBadRecordsToFile(StringBuilder logStrings) {
    if (null == logFilePath) {
        logFilePath = this.storePath + File.separator + this.fileName + CarbonCommonConstants.LOG_FILE_EXTENSION + CarbonCommonConstants.FILE_INPROGRESS_STATUS;
    }
    try {
        if (null == bufferedWriter) {
            FileType fileType = FileFactory.getFileType(storePath);
            if (!FileFactory.isFileExist(this.storePath, fileType)) {
                // create the folders if not exist
                FileFactory.mkdirs(this.storePath, fileType);
                // create the files
                FileFactory.createNewFile(logFilePath, fileType);
            }
            outStream = FileFactory.getDataOutputStream(logFilePath, fileType);
            bufferedWriter = new BufferedWriter(new OutputStreamWriter(outStream, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
        }
        bufferedWriter.write(logStrings.toString());
        bufferedWriter.newLine();
    } catch (FileNotFoundException e) {
        LOGGER.error("Bad Log Files not found");
    } catch (IOException e) {
        LOGGER.error("Error While writing bad log File");
    } finally {
        // if the Bad record file is created means it partially success
        // if any entry present with key that means its have bad record for
        // that key
        badRecordEntry.put(taskKey, "Partially");
    }
}

Example 2 with FileType

use of org.apache.carbondata.core.datastore.impl.FileFactory.FileType in project carbondata by apache.

the class CarbonLoaderUtil method deletePartialLoadDataIfExist.

public static void deletePartialLoadDataIfExist(CarbonLoadModel loadModel, final boolean isCompactionFlow) throws IOException {
    CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
    String metaDataLocation = carbonTable.getMetaDataFilepath();
    final LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(metaDataLocation);
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(loadModel.getStorePath(), carbonTable.getCarbonTableIdentifier());
    //delete folder which metadata no exist in tablestatus
    for (int i = 0; i < carbonTable.getPartitionCount(); i++) {
        final String partitionCount = i + "";
        String partitionPath = carbonTablePath.getPartitionDir(partitionCount);
        FileType fileType = FileFactory.getFileType(partitionPath);
        if (FileFactory.isFileExist(partitionPath, fileType)) {
            CarbonFile carbonFile = FileFactory.getCarbonFile(partitionPath, fileType);
            CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {

                @Override
                public boolean accept(CarbonFile path) {
                    String segmentId = CarbonTablePath.DataPathUtil.getSegmentId(path.getAbsolutePath() + "/dummy");
                    boolean found = false;
                    for (int j = 0; j < details.length; j++) {
                        if (details[j].getLoadName().equals(segmentId) && details[j].getPartitionCount().equals(partitionCount)) {
                            found = true;
                            break;
                        }
                    }
                    return !found;
                }
            });
            for (int k = 0; k < listFiles.length; k++) {
                String segmentId = CarbonTablePath.DataPathUtil.getSegmentId(listFiles[k].getAbsolutePath() + "/dummy");
                if (isCompactionFlow) {
                    if (segmentId.contains(".")) {
                        deleteStorePath(listFiles[k].getAbsolutePath());
                    }
                } else {
                    if (!segmentId.contains(".")) {
                        deleteStorePath(listFiles[k].getAbsolutePath());
                    }
                }
            }
        }
    }
}

Example 3 with FileType

use of org.apache.carbondata.core.datastore.impl.FileFactory.FileType in project carbondata by apache.

the class CarbonLoaderUtil method deleteStorePath.

private static void deleteStorePath(String path) {
    try {
        FileType fileType = FileFactory.getFileType(path);
        if (FileFactory.isFileExist(path, fileType)) {
            CarbonFile carbonFile = FileFactory.getCarbonFile(path, fileType);
            CarbonUtil.deleteFoldersAndFiles(carbonFile);
        }
    } catch (IOException | InterruptedException e) {
        LOGGER.error("Unable to delete the given path :: " + e.getMessage());
    }
}

Example 4 with FileType

use of org.apache.carbondata.core.datastore.impl.FileFactory.FileType in project carbondata by apache.

the class BadRecordsLogger method writeBadRecordsToCSVFile.

/**
   * method will write the row having bad record in the csv file.
   *
   * @param logStrings
   */
private synchronized void writeBadRecordsToCSVFile(StringBuilder logStrings) {
    if (null == csvFilePath) {
        csvFilePath = this.storePath + File.separator + this.fileName + CarbonCommonConstants.CSV_FILE_EXTENSION + CarbonCommonConstants.FILE_INPROGRESS_STATUS;
    }
    try {
        if (null == bufferedCSVWriter) {
            FileType fileType = FileFactory.getFileType(storePath);
            if (!FileFactory.isFileExist(this.storePath, fileType)) {
                // create the folders if not exist
                FileFactory.mkdirs(this.storePath, fileType);
                // create the files
                FileFactory.createNewFile(csvFilePath, fileType);
            }
            outCSVStream = FileFactory.getDataOutputStream(csvFilePath, fileType);
            bufferedCSVWriter = new BufferedWriter(new OutputStreamWriter(outCSVStream, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
        }
        bufferedCSVWriter.write(logStrings.toString());
        bufferedCSVWriter.newLine();
    } catch (FileNotFoundException e) {
        LOGGER.error("Bad record csv Files not found");
    } catch (IOException e) {
        LOGGER.error("Error While writing bad record csv File");
    } finally {
        badRecordEntry.put(taskKey, "Partially");
    }
}

Example 5 with FileType

use of org.apache.carbondata.core.datastore.impl.FileFactory.FileType in project carbondata by apache.

the class CarbonDataProcessorUtil method renameBadRecordsFromInProgressToNormal.

/**
   * @param storeLocation
   */
public static void renameBadRecordsFromInProgressToNormal(String storeLocation) {
    // get the base store location
    String badLogStoreLocation = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC);
    badLogStoreLocation = badLogStoreLocation + File.separator + storeLocation;
    FileType fileType = FileFactory.getFileType(badLogStoreLocation);
    try {
        if (!FileFactory.isFileExist(badLogStoreLocation, fileType)) {
            return;
        }
    } catch (IOException e1) {
        LOGGER.info("bad record folder does not exist");
    }
    CarbonFile carbonFile = FileFactory.getCarbonFile(badLogStoreLocation, fileType);
    CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile pathname) {
            if (pathname.getName().indexOf(CarbonCommonConstants.FILE_INPROGRESS_STATUS) > -1) {
                return true;
            }
            return false;
        }
    });
    String badRecordsInProgressFileName = null;
    String changedFileName = null;
    for (CarbonFile badFiles : listFiles) {
        badRecordsInProgressFileName = badFiles.getName();
        changedFileName = badLogStoreLocation + File.separator + badRecordsInProgressFileName.substring(0, badRecordsInProgressFileName.lastIndexOf('.'));
        badFiles.renameTo(changedFileName);
        if (badFiles.exists()) {
            if (!badFiles.delete()) {
                LOGGER.error("Unable to delete File : " + badFiles.getName());
            }
        }
    }
}