Examples with CarbonFile org.apache.carbondata.core.datastore.filesystem.CarbonFile used on opensource projects

Example 66 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonLoaderUtil method isValidSegment.

/**
 * the method returns true if the segment has carbondata file else returns false.
 *
 * @param loadModel
 * @param currentLoad
 * @return
 */
public static boolean isValidSegment(CarbonLoadModel loadModel, int currentLoad) {
    int fileCount = 0;
    String segmentPath = CarbonTablePath.getSegmentPath(loadModel.getTablePath(), currentLoad + "");
    CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath);
    CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            return file.getName().endsWith(CarbonTablePath.getCarbonIndexExtension()) || file.getName().endsWith(CarbonTablePath.getCarbonDataExtension());
        }
    });
    fileCount += files.length;
    if (files.length > 0) {
        return true;
    }
    if (fileCount == 0) {
        return false;
    }
    return true;
}

Example 67 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonLoaderUtil method deleteNonTransactionalTableForInsertOverwrite.

/**
 * This API deletes the content of the non Transactional Tables when insert overwrite is set true.
 *
 * @param loadModel
 * @throws IOException
 */
public static void deleteNonTransactionalTableForInsertOverwrite(final CarbonLoadModel loadModel) throws IOException {
    // We need to delete the content of the Table Path Folder except the
    // Newly added file.
    List<String> filesToBeDeleted = new ArrayList<>();
    CarbonFile carbonFile = FileFactory.getCarbonFile(loadModel.getTablePath());
    CarbonFile[] filteredList = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile file) {
            return !file.getName().contains(loadModel.getFactTimeStamp() + "");
        }
    });
    for (CarbonFile file : filteredList) {
        filesToBeDeleted.add(file.getAbsolutePath());
    }
    deleteFiles(filesToBeDeleted);
}

Example 68 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonBadRecordUtil method renameBadRecordsFromInProgressToNormal.

/**
 * @param configuration
 * @param storeLocation
 */
private static void renameBadRecordsFromInProgressToNormal(CarbonDataLoadConfiguration configuration, String storeLocation) {
    // get the base store location
    String badLogStoreLocation = (String) configuration.getDataLoadProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH);
    if (null == badLogStoreLocation) {
        badLogStoreLocation = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC);
    }
    badLogStoreLocation = badLogStoreLocation + File.separator + storeLocation;
    try {
        if (!FileFactory.isFileExist(badLogStoreLocation)) {
            return;
        }
    } catch (IOException e1) {
        LOGGER.info("bad record folder does not exist");
    }
    CarbonFile carbonFile = FileFactory.getCarbonFile(badLogStoreLocation);
    CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile pathname) {
            if (pathname.getName().indexOf(CarbonCommonConstants.FILE_INPROGRESS_STATUS) > -1) {
                return true;
            }
            return false;
        }
    });
    String badRecordsInProgressFileName = null;
    String changedFileName = null;
    for (CarbonFile badFiles : listFiles) {
        badRecordsInProgressFileName = badFiles.getName();
        changedFileName = badLogStoreLocation + File.separator + badRecordsInProgressFileName.substring(0, badRecordsInProgressFileName.lastIndexOf('.'));
        badFiles.renameTo(changedFileName);
        if (badFiles.exists()) {
            if (!badFiles.delete()) {
                LOGGER.error("Unable to delete File : " + badFiles.getName());
            }
        }
    }
}

Example 69 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonSchemaReader method readSchema.

/**
 * read schema from path,
 * path can be folder path, carbonindex file path, and carbondata file path
 * and user can decide whether check all files schema
 *
 * @param path           file/folder path
 * @param validateSchema whether check all files schema
 * @param conf           hadoop configuration support, can set s3a AK,SK,
 *                       end point and other conf with this
 * @return schema
 * @throws IOException
 */
public static Schema readSchema(String path, boolean validateSchema, Configuration conf) throws IOException {
    // Check whether it is transational table reads the schema
    String schemaFilePath = CarbonTablePath.getSchemaFilePath(path);
    if (FileFactory.getCarbonFile(schemaFilePath, conf).exists()) {
        return readSchemaInSchemaFile(schemaFilePath, conf);
    }
    if (path.endsWith(INDEX_FILE_EXT)) {
        return readSchemaFromIndexFile(path, conf);
    } else if (path.endsWith(CARBON_DATA_EXT)) {
        return readSchemaFromDataFile(path, conf);
    } else if (validateSchema) {
        CarbonFile[] carbonIndexFiles = getCarbonFile(path, INDEX_FILE_EXT, conf);
        Schema schema;
        if (carbonIndexFiles != null && carbonIndexFiles.length != 0) {
            schema = readSchemaFromIndexFile(carbonIndexFiles[0].getAbsolutePath(), conf);
            for (int i = 1; i < carbonIndexFiles.length; i++) {
                Schema schema2 = readSchemaFromIndexFile(carbonIndexFiles[i].getAbsolutePath(), conf);
                if (!schema.equals(schema2)) {
                    throw new CarbonDataLoadingException("Schema is different between different files.");
                }
            }
            CarbonFile[] carbonDataFiles = getCarbonFile(path, CARBON_DATA_EXT, conf);
            for (int i = 0; i < carbonDataFiles.length; i++) {
                Schema schema2 = readSchemaFromDataFile(carbonDataFiles[i].getAbsolutePath(), conf);
                if (!schema.equals(schema2)) {
                    throw new CarbonDataLoadingException("Schema is different between different files.");
                }
            }
            return schema;
        } else {
            throw new CarbonDataLoadingException("No carbonindex file in this path.");
        }
    } else {
        return readSchemaFromFolder(path, conf);
    }
}

Example 70 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class SDKUtil method extractFilesFromFolder.

public static List<CarbonFile> extractFilesFromFolder(String path, String suf, Configuration hadoopConf) {
    List dataFiles = listFiles(path, suf, hadoopConf);
    List<CarbonFile> carbonFiles = new ArrayList<>();
    for (Object dataFile : dataFiles) {
        carbonFiles.add(FileFactory.getCarbonFile(dataFile.toString(), hadoopConf));
    }
    if (CollectionUtils.isEmpty(dataFiles)) {
        throw new RuntimeException("No file found at given location. Please provide" + "the correct folder location.");
    }
    return carbonFiles;
}