Examples with CarbonFile org.apache.carbondata.core.datastore.filesystem.CarbonFile used on opensource projects

Example 61 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class SegmentFileStore method removeTempFolder.

/**
 * Remove temp stage folder in case of job aborted.
 *
 * @param locationMap
 * @param tmpFolder
 * @param tablePath
 */
public static void removeTempFolder(Map<String, FolderDetails> locationMap, String tmpFolder, String tablePath) {
    if (locationMap == null) {
        return;
    }
    for (Map.Entry<String, SegmentFileStore.FolderDetails> entry : locationMap.entrySet()) {
        String location = entry.getKey();
        if (entry.getValue().isRelative()) {
            location = tablePath + CarbonCommonConstants.FILE_SEPARATOR + location;
        }
        CarbonFile oldFolder = FileFactory.getCarbonFile(location + CarbonCommonConstants.FILE_SEPARATOR + tmpFolder);
        if (oldFolder.exists()) {
            FileFactory.deleteAllCarbonFilesOfDir(oldFolder);
        }
    }
}

Example 62 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonLockUtil method deleteExpiredSegmentLockFiles.

/**
 * Currently the segment lock files are not deleted immediately when unlock,
 * so it needs to delete expired lock files before delete loads.
 */
public static void deleteExpiredSegmentLockFiles(CarbonTable carbonTable) {
    final long currTime = System.currentTimeMillis();
    final long segmentLockFilesPreserveTime = CarbonProperties.getInstance().getSegmentLockFilesPreserveHours();
    AbsoluteTableIdentifier absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier();
    String lockFilesDir = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.LOCK_PATH, CarbonCommonConstants.LOCK_PATH_DEFAULT);
    if (StringUtils.isEmpty(lockFilesDir)) {
        lockFilesDir = CarbonTablePath.getLockFilesDirPath(absoluteTableIdentifier.getTablePath());
    } else {
        lockFilesDir = CarbonTablePath.getLockFilesDirPath(CarbonLockFactory.getLockPath(carbonTable.getTableInfo().getFactTable().getTableId()));
    }
    CarbonFile[] files = FileFactory.getCarbonFile(lockFilesDir).listFiles(new CarbonFileFilter() {

        @Override
        public boolean accept(CarbonFile pathName) {
            if (CarbonTablePath.isSegmentLockFilePath(pathName.getName())) {
                return (currTime - pathName.getLastModifiedTime()) > segmentLockFilesPreserveTime;
            }
            return false;
        }
    });
    for (CarbonFile file : files) {
        file.delete();
    }
}

Example 63 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class CarbonIUD method createEmptyMetadataFile.

private void createEmptyMetadataFile(String path) throws IOException {
    if (!StringUtils.isEmpty(path)) {
        path = path + CarbonCommonConstants.FILE_SEPARATOR + CarbonCommonConstants.CARBON_SDK_EMPTY_METADATA_PATH;
        CarbonFile emptySDKDirectory = FileFactory.getCarbonFile(path);
        if (!emptySDKDirectory.exists()) {
            emptySDKDirectory.mkdirs();
        }
    }
}

Example 64 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class JsonCarbonWriter method setDataFiles.

@Override
public void setDataFiles(CarbonFile[] dataFiles) throws IOException {
    if (dataFiles == null || dataFiles.length == 0) {
        throw new RuntimeException("data files can't be empty.");
    }
    Reader jsonReader = null;
    for (CarbonFile dataFile : dataFiles) {
        try {
            jsonReader = this.buildJsonReader(dataFile, this.configuration);
            new JSONParser().parse(jsonReader);
        } catch (FileNotFoundException ex) {
            throw new FileNotFoundException("File " + dataFile + " not found to build carbon writer.");
        } catch (ParseException ex) {
            throw new RuntimeException("File " + dataFile + " is not in json format.");
        } finally {
            if (jsonReader != null) {
                jsonReader.close();
            }
        }
    }
    this.dataFiles = dataFiles;
}

Example 65 with CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.CarbonFile in project carbondata by apache.

the class ParquetCarbonWriter method setDataFiles.

@Override
public void setDataFiles(CarbonFile[] dataFiles) throws IOException {
    if (dataFiles == null || dataFiles.length == 0) {
        throw new RuntimeException("data files can't be empty.");
    }
    Schema parquetSchema = null;
    for (CarbonFile dataFile : dataFiles) {
        Schema currentFileSchema = extractParquetSchema(dataFile, this.configuration);
        if (parquetSchema == null) {
            parquetSchema = currentFileSchema;
        } else {
            if (!parquetSchema.equals(currentFileSchema)) {
                throw new RuntimeException("All the parquet files must be having the same schema.");
            }
        }
    }
    this.dataFiles = dataFiles;
}