Examples with CarbonTablePath org.apache.carbondata.core.util.path.CarbonTablePath used on opensource projects

Example 46 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonDataProcessorUtil method getLocalDataFolderLocation.

/**
   * This method will form the local data folder store location
   *
   * @param databaseName
   * @param tableName
   * @param taskId
   * @param partitionId
   * @param segmentId
   * @return
   */
public static String getLocalDataFolderLocation(String databaseName, String tableName, String taskId, String partitionId, String segmentId, boolean isCompactionFlow) {
    String tempLocationKey = getTempStoreLocationKey(databaseName, tableName, taskId, isCompactionFlow);
    String baseStorePath = CarbonProperties.getInstance().getProperty(tempLocationKey, CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL);
    CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(databaseName + CarbonCommonConstants.UNDERSCORE + tableName);
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(baseStorePath, carbonTable.getCarbonTableIdentifier());
    String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath(partitionId, segmentId + "");
    return carbonDataDirectoryPath + File.separator + taskId;
}

Example 47 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonInputFormat method getFileStatus.

private void getFileStatus(JobContext job, String[] segmentsToConsider, String[] filesToConsider, List<FileStatus> result) throws IOException {
    String[] partitionsToConsider = getValidPartitions(job);
    if (partitionsToConsider.length == 0) {
        throw new IOException("No partitions/data found");
    }
    PathFilter inputFilter = getDataFileFilter();
    AbsoluteTableIdentifier absIdentifier = getAbsoluteTableIdentifier(job.getConfiguration());
    CarbonTablePath tablePath = getTablePath(absIdentifier);
    // get tokens for all the required FileSystem for table path
    TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { tablePath }, job.getConfiguration());
    //get all data files of valid partitions and segments
    for (int i = 0; i < partitionsToConsider.length; ++i) {
        String partition = partitionsToConsider[i];
        for (int j = 0; j < segmentsToConsider.length; ++j) {
            String segmentId = segmentsToConsider[j];
            String dataDirectoryPath = absIdentifier.appendWithLocalPrefix(tablePath.getCarbonDataDirectoryPath(partition, segmentId));
            if (filesToConsider.length == 0) {
                Path segmentPath = new Path(dataDirectoryPath);
                FileSystem fs = segmentPath.getFileSystem(job.getConfiguration());
                getFileStatusInternal(inputFilter, fs, segmentPath, result);
            } else {
                for (int k = 0; k < filesToConsider.length; ++k) {
                    String dataPath = absIdentifier.appendWithLocalPrefix(tablePath.getCarbonDataDirectoryPath(partition, segmentId) + File.separator + filesToConsider[k]);
                    Path filePath = new Path(dataPath);
                    FileSystem fs = filePath.getFileSystem(job.getConfiguration());
                    getFileStatusInternal(inputFilter, fs, filePath, result);
                }
            }
        }
    }
}