Examples with CarbonTablePath org.apache.carbondata.core.util.path.CarbonTablePath used on opensource projects

Example 16 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonFactDataHandlerModel method getCarbonDataFolderLocation.

/**
   * This method will get the store location for the given path, segment id and partition id
   *
   * @return data directory path
   */
private static String getCarbonDataFolderLocation(CarbonDataLoadConfiguration configuration) {
    String carbonStorePath = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION_HDFS);
    CarbonTableIdentifier tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
    CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableIdentifier.getDatabaseName() + CarbonCommonConstants.UNDERSCORE + tableIdentifier.getTableName());
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonStorePath, carbonTable.getCarbonTableIdentifier());
    return carbonTablePath.getCarbonDataDirectoryPath(configuration.getPartitionId(), configuration.getSegmentId() + "");
}

Example 17 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonDataProcessorUtil method checkAndCreateCarbonStoreLocation.

/**
   * This method will get the store location for the given path, segment id and partition id
   *
   * @return data directory path
   */
public static String checkAndCreateCarbonStoreLocation(String factStoreLocation, String databaseName, String tableName, String partitionId, String segmentId) {
    CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(databaseName + CarbonCommonConstants.UNDERSCORE + tableName);
    CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(factStoreLocation, carbonTableIdentifier);
    String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath(partitionId, segmentId);
    CarbonUtil.checkAndCreateFolder(carbonDataDirectoryPath);
    return carbonDataDirectoryPath;
}

Example 18 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class SchemaReader method readCarbonTableFromStore.

public static CarbonTable readCarbonTableFromStore(AbsoluteTableIdentifier identifier) throws IOException {
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(identifier);
    String schemaFilePath = carbonTablePath.getSchemaFilePath();
    if (FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.LOCAL) || FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.HDFS) || FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.VIEWFS)) {
        String tableName = identifier.getCarbonTableIdentifier().getTableName();
        ThriftReader.TBaseCreator createTBase = new ThriftReader.TBaseCreator() {

            public TBase create() {
                return new org.apache.carbondata.format.TableInfo();
            }
        };
        ThriftReader thriftReader = new ThriftReader(carbonTablePath.getSchemaFilePath(), createTBase);
        thriftReader.open();
        org.apache.carbondata.format.TableInfo tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
        thriftReader.close();
        SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
        TableInfo wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(tableInfo, identifier.getCarbonTableIdentifier().getDatabaseName(), tableName, identifier.getStorePath());
        wrapperTableInfo.setMetaDataFilepath(CarbonTablePath.getFolderContainingFile(schemaFilePath));
        CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo);
        return CarbonMetadata.getInstance().getCarbonTable(identifier.getCarbonTableIdentifier().getTableUniqueName());
    } else {
        throw new IOException("File does not exist: " + schemaFilePath);
    }
}

Example 19 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonDictionaryWriterImplTest method initDictionaryDirPaths.

/**
   * this method will form the dictionary directory paths
   */
private void initDictionaryDirPaths() throws IOException {
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(this.carbonStorePath, carbonTableIdentifier);
    String dictionaryLocation = carbonTablePath.getMetadataDirectoryPath();
    FileFactory.FileType fileType = FileFactory.getFileType(dictionaryLocation);
    if (!FileFactory.isFileExist(dictionaryLocation, fileType)) {
        FileFactory.mkdirs(dictionaryLocation, fileType);
    }
    this.dictionaryFilePath = carbonTablePath.getDictionaryFilePath(columnIdentifier.getColumnId());
    this.dictionaryMetaFilePath = carbonTablePath.getDictionaryMetaFilePath(columnIdentifier.getColumnId());
}

Example 20 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonDataMergerUtil method checkUpdateDeltaFilesInSeg.

/**
   * This method traverses Update Delta Files inside the seg and return true
   * if UpdateDelta Files are more than IUD Compaction threshold.
   *
   * @param seg
   * @param absoluteTableIdentifier
   * @param segmentUpdateStatusManager
   * @param numberDeltaFilesThreshold
   * @return
   */
public static Boolean checkUpdateDeltaFilesInSeg(String seg, AbsoluteTableIdentifier absoluteTableIdentifier, SegmentUpdateStatusManager segmentUpdateStatusManager, int numberDeltaFilesThreshold) {
    CarbonFile[] updateDeltaFiles = null;
    Set<String> uniqueBlocks = new HashSet<String>();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", seg);
    CarbonFile segDir = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
    CarbonFile[] allSegmentFiles = segDir.listFiles();
    updateDeltaFiles = segmentUpdateStatusManager.getUpdateDeltaFilesForSegment(seg, true, CarbonCommonConstants.UPDATE_DELTA_FILE_EXT, false, allSegmentFiles);
    if (updateDeltaFiles == null) {
        return false;
    }
    // Spill Over Blocks will choose files with unique taskID.
    for (CarbonFile blocks : updateDeltaFiles) {
        // Get Task ID and the Timestamp from the Block name for e.g.
        // part-0-3-1481084721319.carbondata => "3-1481084721319"
        String task = CarbonTablePath.DataFileUtil.getTaskNo(blocks.getName());
        String timestamp = CarbonTablePath.DataFileUtil.getTimeStampFromDeleteDeltaFile(blocks.getName());
        String taskAndTimeStamp = task + "-" + timestamp;
        uniqueBlocks.add(taskAndTimeStamp);
    }
    if (uniqueBlocks.size() > numberDeltaFilesThreshold) {
        return true;
    } else {
        return false;
    }
}