Search in sources :

Example 41 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class AbstractDictionaryCache method getDictionaryMetaCarbonFile.

/**
   * This method will return the carbon file objetc based on its type (local, HDFS)
   *
   * @param dictionaryColumnUniqueIdentifier
   * @return
   */
private CarbonFile getDictionaryMetaCarbonFile(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException {
    PathService pathService = CarbonCommonFactory.getPathService();
    CarbonTablePath carbonTablePath = pathService.getCarbonTablePath(carbonStorePath, dictionaryColumnUniqueIdentifier.getCarbonTableIdentifier());
    String dictionaryFilePath = carbonTablePath.getDictionaryMetaFilePath(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId());
    FileFactory.FileType fileType = FileFactory.getFileType(dictionaryFilePath);
    CarbonFile dictFile = FileFactory.getCarbonFile(dictionaryFilePath, fileType);
    // When rename table triggered parallely with select query, dictionary files may not exist
    if (!dictFile.exists()) {
        throw new IOException("Dictionary file does not exist: " + dictionaryFilePath);
    }
    return dictFile;
}
Also used : CarbonFile(org.apache.carbondata.core.datastore.filesystem.CarbonFile) PathService(org.apache.carbondata.core.service.PathService) CarbonTablePath(org.apache.carbondata.core.util.path.CarbonTablePath) IOException(java.io.IOException) FileFactory(org.apache.carbondata.core.datastore.impl.FileFactory)

Example 42 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonTableReader method getFileStatusOfSegments.

private FileSystem getFileStatusOfSegments(String[] segmentsToConsider, CarbonTablePath tablePath, List<FileStatus> result) throws IOException {
    String[] partitionsToConsider = getValidPartitions();
    if (partitionsToConsider.length == 0) {
        throw new IOException("No partitions/data found");
    }
    FileSystem fs = null;
    //get all data files of valid partitions and segments
    for (int i = 0; i < partitionsToConsider.length; ++i) {
        String partition = partitionsToConsider[i];
        for (int j = 0; j < segmentsToConsider.length; ++j) {
            String segmentId = segmentsToConsider[j];
            Path segmentPath = new Path(tablePath.getCarbonDataDirectoryPath(partition, segmentId));
            try {
                Configuration conf = new Configuration();
                fs = segmentPath.getFileSystem(conf);
                RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(segmentPath);
                while (iter.hasNext()) {
                    LocatedFileStatus stat = iter.next();
                    if (DefaultFilter.accept(stat.getPath())) {
                        if (stat.isDirectory()) {
                            addInputPathRecursively(result, fs, stat.getPath(), DefaultFilter);
                        } else {
                            result.add(stat);
                        }
                    }
                }
            } catch (Exception ex) {
                System.out.println(ex.toString());
            }
        }
    }
    return fs;
}
Also used : CarbonTablePath(org.apache.carbondata.core.util.path.CarbonTablePath) Configuration(org.apache.hadoop.conf.Configuration) IOException(java.io.IOException) KeyGenException(org.apache.carbondata.core.keygenerator.KeyGenException) IndexBuilderException(org.apache.carbondata.core.datastore.exception.IndexBuilderException) IOException(java.io.IOException)

Example 43 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonLoaderUtil method checkAndCreateCarbonDataLocation.

/**
   * This method will get the store location for the given path, segment id and partition id
   *
   * @param carbonStorePath
   * @param segmentId
   */
public static void checkAndCreateCarbonDataLocation(String carbonStorePath, String segmentId, CarbonTable carbonTable) {
    CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonStorePath, carbonTableIdentifier);
    String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
    CarbonUtil.checkAndCreateFolder(carbonDataDirectoryPath);
}
Also used : CarbonTableIdentifier(org.apache.carbondata.core.metadata.CarbonTableIdentifier) CarbonTablePath(org.apache.carbondata.core.util.path.CarbonTablePath)

Example 44 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonDataMergerUtil method updateStatusFile.

public static Boolean updateStatusFile(List<CarbonDataMergerUtilResult> updateDataMergerDetailsList, CarbonTable table, String timestamp, SegmentUpdateStatusManager segmentUpdateStatusManager) {
    List<SegmentUpdateDetails> segmentUpdateDetails = new ArrayList<SegmentUpdateDetails>(updateDataMergerDetailsList.size());
    // Check the list output.
    for (CarbonDataMergerUtilResult carbonDataMergerUtilResult : updateDataMergerDetailsList) {
        if (carbonDataMergerUtilResult.getCompactionStatus()) {
            SegmentUpdateDetails tempSegmentUpdateDetails = new SegmentUpdateDetails();
            tempSegmentUpdateDetails.setSegmentName(carbonDataMergerUtilResult.getSegmentName());
            tempSegmentUpdateDetails.setBlockName(carbonDataMergerUtilResult.getBlockName());
            for (SegmentUpdateDetails origDetails : segmentUpdateStatusManager.getUpdateStatusDetails()) {
                if (origDetails.getBlockName().equalsIgnoreCase(carbonDataMergerUtilResult.getBlockName()) && origDetails.getSegmentName().equalsIgnoreCase(carbonDataMergerUtilResult.getSegmentName())) {
                    tempSegmentUpdateDetails.setDeletedRowsInBlock(origDetails.getDeletedRowsInBlock());
                    tempSegmentUpdateDetails.setStatus(origDetails.getStatus());
                    break;
                }
            }
            tempSegmentUpdateDetails.setDeleteDeltaStartTimestamp(carbonDataMergerUtilResult.getDeleteDeltaStartTimestamp());
            tempSegmentUpdateDetails.setDeleteDeltaEndTimestamp(carbonDataMergerUtilResult.getDeleteDeltaEndTimestamp());
            segmentUpdateDetails.add(tempSegmentUpdateDetails);
        } else
            return false;
    }
    CarbonUpdateUtil.updateSegmentStatus(segmentUpdateDetails, table, timestamp, true);
    // Update the Table Status.
    String metaDataFilepath = table.getMetaDataFilepath();
    AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String tableStatusPath = carbonTablePath.getTableStatusFilePath();
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
    boolean lockStatus = false;
    try {
        lockStatus = carbonLock.lockWithRetries();
        if (lockStatus) {
            LOGGER.info("Acquired lock for table" + table.getDatabaseName() + "." + table.getFactTableName() + " for table status updation");
            LoadMetadataDetails[] listOfLoadFolderDetailsArray = segmentStatusManager.readLoadMetadata(metaDataFilepath);
            for (LoadMetadataDetails loadMetadata : listOfLoadFolderDetailsArray) {
                if (loadMetadata.getLoadName().equalsIgnoreCase("0")) {
                    loadMetadata.setUpdateStatusFileName(CarbonUpdateUtil.getUpdateStatusFileName(timestamp));
                }
            }
            try {
                segmentStatusManager.writeLoadDetailsIntoFile(tableStatusPath, listOfLoadFolderDetailsArray);
            } catch (IOException e) {
                return false;
            }
        } else {
            LOGGER.error("Not able to acquire the lock for Table status updation for table " + table.getDatabaseName() + "." + table.getFactTableName());
        }
    } finally {
        if (lockStatus) {
            if (carbonLock.unlock()) {
                LOGGER.info("Table unlocked successfully after table status updation" + table.getDatabaseName() + "." + table.getFactTableName());
            } else {
                LOGGER.error("Unable to unlock Table lock for table" + table.getDatabaseName() + "." + table.getFactTableName() + " during table status updation");
            }
        }
    }
    return true;
}
Also used : ICarbonLock(org.apache.carbondata.core.locks.ICarbonLock) LoadMetadataDetails(org.apache.carbondata.core.statusmanager.LoadMetadataDetails) ArrayList(java.util.ArrayList) SegmentStatusManager(org.apache.carbondata.core.statusmanager.SegmentStatusManager) IOException(java.io.IOException) SegmentUpdateDetails(org.apache.carbondata.core.mutate.SegmentUpdateDetails) CarbonTablePath(org.apache.carbondata.core.util.path.CarbonTablePath) AbsoluteTableIdentifier(org.apache.carbondata.core.metadata.AbsoluteTableIdentifier)

Example 45 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonUpdateUtil method updateSegmentStatus.

/**
   * @param updateDetailsList
   * @param table
   * @param updateStatusFileIdentifier
   * @return
   */
public static boolean updateSegmentStatus(List<SegmentUpdateDetails> updateDetailsList, CarbonTable table, String updateStatusFileIdentifier, boolean isCompaction) {
    boolean status = false;
    SegmentUpdateStatusManager segmentUpdateStatusManager = new SegmentUpdateStatusManager(table.getAbsoluteTableIdentifier());
    ICarbonLock updateLock = segmentUpdateStatusManager.getTableUpdateStatusLock();
    boolean lockStatus = false;
    try {
        lockStatus = updateLock.lockWithRetries();
        if (lockStatus) {
            AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
            CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
            // read the existing file if present and update the same.
            SegmentUpdateDetails[] oldDetails = segmentUpdateStatusManager.getUpdateStatusDetails();
            List<SegmentUpdateDetails> oldList = new ArrayList(Arrays.asList(oldDetails));
            for (SegmentUpdateDetails newBlockEntry : updateDetailsList) {
                int index = oldList.indexOf(newBlockEntry);
                if (index != -1) {
                    // update the element in existing list.
                    SegmentUpdateDetails blockDetail = oldList.get(index);
                    if (blockDetail.getDeleteDeltaStartTimestamp().isEmpty() || (isCompaction == true)) {
                        blockDetail.setDeleteDeltaStartTimestamp(newBlockEntry.getDeleteDeltaStartTimestamp());
                    }
                    blockDetail.setDeleteDeltaEndTimestamp(newBlockEntry.getDeleteDeltaEndTimestamp());
                    blockDetail.setStatus(newBlockEntry.getStatus());
                    blockDetail.setDeletedRowsInBlock(newBlockEntry.getDeletedRowsInBlock());
                } else {
                    // add the new details to the list.
                    oldList.add(newBlockEntry);
                }
            }
            segmentUpdateStatusManager.writeLoadDetailsIntoFile(oldList, updateStatusFileIdentifier);
            status = true;
        } else {
            LOGGER.error("Not able to acquire the segment update lock.");
            status = false;
        }
    } catch (IOException e) {
        status = false;
    } finally {
        if (lockStatus) {
            if (updateLock.unlock()) {
                LOGGER.info("Unlock the segment update lock successfull.");
            } else {
                LOGGER.error("Not able to unlock the segment update lock.");
            }
        }
    }
    return status;
}
Also used : SegmentUpdateStatusManager(org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager) ICarbonLock(org.apache.carbondata.core.locks.ICarbonLock) CarbonTablePath(org.apache.carbondata.core.util.path.CarbonTablePath) AbsoluteTableIdentifier(org.apache.carbondata.core.metadata.AbsoluteTableIdentifier) ArrayList(java.util.ArrayList) IOException(java.io.IOException)

Aggregations

CarbonTablePath (org.apache.carbondata.core.util.path.CarbonTablePath)47 IOException (java.io.IOException)20 ArrayList (java.util.ArrayList)12 CarbonFile (org.apache.carbondata.core.datastore.filesystem.CarbonFile)12 ICarbonLock (org.apache.carbondata.core.locks.ICarbonLock)9 AbsoluteTableIdentifier (org.apache.carbondata.core.metadata.AbsoluteTableIdentifier)9 LoadMetadataDetails (org.apache.carbondata.core.statusmanager.LoadMetadataDetails)8 CarbonFileFilter (org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter)7 PathService (org.apache.carbondata.core.service.PathService)7 SegmentStatusManager (org.apache.carbondata.core.statusmanager.SegmentStatusManager)7 CarbonTableIdentifier (org.apache.carbondata.core.metadata.CarbonTableIdentifier)6 CarbonTable (org.apache.carbondata.core.metadata.schema.table.CarbonTable)6 SegmentUpdateDetails (org.apache.carbondata.core.mutate.SegmentUpdateDetails)6 Gson (com.google.gson.Gson)4 FileFactory (org.apache.carbondata.core.datastore.impl.FileFactory)4 AtomicFileOperations (org.apache.carbondata.core.fileoperations.AtomicFileOperations)4 AtomicFileOperationsImpl (org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl)4 SchemaConverter (org.apache.carbondata.core.metadata.converter.SchemaConverter)3 ThriftWrapperSchemaConverterImpl (org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl)3 TableInfo (org.apache.carbondata.core.metadata.schema.table.TableInfo)3