Examples with CarbonTablePath org.apache.carbondata.core.util.path.CarbonTablePath used on opensource projects

Example 21 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonDataMergerUtil method updateMajorCompactionPropertyInSegment.

/**
   * This will update the property of segments as major compacted.
   * @param model
   * @param changedSegDetails
   */
public static void updateMajorCompactionPropertyInSegment(CarbonLoadModel model, List<LoadMetadataDetails> changedSegDetails, List<LoadMetadataDetails> preservedSegment) throws Exception {
    String metadataPath = model.getCarbonDataLoadSchema().getCarbonTable().getMetaDataFilepath();
    AbsoluteTableIdentifier absoluteTableIdentifier = model.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    LoadMetadataDetails[] details = segmentStatusManager.readLoadMetadata(metadataPath);
    List<LoadMetadataDetails> originalList = Arrays.asList(details);
    for (LoadMetadataDetails segment : changedSegDetails) {
        if (preservedSegment.contains(segment)) {
            continue;
        }
        originalList.get(originalList.indexOf(segment)).setMajorCompacted("true");
    }
    ICarbonLock carbonTableStatusLock = CarbonLockFactory.getCarbonLockObj(model.getCarbonDataLoadSchema().getCarbonTable().getCarbonTableIdentifier(), LockUsage.TABLE_STATUS_LOCK);
    try {
        if (carbonTableStatusLock.lockWithRetries()) {
            LOGGER.info("Acquired lock for the table " + model.getDatabaseName() + "." + model.getTableName() + " for table status updation ");
            CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
            segmentStatusManager.writeLoadDetailsIntoFile(carbonTablePath.getTableStatusFilePath(), originalList.toArray(new LoadMetadataDetails[originalList.size()]));
        } else {
            LOGGER.error("Could not able to obtain lock for table" + model.getDatabaseName() + "." + model.getTableName() + "for table status updation");
            throw new Exception("Failed to update the MajorCompactionStatus.");
        }
    } catch (IOException e) {
        LOGGER.error("Error while writing metadata");
        throw new Exception("Failed to update the MajorCompactionStatus." + e.getMessage());
    } finally {
        if (carbonTableStatusLock.unlock()) {
            LOGGER.info("Table unlocked successfully after table status updation" + model.getDatabaseName() + "." + model.getTableName());
        } else {
            LOGGER.error("Unable to unlock Table lock for table" + model.getDatabaseName() + "." + model.getTableName() + " during table status updation");
        }
    }
}

Example 22 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonDataMergerUtil method updateLoadMetadataIUDUpdateDeltaMergeStatus.

/**
   * Update Both Segment Update Status and Table Status for the case of IUD Delete
   * delta compaction.
   *
   * @param loadsToMerge
   * @param metaDataFilepath
   * @param carbonLoadModel
   * @return
   */
public static boolean updateLoadMetadataIUDUpdateDeltaMergeStatus(List<LoadMetadataDetails> loadsToMerge, String metaDataFilepath, CarbonLoadModel carbonLoadModel) {
    boolean status = false;
    boolean updateLockStatus = false;
    boolean tableLockStatus = false;
    String timestamp = "" + carbonLoadModel.getFactTimeStamp();
    List<String> updatedDeltaFilesList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    // This routine updateLoadMetadataIUDCompactionMergeStatus is suppose to update
    // two files as it is only called during IUD_UPDDEL_DELTA_COMPACTION. Along with
    // Table Status Metadata file (For Update Block Compaction) it has to update the
    // Table Update Status Metadata File (For corresponding Delete Delta File).
    // As the IUD_UPDDEL_DELTA_COMPACTION going to write in the same segment therefore in
    // A) Table Update Status Metadata File (Block Level)
    //      * For each blocks which is being compacted Mark 'Compacted' as the Status.
    // B) Table Status Metadata file (Segment Level)
    //      * loadStatus won't be changed to "compacted'
    //      * UpdateDeltaStartTime and UpdateDeltaEndTime will be both set to current
    //        timestamp (which is being passed from driver)
    // First the Table Update Status Metadata File should be updated as we need to get
    // the updated blocks for the segment from Table Status Metadata Update Delta Start and
    // End Timestamp.
    // Table Update Status Metadata Update.
    AbsoluteTableIdentifier absoluteTableIdentifier = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    SegmentUpdateStatusManager segmentUpdateStatusManager = new SegmentUpdateStatusManager(absoluteTableIdentifier);
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    ICarbonLock updateLock = segmentUpdateStatusManager.getTableUpdateStatusLock();
    ICarbonLock statusLock = segmentStatusManager.getTableStatusLock();
    // Update the Compacted Blocks with Compacted Status.
    try {
        updatedDeltaFilesList = segmentUpdateStatusManager.getUpdateDeltaFiles(loadsToMerge.get(0).getLoadName().toString());
    } catch (Exception e) {
        LOGGER.error("Error while getting the Update Delta Blocks.");
        status = false;
        return status;
    }
    if (updatedDeltaFilesList.size() > 0) {
        try {
            updateLockStatus = updateLock.lockWithRetries();
            tableLockStatus = statusLock.lockWithRetries();
            List<String> blockNames = new ArrayList<>(updatedDeltaFilesList.size());
            for (String compactedBlocks : updatedDeltaFilesList) {
                // Try to BlockName
                int endIndex = compactedBlocks.lastIndexOf(File.separator);
                String blkNoExt = compactedBlocks.substring(endIndex + 1, compactedBlocks.lastIndexOf("-"));
                blockNames.add(blkNoExt);
            }
            if (updateLockStatus && tableLockStatus) {
                SegmentUpdateDetails[] updateLists = segmentUpdateStatusManager.readLoadMetadata();
                for (String compactedBlocks : blockNames) {
                    // Check is the compactedBlocks name matches with oldDetails
                    for (int i = 0; i < updateLists.length; i++) {
                        if (updateLists[i].getBlockName().equalsIgnoreCase(compactedBlocks) && !CarbonCommonConstants.COMPACTED.equalsIgnoreCase(updateLists[i].getStatus()) && !CarbonCommonConstants.MARKED_FOR_DELETE.equalsIgnoreCase(updateLists[i].getStatus())) {
                            updateLists[i].setStatus(CarbonCommonConstants.COMPACTED);
                        }
                    }
                }
                LoadMetadataDetails[] loadDetails = segmentStatusManager.readLoadMetadata(metaDataFilepath);
                for (LoadMetadataDetails loadDetail : loadDetails) {
                    if (loadsToMerge.contains(loadDetail)) {
                        loadDetail.setUpdateDeltaStartTimestamp(timestamp);
                        loadDetail.setUpdateDeltaEndTimestamp(timestamp);
                        if (loadDetail.getLoadName().equalsIgnoreCase("0")) {
                            loadDetail.setUpdateStatusFileName(CarbonUpdateUtil.getUpdateStatusFileName(timestamp));
                        }
                    }
                }
                try {
                    segmentUpdateStatusManager.writeLoadDetailsIntoFile(Arrays.asList(updateLists), timestamp);
                    segmentStatusManager.writeLoadDetailsIntoFile(carbonTablePath.getTableStatusFilePath(), loadDetails);
                    status = true;
                } catch (IOException e) {
                    LOGGER.error("Error while writing metadata. The metadata file path is " + carbonTablePath.getMetadataDirectoryPath());
                    status = false;
                }
            } else {
                LOGGER.error("Not able to acquire the lock.");
                status = false;
            }
        } catch (Exception e) {
            LOGGER.error("Error while updating metadata. The metadata file path is " + carbonTablePath.getMetadataDirectoryPath());
            status = false;
        } finally {
            if (updateLockStatus) {
                if (updateLock.unlock()) {
                    LOGGER.info("Unlock the segment update lock successfully.");
                } else {
                    LOGGER.error("Not able to unlock the segment update lock.");
                }
            }
            if (tableLockStatus) {
                if (statusLock.unlock()) {
                    LOGGER.info("Unlock the table status lock successfully.");
                } else {
                    LOGGER.error("Not able to unlock the table status lock.");
                }
            }
        }
    }
    return status;
}

Example 23 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonDataMergerUtil method updateLoadMetadataWithMergeStatus.

/**
   * method to update table status in case of IUD Update Delta Compaction.
   * @param loadsToMerge
   * @param metaDataFilepath
   * @param MergedLoadName
   * @param carbonLoadModel
   * @param compactionType
   * @return
   */
public static boolean updateLoadMetadataWithMergeStatus(List<LoadMetadataDetails> loadsToMerge, String metaDataFilepath, String MergedLoadName, CarbonLoadModel carbonLoadModel, long mergeLoadStartTime, CompactionType compactionType) {
    boolean tableStatusUpdationStatus = false;
    AbsoluteTableIdentifier absoluteTableIdentifier = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
    try {
        if (carbonLock.lockWithRetries()) {
            LOGGER.info("Acquired lock for the table " + carbonLoadModel.getDatabaseName() + "." + carbonLoadModel.getTableName() + " for table status updation ");
            CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
            String statusFilePath = carbonTablePath.getTableStatusFilePath();
            LoadMetadataDetails[] loadDetails = SegmentStatusManager.readLoadMetadata(metaDataFilepath);
            String mergedLoadNumber = MergedLoadName.substring(MergedLoadName.lastIndexOf(CarbonCommonConstants.LOAD_FOLDER) + CarbonCommonConstants.LOAD_FOLDER.length(), MergedLoadName.length());
            long modificationOrDeletionTimeStamp = CarbonUpdateUtil.readCurrentTime();
            for (LoadMetadataDetails loadDetail : loadDetails) {
                // check if this segment is merged.
                if (loadsToMerge.contains(loadDetail)) {
                    // then need to discard the compaction process and treat it as failed compaction.
                    if (loadDetail.getLoadStatus().equalsIgnoreCase(CarbonCommonConstants.MARKED_FOR_DELETE)) {
                        LOGGER.error("Compaction is aborted as the segment " + loadDetail.getLoadName() + " is deleted after the compaction is started.");
                        return false;
                    }
                    loadDetail.setLoadStatus(CarbonCommonConstants.COMPACTED);
                    loadDetail.setModificationOrdeletionTimesStamp(modificationOrDeletionTimeStamp);
                    loadDetail.setMergedLoadName(mergedLoadNumber);
                }
            }
            // create entry for merged one.
            LoadMetadataDetails loadMetadataDetails = new LoadMetadataDetails();
            loadMetadataDetails.setPartitionCount(carbonLoadModel.getPartitionId());
            loadMetadataDetails.setLoadStatus(CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS);
            long loadEnddate = CarbonUpdateUtil.readCurrentTime();
            loadMetadataDetails.setLoadEndTime(loadEnddate);
            loadMetadataDetails.setLoadName(mergedLoadNumber);
            loadMetadataDetails.setLoadStartTime(mergeLoadStartTime);
            loadMetadataDetails.setPartitionCount("0");
            // if this is a major compaction then set the segment as major compaction.
            if (compactionType == CompactionType.MAJOR_COMPACTION) {
                loadMetadataDetails.setMajorCompacted("true");
            }
            List<LoadMetadataDetails> updatedDetailsList = new ArrayList<>(Arrays.asList(loadDetails));
            // put the merged folder entry
            updatedDetailsList.add(loadMetadataDetails);
            try {
                SegmentStatusManager.writeLoadDetailsIntoFile(statusFilePath, updatedDetailsList.toArray(new LoadMetadataDetails[updatedDetailsList.size()]));
                tableStatusUpdationStatus = true;
            } catch (IOException e) {
                LOGGER.error("Error while writing metadata");
                tableStatusUpdationStatus = false;
            }
        } else {
            LOGGER.error("Could not able to obtain lock for table" + carbonLoadModel.getDatabaseName() + "." + carbonLoadModel.getTableName() + "for table status updation");
        }
    } finally {
        if (carbonLock.unlock()) {
            LOGGER.info("Table unlocked successfully after table status updation" + carbonLoadModel.getDatabaseName() + "." + carbonLoadModel.getTableName());
        } else {
            LOGGER.error("Unable to unlock Table lock for table" + carbonLoadModel.getDatabaseName() + "." + carbonLoadModel.getTableName() + " during table status updation");
        }
    }
    return tableStatusUpdationStatus;
}

Example 24 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonLoaderUtil method writeLoadMetadata.

public static void writeLoadMetadata(String storeLocation, String dbName, String tableName, List<LoadMetadataDetails> listOfLoadFolderDetails) throws IOException {
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(storeLocation, dbName, tableName);
    String dataLoadLocation = carbonTablePath.getTableStatusFilePath();
    DataOutputStream dataOutputStream;
    Gson gsonObjectToWrite = new Gson();
    BufferedWriter brWriter = null;
    AtomicFileOperations writeOperation = new AtomicFileOperationsImpl(dataLoadLocation, FileFactory.getFileType(dataLoadLocation));
    try {
        dataOutputStream = writeOperation.openForWrite(FileWriteOperation.OVERWRITE);
        brWriter = new BufferedWriter(new OutputStreamWriter(dataOutputStream, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
        String metadataInstance = gsonObjectToWrite.toJson(listOfLoadFolderDetails.toArray());
        brWriter.write(metadataInstance);
    } finally {
        try {
            if (null != brWriter) {
                brWriter.flush();
            }
        } catch (Exception e) {
            LOGGER.error("error in  flushing ");
        }
        CarbonUtil.closeStreams(brWriter);
        writeOperation.close();
    }
}

Example 25 with CarbonTablePath

use of org.apache.carbondata.core.util.path.CarbonTablePath in project carbondata by apache.

the class CarbonLoaderUtil method deletePartialLoadDataIfExist.

public static void deletePartialLoadDataIfExist(CarbonLoadModel loadModel, final boolean isCompactionFlow) throws IOException {
    CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
    String metaDataLocation = carbonTable.getMetaDataFilepath();
    final LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(metaDataLocation);
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(loadModel.getStorePath(), carbonTable.getCarbonTableIdentifier());
    //delete folder which metadata no exist in tablestatus
    for (int i = 0; i < carbonTable.getPartitionCount(); i++) {
        final String partitionCount = i + "";
        String partitionPath = carbonTablePath.getPartitionDir(partitionCount);
        FileType fileType = FileFactory.getFileType(partitionPath);
        if (FileFactory.isFileExist(partitionPath, fileType)) {
            CarbonFile carbonFile = FileFactory.getCarbonFile(partitionPath, fileType);
            CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {

                @Override
                public boolean accept(CarbonFile path) {
                    String segmentId = CarbonTablePath.DataPathUtil.getSegmentId(path.getAbsolutePath() + "/dummy");
                    boolean found = false;
                    for (int j = 0; j < details.length; j++) {
                        if (details[j].getLoadName().equals(segmentId) && details[j].getPartitionCount().equals(partitionCount)) {
                            found = true;
                            break;
                        }
                    }
                    return !found;
                }
            });
            for (int k = 0; k < listFiles.length; k++) {
                String segmentId = CarbonTablePath.DataPathUtil.getSegmentId(listFiles[k].getAbsolutePath() + "/dummy");
                if (isCompactionFlow) {
                    if (segmentId.contains(".")) {
                        deleteStorePath(listFiles[k].getAbsolutePath());
                    }
                } else {
                    if (!segmentId.contains(".")) {
                        deleteStorePath(listFiles[k].getAbsolutePath());
                    }
                }
            }
        }
    }
}