Examples with ReadCommittedScope org.apache.carbondata.core.readcommitter.ReadCommittedScope used on opensource projects

Example 1 with ReadCommittedScope

use of org.apache.carbondata.core.readcommitter.ReadCommittedScope in project carbondata by apache.

the class CarbonFileInputFormat method getSplits.

/**
 * get list of block/blocklet and make them to CarbonInputSplit
 * @param job JobContext with Configuration
 * @return list of CarbonInputSplit
 * @throws IOException
 */
@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {
    CarbonTable carbonTable = getOrCreateCarbonTable(job.getConfiguration());
    if (null == carbonTable) {
        throw new IOException("Missing/Corrupt schema file for table.");
    }
    AbsoluteTableIdentifier identifier = carbonTable.getAbsoluteTableIdentifier();
    // get all valid segments and set them into the configuration
    // check for externalTable segment (Segment_null)
    // process and resolve the expression
    ReadCommittedScope readCommittedScope;
    if (carbonTable.isTransactionalTable()) {
        readCommittedScope = new LatestFilesReadCommittedScope(identifier.getTablePath() + "/Fact/Part0/Segment_null/", job.getConfiguration());
    } else {
        readCommittedScope = getReadCommittedScope(job.getConfiguration());
        if (readCommittedScope == null) {
            readCommittedScope = new LatestFilesReadCommittedScope(identifier.getTablePath(), job.getConfiguration());
        } else {
            readCommittedScope.setConfiguration(job.getConfiguration());
        }
    }
    // this will be null in case of corrupt schema file.
    IndexFilter filter = getFilterPredicates(job.getConfiguration());
    // if external table Segments are found, add it to the List
    List<Segment> externalTableSegments = new ArrayList<>();
    Segment seg;
    if (carbonTable.isTransactionalTable()) {
        // SDK some cases write into the Segment Path instead of Table Path i.e. inside
        // the "Fact/Part0/Segment_null". The segment in this case is named as "null".
        // The table is denoted by default as a transactional table and goes through
        // the path of CarbonFileInputFormat. The above scenario is handled in the below code.
        seg = new Segment("null", null, readCommittedScope);
        externalTableSegments.add(seg);
    } else {
        LoadMetadataDetails[] loadMetadataDetails = readCommittedScope.getSegmentList();
        for (LoadMetadataDetails load : loadMetadataDetails) {
            seg = new Segment(load.getLoadName(), null, readCommittedScope);
            if (fileLists != null) {
                for (Object fileList : fileLists) {
                    String timestamp = CarbonTablePath.DataFileUtil.getTimeStampFromFileName(fileList.toString());
                    if (timestamp.equals(seg.getSegmentNo())) {
                        externalTableSegments.add(seg);
                        break;
                    }
                }
            } else {
                externalTableSegments.add(seg);
            }
        }
    }
    List<InputSplit> splits = new ArrayList<>();
    boolean useBlockIndex = job.getConfiguration().getBoolean("filter_blocks", true);
    // scenarios
    if (filter != null) {
        filter.resolve(false);
    }
    if (useBlockIndex) {
        // do block filtering and get split
        splits = getSplits(job, filter, externalTableSegments);
    } else {
        List<CarbonFile> carbonFiles;
        if (null != this.fileLists) {
            carbonFiles = getAllCarbonDataFiles(this.fileLists);
        } else {
            carbonFiles = getAllCarbonDataFiles(carbonTable.getTablePath());
        }
        List<String> allDeleteDeltaFiles = getAllDeleteDeltaFiles(carbonTable.getTablePath());
        for (CarbonFile carbonFile : carbonFiles) {
            // Segment id is set to null because SDK does not write carbondata files with respect
            // to segments. So no specific name is present for this load.
            CarbonInputSplit split = new CarbonInputSplit("null", carbonFile.getAbsolutePath(), 0, carbonFile.getLength(), carbonFile.getLocations(), FileFormat.COLUMNAR_V3);
            split.setVersion(ColumnarFormatVersion.V3);
            BlockletDetailInfo info = new BlockletDetailInfo();
            split.setDetailInfo(info);
            info.setBlockSize(carbonFile.getLength());
            info.setVersionNumber(split.getVersion().number());
            info.setUseMinMaxForPruning(false);
            if (CollectionUtils.isNotEmpty(allDeleteDeltaFiles)) {
                split.setDeleteDeltaFiles(getDeleteDeltaFiles(carbonFile.getAbsolutePath(), allDeleteDeltaFiles));
            }
            splits.add(split);
        }
        splits.sort(Comparator.comparing(o -> ((CarbonInputSplit) o).getFilePath()));
    }
    setAllColumnProjectionIfNotConfigured(job, carbonTable);
    return splits;
}

Example 2 with ReadCommittedScope

use of org.apache.carbondata.core.readcommitter.ReadCommittedScope in project carbondata by apache.

the class IndexUtil method executeClearIndexJob.

/**
 * This method gets the indexJob and call execute , this job will be launched before clearing
 * indexes from driver side during drop table and drop index and clears the index in executor
 * side
 * @param carbonTable
 * @throws IOException
 */
private static void executeClearIndexJob(IndexJob indexJob, CarbonTable carbonTable, String indexToClear) throws IOException {
    IndexInputFormat indexInputFormat;
    if (!carbonTable.isTransactionalTable()) {
        ReadCommittedScope readCommittedScope = new LatestFilesReadCommittedScope(carbonTable.getTablePath(), FileFactory.getConfiguration());
        LoadMetadataDetails[] loadMetadataDetails = readCommittedScope.getSegmentList();
        List<Segment> listOfValidSegments = new ArrayList<>(loadMetadataDetails.length);
        Arrays.stream(loadMetadataDetails).forEach(segment -> {
            Segment seg = new Segment(segment.getLoadName(), segment.getSegmentFile());
            seg.setLoadMetadataDetails(segment);
            listOfValidSegments.add(seg);
        });
        indexInputFormat = new IndexInputFormat(carbonTable, listOfValidSegments, new ArrayList<>(0), true, indexToClear);
    } else {
        SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegmentsInfo = getValidAndInvalidSegments(carbonTable, FileFactory.getConfiguration());
        List<String> invalidSegment = new ArrayList<>();
        validAndInvalidSegmentsInfo.getInvalidSegments().forEach(segment -> invalidSegment.add(segment.getSegmentNo()));
        indexInputFormat = new IndexInputFormat(carbonTable, validAndInvalidSegmentsInfo.getValidSegments(), invalidSegment, true, indexToClear);
    }
    try {
        indexJob.execute(indexInputFormat, null);
    } catch (Exception e) {
        // Consider a scenario where clear index job is called from drop table
        // and index server crashes, in this no exception should be thrown and
        // drop table should complete.
        LOGGER.error("Failed to execute Index clear Job", e);
    }
}

Example 3 with ReadCommittedScope

use of org.apache.carbondata.core.readcommitter.ReadCommittedScope in project carbondata by apache.

the class CarbonTableInputFormat method getFilteredSegment.

/**
 * Return segment list after filtering out valid segments and segments set by user by
 * `INPUT_SEGMENT_NUMBERS` in job configuration
 */
private List<Segment> getFilteredSegment(JobContext job, List<Segment> validSegments, boolean validationRequired, ReadCommittedScope readCommittedScope) throws IOException {
    Segment[] segmentsToAccess = getSegmentsToAccess(job, readCommittedScope);
    if (segmentsToAccess.length == 0 || segmentsToAccess[0].getSegmentNo().equalsIgnoreCase("*")) {
        return validSegments;
    }
    Map<String, Segment> segmentToAccessMap = Arrays.stream(segmentsToAccess).collect(Collectors.toMap(Segment::getSegmentNo, segment -> segment, (e1, e2) -> e1));
    Map<String, Segment> filteredSegmentToAccess = new HashMap<>(segmentToAccessMap.size());
    for (Segment validSegment : validSegments) {
        String segmentNoOfValidSegment = validSegment.getSegmentNo();
        if (segmentToAccessMap.containsKey(segmentNoOfValidSegment)) {
            Segment segmentToAccess = segmentToAccessMap.get(segmentNoOfValidSegment);
            if (segmentToAccess.getSegmentFileName() != null && validSegment.getSegmentFileName() == null) {
                validSegment = segmentToAccess;
            }
            filteredSegmentToAccess.put(segmentNoOfValidSegment, validSegment);
        }
    }
    if (!validationRequired && filteredSegmentToAccess.size() != segmentToAccessMap.size()) {
        for (Segment segment : segmentToAccessMap.values()) {
            boolean isSegmentValid = true;
            LoadMetadataDetails[] segmentList = readCommittedScope.getSegmentList();
            for (LoadMetadataDetails validSegment : segmentList) {
                if (validSegment.getLoadName().equals(segment.getSegmentNo()) && (validSegment.getSegmentStatus().equals(SegmentStatus.MARKED_FOR_DELETE) || validSegment.getSegmentStatus().equals(SegmentStatus.COMPACTED))) {
                    isSegmentValid = false;
                    break;
                }
            }
            if (isSegmentValid && !filteredSegmentToAccess.containsKey(segment.getSegmentNo())) {
                filteredSegmentToAccess.put(segment.getSegmentNo(), segment);
            }
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Segments ignored are : " + Arrays.toString(Sets.difference(new HashSet<>(filteredSegmentToAccess.values()), new HashSet<>(segmentToAccessMap.values())).toArray()));
    }
    return new ArrayList<>(filteredSegmentToAccess.values());
}

Example 4 with ReadCommittedScope

use of org.apache.carbondata.core.readcommitter.ReadCommittedScope in project carbondata by apache.

the class CarbonTableInputFormat method getBlockRowCount.

/**
 * Get the row count of the Block and mapping of segment and Block count.
 */
public BlockMappingVO getBlockRowCount(Job job, CarbonTable table, List<PartitionSpec> partitions, boolean isUpdateFlow) throws IOException {
    // Normal query flow goes to CarbonInputFormat#getPrunedBlocklets and initialize the
    // pruning info for table we queried. But here count star query without filter uses a different
    // query plan, and no pruning info is initialized. When it calls default index to
    // prune(with a null filter), exception will occur during setting pruning info.
    // Considering no useful information about block/blocklet pruning for such query
    // (actually no pruning), so we disable explain collector here
    ExplainCollector.remove();
    AbsoluteTableIdentifier identifier = table.getAbsoluteTableIdentifier();
    ReadCommittedScope readCommittedScope = getReadCommitted(job, identifier);
    LoadMetadataDetails[] loadMetadataDetails = readCommittedScope.getSegmentList();
    SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(table, loadMetadataDetails);
    SegmentStatusManager.ValidAndInvalidSegmentsInfo allSegments = new SegmentStatusManager(identifier, readCommittedScope.getConfiguration()).getValidAndInvalidSegments(table.isMV(), loadMetadataDetails, readCommittedScope);
    Map<String, Long> blockRowCountMapping = new HashMap<>();
    Map<String, Long> segmentAndBlockCountMapping = new HashMap<>();
    Map<String, String> blockToSegmentMapping = new HashMap<>();
    // TODO: currently only batch segment is supported, add support for streaming table
    List<Segment> filteredSegment = getFilteredSegment(job, allSegments.getValidSegments(), false, readCommittedScope);
    boolean isIUDTable = (updateStatusManager.getUpdateStatusDetails().length != 0);
    /* In the select * flow, getSplits() method was clearing the segmentMap if,
    segment needs refreshing. same thing need for select count(*) flow also.
    For NonTransactional table, one of the reason for a segment refresh is below scenario.
    SDK is written one set of files with UUID, with same UUID it can write again.
    So, latest files content should reflect the new count by refreshing the segment */
    List<String> toBeCleanedSegments = new ArrayList<>();
    for (Segment segment : filteredSegment) {
        boolean refreshNeeded = IndexStoreManager.getInstance().getTableSegmentRefresher(getOrCreateCarbonTable(job.getConfiguration())).isRefreshNeeded(segment, SegmentUpdateStatusManager.getInvalidTimestampRange(segment.getLoadMetadataDetails()));
        if (refreshNeeded) {
            toBeCleanedSegments.add(segment.getSegmentNo());
        }
    }
    for (Segment segment : allSegments.getInvalidSegments()) {
        // remove entry in the segment index if there are invalid segments
        toBeCleanedSegments.add(segment.getSegmentNo());
    }
    if (toBeCleanedSegments.size() > 0) {
        IndexStoreManager.getInstance().clearInvalidSegments(getOrCreateCarbonTable(job.getConfiguration()), toBeCleanedSegments);
    }
    IndexExprWrapper indexExprWrapper = IndexChooser.getDefaultIndex(getOrCreateCarbonTable(job.getConfiguration()), null);
    IndexUtil.loadIndexes(table, indexExprWrapper, filteredSegment);
    if (isIUDTable || isUpdateFlow) {
        Map<String, Long> blockletToRowCountMap = new HashMap<>();
        if (CarbonProperties.getInstance().isDistributedPruningEnabled(table.getDatabaseName(), table.getTableName())) {
            try {
                List<ExtendedBlocklet> extendedBlocklets = getDistributedBlockRowCount(table, partitions, filteredSegment, allSegments.getInvalidSegments(), toBeCleanedSegments, job.getConfiguration());
                for (ExtendedBlocklet blocklet : extendedBlocklets) {
                    String filePath = blocklet.getFilePath().replace("\\", "/");
                    String blockName = filePath.substring(filePath.lastIndexOf("/") + 1);
                    blockletToRowCountMap.put(blocklet.getSegmentId() + "," + blockName, blocklet.getRowCount());
                }
            } catch (Exception e) {
                // pruning.
                if (CarbonProperties.getInstance().isFallBackDisabled()) {
                    throw e;
                }
                TableIndex defaultIndex = IndexStoreManager.getInstance().getDefaultIndex(table);
                blockletToRowCountMap.putAll(defaultIndex.getBlockRowCount(filteredSegment, partitions, defaultIndex));
            }
        } else {
            TableIndex defaultIndex = IndexStoreManager.getInstance().getDefaultIndex(table);
            blockletToRowCountMap.putAll(defaultIndex.getBlockRowCount(filteredSegment, partitions, defaultIndex));
        }
        // key is the (segmentId","+blockletPath) and key is the row count of that blocklet
        for (Map.Entry<String, Long> eachBlocklet : blockletToRowCountMap.entrySet()) {
            String[] segmentIdAndPath = eachBlocklet.getKey().split(",", 2);
            String segmentId = segmentIdAndPath[0];
            String blockName = segmentIdAndPath[1];
            long rowCount = eachBlocklet.getValue();
            String key = CarbonUpdateUtil.getSegmentBlockNameKey(segmentId, blockName, table.isHivePartitionTable());
            // if block is invalid then don't add the count
            SegmentUpdateDetails details = updateStatusManager.getDetailsForABlock(key);
            if (null == details || !CarbonUpdateUtil.isBlockInvalid(details.getSegmentStatus())) {
                Long blockCount = blockRowCountMapping.get(key);
                if (blockCount == null) {
                    blockCount = 0L;
                    Long count = segmentAndBlockCountMapping.get(segmentId);
                    if (count == null) {
                        count = 0L;
                    }
                    segmentAndBlockCountMapping.put(segmentId, count + 1);
                }
                blockToSegmentMapping.put(key, segmentId);
                blockCount += rowCount;
                blockRowCountMapping.put(key, blockCount);
            }
        }
    } else {
        long totalRowCount;
        if (CarbonProperties.getInstance().isDistributedPruningEnabled(table.getDatabaseName(), table.getTableName())) {
            totalRowCount = getDistributedCount(table, partitions, filteredSegment, job.getConfiguration());
        } else {
            TableIndex defaultIndex = IndexStoreManager.getInstance().getDefaultIndex(table);
            totalRowCount = defaultIndex.getRowCount(filteredSegment, partitions, defaultIndex);
        }
        blockRowCountMapping.put(CarbonCommonConstantsInternal.ROW_COUNT, totalRowCount);
    }
    BlockMappingVO blockMappingVO = new BlockMappingVO(blockRowCountMapping, segmentAndBlockCountMapping);
    blockMappingVO.setBlockToSegmentMapping(blockToSegmentMapping);
    return blockMappingVO;
}

Example 5 with ReadCommittedScope

use of org.apache.carbondata.core.readcommitter.ReadCommittedScope in project carbondata by apache.

the class CarbonTableInputFormat method getReadCommitted.

public ReadCommittedScope getReadCommitted(JobContext job, AbsoluteTableIdentifier identifier) throws IOException {
    if (readCommittedScope == null) {
        ReadCommittedScope readCommittedScope;
        if (job.getConfiguration().getBoolean(CARBON_TRANSACTIONAL_TABLE, true)) {
            readCommittedScope = new TableStatusReadCommittedScope(identifier, job.getConfiguration());
        } else {
            readCommittedScope = getReadCommittedScope(job.getConfiguration());
            if (readCommittedScope == null) {
                readCommittedScope = new LatestFilesReadCommittedScope(identifier.getTablePath(), job.getConfiguration());
            }
        }
        this.readCommittedScope = readCommittedScope;
    }
    return readCommittedScope;
}