Examples with TableStatusReadCommittedScope org.apache.carbondata.core.readcommitter.TableStatusReadCommittedScope used on opensource projects

Example 1 with TableStatusReadCommittedScope

use of org.apache.carbondata.core.readcommitter.TableStatusReadCommittedScope in project carbondata by apache.

the class CarbonTableInputFormat method getReadCommitted.

public ReadCommittedScope getReadCommitted(JobContext job, AbsoluteTableIdentifier identifier) throws IOException {
    if (readCommittedScope == null) {
        ReadCommittedScope readCommittedScope;
        if (job.getConfiguration().getBoolean(CARBON_TRANSACTIONAL_TABLE, true)) {
            readCommittedScope = new TableStatusReadCommittedScope(identifier, job.getConfiguration());
        } else {
            readCommittedScope = getReadCommittedScope(job.getConfiguration());
            if (readCommittedScope == null) {
                readCommittedScope = new LatestFilesReadCommittedScope(identifier.getTablePath(), job.getConfiguration());
            }
        }
        this.readCommittedScope = readCommittedScope;
    }
    return readCommittedScope;
}

Example 2 with TableStatusReadCommittedScope

use of org.apache.carbondata.core.readcommitter.TableStatusReadCommittedScope in project carbondata by apache.

the class TestBlockletIndexFactory method getValidDistributables.

@Test
public void getValidDistributables() throws IOException {
    BlockletIndexInputSplit blockletIndexInputSplit = new BlockletIndexInputSplit("/opt/store/default/carbon_table/Fact/Part0/Segment_0/0_batchno0-0-1521012756709.carbonindex");
    Segment segment = new Segment("0", null, new TableStatusReadCommittedScope(carbonTable.getAbsoluteTableIdentifier(), new Configuration(false)));
    blockletIndexInputSplit.setSegment(segment);
    BlockletIndexInputSplit indexInputSplit = new BlockletIndexInputSplit("/opt/store/default/carbon_table/Fact/Part0/Segment_0/0_batchno0-0-1521012756701.carbonindex");
    indexInputSplit.setSegment(segment);
    List<IndexInputSplit> indexInputSplits = new ArrayList<>(2);
    indexInputSplits.add(blockletIndexInputSplit);
    indexInputSplits.add(indexInputSplit);
    new MockUp<BlockletIndexFactory>() {

        @Mock
        Set<TableBlockIndexUniqueIdentifier> getTableBlockIndexUniqueIdentifiers(Segment segment) {
            TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier1 = new TableBlockIndexUniqueIdentifier("/opt/store/default/carbon_table/Fact/Part0/Segment_0", "0_batchno0-0-1521012756701.carbonindex", null, "0");
            Set<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers = new HashSet<>(3);
            tableBlockIndexUniqueIdentifiers.add(tableBlockIndexUniqueIdentifier);
            tableBlockIndexUniqueIdentifiers.add(tableBlockIndexUniqueIdentifier1);
            return tableBlockIndexUniqueIdentifiers;
        }
    };
    List<IndexInputSplit> validDistributables = blockletIndexFactory.getAllUncached(indexInputSplits);
    assert 1 == validDistributables.size();
}

Example 3 with TableStatusReadCommittedScope

use of org.apache.carbondata.core.readcommitter.TableStatusReadCommittedScope in project carbondata by apache.

the class SegmentStatusManager method getValidAndInvalidSegments.

/**
 * get valid segment for given load status details.
 */
public ValidAndInvalidSegmentsInfo getValidAndInvalidSegments(Boolean isChildTable, LoadMetadataDetails[] loadMetadataDetails, ReadCommittedScope readCommittedScope) throws IOException {
    // @TODO: move reading LoadStatus file to separate class
    List<Segment> listOfValidSegments = new ArrayList<>(10);
    List<Segment> listOfValidUpdatedSegments = new ArrayList<>(10);
    List<Segment> listOfInvalidSegments = new ArrayList<>(10);
    List<Segment> listOfStreamSegments = new ArrayList<>(10);
    List<Segment> listOfInProgressSegments = new ArrayList<>(10);
    Map<String, List<String>> mergedLoadMapping = new HashMap<>();
    try {
        if (loadMetadataDetails == null) {
            loadMetadataDetails = readTableStatusFile(CarbonTablePath.getTableStatusFilePath(identifier.getTablePath()));
        }
        if (readCommittedScope == null) {
            readCommittedScope = new TableStatusReadCommittedScope(identifier, loadMetadataDetails, configuration);
        }
        // just directly iterate Array
        for (LoadMetadataDetails segment : loadMetadataDetails) {
            if (SegmentStatus.SUCCESS == segment.getSegmentStatus() || SegmentStatus.MARKED_FOR_UPDATE == segment.getSegmentStatus() || SegmentStatus.LOAD_PARTIAL_SUCCESS == segment.getSegmentStatus() || SegmentStatus.STREAMING == segment.getSegmentStatus() || SegmentStatus.STREAMING_FINISH == segment.getSegmentStatus()) {
                // check for merged loads.
                if (null != segment.getMergedLoadName()) {
                    Segment seg = new Segment(segment.getMergedLoadName(), segment.getSegmentFile(), readCommittedScope, segment);
                    if (!listOfValidSegments.contains(seg)) {
                        listOfValidSegments.add(seg);
                    }
                    // if merged load is updated then put it in updated list
                    if (SegmentStatus.MARKED_FOR_UPDATE == segment.getSegmentStatus()) {
                        listOfValidUpdatedSegments.add(seg);
                    }
                    continue;
                }
                if (SegmentStatus.MARKED_FOR_UPDATE == segment.getSegmentStatus()) {
                    listOfValidUpdatedSegments.add(new Segment(segment.getLoadName(), segment.getSegmentFile(), readCommittedScope));
                }
                if (SegmentStatus.STREAMING == segment.getSegmentStatus() || SegmentStatus.STREAMING_FINISH == segment.getSegmentStatus()) {
                    listOfStreamSegments.add(new Segment(segment.getLoadName(), segment.getSegmentFile(), readCommittedScope));
                    continue;
                }
                // to validSegment list, as segment does not exists
                if (isChildTable) {
                    if (!segment.getDataSize().equalsIgnoreCase("0") && !segment.getIndexSize().equalsIgnoreCase("0")) {
                        listOfValidSegments.add(new Segment(segment.getLoadName(), segment.getSegmentFile(), readCommittedScope, segment));
                    }
                } else {
                    listOfValidSegments.add(new Segment(segment.getLoadName(), segment.getSegmentFile(), readCommittedScope, segment));
                }
            } else if ((SegmentStatus.LOAD_FAILURE == segment.getSegmentStatus() || SegmentStatus.COMPACTED == segment.getSegmentStatus() || SegmentStatus.MARKED_FOR_DELETE == segment.getSegmentStatus())) {
                listOfInvalidSegments.add(new Segment(segment.getLoadName(), segment.getSegmentFile()));
                if (SegmentStatus.COMPACTED == segment.getSegmentStatus()) {
                    // check the main table's merged segment's map. ex: {0.1 -> 0,1,2,3}
                    if (null != segment.getMergedLoadName()) {
                        if (mergedLoadMapping.containsKey(segment.getMergedLoadName())) {
                            mergedLoadMapping.get(segment.getMergedLoadName()).add(segment.getLoadName());
                        } else {
                            List<String> mergedLoads = new ArrayList<>();
                            mergedLoads.add(segment.getLoadName());
                            mergedLoadMapping.put(segment.getMergedLoadName(), mergedLoads);
                        }
                    }
                }
            } else if (SegmentStatus.INSERT_IN_PROGRESS == segment.getSegmentStatus() || SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == segment.getSegmentStatus()) {
                listOfInProgressSegments.add(new Segment(segment.getLoadName(), segment.getSegmentFile(), readCommittedScope));
            }
        }
    } catch (IOException e) {
        LOG.error(e.getMessage(), e);
        throw e;
    }
    return new ValidAndInvalidSegmentsInfo(listOfValidSegments, listOfValidUpdatedSegments, listOfInvalidSegments, listOfStreamSegments, listOfInProgressSegments, mergedLoadMapping);
}