Examples with LoadMetadataDetails org.apache.carbondata.core.statusmanager.LoadMetadataDetails used on opensource projects

Example 6 with LoadMetadataDetails

use of org.apache.carbondata.core.statusmanager.LoadMetadataDetails in project carbondata by apache.

the class LoadMetadataDetailsUnitTest method testEqualsObjectIsLoadMetadataDetails.

@Test
public void testEqualsObjectIsLoadMetadataDetails() throws Exception {
    loadMetadataDetails.setLoadName("test");
    LoadMetadataDetails obj = new LoadMetadataDetails();
    boolean result = loadMetadataDetails.equals(obj);
    assertEquals(false, result);
}

Example 7 with LoadMetadataDetails

use of org.apache.carbondata.core.statusmanager.LoadMetadataDetails in project carbondata by apache.

the class LoadMetadataDetailsUnitTest method testEqualsObjectIsLoadMetadataDetailsLoadNameEqualsObjectLoadName.

@Test
public void testEqualsObjectIsLoadMetadataDetailsLoadNameEqualsObjectLoadName() throws Exception {
    loadMetadataDetails.setLoadName("test");
    LoadMetadataDetails obj = new LoadMetadataDetails();
    obj.setLoadName("test");
    boolean result = loadMetadataDetails.equals(obj);
    assertEquals(true, result);
}

Example 8 with LoadMetadataDetails

use of org.apache.carbondata.core.statusmanager.LoadMetadataDetails in project carbondata by apache.

the class LoadMetadataDetailsUnitTest method testEqualsObjectIsLoadMetadataDetailsWithoutLoadName.

@Test
public void testEqualsObjectIsLoadMetadataDetailsWithoutLoadName() throws Exception {
    LoadMetadataDetails obj = new LoadMetadataDetails();
    boolean result = loadMetadataDetails.equals(obj);
    assertEquals(true, result);
}

Example 9 with LoadMetadataDetails

use of org.apache.carbondata.core.statusmanager.LoadMetadataDetails in project carbondata by apache.

the class CarbonDataMergerUtil method updateMajorCompactionPropertyInSegment.

/**
   * This will update the property of segments as major compacted.
   * @param model
   * @param changedSegDetails
   */
public static void updateMajorCompactionPropertyInSegment(CarbonLoadModel model, List<LoadMetadataDetails> changedSegDetails, List<LoadMetadataDetails> preservedSegment) throws Exception {
    String metadataPath = model.getCarbonDataLoadSchema().getCarbonTable().getMetaDataFilepath();
    AbsoluteTableIdentifier absoluteTableIdentifier = model.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
    LoadMetadataDetails[] details = segmentStatusManager.readLoadMetadata(metadataPath);
    List<LoadMetadataDetails> originalList = Arrays.asList(details);
    for (LoadMetadataDetails segment : changedSegDetails) {
        if (preservedSegment.contains(segment)) {
            continue;
        }
        originalList.get(originalList.indexOf(segment)).setMajorCompacted("true");
    }
    ICarbonLock carbonTableStatusLock = CarbonLockFactory.getCarbonLockObj(model.getCarbonDataLoadSchema().getCarbonTable().getCarbonTableIdentifier(), LockUsage.TABLE_STATUS_LOCK);
    try {
        if (carbonTableStatusLock.lockWithRetries()) {
            LOGGER.info("Acquired lock for the table " + model.getDatabaseName() + "." + model.getTableName() + " for table status updation ");
            CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
            segmentStatusManager.writeLoadDetailsIntoFile(carbonTablePath.getTableStatusFilePath(), originalList.toArray(new LoadMetadataDetails[originalList.size()]));
        } else {
            LOGGER.error("Could not able to obtain lock for table" + model.getDatabaseName() + "." + model.getTableName() + "for table status updation");
            throw new Exception("Failed to update the MajorCompactionStatus.");
        }
    } catch (IOException e) {
        LOGGER.error("Error while writing metadata");
        throw new Exception("Failed to update the MajorCompactionStatus." + e.getMessage());
    } finally {
        if (carbonTableStatusLock.unlock()) {
            LOGGER.info("Table unlocked successfully after table status updation" + model.getDatabaseName() + "." + model.getTableName());
        } else {
            LOGGER.error("Unable to unlock Table lock for table" + model.getDatabaseName() + "." + model.getTableName() + " during table status updation");
        }
    }
}

Example 10 with LoadMetadataDetails

use of org.apache.carbondata.core.statusmanager.LoadMetadataDetails in project carbondata by apache.

the class CarbonDataMergerUtil method identifySegmentsToBeMergedBasedOnIUD.

/**
   * method to identify the segments qualified for merging in case of IUD Compaction.
   *
   * @param segments
   * @param carbonLoadModel
   * @return
   */
private static List<LoadMetadataDetails> identifySegmentsToBeMergedBasedOnIUD(List<LoadMetadataDetails> segments, CarbonLoadModel carbonLoadModel) {
    List<LoadMetadataDetails> validSegments = new ArrayList<>(segments.size());
    AbsoluteTableIdentifier absoluteTableIdentifier = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
    int numberUpdateDeltaFilesThreshold = CarbonProperties.getInstance().getNoUpdateDeltaFilesThresholdForIUDCompaction();
    for (LoadMetadataDetails seg : segments) {
        if ((isSegmentValid(seg)) && checkUpdateDeltaFilesInSeg(seg.getLoadName(), absoluteTableIdentifier, carbonLoadModel.getSegmentUpdateStatusManager(), numberUpdateDeltaFilesThreshold)) {
            validSegments.add(seg);
        }
    }
    return validSegments;
}