Examples with TableIndex org.apache.carbondata.core.index.TableIndex used on opensource projects

Example 6 with TableIndex

use of org.apache.carbondata.core.index.TableIndex in project carbondata by apache.

the class DeleteLoadFolders method physicalFactAndMeasureMetadataDeletion.

/**
 * Delete the invalid data physically from table.
 * @param carbonTable table
 * @param loadDetails Load details which need clean up
 * @param isForceDelete Force delete Compacted and MFD segments. it will empty the trash folder
 * @param specs Partition specs
 * @param currLoadDetails Current table status load details which are required for update manager.
 */
private static void physicalFactAndMeasureMetadataDeletion(CarbonTable carbonTable, LoadMetadataDetails[] loadDetails, boolean isForceDelete, List<PartitionSpec> specs, LoadMetadataDetails[] currLoadDetails, boolean cleanStaleInProgress, Set<String> loadsToDelete) {
    List<TableIndex> indexes = new ArrayList<>();
    try {
        for (TableIndex index : IndexStoreManager.getInstance().getAllCGAndFGIndexes(carbonTable)) {
            if (index.getIndexSchema().isIndex()) {
                indexes.add(index);
            }
        }
    } catch (IOException e) {
        LOGGER.warn(String.format("Failed to get indexes for %s.%s, therefore the index files could not be cleaned.", carbonTable.getAbsoluteTableIdentifier().getDatabaseName(), carbonTable.getAbsoluteTableIdentifier().getTableName()));
    }
    SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(carbonTable, currLoadDetails);
    for (final LoadMetadataDetails oneLoad : loadDetails) {
        if (loadsToDelete.contains(oneLoad.getLoadName())) {
            try {
                if (oneLoad.getSegmentFile() != null) {
                    String tablePath = carbonTable.getAbsoluteTableIdentifier().getTablePath();
                    Segment segment = new Segment(oneLoad.getLoadName(), oneLoad.getSegmentFile());
                    // No need to delete physical data for external segments.
                    if (oneLoad.getPath() == null || oneLoad.getPath().equalsIgnoreCase("NA")) {
                        SegmentFileStore.deleteSegment(tablePath, segment, specs, updateStatusManager);
                    }
                    // delete segment files for all segments.
                    SegmentFileStore.deleteSegmentFile(tablePath, segment);
                } else {
                    String path = getSegmentPath(carbonTable.getAbsoluteTableIdentifier(), oneLoad);
                    boolean status = false;
                    if (FileFactory.isFileExist(path)) {
                        CarbonFile file = FileFactory.getCarbonFile(path);
                        CarbonFile[] filesToBeDeleted = file.listFiles(new CarbonFileFilter() {

                            @Override
                            public boolean accept(CarbonFile file) {
                                return (CarbonTablePath.isCarbonDataFile(file.getName()) || CarbonTablePath.isCarbonIndexFile(file.getName()));
                            }
                        });
                        // entry in metadata.
                        if (filesToBeDeleted.length == 0) {
                            status = true;
                        } else {
                            for (CarbonFile eachFile : filesToBeDeleted) {
                                if (!eachFile.delete()) {
                                    LOGGER.warn("Unable to delete the file as per delete command " + eachFile.getAbsolutePath());
                                    status = false;
                                } else {
                                    status = true;
                                }
                            }
                        }
                        // need to delete the complete folder.
                        if (status) {
                            if (!file.delete()) {
                                LOGGER.warn("Unable to delete the folder as per delete command " + file.getAbsolutePath());
                            }
                        }
                    }
                }
                List<Segment> segments = new ArrayList<>(1);
                for (TableIndex index : indexes) {
                    segments.clear();
                    segments.add(new Segment(oneLoad.getLoadName()));
                    index.deleteIndexData(segments);
                }
            } catch (Exception e) {
                LOGGER.warn("Unable to delete the file as per delete command " + oneLoad.getLoadName());
            }
        }
    }
}

Example 7 with TableIndex

use of org.apache.carbondata.core.index.TableIndex in project carbondata by apache.

the class LuceneIndexFactoryBase method getAllIndexDirs.

/**
 * returns all the directories of lucene index files for query
 * @param tablePath
 * @param segmentId
 * @return
 */
private CarbonFile[] getAllIndexDirs(String tablePath, String segmentId) {
    List<CarbonFile> indexDirs = new ArrayList<>();
    List<TableIndex> indexes = new ArrayList<>();
    try {
        // there can be multiple lucene index present on a table, so get all indexes and form
        // the path till the index file directories in all index folders present in each segment
        indexes = IndexStoreManager.getInstance().getAllCGAndFGIndexes(getCarbonTable());
    } catch (IOException ex) {
        LOGGER.error("failed to get indexes");
    }
    if (indexes.size() > 0) {
        for (TableIndex index : indexes) {
            if (index.getIndexSchema().getIndexName().equals(this.indexName)) {
                List<CarbonFile> indexFiles;
                String dmPath = CarbonTablePath.getIndexesStorePath(tablePath, segmentId, index.getIndexSchema().getIndexName());
                final CarbonFile dirPath = FileFactory.getCarbonFile(dmPath);
                indexFiles = Arrays.asList(dirPath.listFiles(new CarbonFileFilter() {

                    @Override
                    public boolean accept(CarbonFile file) {
                        return file.isDirectory();
                    }
                }));
                indexDirs.addAll(indexFiles);
            }
        }
    }
    return indexDirs.toArray(new CarbonFile[0]);
}