Examples with IndexClient io.prestosql.spi.heuristicindex.IndexClient used on opensource projects

Example 1 with IndexClient

use of io.prestosql.spi.heuristicindex.IndexClient in project hetu-core by openlookeng.

the class CreateIndexOperator method finish.

@Override
public void finish() {
    // i.e. index is PERSISTING , FINISHED_PERSISTING or FINISHED
    if (state != State.NEEDS_INPUT) {
        return;
    }
    // start PERSISTING
    // if this operator is responsible for PERSISTING an IndexWriter it will call persist()
    // otherwise the operator will simply go from PERSISTING to FINISHED_PERSISTING without calling persist()
    state = State.PERSISTING;
    // mark current operator as finished
    finished.put(this, true);
    // wait for all operators to have their finish() method called, i.e. there's no more inputs expected
    while (finished.containsValue(false)) {
        try {
            TimeUnit.MILLISECONDS.sleep(50);
        } catch (InterruptedException e) {
            throw new RuntimeException("CreateIndexOperator unexpectedly interrupted while waiting for all operators to finish: ", e);
        }
    }
    // persist index to disk if this operator is responsible for persisting a writer
    try {
        Iterator<Map.Entry<String, IndexWriter>> iterator = levelWriter.entrySet().iterator();
        while (iterator.hasNext()) {
            Map.Entry<String, IndexWriter> entry = iterator.next();
            if (persistBy.get(entry.getValue()) == this) {
                String writerKey = entry.getKey();
                entry.getValue().persist();
                // remove reference to writer once persisted so it can be GCed
                iterator.remove();
                LOG.debug("Writer for %s has finished persisting. Remaining: %d", writerKey, levelWriter.size());
            }
        }
    } catch (IOException e) {
        throw new UncheckedIOException("Persisting index failed: " + e.getMessage(), e);
    }
    synchronized (levelWriter) {
        // All writers have finished persisting
        if (levelWriter.isEmpty()) {
            LOG.debug("Writing index record by %s", this);
            if (persistBy.isEmpty()) {
                // table scan is empty. no data scanned from table. addInput() has never been called.
                throw new IllegalStateException("The table is empty. No index will be created.");
            }
            // update metadata
            IndexClient indexClient = heuristicIndexerManager.getIndexClient();
            try {
                createIndexMetadata.setIndexSize(heuristicIndexerManager.getIndexClient().getIndexSize(createIndexMetadata.getIndexName()));
                IndexClient.RecordStatus recordStatus = indexClient.lookUpIndexRecord(createIndexMetadata);
                LOG.debug("Current record status: %s", recordStatus);
                switch(recordStatus) {
                    case SAME_NAME:
                    case IN_PROGRESS_SAME_NAME:
                    case IN_PROGRESS_SAME_CONTENT:
                    case IN_PROGRESS_SAME_INDEX_PART_CONFLICT:
                    case IN_PROGRESS_SAME_INDEX_PART_CAN_MERGE:
                        indexClient.deleteIndexRecord(createIndexMetadata.getIndexName(), Collections.emptyList());
                        indexClient.addIndexRecord(createIndexMetadata);
                        break;
                    case NOT_FOUND:
                    case SAME_INDEX_PART_CAN_MERGE:
                        indexClient.addIndexRecord(createIndexMetadata);
                        break;
                    default:
                }
            } catch (IOException e) {
                throw new UncheckedIOException("Unable to update index records: ", e);
            }
        }
    }
    state = State.FINISHED_PERSISTING;
}

Example 2 with IndexClient

use of io.prestosql.spi.heuristicindex.IndexClient in project hetu-core by openlookeng.

the class TestIndexCacheLoader method testNoValidIndexFilesFoundException.

@Test(expectedExceptions = Exception.class)
public void testNoValidIndexFilesFoundException() throws Exception {
    IndexClient indexclient = mock(IndexClient.class);
    IndexCacheLoader indexCacheLoader = new IndexCacheLoader(indexclient);
    long lastModifiedTime = 1L;
    IndexCacheKey indexCacheKey = new IndexCacheKey("/path/to/split", lastModifiedTime);
    when(indexclient.getLastModifiedTime((indexCacheKey.getPath()))).thenReturn(lastModifiedTime);
    when(indexclient.readSplitIndex((indexCacheKey.getPath()))).thenThrow(Exception.class);
    indexCacheLoader.load(indexCacheKey);
}

Example 3 with IndexClient

use of io.prestosql.spi.heuristicindex.IndexClient in project hetu-core by openlookeng.

the class TestIndexCacheLoader method testNoMatchingLastModifiedTime.

@Test(expectedExceptions = Exception.class)
public void testNoMatchingLastModifiedTime() throws Exception {
    IndexClient indexclient = mock(IndexClient.class);
    IndexCacheLoader indexCacheLoader = new IndexCacheLoader(indexclient);
    IndexCacheKey indexCacheKey = new IndexCacheKey("/path/to/split", 1L);
    // return different last modified time to simulate expired index
    when(indexclient.getLastModifiedTime((indexCacheKey.getPath()))).thenReturn(2L);
    indexCacheLoader.load(indexCacheKey);
}

Example 4 with IndexClient

use of io.prestosql.spi.heuristicindex.IndexClient in project hetu-core by openlookeng.

the class TestIndexCacheLoader method testIndexFound.

@Test
public void testIndexFound() throws Exception {
    IndexClient indexclient = mock(IndexClient.class);
    IndexCacheLoader indexCacheLoader = new IndexCacheLoader(indexclient);
    List<IndexMetadata> expectedSplitIndexes = new LinkedList<>();
    expectedSplitIndexes.add(mock(IndexMetadata.class));
    long lastModifiedTime = 1L;
    IndexCacheKey indexCacheKey = new IndexCacheKey("/path/to/split", lastModifiedTime);
    when(indexclient.getLastModifiedTime((indexCacheKey.getPath()))).thenReturn(lastModifiedTime);
    when(indexclient.readSplitIndex((indexCacheKey.getPath()))).thenReturn(expectedSplitIndexes);
    List<IndexMetadata> actualSplitIndexes = indexCacheLoader.load(indexCacheKey);
    assertEquals(expectedSplitIndexes.size(), actualSplitIndexes.size());
}

Example 5 with IndexClient

use of io.prestosql.spi.heuristicindex.IndexClient in project hetu-core by openlookeng.

the class TestIndexCacheLoader method testIndexFound.

@Test
public void testIndexFound() throws Exception {
    IndexClient indexclient = mock(IndexClient.class);
    IndexCacheLoader indexCacheLoader = new IndexCacheLoader(indexclient);
    List<IndexMetadata> expectedSplitIndexes = new LinkedList<>();
    expectedSplitIndexes.add(mock(IndexMetadata.class));
    long lastModifiedTime = 1L;
    IndexCacheKey indexCacheKey = new IndexCacheKey("/path/to/split", lastModifiedTime);
    when(indexclient.getLastModifiedTime((indexCacheKey.getPath()))).thenReturn(lastModifiedTime);
    when(indexclient.readSplitIndex((indexCacheKey.getPath()))).thenReturn(expectedSplitIndexes);
    List<IndexMetadata> actualSplitIndexes = indexCacheLoader.load(indexCacheKey);
    assertEquals(expectedSplitIndexes.size(), actualSplitIndexes.size());
}