Examples with Archiver com.yelp.nrtsearch.server.backup.Archiver used on opensource projects

Example 1 with Archiver

use of com.yelp.nrtsearch.server.backup.Archiver in project nrtsearch by Yelp.

the class IndexState method commit.

/**
 * Commit all state and shards. If backupFromIncArchiver is passed in it will also attempt to use
 * IndexArchiver to upload files to remote storage
 */
public synchronized long commit(boolean backupFromIncArchiver) throws IOException {
    if (saveLoadState == null) {
        initSaveLoadState();
    }
    // nocommit this does nothing on replica?  make a failing test!
    long gen = -1;
    for (ShardState shard : shards.values()) {
        gen = shard.commit();
    }
    for (Lookup suggester : suggesters.values()) {
        if (suggester instanceof AnalyzingInfixSuggester) {
            ((AnalyzingInfixSuggester) suggester).commit();
        }
    }
    // nocommit needs test case that creates index, changes
    // some settings, closes it w/o ever starting it:
    // settings changes are lost then?
    JsonObject saveState = new JsonObject();
    saveState.add("state", getSaveState());
    saveLoadState.save(saveState);
    SnapshotId snapshotId = null;
    try {
        if (this.getShard(0).isPrimary() && globalState.getIncArchiver().isPresent() && backupFromIncArchiver) {
            CreateSnapshotRequest createSnapshotRequest = CreateSnapshotRequest.newBuilder().setIndexName(this.name).build();
            snapshotId = new CreateSnapshotHandler().createSnapshot(this, createSnapshotRequest).getSnapshotId();
            // upload data
            Collection<String> segmentFiles = getSegmentFilesInSnapshot(this, snapshotId);
            String resourceData = IndexBackupUtils.getResourceData(this.name);
            Archiver incArchiver = globalState.getIncArchiver().get();
            String versionHash = incArchiver.upload(globalState.getConfiguration().getServiceName(), resourceData, this.rootDir, segmentFiles, Collections.emptyList(), true);
            incArchiver.blessVersion(globalState.getConfiguration().getServiceName(), resourceData, versionHash);
            // upload metadata
            String resourceMetadata = IndexBackupUtils.getResourceMetadata(this.name);
            versionHash = incArchiver.upload(globalState.getConfiguration().getServiceName(), resourceMetadata, this.globalState.getStateDir(), Collections.emptyList(), Collections.emptyList(), true);
            incArchiver.blessVersion(globalState.getConfiguration().getServiceName(), resourceMetadata, versionHash);
        }
    } finally {
        if (snapshotId != null) {
            releaseSnapshot(this, this.name, snapshotId);
        }
    }
    return gen;
}

Example 2 with Archiver

use of com.yelp.nrtsearch.server.backup.Archiver in project nrtsearch by Yelp.

the class BackupHelper method call.

@Override
public Integer call() throws Exception {
    Archiver archiver = baseCmd.getArchiver();
    List<String> files = Files.list(IndexArchiver.getIndexDataDir(Path.of(this.getIndexDir()))).filter(Files::isRegularFile).map(Path::getFileName).map(Path::toString).collect(Collectors.toList());
    long t1 = System.nanoTime();
    String versionHash = archiver.upload(baseCmd.getServiceName(), IndexBackupUtils.getResourceData(baseCmd.getResourceName()), Path.of(this.getIndexDir()), files, Collections.emptyList(), true);
    long t2 = System.nanoTime();
    logger.info(String.format("Time taken to upload data %s milliseconds, versionHash uploaded: %s", (t2 - t1) / (1000 * 1000), versionHash));
    boolean result = archiver.blessVersion(baseCmd.getServiceName(), IndexBackupUtils.getResourceData(baseCmd.getResourceName()), versionHash);
    t1 = System.nanoTime();
    String versionHashMetadata = archiver.upload(baseCmd.getServiceName(), IndexBackupUtils.getResourceMetadata(baseCmd.getResourceName()), Path.of(this.getStateDir()), Collections.emptyList(), Collections.emptyList(), true);
    t2 = System.nanoTime();
    logger.info("Time taken to upload data %s milliseconds, versionHash uploaded: %s", (t2 - t1) / (1000 * 1000), versionHashMetadata);
    boolean resultMetadata = archiver.blessVersion(baseCmd.getServiceName(), IndexBackupUtils.getResourceMetadata(baseCmd.getResourceName()), versionHashMetadata);
    if (result && resultMetadata) {
        logger.info(String.format("Blessed, service: %s, resource: %s, version: %s", baseCmd.getServiceName(), IndexBackupUtils.getResourceData(baseCmd.getResourceName()), versionHash));
        logger.info(String.format("Blessed, service: %s, resource: %s, version: %s", baseCmd.getServiceName(), IndexBackupUtils.getResourceMetadata(baseCmd.getResourceName()), versionHashMetadata));
    } else {
        logger.info(String.format("Failed to bless: service: %s, resource: %s, version: %s", baseCmd.getServiceName(), IndexBackupUtils.getResourceData(baseCmd.getResourceName()), versionHash));
    }
    return 0;
}

Example 3 with Archiver

use of com.yelp.nrtsearch.server.backup.Archiver in project nrtsearch by Yelp.

the class BackendGlobalStateTest method testUseRemoteBackend.

@Test
public void testUseRemoteBackend() throws IOException {
    String configFile = String.join("\n", "stateConfig:", "  backendType: REMOTE", "stateDir: " + folder.newFolder("state").getAbsolutePath(), "indexDir: " + folder.newFolder("index").getAbsolutePath());
    LuceneServerConfiguration config = new LuceneServerConfiguration(new ByteArrayInputStream(configFile.getBytes()));
    Path tmpStateFolder = Paths.get(folder.getRoot().getAbsolutePath(), StateUtils.GLOBAL_STATE_FOLDER);
    StateUtils.ensureDirectory(tmpStateFolder);
    StateUtils.writeStateToFile(new PersistentGlobalState(), tmpStateFolder, StateUtils.GLOBAL_STATE_FILE);
    Archiver archiver = mock(Archiver.class);
    when(archiver.download(any(), any())).thenReturn(Paths.get(folder.getRoot().getAbsolutePath()));
    BackendGlobalState backendGlobalState = new BackendGlobalState(config, archiver);
    assertTrue(backendGlobalState.getStateBackend() instanceof RemoteStateBackend);
}

Example 4 with Archiver

use of com.yelp.nrtsearch.server.backup.Archiver in project nrtsearch by Yelp.

the class DeleteIndexBackupHandlerTest method testDeleteIndexBackupHandlerHandle.

@Test
public void testDeleteIndexBackupHandlerHandle() throws HandlerException, IOException {
    String indexName = "testindex";
    String serviceName = "testservice";
    String resourceName = "testresource";
    int nDays = 30;
    Archiver archiver = mock(Archiver.class);
    DeleteIndexBackupHandler handler = new DeleteIndexBackupHandler(archiver);
    DeleteIndexBackupRequest request = DeleteIndexBackupRequest.newBuilder().setIndexName(indexName).setServiceName(serviceName).setResourceName(resourceName).setNDays(nDays).build();
    IndexState testIndex = mock(IndexState.class);
    Instant now = Instant.now();
    Instant date1 = now.minus(20, ChronoUnit.DAYS);
    Instant date2 = now.minus(40, ChronoUnit.DAYS);
    String versionHash1 = "hash_1";
    String versionHash2 = "hash_2";
    String resourceData = IndexBackupUtils.getResourceData(resourceName);
    String resourceMetadata = IndexBackupUtils.getResourceMetadata(resourceName);
    String resourceVersionData = IndexBackupUtils.getResourceVersionData(resourceName);
    String resourceVersionMetadata = IndexBackupUtils.getResourceVersionMetadata(resourceName);
    VersionedResource resourceData1 = buildVersionedResource(serviceName, resourceData, date1, versionHash1);
    VersionedResource resourceMetadata1 = buildVersionedResource(serviceName, resourceMetadata, date1, versionHash1);
    VersionedResource resourceVersionData1 = buildVersionedResource(serviceName, resourceVersionData, date1, versionHash1);
    VersionedResource resourceVersionMetadata1 = buildVersionedResource(serviceName, resourceVersionMetadata, date1, versionHash1);
    VersionedResource resourceData2 = buildVersionedResource(serviceName, resourceData, date2, versionHash2);
    VersionedResource resourceMetadata2 = buildVersionedResource(serviceName, resourceMetadata, date2, versionHash2);
    VersionedResource resourceVersionData2 = buildVersionedResource(serviceName, resourceVersionData, date2, versionHash2);
    VersionedResource resourceVersionMetadata2 = buildVersionedResource(serviceName, resourceVersionMetadata, date2, versionHash2);
    // data from {resource}_data folder
    when(archiver.getVersionedResource(serviceName, resourceData)).thenReturn(Arrays.asList(resourceData1, resourceData2));
    // data from {resource}_metadata folder
    when(archiver.getVersionedResource(serviceName, resourceMetadata)).thenReturn(Arrays.asList(resourceMetadata1, resourceMetadata2));
    // data from _version/{resource}_data folder
    when(archiver.getVersionedResource(serviceName, resourceVersionData)).thenReturn(Arrays.asList(resourceVersionData1, resourceVersionData2));
    // data from _version/{resource}_metadata folder
    when(archiver.getVersionedResource(serviceName, resourceVersionMetadata)).thenReturn(Arrays.asList(resourceVersionMetadata1, resourceVersionMetadata2));
    DeleteIndexBackupResponse response = handler.handle(testIndex, request);
    // verify that a 20 days old backup doesn't get deleted
    verify(archiver, never()).deleteVersion(serviceName, resourceData, versionHash1);
    verify(archiver, never()).deleteVersion(serviceName, resourceMetadata, versionHash1);
    verify(archiver, never()).deleteVersion(serviceName, resourceVersionData, versionHash1);
    verify(archiver, never()).deleteVersion(serviceName, resourceVersionMetadata, versionHash1);
    // verify that a 40 days old backup gets deleted completely
    verify(archiver).deleteVersion(serviceName, resourceData, versionHash2);
    verify(archiver).deleteVersion(serviceName, resourceMetadata, versionHash2);
    verify(archiver).deleteVersion(serviceName, resourceVersionData, versionHash2);
    verify(archiver).deleteVersion(serviceName, resourceVersionMetadata, versionHash2);
    // verify the response format: data from all four locations should be deleted. The locations:
    // {resource}_data
    Assert.assertEquals(response.getDeletedResourceDataHashesList(), Arrays.asList(versionHash2));
    // {resource}_metadata
    Assert.assertEquals(response.getDeletedResourceMetadataHashesList(), Arrays.asList(versionHash2));
    // _version/{resource}_data
    Assert.assertEquals(response.getDeletedDataVersionsList(), Arrays.asList(versionHash2));
    // _version/{resource}_metadata
    Assert.assertEquals(response.getDeletedMetadataVersionsList(), Arrays.asList(versionHash2));
}