The status file will be created if it doesn't exist. * * @param appSearchDir The directory where the migration status file should be created. * @param migrationStats The migration stats to be written to the file. If null, the write will * be skipped. */ public static void writeMigrationStatus( @NonNull File appSearchDir, @Nullable DataMigrationStats migrationStats) { try { // TODO(b/407815165) right now we just create this file // to mark migration done successfully. // In the future we might want to create the file // earlier and put different statuses for // different stages during migration. File icingMigrationStatus = new File( appSearchDir, DATA_MIGRATION_STATUS_FILE); icingMigrationStatus.createNewFile(); // Also dump the migration stats if (migrationStats != null) { IndexerSettings.writeBundle(icingMigrationStatus, migrationStats.getBundle()); } } catch (IOException e) { Log.e(TAG, "Failed to write migration status file", e); } } /** * Wipes source Icing directory after a successful migration. * *
Currently everything will be wiped except for blob_dir. * @param icingDir */ public static void wipeSourceIcingDir(@NonNull File icingDir) { if (icingDir.exists() && icingDir.isDirectory()) { File[] files = icingDir.listFiles(); boolean all_deleted = true; if (files != null) { for (int i = 0; i < files.length; ++i) { File file = files[i]; // blob_dir will still be used no matter which backend we are // connecting. So we need to keep it. if (!file.getName().equals("blob_dir")) { if (!deleteDirectoryRecursively(file)) { Log.e(TAG, "Data migration: failed to delete " + file.getName()); all_deleted = false; } } else { all_deleted = false; } } } // icingDir is now empty. We can just delete it. if (all_deleted) { icingDir.delete(); } } } // Returns true if file/directory gets deleted. false otherwise. private static boolean deleteDirectoryRecursively(@NonNull File dir) { if (!dir.exists()) { return true; } if (dir.isDirectory()) { File[] files = dir.listFiles(); if (files != null) { for (int i = 0; i < files.length; ++i) { deleteDirectoryRecursively(files[i]); } } } return dir.delete(); } /** * Copies all the data from the source to the destination. * *
{@code source} won't be modified by this method. * *
This method calls AppSearchImpl's thread-safe APIs for source.
*/
private static StatusProto copyData(
@NonNull AppSearchImpl source,
@NonNull IcingSearchEngineInterface destination,
boolean resetDestination,
boolean forceOverride,
@NonNull DataMigrationStats migrationStats) {
// TODO(b/407815165): Either remove this limit or make it configurable in future.
int maxBytesPerPage =
IsolatedStorageServiceManager.DEFAULT_MAX_PAGE_BYTES_LIMIT_FOR_ISOLATED_STORAGE;
if (resetDestination) {
// Clear all current data from icing instance and reinitialize it.
ResetResultProto resetResult = destination.reset();
migrationStats.setResetStatus(resetResult.getStatus().getCode().getNumber());
if (resetResult.getStatus().getCode() != StatusProto.Code.OK) {
return resetResult.getStatus();
}
Log.i(TAG, "Destination reset during data migration successful");
}
// Step-1 Set schema in destination by getting schema from source.
SchemaProto rawSchema = source.rawGetSchema();
if (!source.useDatabaseScopedSchemaOperations()) {
rawSchema = getSchemaProtoWithDatabase(rawSchema);
}
SetSchemaResultProto setSchemaResult = destination.setSchema(rawSchema, forceOverride);
migrationStats.setSetSchemaStatus(setSchemaResult.getStatus().getCode().getNumber());
if (setSchemaResult.getStatus().getCode() != StatusProto.Code.OK) {
return setSchemaResult.getStatus();
}
Log.i(TAG, "Set Schema during data migration successful");
// Step-2 Query all documents from source using an empty query.
//
// TODO(b/407815165) Add exception handling for any exceptions while querying.
//
// TODO(b/407815165) Add logic to queue any incoming changes during copy.
SearchSpecProto searchSpec =
SearchSpecProto.newBuilder()
.setQuery("") // an empty query will return all docs.
.setTermMatchType(TermMatchType.Code.PREFIX)
.build();
SearchResultProto searchResult =
source.rawSearch(
searchSpec,
ScoringSpecProto.newBuilder().build(),
ResultSpecProto.newBuilder()
.setNumTotalBytesPerPageThreshold(maxBytesPerPage)
// median doc size is 700b.
// half of the maxBytesPerPage is 64_000b.
// so 64_000 / 700 ~= 95 results per page.
.setNumPerPage(95)
.setNumToScore(Integer.MAX_VALUE)
.build());
migrationStats.setQueryStatus(searchResult.getStatus().getCode().getNumber());
if (searchResult.getStatus().getCode() != StatusProto.Code.OK) {
return searchResult.getStatus();
}
Log.i(TAG, "Querying documents from source for data migration successful");
// Step-3 Put all documents in searchResult in destination using a batchPut call.
long nextPageToken = searchResult.getNextPageToken();
ArraySet {@code isolatedIcingInterface} must be instantiated with all VM options enabled.
*
* @param userContext User Context
* @param userHandle User to run migration for
* @param appSearchImpl {@link AppSearchImpl} instance for the source.
* @param isolatedIcingInterface {@link IcingSearchEngineInterface} instance for the VM.
* @param logger {@link AppSearchLogger} to log the migration stats if available.
*/
@NonNull
public static DataMigrationStats runDataMigrationForUser(
@NonNull Context userContext,
@NonNull UserHandle userHandle,
@NonNull AppSearchImpl appSearchImpl,
@NonNull IcingSearchEngineInterface isolatedIcingInterface,
@Nullable AppSearchLogger logger) {
long totalLatencyStartTimeMillis =
SystemClock.elapsedRealtime();
@AppSearchResult.ResultCode int statusCode =
RESULT_OK;
File appSearchDir =
AppSearchEnvironmentFactory.getEnvironmentInstance()
.getAppSearchDir(userContext, userHandle);
DataMigrationStats migrationStats =
new DataMigrationStats();
int prevNumberOfMigrationRun = 0;
try {
// Check data migration file status.
File dataMigrationStatusFile = new File(
appSearchDir,
DataMigrationUtil.DATA_MIGRATION_STATUS_FILE);
DataMigrationStats prevDataMigrationStats = new DataMigrationStats();
prevDataMigrationStats.setDataMigrationRunCounter(0);
if (dataMigrationStatusFile.exists()) {
try {
PersistableBundle bundle = IndexerSettings.readBundle(dataMigrationStatusFile);
if (!bundle.isEmpty()) {
prevDataMigrationStats.setBundle(bundle);
}
} catch (Exception e) {
Log.e(TAG, "Fail to read previous data migration stats.", e);
}
}
prevNumberOfMigrationRun = prevDataMigrationStats.getDataMigrationRunCounter();
migrationStats.setDataMigrationRunCounter(
prevNumberOfMigrationRun + 1);
DataMigrationUtil.runDataMigrationForUserImpl(
userContext,
userHandle,
appSearchImpl,
isolatedIcingInterface,
migrationStats);
statusCode =
ResultCodeToProtoConverter.toResultCode(
StatusProto.Code.forNumber(
migrationStats.getDataMigrationStatus()));
} catch (Exception e) {
Log.e(TAG, "Fail to migrate the data.", e);
statusCode = AppSearchResult
.throwableToFailedResult(e)
.getResultCode();
} finally {
// Always dump the migration stats if method gets called.
DataMigrationUtil.writeMigrationStatus(appSearchDir, migrationStats);
if (logger != null) {
int totalLatencyMillis =
(int) (SystemClock.elapsedRealtime()
- totalLatencyStartTimeMillis);
CallStats.Builder callStatsBuilder =
new CallStats.Builder()
.setStatusCode(statusCode)
.setCallReceivedTimestampMillis(
totalLatencyStartTimeMillis)
.setTotalLatencyMillis(
totalLatencyMillis)
// We re-purpose this to be the counter for previous runs.
.setEstimatedBinderLatencyMillis(prevNumberOfMigrationRun)
.setCallType(
CallStats
.INTERNAL_CALL_TYPE_ISOLATED_STORAGE_DATA_MIGRATION)
.setLaunchVMEnabled(appSearchImpl.isVMEnabled());
if (migrationStats != null) {
callStatsBuilder.setNumOperationsSucceeded(
(int) migrationStats.getNumberOfDocsSucceeded());
callStatsBuilder.setNumOperationsFailed(
(int) migrationStats.getNumberOfDocsFailed());
}
logger.logStats(
callStatsBuilder.build());
}
}
return migrationStats;
}
/**
* Runs data migration for the specified user to move data from {@code source} to {@code dest}.
*
* {@code dest} must be instantiated with all VM options enabled.
*
* @param context User Context
* @param userHandle User to run migration for
* @param source {@link AppSearchImpl} instance for the source.
* @param dest {@link IcingSearchEngineInterface} instance for the destination.
* @param migrationStats out object to hold migration stats.
*/
@NonNull
private static void runDataMigrationForUserImpl(
@NonNull Context context,
@NonNull UserHandle userHandle,
@NonNull AppSearchImpl source,
@NonNull IcingSearchEngineInterface dest,
@NonNull DataMigrationStats migrationStats) {
InitializeResultProto initResult = dest.initialize();
migrationStats.setVMInitStatus(initResult.getStatus().getCode().getNumber());
if (initResult.getStatus().getCode() != StatusProto.Code.OK) {
// TODO(b/430610163) we can dump the migration stats for failure cases.
Log.e(TAG,
"Failed to initialize Isolated Storage Icing!"
+ " Status code: "
+ initResult.getStatus().getCode().getNumber());
return;
}
// TODO(b/407815165) probably this should be moved to AppSearchImpl so lock can be
// grabbed there.
StatusProto status =
DataMigrationUtil.copyData(source, dest,
/* resetDestination= */ true,
/* forceOverride= */ true,
migrationStats);
migrationStats.setDataMigrationStatus(status.getCode().getNumber());
// TODO(b/407815165) Rework those logging and put useful information in the marker file, so
// dumpsys can read and print those information.
Log.i(TAG, "Data migration status: " + status);
if (status.getCode() != StatusProto.Code.OK) {
Log.e(TAG,
"Data migration: failed with status code: "
+ status.getCode().getNumber()
+ ", status message: "
+ status.getMessage());
return;
}
// Migration succeeded. Switch to the isolated instance.
IcingSearchEngineInterface priorIcingSearchEngine = source.swapIcingSearchEngineLocked(
dest, /*isVMEnabled=*/ true);
// Destroy the current instance.
priorIcingSearchEngine.close();
if (LogUtil.INFO) {
Log.i(TAG, "Data migration: wiping source directory.");
}
File appSearchDir =
AppSearchEnvironmentFactory.getEnvironmentInstance()
.getAppSearchDir(context, userHandle);
File icingDir = new File(appSearchDir, "icing");
DataMigrationUtil.wipeSourceIcingDir(icingDir);
}
}