/*
* Copyright 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.appsearch.external.localstorage;
import static android.app.appsearch.AppSearchResult.RESULT_SECURITY_ERROR;
import static android.app.appsearch.AppSearchResult.throwableToFailedResult;
import static android.app.appsearch.InternalSetSchemaResponse.newFailedSetSchemaResponse;
import static android.app.appsearch.InternalSetSchemaResponse.newSuccessfulSetSchemaResponse;
import static com.android.server.appsearch.external.localstorage.util.PrefixUtil.addPrefixToDocument;
import static com.android.server.appsearch.external.localstorage.util.PrefixUtil.createPrefix;
import static com.android.server.appsearch.external.localstorage.util.PrefixUtil.getDatabaseName;
import static com.android.server.appsearch.external.localstorage.util.PrefixUtil.getPackageName;
import static com.android.server.appsearch.external.localstorage.util.PrefixUtil.getPrefix;
import static com.android.server.appsearch.external.localstorage.util.PrefixUtil.removePrefix;
import static com.android.server.appsearch.external.localstorage.util.PrefixUtil.removePrefixesFromDocument;
import static com.android.server.appsearch.external.localstorage.util.PrefixUtil.removePrefixesFromSchemaType;
import android.annotation.WorkerThread;
import android.app.appsearch.AppSearchBatchResult;
import android.app.appsearch.AppSearchBlobHandle;
import android.app.appsearch.AppSearchResult;
import android.app.appsearch.AppSearchSchema;
import android.app.appsearch.GenericDocument;
import android.app.appsearch.GetByDocumentIdRequest;
import android.app.appsearch.GetSchemaResponse;
import android.app.appsearch.InternalSetSchemaResponse;
import android.app.appsearch.InternalVisibilityConfig;
import android.app.appsearch.JoinSpec;
import android.app.appsearch.PackageIdentifier;
import android.app.appsearch.SchemaVisibilityConfig;
import android.app.appsearch.SearchResult;
import android.app.appsearch.SearchResultPage;
import android.app.appsearch.SearchSpec;
import android.app.appsearch.SearchSuggestionResult;
import android.app.appsearch.SearchSuggestionSpec;
import android.app.appsearch.SetSchemaResponse;
import android.app.appsearch.StorageInfo;
import android.app.appsearch.exceptions.AppSearchException;
import android.app.appsearch.observer.DocumentChangeInfo;
import android.app.appsearch.observer.ObserverCallback;
import android.app.appsearch.observer.ObserverSpec;
import android.app.appsearch.stats.BaseStats;
import android.app.appsearch.util.LogUtil;
import android.os.ParcelFileDescriptor;
import android.os.SystemClock;
import android.util.ArrayMap;
import android.util.ArraySet;
import android.util.Log;
import android.util.Pair;
import com.android.appsearch.flags.Flags;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.annotations.VisibleForTesting;
import com.android.internal.util.Preconditions;
import com.android.server.appsearch.external.localstorage.converter.BlobHandleToProtoConverter;
import com.android.server.appsearch.external.localstorage.converter.GenericDocumentToProtoConverter;
import com.android.server.appsearch.external.localstorage.converter.ResultCodeToProtoConverter;
import com.android.server.appsearch.external.localstorage.converter.SchemaToProtoConverter;
import com.android.server.appsearch.external.localstorage.converter.SearchResultToProtoConverter;
import com.android.server.appsearch.external.localstorage.converter.SearchSpecToProtoConverter;
import com.android.server.appsearch.external.localstorage.converter.SearchSuggestionSpecToProtoConverter;
import com.android.server.appsearch.external.localstorage.converter.SetSchemaResponseToProtoConverter;
import com.android.server.appsearch.external.localstorage.converter.TypePropertyPathToProtoConverter;
import com.android.server.appsearch.external.localstorage.stats.CallStats;
import com.android.server.appsearch.external.localstorage.stats.InitializeStats;
import com.android.server.appsearch.external.localstorage.stats.OptimizeStats;
import com.android.server.appsearch.external.localstorage.stats.PersistToDiskStats;
import com.android.server.appsearch.external.localstorage.stats.PutDocumentStats;
import com.android.server.appsearch.external.localstorage.stats.QueryStats;
import com.android.server.appsearch.external.localstorage.stats.RemoveStats;
import com.android.server.appsearch.external.localstorage.stats.SetSchemaStats;
import com.android.server.appsearch.external.localstorage.visibilitystore.CallerAccess;
import com.android.server.appsearch.external.localstorage.visibilitystore.VisibilityChecker;
import com.android.server.appsearch.external.localstorage.visibilitystore.VisibilityStore;
import com.android.server.appsearch.external.localstorage.visibilitystore.VisibilityUtil;
import com.google.android.icing.IcingSearchEngine;
import com.google.android.icing.IcingSearchEngineInterface;
import com.google.android.icing.proto.BatchGetResultProto;
import com.google.android.icing.proto.BatchPutResultProto;
import com.google.android.icing.proto.BlobProto;
import com.google.android.icing.proto.DebugInfoProto;
import com.google.android.icing.proto.DebugInfoResultProto;
import com.google.android.icing.proto.DebugInfoVerbosity;
import com.google.android.icing.proto.DeleteByQueryResultProto;
import com.google.android.icing.proto.DeleteResultProto;
import com.google.android.icing.proto.DocumentProto;
import com.google.android.icing.proto.DocumentStorageInfoProto;
import com.google.android.icing.proto.GetAllNamespacesResultProto;
import com.google.android.icing.proto.GetNextPageRequestProto;
import com.google.android.icing.proto.GetOptimizeInfoResultProto;
import com.google.android.icing.proto.GetResultProto;
import com.google.android.icing.proto.GetResultSpecProto;
import com.google.android.icing.proto.GetSchemaResultProto;
import com.google.android.icing.proto.IcingSearchEngineOptions;
import com.google.android.icing.proto.InitializeResultProto;
import com.google.android.icing.proto.InitializeStatsProto;
import com.google.android.icing.proto.LogSeverity;
import com.google.android.icing.proto.NamespaceBlobStorageInfoProto;
import com.google.android.icing.proto.NamespaceStorageInfoProto;
import com.google.android.icing.proto.OptimizeResultProto;
import com.google.android.icing.proto.PersistToDiskResultProto;
import com.google.android.icing.proto.PersistType;
import com.google.android.icing.proto.PropertyConfigProto;
import com.google.android.icing.proto.PropertyProto;
import com.google.android.icing.proto.PutDocumentRequest;
import com.google.android.icing.proto.PutResultProto;
import com.google.android.icing.proto.QueryStatsProto;
import com.google.android.icing.proto.ReportUsageResultProto;
import com.google.android.icing.proto.ResetResultProto;
import com.google.android.icing.proto.ResultSpecProto;
import com.google.android.icing.proto.SchemaProto;
import com.google.android.icing.proto.SchemaTypeConfigProto;
import com.google.android.icing.proto.ScoringSpecProto;
import com.google.android.icing.proto.SearchResultProto;
import com.google.android.icing.proto.SearchSpecProto;
import com.google.android.icing.proto.SetSchemaRequestProto;
import com.google.android.icing.proto.SetSchemaResultProto;
import com.google.android.icing.proto.StatusProto;
import com.google.android.icing.proto.StorageInfoProto;
import com.google.android.icing.proto.StorageInfoResultProto;
import com.google.android.icing.proto.SuggestionResponse;
import com.google.android.icing.proto.TypePropertyMask;
import com.google.android.icing.proto.UsageReport;
import org.jspecify.annotations.NonNull;
import org.jspecify.annotations.Nullable;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Executor;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* Manages interaction with the native IcingSearchEngine and other components to implement AppSearch
* functionality.
*
* Never create two instances using the same folder.
*
*
A single instance of {@link AppSearchImpl} can support all packages and databases. This is
* done by combining the package and database name into a unique prefix and prefixing the schemas
* and documents stored under that owner. Schemas and documents are physically saved together in
* {@link IcingSearchEngine}, but logically isolated:
*
*
* - Rewrite SchemaType in SchemaProto by adding the package-database prefix and save into
* SchemaTypes set in {@link #setSchema}.
*
- Rewrite namespace and SchemaType in DocumentProto by adding package-database prefix and
* save to namespaces set in {@link #putDocument}.
*
- Remove package-database prefix when retrieving documents in {@link #getDocument} and {@link
* #query}.
*
- Rewrite filters in {@link SearchSpecProto} to have all namespaces and schema types of the
* queried database when user using empty filters in {@link #query}.
*
*
* Methods in this class belong to two groups, the query group and the mutate group.
*
*
* - All methods are going to modify global parameters and data in Icing are executed under
* WRITE lock to keep thread safety.
*
- All methods are going to access global parameters or query data from Icing are executed
* under READ lock to improve query performance.
*
*
* This class is thread safe.
*
* @hide
*/
@WorkerThread
public final class AppSearchImpl implements Closeable {
private static final String TAG = "AppSearchImpl";
@VisibleForTesting static final int CHECK_OPTIMIZE_INTERVAL = 100;
@VisibleForTesting static final int PRUNE_PACKAGE_USING_FULL_SET_SCHEMA_THRESHOLD = 20;
/** A GetResultSpec that uses projection to skip all properties. */
private static final GetResultSpecProto GET_RESULT_SPEC_NO_PROPERTIES =
GetResultSpecProto.newBuilder()
.addTypePropertyMasks(
TypePropertyMask.newBuilder()
.setSchemaType(
GetByDocumentIdRequest.PROJECTION_SCHEMA_TYPE_WILDCARD))
.build();
private final ReadWriteLock mReadWriteLock = new ReentrantReadWriteLock();
private final OptimizeStrategy mOptimizeStrategy;
private final AppSearchConfig mConfig;
private final File mBlobFilesDir;
@GuardedBy("mReadWriteLock")
@VisibleForTesting
IcingSearchEngineInterface mIcingSearchEngineLocked;
private boolean mIsVMEnabled;
private boolean mResetVisibilityStore;
private boolean mIsIcingSchemaDatabaseEnabled;
@GuardedBy("mReadWriteLock")
private final SchemaCache mSchemaCacheLocked = new SchemaCache();
@GuardedBy("mReadWriteLock")
private final NamespaceCache mNamespaceCacheLocked = new NamespaceCache();
// Marked as volatile because a new instance may be assigned during resetLocked.
@GuardedBy("mReadWriteLock")
private volatile DocumentLimiter mDocumentLimiterLocked;
// Maps packages to the set of valid nextPageTokens that the package can manipulate. A token
// is unique and constant per query (i.e. the same token '123' is used to iterate through
// pages of search results). The tokens themselves are generated and tracked by
// IcingSearchEngine. IcingSearchEngine considers a token valid and won't be reused
// until we call invalidateNextPageToken on the token.
//
// Note that we synchronize on itself because the nextPageToken cache is checked at
// query-time, and queries are done in parallel with a read lock. Ideally, this would be
// guarded by the normal mReadWriteLock.writeLock, but ReentrantReadWriteLocks can't upgrade
// read to write locks. This lock should be acquired at the smallest scope possible.
// mReadWriteLock is a higher-level lock, so calls shouldn't be made out
// to any functions that grab the lock.
@GuardedBy("mNextPageTokensLocked")
private final Map> mNextPageTokensLocked = new ArrayMap<>();
private final ObserverManager mObserverManager = new ObserverManager();
/**
* VisibilityStore will be used in {@link #setSchema} and {@link #getSchema} to store and query
* visibility information. But to create a {@link VisibilityStore}, it will call {@link
* #setSchema} and {@link #getSchema} to get the visibility schema. Make it nullable to avoid
* call it before we actually create it.
*/
@VisibleForTesting
@GuardedBy("mReadWriteLock")
final @Nullable VisibilityStore mDocumentVisibilityStoreLocked;
@VisibleForTesting
@GuardedBy("mReadWriteLock")
final @Nullable VisibilityStore mBlobVisibilityStoreLocked;
@GuardedBy("mReadWriteLock")
private final @Nullable VisibilityChecker mVisibilityCheckerLocked;
/**
* The counter to check when to call {@link #checkForOptimize}. The interval is {@link
* #CHECK_OPTIMIZE_INTERVAL}.
*/
@GuardedBy("mReadWriteLock")
private int mOptimizeIntervalCountLocked = 0;
/**
* The call type of the last mutation call holds the write lock. This value will be logged in
* the following call to analysis what the on-going call have to wait for.
*/
@BaseStats.CallType
@GuardedBy("mReadWriteLock")
private int mLastWriteOperationLocked;
/**
* The call type of the last read call that holds the read lock. This value will be logged in
* the following call to analysis what the on-going call have to wait for.
*/
@CallStats.CallType
@GuardedBy("mReadWriteLock")
private int mLastReadOrWriteOperationLocked;
/**
* Time in millis that the last mutation call holds the write lock. This value will be logged in
* the following call to analysis what the on-going call have to wait for.
*/
@GuardedBy("mReadWriteLock")
private int mLastWriteOperationLatencyMillisLocked;
/**
* Time in millis that the last read call blocks both read and write operation in AppSearch.
* This value will be logged in the following call to analysis what the on-going call have to
* wait for.
*/
@GuardedBy("mReadWriteLock")
private int mLastReadOrWriteOperationLatencyMillisLocked;
private final @Nullable RevocableFileDescriptorStore mRevocableFileDescriptorStore;
/** Whether this instance has been closed, and therefore unusable. */
@GuardedBy("mReadWriteLock")
private boolean mClosedLocked = false;
/**
* Whether AppSearchImpl has mutated the database and needs to call {@link #persistToDisk}.
*
* Since there are some internal logic (e.g. visibility document changes) here which
* potentially mutates the database, {@link AppSearchImpl} has to provide a public method for
* the upper level caller to check whether flush ({@link #persistToDisk} is needed or not.
*/
// TODO(b/417463182): this is a temporary solution. In the future we might want to consider
// some other better implementations, e.g. listener or return value.
private final AtomicReference mNeedsPersistToDisk = new AtomicReference<>();
/**
* Creates and initializes an instance of {@link AppSearchImpl} which writes data to the given
* folder.
*
* Clients can pass a {@link AppSearchLogger} here through their AppSearchSession, but it
* can't be saved inside {@link AppSearchImpl}, because the impl will be shared by all the
* sessions for the same package in JetPack.
*
*
Instead, logger instance needs to be passed to each individual method, like create, query
* and putDocument.
*
* @param initStatsBuilder collects stats for initialization if provided.
* @param visibilityChecker The {@link VisibilityChecker} that check whether the caller has
* access to aa specific schema. Pass null will lost that ability and global querier could
* only get their own data.
* @param icingSearchEngine the underlying icing instance to use. If not provided, a new {@link
* IcingSearchEngine} instance will be created and used.
*/
public static @NonNull AppSearchImpl create(
@NonNull File icingDir,
@NonNull AppSearchConfig config,
InitializeStats.@Nullable Builder initStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder,
@Nullable VisibilityChecker visibilityChecker,
@Nullable RevocableFileDescriptorStore revocableFileDescriptorStore,
@Nullable IcingSearchEngineInterface icingSearchEngine,
@NonNull OptimizeStrategy optimizeStrategy)
throws AppSearchException {
return new AppSearchImpl(
icingDir,
config,
initStatsBuilder,
callStatsBuilder,
visibilityChecker,
revocableFileDescriptorStore,
icingSearchEngine,
optimizeStrategy);
}
/**
* @param initStatsBuilder collects stats for initialization if provided.
*/
private AppSearchImpl(
@NonNull File icingDir,
@NonNull AppSearchConfig config,
InitializeStats.@Nullable Builder initStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder,
@Nullable VisibilityChecker visibilityChecker,
@Nullable RevocableFileDescriptorStore revocableFileDescriptorStore,
@Nullable IcingSearchEngineInterface icingSearchEngine,
@NonNull OptimizeStrategy optimizeStrategy)
throws AppSearchException {
Objects.requireNonNull(icingDir);
// This directory stores blob files. It is the same directory that Icing used to manage
// blob files when Flags.enableAppSearchManageBlobFiles() was false. After the rollout of
// this flag, AppSearch will continue to manage blob files in this same directory within
// Icing's directory. The location remains unchanged to ensure that the flag does not
// introduce any behavioral changes.
mBlobFilesDir = new File(icingDir, "blob_dir/blob_files");
mConfig = Objects.requireNonNull(config);
mOptimizeStrategy = Objects.requireNonNull(optimizeStrategy);
mVisibilityCheckerLocked = visibilityChecker;
mRevocableFileDescriptorStore = revocableFileDescriptorStore;
mNeedsPersistToDisk.set(false);
// By default, we don't perform any retries.
int maxInitRetries = 0;
Map visibilityStoreMap = new ArrayMap<>();
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.writeLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
// We synchronize here because we don't want to call IcingSearchEngine.initialize() more
// than once. It's unnecessary and can be a costly operation.
if (icingSearchEngine == null) {
mIsVMEnabled = false;
if (Flags.enableInitializationRetriesBeforeReset()) {
maxInitRetries = 2;
}
IcingSearchEngineOptions options =
mConfig.toIcingSearchEngineOptions(
icingDir.getAbsolutePath(), mIsVMEnabled);
LogUtil.piiTrace(TAG, "Constructing IcingSearchEngine, request", options);
mIcingSearchEngineLocked = new IcingSearchEngine(options);
mIsIcingSchemaDatabaseEnabled = options.getEnableSchemaDatabase();
LogUtil.piiTrace(
TAG,
"Constructing IcingSearchEngine, response",
Objects.hashCode(mIcingSearchEngineLocked));
} else {
mIcingSearchEngineLocked = icingSearchEngine;
mIsVMEnabled = true;
mIsIcingSchemaDatabaseEnabled = true;
maxInitRetries = 2;
}
mResetVisibilityStore = Flags.enableResetVisibilityStore() || mIsVMEnabled;
// The core initialization procedure. If any part of this fails, we bail into
// resetLocked(), deleting all data (but hopefully allowing AppSearchImpl to come up).
try {
LogUtil.piiTrace(TAG, "icingSearchEngine.initialize, request");
InitializeResultProto initializeResultProto = mIcingSearchEngineLocked.initialize();
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(
initializeResultProto.getGetVmLatencyMs());
}
while (maxInitRetries > 0 && !isSuccess(initializeResultProto.getStatus())) {
Log.e(
TAG,
String.format(
"INIT: Initialize failed with status (%d:%s). %d retries left!",
initializeResultProto.getStatus().getCode().getNumber(),
initializeResultProto.getStatus().getMessage(),
maxInitRetries));
--maxInitRetries;
initializeResultProto = mIcingSearchEngineLocked.initialize();
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(
initializeResultProto.getGetVmLatencyMs());
}
}
LogUtil.piiTrace(
TAG,
"icingSearchEngine.initialize, response",
initializeResultProto.getStatus(),
initializeResultProto);
if (initStatsBuilder != null) {
initStatsBuilder
.setJavaLockAcquisitionLatencyMillis(
(int)
(javaLockAcquisitionEndTimeMillis
- totalLatencyStartMillis))
.setStatusCode(
statusProtoToResultCode(initializeResultProto.getStatus()))
// TODO(b/173532925) how to get DeSyncs value
.setHasDeSync(false)
.setLaunchVMEnabled(mIsVMEnabled)
.addGetVmLatencyMillis(initializeResultProto.getGetVmLatencyMs());
AppSearchLoggerHelper.copyNativeStats(
initializeResultProto.getInitializeStats(), initStatsBuilder);
if (isVMEnabled()) {
// TODO(b/415387509): Add an actual atom field to capture this value.
// Hack to propagate the failure cause early
// Store value in IcuDataStatus because that field doesn't matter in
// platform. Add 100 to separate from the range of possible values that
// would otherwise be set in this field.
initStatsBuilder.setNativeInitializeIcuDataStatusCode(
100
+ initializeResultProto
.getInitializeStats()
.getFailureStage()
.getNumber());
}
}
checkSuccess(initializeResultProto.getStatus());
if (hasDatabaseStateChangedAfterInit(initializeResultProto)) {
mNeedsPersistToDisk.set(true);
}
if (Flags.enableAppSearchManageBlobFiles()
&& !mBlobFilesDir.exists()
&& !mBlobFilesDir.mkdirs()) {
throw new AppSearchException(
AppSearchResult.RESULT_IO_ERROR,
"Cannot create the blob file directory: "
+ mBlobFilesDir.getAbsolutePath());
}
// Read all protos we need to construct AppSearchImpl's cache maps
long prepareSchemaAndNamespacesLatencyStartMillis = SystemClock.elapsedRealtime();
LogUtil.piiTrace(TAG, "getSchema, request");
GetSchemaResultProto schemaResultProto = mIcingSearchEngineLocked.getSchema();
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(schemaResultProto.getGetVmLatencyMs());
}
// GetSchema may return NOT_FOUND if we've initialized an empty instance.
while (maxInitRetries > 0
&& !isCodeOneOf(
schemaResultProto.getStatus(),
StatusProto.Code.OK,
StatusProto.Code.NOT_FOUND)) {
Log.e(
TAG,
String.format(
"INIT: GetSchema failed with status (%d:%s). %d retries left!",
schemaResultProto.getStatus().getCode().getNumber(),
schemaResultProto.getStatus().getMessage(),
maxInitRetries));
--maxInitRetries;
schemaResultProto = mIcingSearchEngineLocked.getSchema();
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(
schemaResultProto.getGetVmLatencyMs());
}
}
LogUtil.piiTrace(
TAG,
"getSchema, response",
schemaResultProto.getStatus(),
schemaResultProto);
checkCodeOneOf(
schemaResultProto.getStatus(),
StatusProto.Code.OK,
StatusProto.Code.NOT_FOUND);
SchemaProto schemaProto = schemaResultProto.getSchema();
LogUtil.piiTrace(TAG, "getStorageInfo, request");
StorageInfoResultProto storageInfoResult =
mIcingSearchEngineLocked.getStorageInfo();
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(storageInfoResult.getGetVmLatencyMs());
}
while (maxInitRetries > 0 && !isSuccess(storageInfoResult.getStatus())) {
Log.e(
TAG,
String.format(
"INIT: GetStorageInfo failed with status (%d:%s). %d retries"
+ " left!",
storageInfoResult.getStatus().getCode().getNumber(),
storageInfoResult.getStatus().getMessage(),
maxInitRetries));
--maxInitRetries;
storageInfoResult = mIcingSearchEngineLocked.getStorageInfo();
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(
storageInfoResult.getGetVmLatencyMs());
}
}
LogUtil.piiTrace(
TAG,
"getStorageInfo, response",
storageInfoResult.getStatus(),
storageInfoResult);
checkSuccess(storageInfoResult.getStatus());
StorageInfoProto storageInfoProto = storageInfoResult.getStorageInfo();
// Log the time it took to read the data that goes into the cache maps
if (initStatsBuilder != null) {
// In case there is some error for getAllNamespaces, we can still
// set the latency for preparation.
// If there is no error, the value will be overridden by the actual one later.
initStatsBuilder
.setStatusCode(AppSearchResult.RESULT_OK)
.setPrepareSchemaAndNamespacesLatencyMillis(
(int)
(SystemClock.elapsedRealtime()
- prepareSchemaAndNamespacesLatencyStartMillis));
}
// Populate schema map
List schemaProtoTypesList = schemaProto.getTypesList();
for (int i = 0; i < schemaProtoTypesList.size(); i++) {
SchemaTypeConfigProto schema = schemaProtoTypesList.get(i);
String prefixedSchemaType = schema.getSchemaType();
mSchemaCacheLocked.addToSchemaMap(getPrefix(prefixedSchemaType), schema);
}
// Populate schema parent-to-children map
mSchemaCacheLocked.rebuildCache();
// Populate namespace map
List namespaceInfos =
storageInfoProto.getDocumentStorageInfo().getNamespaceStorageInfoList();
for (int i = 0; i < namespaceInfos.size(); i++) {
String prefixedNamespace = namespaceInfos.get(i).getNamespace();
mNamespaceCacheLocked.addToDocumentNamespaceMap(
getPrefix(prefixedNamespace), prefixedNamespace);
}
// Populate blob namespace map
if (mRevocableFileDescriptorStore != null) {
List namespaceBlobStorageInfoProto =
storageInfoProto.getNamespaceBlobStorageInfoList();
for (int i = 0; i < namespaceBlobStorageInfoProto.size(); i++) {
String prefixedNamespace =
namespaceBlobStorageInfoProto.get(i).getNamespace();
mNamespaceCacheLocked.addToBlobNamespaceMap(
getPrefix(prefixedNamespace), prefixedNamespace);
}
}
// Populate document count map
mDocumentLimiterLocked =
new DocumentLimiter(
mConfig.getDocumentCountLimitStartThreshold(),
mConfig.getPerPackageDocumentCountLimit(),
storageInfoProto
.getDocumentStorageInfo()
.getNamespaceStorageInfoList());
// logging prepare_schema_and_namespaces latency
if (initStatsBuilder != null) {
initStatsBuilder.setPrepareSchemaAndNamespacesLatencyMillis(
(int)
(SystemClock.elapsedRealtime()
- prepareSchemaAndNamespacesLatencyStartMillis));
}
if (mResetVisibilityStore) {
// Move initialize Visibility Store in the try-catch reset block. We will
// trigger reset if we cannot create Visibility Store properly.
visibilityStoreMap =
initializeVisibilityStore(
mRevocableFileDescriptorStore,
initStatsBuilder,
callStatsBuilder);
}
LogUtil.piiTrace(TAG, "Init completed successfully");
} catch (AppSearchException e) {
// Some error. Reset and see if it fixes it.
LogUtil.criticalError(
TAG, "Error initializing, attempting to reset IcingSearchEngine.", e);
if (initStatsBuilder != null) {
initStatsBuilder.setStatusCode(e.getResultCode());
}
resetLocked(initStatsBuilder, callStatsBuilder);
if (mResetVisibilityStore) {
// After reset Icing, we should build and initialize VisibilityStore as well.
visibilityStoreMap =
initializeVisibilityStore(
mRevocableFileDescriptorStore,
initStatsBuilder,
callStatsBuilder);
}
}
if (!mResetVisibilityStore) {
// Keep the old behaviour when flags are off.
visibilityStoreMap =
initializeVisibilityStore(
mRevocableFileDescriptorStore, initStatsBuilder, callStatsBuilder);
}
mDocumentVisibilityStoreLocked =
visibilityStoreMap.get(VisibilityStore.DOCUMENT_VISIBILITY_DATABASE_NAME);
mBlobVisibilityStoreLocked =
visibilityStoreMap.get(VisibilityStore.BLOB_VISIBILITY_DATABASE_NAME);
} finally {
mLastWriteOperationLocked = BaseStats.CALL_TYPE_INITIALIZE;
mLastWriteOperationLatencyMillisLocked =
(int) (SystemClock.elapsedRealtime() - javaLockAcquisitionEndTimeMillis);
mLastReadOrWriteOperationLocked = mLastWriteOperationLocked;
mLastReadOrWriteOperationLatencyMillisLocked = mLastWriteOperationLatencyMillisLocked;
mReadWriteLock.writeLock().unlock();
}
}
private Map initializeVisibilityStore(
@Nullable RevocableFileDescriptorStore revocableFileDescriptorStore,
InitializeStats.@Nullable Builder initStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long prepareVisibilityStoreLatencyStartMillis = SystemClock.elapsedRealtime();
Map visibilityStoreMap = new ArrayMap<>();
visibilityStoreMap.put(
VisibilityStore.DOCUMENT_VISIBILITY_DATABASE_NAME,
VisibilityStore.createDocumentVisibilityStore(this, callStatsBuilder));
VisibilityStore blobVisibilityStore = null;
if (revocableFileDescriptorStore != null) {
blobVisibilityStore = VisibilityStore.createBlobVisibilityStore(this, callStatsBuilder);
}
visibilityStoreMap.put(VisibilityStore.BLOB_VISIBILITY_DATABASE_NAME, blobVisibilityStore);
long prepareVisibilityStoreLatencyEndMillis = SystemClock.elapsedRealtime();
if (initStatsBuilder != null) {
initStatsBuilder.setPrepareVisibilityStoreLatencyMillis(
(int)
(prepareVisibilityStoreLatencyEndMillis
- prepareVisibilityStoreLatencyStartMillis));
}
return visibilityStoreMap;
}
@GuardedBy("mReadWriteLock")
private void throwIfClosedLocked() {
if (mClosedLocked) {
throw new IllegalStateException("Trying to use a closed AppSearchImpl instance.");
}
}
/**
* Persists data to disk and closes the instance.
*
* This instance is no longer usable after it's been closed. Call {@link #create} to create a
* new, usable instance.
*/
@Override
public void close() {
mReadWriteLock.writeLock().lock();
try {
if (mClosedLocked) {
return;
}
persistToDisk(
/* callingPackageName= */ null,
BaseStats.INTERNAL_CALL_TYPE_CLOSE,
PersistType.Code.FULL,
/* logger= */ null,
/* callStatsBuilder= */ null);
LogUtil.piiTrace(TAG, "icingSearchEngine.close, request");
mIcingSearchEngineLocked.close();
LogUtil.piiTrace(TAG, "icingSearchEngine.close, response");
if (mRevocableFileDescriptorStore != null) {
mRevocableFileDescriptorStore.revokeAll();
}
mClosedLocked = true;
} catch (AppSearchException | IOException e) {
Log.w(TAG, "Error when closing AppSearchImpl.", e);
} finally {
mReadWriteLock.writeLock().unlock();
}
}
/** Returns the instance of AppSearchConfig used by this instance of AppSearchImpl. */
public @NonNull AppSearchConfig getConfig() {
return mConfig;
}
/** Returns whether pVM is enabled in this AppSearchImpl instance. */
public boolean isVMEnabled() {
return mIsVMEnabled;
}
/** Returns whether this AppSearchImpl instance should use database-scoped set and get schema */
public boolean useDatabaseScopedSchemaOperations() {
return mIsIcingSchemaDatabaseEnabled;
}
public boolean enableEarlySetSchemaExit() {
return Flags.enableEarlySetSchemaExit() || isVMEnabled();
}
/** Atomic method to set a new icing search engine and return the previous engine. */
@GuardedBy("mReadWriteLock")
public @NonNull IcingSearchEngineInterface swapIcingSearchEngineLocked(
@NonNull IcingSearchEngineInterface icingSearchEngineLocked, boolean isVmEnabled) {
Objects.requireNonNull(icingSearchEngineLocked);
mReadWriteLock.writeLock().lock();
try {
IcingSearchEngineInterface previousIcingSearchEngine = mIcingSearchEngineLocked;
mIcingSearchEngineLocked = icingSearchEngineLocked;
mIsVMEnabled = isVmEnabled;
mIsIcingSchemaDatabaseEnabled =
Flags.enableDatabaseScopedSchemaOperations() || isVmEnabled;
mResetVisibilityStore = Flags.enableResetVisibilityStore() || isVmEnabled;
return previousIcingSearchEngine;
} finally {
mReadWriteLock.writeLock().unlock();
}
}
/**
* Gets whether AppSearch data needs to be flushed, and resets the flag to false atomically.
*
*
The caller should call this method after any write APIs, and invoke {@link #persistToDisk}
* or schedule a background job for it if this method returns true.
*/
public boolean getAndResetNeedPersistToDisk() {
return mNeedsPersistToDisk.getAndSet(false);
}
/**
* Clears all data from the current icing instance and close this AppSearchImpl instance. The
* caller must not use this AppSearchImpl instance anymore.
*
*
The following actions are taken:
*
*
* - Reset all Icing internal members.
*
- Delete Icing directory.
*
- Close AppSearchImpl.
*
*/
public void clearAndDestroy() {
mReadWriteLock.writeLock().lock();
try {
throwIfClosedLocked();
if (mRevocableFileDescriptorStore != null) {
mRevocableFileDescriptorStore.revokeAll();
}
mIcingSearchEngineLocked.clearAndDestroy();
mNeedsPersistToDisk.set(false);
mClosedLocked = true;
} catch (IOException e) {
Log.w(TAG, "Error when clearAndDestroy AppSearchImpl.", e);
} finally {
mReadWriteLock.writeLock().unlock();
}
}
/**
* Updates the AppSearch schema for this app.
*
* This method belongs to mutate group.
*
* @param packageName The package name that owns the schemas.
* @param databaseName The name of the database where this schema lives.
* @param schemas Schemas to set for this app.
* @param visibilityConfigs {@link InternalVisibilityConfig}s that contain all visibility
* setting information for those schemas has user custom settings. Other schemas in the list
* that don't has a {@link InternalVisibilityConfig} will be treated as having the default
* visibility, which is accessible by the system and no other packages.
* @param forceOverride Whether to force-apply the schema even if it is incompatible. Documents
* which do not comply with the new schema will be deleted.
* @param version The overall version number of the request.
* @param setSchemaStatsBuilder Builder for {@link SetSchemaStats} to hold stats for setSchema
* @return A success {@link InternalSetSchemaResponse} with a {@link SetSchemaResponse}. Or a
* failed {@link InternalSetSchemaResponse} if this call contains incompatible change. The
* {@link SetSchemaResponse} in the failed {@link InternalSetSchemaResponse} contains which
* type is incompatible. You need to check the status by {@link
* InternalSetSchemaResponse#isSuccess()}.
* @throws AppSearchException On IcingSearchEngine error. If the status code is
* FAILED_PRECONDITION for the incompatible change, the exception will be converted to the
* SetSchemaResponse.
*/
public @NonNull InternalSetSchemaResponse setSchema(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull List schemas,
@NonNull List visibilityConfigs,
boolean forceOverride,
int version,
SetSchemaStats.@Nullable Builder setSchemaStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.writeLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked);
}
if (setSchemaStatsBuilder != null) {
setSchemaStatsBuilder
.setJavaLockAcquisitionLatencyMillis(
(int) (javaLockAcquisitionEndTimeMillis - totalLatencyStartMillis))
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked)
.setLaunchVMEnabled(mIsVMEnabled);
}
if (mObserverManager.isPackageObserved(packageName)) {
if (useDatabaseScopedSchemaOperations()) {
return doSetSchemaWithChangeNotificationNoGetSchemaLocked(
packageName,
databaseName,
schemas,
visibilityConfigs,
forceOverride,
version,
setSchemaStatsBuilder,
callStatsBuilder);
} else {
return doSetSchemaWithChangeNotificationLocked(
packageName,
databaseName,
schemas,
visibilityConfigs,
forceOverride,
version,
setSchemaStatsBuilder,
callStatsBuilder);
}
} else {
return doSetSchemaNoChangeNotificationLocked(
packageName,
databaseName,
schemas,
visibilityConfigs,
forceOverride,
version,
setSchemaStatsBuilder,
callStatsBuilder)
.first;
}
} finally {
logWriteOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_SET_SCHEMA,
callStatsBuilder);
mReadWriteLock.writeLock().unlock();
}
}
/**
* Updates the AppSearch schema for this app, dispatching change notifications. This method
* calls the getSchema API in the process.
*
* @see #setSchema
* @see #doSetSchemaNoChangeNotificationLocked
*/
@GuardedBy("mReadWriteLock")
private @NonNull InternalSetSchemaResponse doSetSchemaWithChangeNotificationLocked(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull List schemas,
@NonNull List visibilityConfigs,
boolean forceOverride,
int version,
SetSchemaStats.@Nullable Builder setSchemaStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
// First, capture the old state of the system. This includes the old schema as well as
// whether each registered observer can access each type. Once VisibilityStore is updated
// by the setSchema call, the information of which observers could see which types will be
// lost.
long getOldSchemaStartTimeMillis = SystemClock.elapsedRealtime();
GetSchemaResponse oldSchema =
getSchema(
packageName,
databaseName,
// A CallerAccess object for internal use that has local access to this
// database.
new CallerAccess(/* callingPackageName= */ packageName),
callStatsBuilder);
long getOldSchemaEndTimeMillis = SystemClock.elapsedRealtime();
if (setSchemaStatsBuilder != null) {
setSchemaStatsBuilder
.setIsPackageObserved(true)
.setGetOldSchemaLatencyMillis(
(int) (getOldSchemaEndTimeMillis - getOldSchemaStartTimeMillis));
}
long getOldSchemaObserverStartTimeMillis = SystemClock.elapsedRealtime();
// Cache some lookup tables to help us work with the old schema
Set oldSchemaTypes = oldSchema.getSchemas();
Map oldSchemaNameToType = new ArrayMap<>(oldSchemaTypes.size());
// Maps unprefixed schema name to the set of listening packages that had visibility into
// that type under the old schema.
Map> oldSchemaNameToVisibleListeningPackage =
new ArrayMap<>(oldSchemaTypes.size());
for (AppSearchSchema oldSchemaType : oldSchemaTypes) {
String oldSchemaName = oldSchemaType.getSchemaType();
oldSchemaNameToType.put(oldSchemaName, oldSchemaType);
oldSchemaNameToVisibleListeningPackage.put(
oldSchemaName,
mObserverManager.getObserversForSchemaType(
packageName,
databaseName,
oldSchemaName,
mDocumentVisibilityStoreLocked,
mVisibilityCheckerLocked));
}
int getOldSchemaObserverLatencyMillis =
(int) (SystemClock.elapsedRealtime() - getOldSchemaObserverStartTimeMillis);
// Apply the new schema
InternalSetSchemaResponse internalSetSchemaResponse =
doSetSchemaNoChangeNotificationLocked(
packageName,
databaseName,
schemas,
visibilityConfigs,
forceOverride,
version,
setSchemaStatsBuilder,
callStatsBuilder)
.first;
// This check is needed wherever setSchema is called to detect soft errors which do not
// throw an exception but also prevent the schema from actually being applied.
if (!internalSetSchemaResponse.isSuccess()) {
return internalSetSchemaResponse;
}
long getNewSchemaObserverStartTimeMillis = SystemClock.elapsedRealtime();
// Cache some lookup tables to help us work with the new schema
Map newSchemaNameToType = new ArrayMap<>(schemas.size());
// Maps unprefixed schema name to the set of listening packages that have visibility into
// that type under the new schema.
Map> newSchemaNameToVisibleListeningPackage =
new ArrayMap<>(schemas.size());
for (AppSearchSchema newSchemaType : schemas) {
String newSchemaName = newSchemaType.getSchemaType();
newSchemaNameToType.put(newSchemaName, newSchemaType);
newSchemaNameToVisibleListeningPackage.put(
newSchemaName,
mObserverManager.getObserversForSchemaType(
packageName,
databaseName,
newSchemaName,
mDocumentVisibilityStoreLocked,
mVisibilityCheckerLocked));
}
long getNewSchemaObserverEndTimeMillis = SystemClock.elapsedRealtime();
if (setSchemaStatsBuilder != null) {
setSchemaStatsBuilder.setGetObserverLatencyMillis(
getOldSchemaObserverLatencyMillis
+ (int)
(getNewSchemaObserverEndTimeMillis
- getNewSchemaObserverStartTimeMillis));
}
long preparingChangeNotificationStartTimeMillis = SystemClock.elapsedRealtime();
// Create a unified set of all schema names mentioned in either the old or new schema.
Set allSchemaNames = new ArraySet<>(oldSchemaNameToType.keySet());
allSchemaNames.addAll(newSchemaNameToType.keySet());
// Perform the diff between the old and new schema.
for (String schemaName : allSchemaNames) {
final AppSearchSchema contentBefore = oldSchemaNameToType.get(schemaName);
final AppSearchSchema contentAfter = newSchemaNameToType.get(schemaName);
final boolean existBefore = (contentBefore != null);
final boolean existAfter = (contentAfter != null);
// This should never happen
if (!existBefore && !existAfter) {
continue;
}
boolean contentsChanged = true;
if (contentBefore != null && contentBefore.equals(contentAfter)) {
contentsChanged = false;
}
Set oldVisibleListeners =
oldSchemaNameToVisibleListeningPackage.get(schemaName);
Set newVisibleListeners =
newSchemaNameToVisibleListeningPackage.get(schemaName);
Set allListeningPackages = new ArraySet<>(oldVisibleListeners);
if (newVisibleListeners != null) {
allListeningPackages.addAll(newVisibleListeners);
}
// Now that we've computed the relationship between the old and new schema, we go
// observer by observer and consider the observer's own personal view of the schema.
for (String listeningPackageName : allListeningPackages) {
// Figure out the visibility
final boolean visibleBefore =
(existBefore
&& oldVisibleListeners != null
&& oldVisibleListeners.contains(listeningPackageName));
final boolean visibleAfter =
(existAfter
&& newVisibleListeners != null
&& newVisibleListeners.contains(listeningPackageName));
// Now go through the truth table of all the relevant flags.
// visibleBefore and visibleAfter take into account existBefore and existAfter, so
// we can stop worrying about existBefore and existAfter.
boolean sendNotification = false;
if (visibleBefore && visibleAfter && contentsChanged) {
sendNotification = true; // Type configuration was modified
} else if (!visibleBefore && visibleAfter) {
sendNotification = true; // Newly granted visibility or type was created
} else if (visibleBefore && !visibleAfter) {
sendNotification = true; // Revoked visibility or type was deleted
} else {
// No visibility before and no visibility after. Nothing to dispatch.
}
if (sendNotification) {
mObserverManager.onSchemaChange(
/* listeningPackageName= */ listeningPackageName,
/* targetPackageName= */ packageName,
/* databaseName= */ databaseName,
/* schemaName= */ schemaName);
}
}
}
if (setSchemaStatsBuilder != null) {
setSchemaStatsBuilder.setPreparingChangeNotificationLatencyMillis(
(int)
(SystemClock.elapsedRealtime()
- preparingChangeNotificationStartTimeMillis));
}
return internalSetSchemaResponse;
}
/**
* Updates the AppSearch schema for this app and dispatches change notifications, without
* calling the getSchema API.
*
* @see #setSchema
* @see #doSetSchemaNoChangeNotificationLocked
*/
@GuardedBy("mReadWriteLock")
private @NonNull InternalSetSchemaResponse doSetSchemaWithChangeNotificationNoGetSchemaLocked(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull List schemas,
@NonNull List visibilityConfigs,
boolean forceOverride,
int version,
SetSchemaStats.@Nullable Builder setSchemaStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
// Get the old schema map and cache the listening packages for the database
long getOldSchemaObserverStartTimeMillis = SystemClock.elapsedRealtime();
Set oldPrefixedTypesForPrefix =
mSchemaCacheLocked
.getSchemaMapForPrefix(createPrefix(packageName, databaseName))
.keySet();
Map> oldSchemaNameToVisibleListeningPackage =
new ArrayMap<>(oldPrefixedTypesForPrefix.size());
for (String prefixedTypeName : oldPrefixedTypesForPrefix) {
String unprefixedTypeName = removePrefix(prefixedTypeName);
oldSchemaNameToVisibleListeningPackage.put(
unprefixedTypeName,
mObserverManager.getObserversForSchemaType(
packageName,
databaseName,
unprefixedTypeName,
mDocumentVisibilityStoreLocked,
mVisibilityCheckerLocked));
}
int getOldSchemaObserverLatencyMillis =
(int) (SystemClock.elapsedRealtime() - getOldSchemaObserverStartTimeMillis);
// Apply the new schema
Pair setSchemaResponsePair =
doSetSchemaNoChangeNotificationLocked(
packageName,
databaseName,
schemas,
visibilityConfigs,
forceOverride,
version,
setSchemaStatsBuilder,
callStatsBuilder);
// This check is needed wherever setSchema is called to detect soft errors which do not
// throw an exception but also prevent the schema from actually being applied.
if (!setSchemaResponsePair.first.isSuccess()) {
return setSchemaResponsePair.first;
}
long getNewSchemaObserverStartTimeMillis = SystemClock.elapsedRealtime();
// Maps unprefixed schema name to the set of listening packages that have visibility into
// that type under the new schema.
Map> requestSchemaNameToVisibleListeningPackage =
new ArrayMap<>(schemas.size());
Set requestUnprefixedSchemaNames = new ArraySet<>();
for (AppSearchSchema requestSchemaType : schemas) {
String requestSchemaName = requestSchemaType.getSchemaType();
requestUnprefixedSchemaNames.add(requestSchemaName);
requestSchemaNameToVisibleListeningPackage.put(
requestSchemaName,
mObserverManager.getObserversForSchemaType(
packageName,
databaseName,
requestSchemaName,
mDocumentVisibilityStoreLocked,
mVisibilityCheckerLocked));
}
long getNewSchemaObserverEndTimeMillis = SystemClock.elapsedRealtime();
if (setSchemaStatsBuilder != null) {
setSchemaStatsBuilder.setGetObserverLatencyMillis(
getOldSchemaObserverLatencyMillis
+ (int)
(getNewSchemaObserverEndTimeMillis
- getNewSchemaObserverStartTimeMillis));
}
long preparingChangeNotificationStartTimeMillis = SystemClock.elapsedRealtime();
SetSchemaResultProto setSchemaResultProto = setSchemaResponsePair.second;
// Send notifications for all old listeners of deleted types
sendDeletedTypeNotificationsLocked(
packageName,
databaseName,
setSchemaResultProto.getDeletedSchemaTypesList(),
oldSchemaNameToVisibleListeningPackage);
// Send notifications for types in the request schema.
sendRequestSchemaTypesNotificationsLocked(
packageName,
databaseName,
setSchemaResultProto,
requestUnprefixedSchemaNames,
oldSchemaNameToVisibleListeningPackage,
requestSchemaNameToVisibleListeningPackage);
if (setSchemaStatsBuilder != null) {
setSchemaStatsBuilder.setPreparingChangeNotificationLatencyMillis(
(int)
(SystemClock.elapsedRealtime()
- preparingChangeNotificationStartTimeMillis));
}
return setSchemaResponsePair.first;
}
/**
* Schedule observer notifications for schema types that have been deleted.
*
* @param targetPackageName The package of the deleted types.
* @param databaseName The database of the deleted types.
* @param prefixedDeletedTypes A list of prefixed deleted schema type names.
* @param unprefixedSchemaNameToObserversMap A map from unprefixed schema type names to the set
* of observer package names that should be notified.
*/
private void sendDeletedTypeNotificationsLocked(
String targetPackageName,
String databaseName,
List prefixedDeletedTypes,
Map> unprefixedSchemaNameToObserversMap)
throws AppSearchException {
for (int i = 0; i < prefixedDeletedTypes.size(); ++i) {
String deletedType = removePrefix(prefixedDeletedTypes.get(i));
Set visibleListeners = unprefixedSchemaNameToObserversMap.get(deletedType);
if (visibleListeners != null) {
for (String listeningPackageName : visibleListeners) {
mObserverManager.onSchemaChange(
listeningPackageName, targetPackageName, databaseName, deletedType);
}
}
}
}
/**
* Schedule observer notifications for schema types in the request schema. Notifications are
* scheduled for a type if: 1. The type is either a new type, or its definition has changed from
* before. 2. There is a change in the type's visibility from its old visibility for the
* observer.
*
* @param targetPackageName The package of the deleted types.
* @param databaseName The database of the deleted types.
* @param setSchemaResultProto Result proto from the set schema request
* @param unprefixedRequestSchemaNames Set of unprefixed schema type names for the set schema
* request
* @param unprefixedPriorSchemaNameToObserversMap A map from the prior unprefixed schema type
* names to the set of observer package names that should be notified.
* @param unprefixedRequestSchemaNameToObserversMap A map from the request's unprefixed schema
* type names to the set of observer package names that should be notified.
*/
private void sendRequestSchemaTypesNotificationsLocked(
String targetPackageName,
String databaseName,
SetSchemaResultProto setSchemaResultProto,
Set unprefixedRequestSchemaNames,
Map> unprefixedPriorSchemaNameToObserversMap,
Map> unprefixedRequestSchemaNameToObserversMap)
throws AppSearchException {
// Get new or changed types from the setSchemaResultProto
Set unprefixedNewAndChangedTypes = new ArraySet<>();
addUnprefixedTypeNames(
setSchemaResultProto.getNewSchemaTypesList(), unprefixedNewAndChangedTypes);
addUnprefixedTypeNames(
setSchemaResultProto.getIncompatibleSchemaTypesList(),
unprefixedNewAndChangedTypes);
addUnprefixedTypeNames(
setSchemaResultProto.getFullyCompatibleChangedSchemaTypesList(),
unprefixedNewAndChangedTypes);
addUnprefixedTypeNames(
setSchemaResultProto.getIndexIncompatibleChangedSchemaTypesList(),
unprefixedNewAndChangedTypes);
addUnprefixedTypeNames(
setSchemaResultProto.getJoinIncompatibleChangedSchemaTypesList(),
unprefixedNewAndChangedTypes);
// Iterate through each type in the request schema and send notifications
for (String schemaName : unprefixedRequestSchemaNames) {
Set priorVisibleListeners =
unprefixedPriorSchemaNameToObserversMap.get(schemaName);
Set requestVisibleListeners =
unprefixedRequestSchemaNameToObserversMap.get(schemaName);
// Iterate through each observer in the prior and current listeners and consider its
// view of the type to send notifications
if (priorVisibleListeners != null) {
for (String priorListeningPackage : priorVisibleListeners) {
if (requestVisibleListeners != null
&& requestVisibleListeners.contains(priorListeningPackage)
&& !unprefixedNewAndChangedTypes.contains(schemaName)) {
// Neither the listener's view nor the type itself has changed -- no need to
// notify
continue;
}
mObserverManager.onSchemaChange(
priorListeningPackage, targetPackageName, databaseName, schemaName);
}
}
if (requestVisibleListeners != null) {
for (String currentListeningPackage : requestVisibleListeners) {
// At this point we only need to notify if the listener is not a visible
// listener prior to the request.
// For other scenarios, we've already checked and notified above while
// notifying prior listeners.
if (priorVisibleListeners == null
|| !priorVisibleListeners.contains(currentListeningPackage)) {
mObserverManager.onSchemaChange(
currentListeningPackage,
targetPackageName,
databaseName,
schemaName);
}
}
}
}
}
/**
* Extracts unprefixed type names from a list of prefixed type names and adds them to the given
* set.
*/
private void addUnprefixedTypeNames(List prefixedTypes, Set unprefixedTypeSet)
throws AppSearchException {
for (int i = 0; i < prefixedTypes.size(); ++i) {
unprefixedTypeSet.add(removePrefix(prefixedTypes.get(i)));
}
}
/**
* Updates the AppSearch schema for this app, without dispatching change notifications.
*
* This method can be used only when no one is observing {@code packageName}.
*
* @see #setSchema
* @see #doSetSchemaWithChangeNotificationLocked
*/
@GuardedBy("mReadWriteLock")
private @NonNull Pair
doSetSchemaNoChangeNotificationLocked(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull List schemas,
@NonNull List visibilityConfigs,
boolean forceOverride,
int version,
SetSchemaStats.@Nullable Builder setSchemaStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long rewriteSchemaStartTimeMillis = SystemClock.elapsedRealtime();
String prefix = createPrefix(packageName, databaseName);
SchemaProto.Builder newSchemaBuilder = SchemaProto.newBuilder();
for (int i = 0; i < schemas.size(); i++) {
AppSearchSchema schema = schemas.get(i);
SchemaTypeConfigProto schemaTypeProto =
SchemaToProtoConverter.toSchemaTypeConfigProto(schema, version);
newSchemaBuilder.addTypes(schemaTypeProto);
}
Set deletedPrefixedTypes;
Map rewrittenPrefixedTypes;
SetSchemaResultProto setSchemaResultProto;
long rewriteSchemaEndTimeMillis;
long nativeLatencyStartTimeMillis = SystemClock.elapsedRealtime();
boolean containsSchemaChange = true; // Presumed true, by default.
// Rewrite and apply schema
if (useDatabaseScopedSchemaOperations()) {
rewrittenPrefixedTypes =
getRewrittenPrefixedTypes(
prefix, newSchemaBuilder.build(), /* populateDatabase= */ true);
rewriteSchemaEndTimeMillis = SystemClock.elapsedRealtime();
if (enableEarlySetSchemaExit()) {
containsSchemaChange =
doesSchemaContainChangeLocked(prefix, rewrittenPrefixedTypes);
}
if (containsSchemaChange) {
SchemaProto finalSchema =
SchemaProto.newBuilder()
.addAllTypes(rewrittenPrefixedTypes.values())
.build();
SetSchemaRequestProto setSchemaRequestProto =
SetSchemaRequestProto.newBuilder()
.setSchema(finalSchema)
.setDatabase(prefix)
.setIgnoreErrorsAndDeleteDocuments(forceOverride)
.build();
LogUtil.piiTrace(
TAG,
"setSchema, request",
finalSchema.getTypesCount(),
setSchemaRequestProto);
setSchemaResultProto =
mIcingSearchEngineLocked.setSchemaWithRequestProto(setSchemaRequestProto);
deletedPrefixedTypes =
new ArraySet<>(setSchemaResultProto.getDeletedSchemaTypesList());
} else {
// Schema was a no-op. Skip interaction with Icing.
setSchemaResultProto =
SetSchemaResultProto.newBuilder()
.setStatus(StatusProto.newBuilder().setCode(StatusProto.Code.OK))
.build();
deletedPrefixedTypes = Collections.emptySet();
}
} else {
SchemaProto.Builder existingSchemaBuilder =
getSchemaProtoLocked(callStatsBuilder).toBuilder();
// Combine the existing schema (which may have types from other prefixes) with this
// prefix's new schema. Modifies the existingSchemaBuilder.
RewrittenSchemaResults rewrittenSchemaResults =
rewriteSchema(
prefix,
existingSchemaBuilder,
newSchemaBuilder.build(),
/* populateDatabase= */ false);
rewriteSchemaEndTimeMillis = SystemClock.elapsedRealtime();
deletedPrefixedTypes = rewrittenSchemaResults.mDeletedPrefixedTypes;
rewrittenPrefixedTypes = rewrittenSchemaResults.mRewrittenPrefixedTypes;
SchemaProto finalSchema = existingSchemaBuilder.build();
LogUtil.piiTrace(TAG, "setSchema, request", finalSchema.getTypesCount(), finalSchema);
setSchemaResultProto = mIcingSearchEngineLocked.setSchema(finalSchema, forceOverride);
}
LogUtil.piiTrace(
TAG, "setSchema, response", setSchemaResultProto.getStatus(), setSchemaResultProto);
long nativeLatencyEndTimeMillis = SystemClock.elapsedRealtime();
// Populate logging stats
if (setSchemaStatsBuilder != null) {
setSchemaStatsBuilder.setRewriteSchemaLatencyMillis(
(int) (rewriteSchemaEndTimeMillis - rewriteSchemaStartTimeMillis));
setSchemaStatsBuilder
.setTotalNativeLatencyMillis(
(int) (nativeLatencyEndTimeMillis - nativeLatencyStartTimeMillis))
.setStatusCode(statusProtoToResultCode(setSchemaResultProto.getStatus()))
.setSkippedIcingInteraction(!containsSchemaChange);
AppSearchLoggerHelper.copyNativeStats(setSchemaResultProto, setSchemaStatsBuilder);
}
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(setSchemaResultProto.getGetVmLatencyMs());
}
boolean isFailedPrecondition =
setSchemaResultProto.getStatus().getCode() == StatusProto.Code.FAILED_PRECONDITION;
// Determine whether it succeeded.
try {
checkSuccess(setSchemaResultProto.getStatus());
// TODO(b/417463182): add boolean field(s) into SetSchemaResultProto indicating whether
// ground truths and derived files have changed or not, and we can have a simpler way
// here to decide if persistToDisk is needed or not.
if (setSchemaResultProto.getNewSchemaTypesCount() > 0
|| setSchemaResultProto.getDeletedSchemaTypesCount() > 0
|| setSchemaResultProto.getIncompatibleSchemaTypesCount() > 0
|| setSchemaResultProto.getDeletedDocumentCount() > 0
|| setSchemaResultProto.getScorablePropertyIncompatibleChangedSchemaTypesCount()
> 0
|| setSchemaResultProto.getHasTermIndexRestored()
|| setSchemaResultProto.getHasIntegerIndexRestored()
|| setSchemaResultProto.getHasQualifiedIdJoinIndexRestored()
|| setSchemaResultProto.getHasEmbeddingIndexRestored()) {
mNeedsPersistToDisk.set(true);
}
} catch (AppSearchException e) {
// Swallow the exception for the incompatible change case. We will generate a failed
// InternalSetSchemaResponse for this case.
int deletedTypes = setSchemaResultProto.getDeletedSchemaTypesCount();
int incompatibleTypes = setSchemaResultProto.getIncompatibleSchemaTypesCount();
boolean isIncompatible = deletedTypes > 0 || incompatibleTypes > 0;
if (isFailedPrecondition && !forceOverride && isIncompatible) {
SetSchemaResponse setSchemaResponse =
SetSchemaResponseToProtoConverter.toSetSchemaResponse(
setSchemaResultProto, prefix);
String errorMessage =
"Schema is incompatible."
+ "\n Deleted types: "
+ setSchemaResponse.getDeletedTypes()
+ "\n Incompatible types: "
+ setSchemaResponse.getIncompatibleTypes();
return new Pair<>(
newFailedSetSchemaResponse(setSchemaResponse, errorMessage),
setSchemaResultProto);
} else {
throw e;
}
}
long saveVisibilitySettingStartTimeMillis = SystemClock.elapsedRealtime();
if (containsSchemaChange) {
// Update derived data structures.
for (SchemaTypeConfigProto schemaTypeConfigProto : rewrittenPrefixedTypes.values()) {
mSchemaCacheLocked.addToSchemaMap(prefix, schemaTypeConfigProto);
}
for (String schemaType : deletedPrefixedTypes) {
mSchemaCacheLocked.removeFromSchemaMap(prefix, schemaType);
}
mSchemaCacheLocked.rebuildCacheForPrefix(prefix);
}
// Since the constructor of VisibilityStore will set schema. Avoid call visibility
// store before we have already created it.
if (mDocumentVisibilityStoreLocked != null) {
// Add prefix to all visibility documents.
// Find out which Visibility document is deleted or changed to all-default settings.
// We need to remove them from Visibility Store.
Set deprecatedVisibilityDocuments =
new ArraySet<>(rewrittenPrefixedTypes.keySet());
List prefixedVisibilityConfigs =
rewriteVisibilityConfigs(
prefix, visibilityConfigs, deprecatedVisibilityDocuments);
// Now deprecatedVisibilityDocuments contains those existing schemas that has
// all-default visibility settings, add deleted schemas. That's all we need to
// remove.
deprecatedVisibilityDocuments.addAll(deletedPrefixedTypes);
mDocumentVisibilityStoreLocked.removeVisibility(
deprecatedVisibilityDocuments, callStatsBuilder);
mDocumentVisibilityStoreLocked.setVisibility(
prefixedVisibilityConfigs, callStatsBuilder);
}
long saveVisibilitySettingEndTimeMillis = SystemClock.elapsedRealtime();
if (setSchemaStatsBuilder != null) {
setSchemaStatsBuilder.setVisibilitySettingLatencyMillis(
(int)
(saveVisibilitySettingEndTimeMillis
- saveVisibilitySettingStartTimeMillis));
}
long convertToResponseStartTimeMillis = SystemClock.elapsedRealtime();
InternalSetSchemaResponse setSchemaResponse =
newSuccessfulSetSchemaResponse(
SetSchemaResponseToProtoConverter.toSetSchemaResponse(
setSchemaResultProto, prefix));
long convertToResponseEndTimeMillis = SystemClock.elapsedRealtime();
if (setSchemaStatsBuilder != null) {
setSchemaStatsBuilder.setConvertToResponseLatencyMillis(
(int) (convertToResponseEndTimeMillis - convertToResponseStartTimeMillis));
}
return new Pair<>(setSchemaResponse, setSchemaResultProto);
}
private boolean doesSchemaContainChangeLocked(
@NonNull String prefix,
@NonNull Map rewrittenPrefixedTypes) {
// RewrittenPrefixedTypes maps from prefixed schema type to SchemaTypeConfigProto.
Map previousTypes =
mSchemaCacheLocked.getSchemaMapForPrefix(prefix);
if (previousTypes.size() != rewrittenPrefixedTypes.size()) {
return true; // A type was added or deleted.
}
Set previousTypeNames = new ArraySet<>(previousTypes.keySet());
for (SchemaTypeConfigProto typeConfig : rewrittenPrefixedTypes.values()) {
SchemaTypeConfigProto oldTypeConfig = previousTypes.get(typeConfig.getSchemaType());
if (oldTypeConfig == null || !typeConfig.equals(oldTypeConfig)) {
return true; // Type definition has changed in some way. Let Icing sort out how.
}
previousTypeNames.remove(typeConfig.getSchemaType());
}
return !previousTypeNames.isEmpty(); // A type was deleted.
}
/**
* Retrieves the AppSearch schema for this package name, database.
*
* This method belongs to query group.
*
* @param packageName Package that owns the requested {@link AppSearchSchema} instances.
* @param databaseName Database that owns the requested {@link AppSearchSchema} instances.
* @param callerAccess Visibility access info of the calling app
* @throws AppSearchException on IcingSearchEngine error.
*/
public @NonNull GetSchemaResponse getSchema(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull CallerAccess callerAccess,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.readLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastWriteOperationLatencyMillisLocked);
}
throwIfClosedLocked();
// Get the schema from IcingLib.
// If database-scoped schema operations is enabled, the schema that is retrieved would
// only contain schema types corresponding to the package-database prefix. Otherwise,
// the full schema containing all types (across all packages and databases) will be
// retrieved.
SchemaProto icingSchema;
String prefix = createPrefix(packageName, databaseName);
if (useDatabaseScopedSchemaOperations()) {
icingSchema = getSchemaProtoForPrefixLocked(prefix);
} else {
icingSchema = getSchemaProtoLocked(callStatsBuilder);
}
GetSchemaResponse.Builder responseBuilder = new GetSchemaResponse.Builder();
for (int i = 0; i < icingSchema.getTypesCount(); i++) {
// Check that this type belongs to the requested app and that the caller has
// access to it.
SchemaTypeConfigProto typeConfig = icingSchema.getTypes(i);
String prefixedSchemaType = typeConfig.getSchemaType();
String typePrefix = getPrefix(prefixedSchemaType);
if (!prefix.equals(typePrefix)) {
// TODO: Remove this unnecessary check once
// enableDatabaseScopedSchemaOperations is rolled out.
// This schema type doesn't belong to the database we're querying for.
continue;
}
if (!VisibilityUtil.isSchemaSearchableByCaller(
callerAccess,
packageName,
prefixedSchemaType,
mDocumentVisibilityStoreLocked,
mVisibilityCheckerLocked)) {
// Caller doesn't have access to this type.
continue;
}
// Rewrite SchemaProto.types.schema_type
SchemaTypeConfigProto.Builder typeConfigBuilder = typeConfig.toBuilder();
removePrefixesFromSchemaType(typeConfigBuilder);
AppSearchSchema schema =
SchemaToProtoConverter.toAppSearchSchema(typeConfigBuilder);
responseBuilder.setVersion(typeConfig.getVersion());
responseBuilder.addSchema(schema);
// Populate visibility info. Since the constructor of VisibilityStore will get
// schema. Avoid call visibility store before we have already created it.
if (mDocumentVisibilityStoreLocked != null) {
String typeName = typeConfig.getSchemaType().substring(typePrefix.length());
InternalVisibilityConfig visibilityConfig =
mDocumentVisibilityStoreLocked.getVisibility(prefixedSchemaType);
if (visibilityConfig != null) {
if (visibilityConfig.isNotDisplayedBySystem()) {
responseBuilder.addSchemaTypeNotDisplayedBySystem(typeName);
}
List packageIdentifiers =
visibilityConfig.getVisibilityConfig().getAllowedPackages();
if (!packageIdentifiers.isEmpty()) {
responseBuilder.setSchemaTypeVisibleToPackages(
typeName, new ArraySet<>(packageIdentifiers));
}
Set> visibleToPermissions =
visibilityConfig.getVisibilityConfig().getRequiredPermissions();
if (!visibleToPermissions.isEmpty()) {
Set> visibleToPermissionsSet =
new ArraySet<>(visibleToPermissions.size());
for (Set permissionList : visibleToPermissions) {
visibleToPermissionsSet.add(new ArraySet<>(permissionList));
}
responseBuilder.setRequiredPermissionsForSchemaTypeVisibility(
typeName, visibleToPermissionsSet);
}
// Check for Visibility properties from the overlay
PackageIdentifier publiclyVisibleFromPackage =
visibilityConfig
.getVisibilityConfig()
.getPubliclyVisibleTargetPackage();
if (publiclyVisibleFromPackage != null) {
responseBuilder.setPubliclyVisibleSchema(
typeName, publiclyVisibleFromPackage);
}
Set visibleToConfigs =
visibilityConfig.getVisibleToConfigs();
if (!visibleToConfigs.isEmpty()) {
responseBuilder.setSchemaTypeVisibleToConfigs(
typeName, visibleToConfigs);
}
}
}
}
return responseBuilder.build();
} finally {
logReadOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_GET_SCHEMA,
callStatsBuilder);
mReadWriteLock.readLock().unlock();
}
}
/**
* Retrieves the list of namespaces with at least one document for this package name, database.
*
* This method belongs to query group.
*
* @param packageName Package name that owns this schema
* @param databaseName The name of the database where this schema lives.
* @throws AppSearchException on IcingSearchEngine error.
*/
public @NonNull List getNamespaces(
@NonNull String packageName,
@NonNull String databaseName,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.readLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastWriteOperationLatencyMillisLocked);
}
throwIfClosedLocked();
LogUtil.piiTrace(TAG, "getAllNamespaces, request");
// We can't just use mNamespaceMap here because we have no way to prune namespaces from
// mNamespaceMap when they have no more documents (e.g. after setting schema to empty or
// using deleteByQuery).
GetAllNamespacesResultProto getAllNamespacesResultProto =
mIcingSearchEngineLocked.getAllNamespaces();
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(
getAllNamespacesResultProto.getGetVmLatencyMs());
}
LogUtil.piiTrace(
TAG,
"getAllNamespaces, response",
getAllNamespacesResultProto.getNamespacesCount(),
getAllNamespacesResultProto);
checkSuccess(getAllNamespacesResultProto.getStatus());
String prefix = createPrefix(packageName, databaseName);
List results = new ArrayList<>();
for (int i = 0; i < getAllNamespacesResultProto.getNamespacesCount(); i++) {
String prefixedNamespace = getAllNamespacesResultProto.getNamespaces(i);
if (prefixedNamespace.startsWith(prefix)) {
results.add(prefixedNamespace.substring(prefix.length()));
}
}
return results;
} finally {
logReadOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_GET_NAMESPACES,
callStatsBuilder);
mReadWriteLock.readLock().unlock();
}
}
/**
* Creates a {@link DocumentProto} from a {@link GenericDocument}.
*
* @param packageName name of the calling package
* @param databaseName database name
* @param doc a {@link GenericDocument}
* @param statsBuilder {@link PutDocumentStats.Builder} to hold the stats for creating {@link
* DocumentProto}.
*/
@NonNull
private static DocumentProto createDocumentProto(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull GenericDocument doc,
PutDocumentStats.@Nullable Builder statsBuilder) {
String prefix = createPrefix(packageName, databaseName);
// Generate Document Proto
long generateDocumentProtoStartTimeMillis = SystemClock.elapsedRealtime();
DocumentProto.Builder documentBuilder =
GenericDocumentToProtoConverter.toDocumentProto(doc).toBuilder();
long generateDocumentProtoEndTimeMillis = SystemClock.elapsedRealtime();
// Rewrite Document Type
long rewriteDocumentTypeStartTimeMillis = SystemClock.elapsedRealtime();
addPrefixToDocument(documentBuilder, prefix);
long rewriteDocumentTypeEndTimeMillis = SystemClock.elapsedRealtime();
DocumentProto finalDocument = documentBuilder.build();
if (statsBuilder != null) {
statsBuilder
.setGenerateDocumentProtoLatencyMillis(
(int)
(generateDocumentProtoEndTimeMillis
- generateDocumentProtoStartTimeMillis))
.setRewriteDocumentTypesLatencyMillis(
(int)
(rewriteDocumentTypeEndTimeMillis
- rewriteDocumentTypeStartTimeMillis));
}
return finalDocument;
}
/**
* Adds a list of {@link GenericDocument} to the AppSearch index.
*
* This method belongs to mutate group.
*
* @param packageName The package name that owns this document.
* @param databaseName The databaseName this document resides in.
* @param documents A list of documents to index.
* @param batchResultBuilder The builder for returning the batch result.
* @param sendChangeNotifications Whether to dispatch {@link DocumentChangeInfo} messages to
* observers for this change.
* @param persistType The persist type used to call PersistToDisk inside Icing at the end of the
* Put request. If UNKNOWN, PersistToDisk will not be called. See also {@link
* #persistToDisk(PersistType.Code)}.
* @throws AppSearchException on IcingSearchEngine error.
*/
public void batchPutDocuments(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull List documents,
AppSearchBatchResult.@Nullable Builder batchResultBuilder,
boolean sendChangeNotifications,
@Nullable AppSearchLogger logger,
PersistType.@NonNull Code persistType,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
int maxBufferedBytes = mConfig.getMaxByteLimitForBatchPut();
List docProtos = new ArrayList<>();
List statsBuilders = new ArrayList<>();
int currentTotalBytes = 0;
for (int i = 0; i < documents.size(); ++i) {
PutDocumentStats.Builder pStatsBuilder =
new PutDocumentStats.Builder(packageName, databaseName);
DocumentProto docProto =
createDocumentProto(packageName, databaseName, documents.get(i), pStatsBuilder);
int serializedSizeBytes = docProto.getSerializedSize();
if ((serializedSizeBytes > maxBufferedBytes - currentTotalBytes)
&& !docProtos.isEmpty()) {
try {
batchPutDocuments(
packageName,
databaseName,
docProtos,
statsBuilders,
batchResultBuilder,
/* sendChangeNotifications= */ true,
logger,
PersistType.Code.UNKNOWN,
callStatsBuilder);
} catch (AppSearchException e) {
// Catch the AppSearchException so we can move on to the next batch.
// For other exceptions, we allow it throw and stop indexing.
Log.e(TAG, "BatchPut failed.", e);
}
currentTotalBytes = 0;
docProtos.clear();
statsBuilders.clear();
}
currentTotalBytes += serializedSizeBytes;
docProtos.add(docProto);
statsBuilders.add(pStatsBuilder);
}
// Do last flush with persistType passed in, even if docProtos is empty.
batchPutDocuments(
packageName,
databaseName,
docProtos,
statsBuilders,
batchResultBuilder,
sendChangeNotifications,
logger,
persistType,
callStatsBuilder);
}
/**
* Adds a list of {@link DocumentProto} to the AppSearch index.
*
* This method belongs to mutate group.
*
* @param packageName The package name that owns this document.
* @param databaseName The databaseName this document resides in.
* @param documents A list of {@link DocumentProto} to index.
* @param statsBuilders The corresponding {@link PutDocumentStats.Builder} to the documents.
* @param batchResultBuilder The builder for returning the batch result.
* @param sendChangeNotifications Whether to dispatch {@link DocumentChangeInfo} messages to
* observers for this change.
* @param persistType The persist type used to call PersistToDisk inside Icing at the end of the
* Put request. If UNKNOWN, PersistToDisk will not be called. See also {@link
* #persistToDisk(PersistType.Code)}.
* @throws AppSearchException on IcingSearchEngine error.
*/
private void batchPutDocuments(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull List documents,
@NonNull List statsBuilders,
AppSearchBatchResult.@Nullable Builder batchResultBuilder,
boolean sendChangeNotifications,
@Nullable AppSearchLogger logger,
PersistType.@NonNull Code persistType,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
Preconditions.checkArgument(
documents.size() == statsBuilders.size(),
"documents and statsBuilders should have same size");
List statsNotFilteredOut = new ArrayList<>();
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
String prefix = createPrefix(packageName, databaseName);
mReadWriteLock.writeLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked);
}
throwIfClosedLocked();
PutDocumentRequest.Builder putRequestBuilder =
PutDocumentRequest.newBuilder().setPersistType(persistType);
for (int i = 0; i < documents.size(); ++i) {
DocumentProto finalDocument = documents.get(i);
String docId = finalDocument.getUri();
PutDocumentStats.Builder pStatsBuilder =
statsBuilders
.get(i)
.setLaunchVMEnabled(mIsVMEnabled)
.setJavaLockAcquisitionLatencyMillis(
(int)
(javaLockAcquisitionEndTimeMillis
- totalLatencyStartMillis))
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked);
try {
// Check limits
int serializedSizeBytes = finalDocument.getSerializedSize();
enforceLimitConfigLocked(
packageName, docId, serializedSizeBytes, callStatsBuilder);
putRequestBuilder.addDocuments(finalDocument);
statsNotFilteredOut.add(pStatsBuilder);
} catch (Throwable t) {
if (batchResultBuilder != null) {
batchResultBuilder.setResult(docId, throwableToFailedResult(t));
}
}
}
// Put documents
PutDocumentRequest requestProto = putRequestBuilder.build();
LogUtil.piiTrace(
TAG,
"batchPutDocument, request",
requestProto.getDocumentsCount(),
requestProto);
BatchPutResultProto batchPutResultProto =
mIcingSearchEngineLocked.batchPut(requestProto);
// Put API may fail, but Icing has internal logic to delete the proto or rebuild index
// after failure so we need PersistToDisk regardless of the result.
// TODO(b/417463182): add boolean field(s) into BatchPutResultProto indicating whether
// ground truths and derived files have changed or not.
mNeedsPersistToDisk.set(true);
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(batchPutResultProto.getGetVmLatencyMs());
}
// TODO(b/394875109) We can provide a better debug information for fast trace here.
LogUtil.piiTrace(
TAG, "batchPutDocument", /* fastTraceObj= */ null, batchPutResultProto);
// Create Results
List putResultProtoList = batchPutResultProto.getPutResultProtosList();
for (int i = 0; i < putResultProtoList.size(); ++i) {
PutResultProto putResultProto = putResultProtoList.get(i);
String docId = putResultProto.getUri();
try {
if (i < statsNotFilteredOut.size()) {
PutDocumentStats.Builder pStatsBuilder = statsNotFilteredOut.get(i);
pStatsBuilder
.setStatusCode(statusProtoToResultCode(putResultProto.getStatus()))
.addGetVmLatencyMillis(putResultProto.getGetVmLatencyMs());
AppSearchLoggerHelper.copyNativeStats(
putResultProto.getPutDocumentStats(), pStatsBuilder);
} else {
// since it is just stats, we just log the debug message if
// something goes wrong.
LogUtil.piiTrace(
TAG,
"batchPutDocument",
"index out of boundary for stats",
statsNotFilteredOut);
}
// If it is a failure, it will throw and the catch section will
// set generated result
checkSuccess(putResultProto.getStatus());
if (batchResultBuilder != null) {
batchResultBuilder.setSuccess(docId, /* value= */ null);
}
// Don't need to check the index here, as request doc list size should
// definitely be bigger than response doc list size.
DocumentProto documentProto = requestProto.getDocuments(i);
if (!docId.equals(documentProto.getUri())) {
// This shouldn't happen if native code implemented correctly.
// Have a check here just in case something unexpected happens.
Log.w(TAG, "id mismatch between request and response for batchPut");
continue;
}
// Only update caches if the document is successfully put to Icing.
// Prefixed namespace needed here.
mNamespaceCacheLocked.addToDocumentNamespaceMap(
prefix, documentProto.getNamespace());
if (!Flags.enableDocumentLimiterReplaceTracking()
|| !putResultProto.getWasReplacement()) {
// If the document was a replacement, then there is no need to report it
// because the number of documents has not changed. We only need to
// report "true" additions to the DocumentLimiter.
// Although a replacement document will consume a document id,
// the limit is only intended to apply to "living" documents.
// It is the responsibility of AppSearch's optimization task to reclaim
// space when needed.
mDocumentLimiterLocked.reportDocumentAdded(
packageName,
() ->
getRawStorageInfoProto(callStatsBuilder)
.getDocumentStorageInfo()
.getNamespaceStorageInfoList());
}
// Prepare notifications
if (sendChangeNotifications) {
mObserverManager.onDocumentChange(
packageName,
databaseName,
removePrefix(documentProto.getNamespace()),
removePrefix(documentProto.getSchema()),
documentProto.getUri(),
mDocumentVisibilityStoreLocked,
mVisibilityCheckerLocked);
}
} catch (Throwable t) {
if (batchResultBuilder != null) {
batchResultBuilder.setResult(docId, throwableToFailedResult(t));
} else {
throw t;
}
}
}
if (requestProto.getPersistType() != PersistType.Code.UNKNOWN) {
checkSuccess(batchPutResultProto.getPersistToDiskResultProto().getStatus());
}
} finally {
logWriteOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_PUT_DOCUMENTS,
callStatsBuilder);
mReadWriteLock.writeLock().unlock();
if (logger != null && !statsBuilders.isEmpty()) {
// This seems broken and no easy way to get accurate number.
int avgTotalLatencyMs =
(int)
((SystemClock.elapsedRealtime() - totalLatencyStartMillis)
/ statsBuilders.size());
for (int i = 0; i < statsBuilders.size(); ++i) {
PutDocumentStats.Builder pStatsBuilder = statsBuilders.get(i);
pStatsBuilder.setTotalLatencyMillis(avgTotalLatencyMs);
logger.logStats(pStatsBuilder.build());
}
}
}
}
/**
* Adds a document to the AppSearch index.
*
* This method belongs to mutate group.
*
* @param packageName The package name that owns this document.
* @param databaseName The databaseName this document resides in.
* @param document The document to index.
* @param sendChangeNotifications Whether to dispatch {@link DocumentChangeInfo} messages to
* observers for this change.
* @throws AppSearchException on IcingSearchEngine error.
* @deprecated use {@link #batchPutDocuments(String, String, List, AppSearchBatchResult.Builder,
* boolean, AppSearchLogger)}
*/
// TODO(b/394875109) keep this for now to make code sync easier.
@Deprecated
public void putDocument(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull GenericDocument document,
boolean sendChangeNotifications,
@Nullable AppSearchLogger logger,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
PutDocumentStats.Builder pStatsBuilder = null;
if (logger != null) {
pStatsBuilder =
new PutDocumentStats.Builder(packageName, databaseName)
.setLaunchVMEnabled(mIsVMEnabled);
}
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.writeLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked);
}
throwIfClosedLocked();
// Generate Document Proto
long generateDocumentProtoStartTimeMillis = SystemClock.elapsedRealtime();
DocumentProto.Builder documentBuilder =
GenericDocumentToProtoConverter.toDocumentProto(document).toBuilder();
long generateDocumentProtoEndTimeMillis = SystemClock.elapsedRealtime();
// Rewrite Document Type
long rewriteDocumentTypeStartTimeMillis = SystemClock.elapsedRealtime();
String prefix = createPrefix(packageName, databaseName);
addPrefixToDocument(documentBuilder, prefix);
long rewriteDocumentTypeEndTimeMillis = SystemClock.elapsedRealtime();
DocumentProto finalDocument = documentBuilder.build();
// Check limits
enforceLimitConfigLocked(
packageName,
finalDocument.getUri(),
finalDocument.getSerializedSize(),
callStatsBuilder);
// Insert document
LogUtil.piiTrace(TAG, "putDocument, request", finalDocument.getUri(), finalDocument);
PutResultProto putResultProto = mIcingSearchEngineLocked.put(finalDocument);
// Put API may fail, but Icing has internal logic to delete the proto or rebuild index
// after failure so we need PersistToDisk regardless of the result.
// TODO(b/417463182): add boolean field(s) into PutResultProto indicating whether
// ground truths and derived files have changed or not.
mNeedsPersistToDisk.set(true);
LogUtil.piiTrace(
TAG, "putDocument, response", putResultProto.getStatus(), putResultProto);
// Logging stats
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(putResultProto.getGetVmLatencyMs());
}
if (pStatsBuilder != null) {
pStatsBuilder
.setJavaLockAcquisitionLatencyMillis(
(int) (javaLockAcquisitionEndTimeMillis - totalLatencyStartMillis))
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked)
.setStatusCode(statusProtoToResultCode(putResultProto.getStatus()))
.setGenerateDocumentProtoLatencyMillis(
(int)
(generateDocumentProtoEndTimeMillis
- generateDocumentProtoStartTimeMillis))
.setRewriteDocumentTypesLatencyMillis(
(int)
(rewriteDocumentTypeEndTimeMillis
- rewriteDocumentTypeStartTimeMillis))
.addGetVmLatencyMillis(putResultProto.getGetVmLatencyMs());
AppSearchLoggerHelper.copyNativeStats(
putResultProto.getPutDocumentStats(), pStatsBuilder);
}
checkSuccess(putResultProto.getStatus());
// Only update caches if the document is successfully put to Icing.
mNamespaceCacheLocked.addToDocumentNamespaceMap(prefix, finalDocument.getNamespace());
if (!Flags.enableDocumentLimiterReplaceTracking()
|| !putResultProto.getWasReplacement()) {
// If the document was a replacement, then there is no need to report it because the
// number of documents has not changed. We only need to report "true" additions to
// the DocumentLimiter.
// Although a replacement document will consume a document id, the limit is only
// intended to apply to "living" documents. It is the responsibility of AppSearch's
// optimization task to reclaim space when needed.
mDocumentLimiterLocked.reportDocumentAdded(
packageName,
() ->
getRawStorageInfoProto(callStatsBuilder)
.getDocumentStorageInfo()
.getNamespaceStorageInfoList());
}
// Prepare notifications
if (sendChangeNotifications) {
mObserverManager.onDocumentChange(
packageName,
databaseName,
document.getNamespace(),
document.getSchemaType(),
document.getId(),
mDocumentVisibilityStoreLocked,
mVisibilityCheckerLocked);
}
} finally {
logWriteOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_PUT_DOCUMENT,
callStatsBuilder);
mReadWriteLock.writeLock().unlock();
if (pStatsBuilder != null && logger != null) {
long totalEndTimeMillis = SystemClock.elapsedRealtime();
pStatsBuilder.setTotalLatencyMillis(
(int) (totalEndTimeMillis - totalLatencyStartMillis));
logger.logStats(pStatsBuilder.build());
}
}
}
/**
* Gets the {@link ParcelFileDescriptor} for write purpose of the given {@link
* AppSearchBlobHandle}.
*
*
Only one opened {@link ParcelFileDescriptor} is allowed for each {@link
* AppSearchBlobHandle}. The same {@link ParcelFileDescriptor} will be returned if it is not
* closed by caller.
*
* @param packageName The package name that owns this blob.
* @param databaseName The databaseName this blob resides in.
* @param handle The {@link AppSearchBlobHandle} represent the blob.
*/
public @NonNull ParcelFileDescriptor openWriteBlob(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull AppSearchBlobHandle handle,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException, IOException {
if (mRevocableFileDescriptorStore == null) {
throw new UnsupportedOperationException(
"BLOB_STORAGE is not available on this AppSearch implementation.");
}
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.writeLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked);
}
throwIfClosedLocked();
verifyCallingBlobHandle(packageName, databaseName, handle);
ParcelFileDescriptor pfd =
mRevocableFileDescriptorStore.getOpenedRevocableFileDescriptorForWrite(
packageName, handle);
if (pfd != null) {
// There is already an opened pfd for write with same blob handle, just return the
// already opened one.
return pfd;
}
mRevocableFileDescriptorStore.checkBlobStoreLimit(packageName);
PropertyProto.BlobHandleProto blobHandleProto =
BlobHandleToProtoConverter.toBlobHandleProto(handle);
BlobProto result = mIcingSearchEngineLocked.openWriteBlob(blobHandleProto);
mNeedsPersistToDisk.set(true);
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(result.getGetVmLatencyMs());
}
pfd =
retrieveFileDescriptorLocked(
result,
ParcelFileDescriptor.MODE_CREATE
| ParcelFileDescriptor.MODE_READ_WRITE);
mNamespaceCacheLocked.addToBlobNamespaceMap(
createPrefix(packageName, databaseName), blobHandleProto.getNamespace());
return mRevocableFileDescriptorStore.wrapToRevocableFileDescriptor(
packageName, handle, pfd, ParcelFileDescriptor.MODE_READ_WRITE);
} finally {
logWriteOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_OPEN_WRITE_BLOB,
callStatsBuilder);
mReadWriteLock.writeLock().unlock();
}
}
/**
* Remove and delete the blob file of given {@link AppSearchBlobHandle} from AppSearch storage.
*
*
This method will delete pending blob or committed blobs. Remove blobs that have reference
* documents linked to it will make those reference document has nothing to read.
*
* @param packageName The package name that owns this blob.
* @param databaseName The databaseName this blob resides in.
* @param handle The {@link AppSearchBlobHandle} represent the blob.
*/
public void removeBlob(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull AppSearchBlobHandle handle,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException, IOException {
if (mRevocableFileDescriptorStore == null) {
throw new UnsupportedOperationException(
"BLOB_STORAGE is not available on this AppSearch implementation.");
}
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.writeLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked);
}
verifyCallingBlobHandle(packageName, databaseName, handle);
BlobProto result =
mIcingSearchEngineLocked.removeBlob(
BlobHandleToProtoConverter.toBlobHandleProto(handle));
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(result.getGetVmLatencyMs());
}
checkSuccess(result.getStatus());
mNeedsPersistToDisk.set(true);
if (Flags.enableAppSearchManageBlobFiles()) {
File blobFileToRemove = new File(mBlobFilesDir, result.getFileName());
if (!blobFileToRemove.delete()) {
throw new AppSearchException(
AppSearchResult.RESULT_IO_ERROR,
"Cannot delete the blob file: " + blobFileToRemove.getName());
}
}
mRevocableFileDescriptorStore.revokeFdForWrite(packageName, handle);
} finally {
logWriteOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_REMOVE_BLOB,
callStatsBuilder);
mReadWriteLock.writeLock().unlock();
}
}
/**
* Verifies the integrity of a blob file by comparing its SHA-256 digest with the expected
* digest.
*
*
This method is used when AppSearch manages blob files directly. It opens the blob file
* associated with the given {@link AppSearchBlobHandle}, calculates its SHA-256 digest, and
* compares it with the digest provided in the handle. If the file does not exist or the
* calculated digest does not match the expected digest, the blob is considered invalid and is
* removed.
*
* @param handle The {@link AppSearchBlobHandle} representing the blob to verify.
* @throws AppSearchException if the blob file does not exist, the calculated digest does not
* match the expected digest, or if there is an error removing the invalid blob.
* @throws IOException if there is an error opening or reading the blob file.
*/
@GuardedBy("mReadWriteLock")
private void verifyBlobIntegrityLocked(@NonNull AppSearchBlobHandle handle)
throws AppSearchException, IOException {
// Since the blob has not yet been committed, we open the blob for *write* again to
// get the file name.
BlobProto result =
mIcingSearchEngineLocked.openWriteBlob(
BlobHandleToProtoConverter.toBlobHandleProto(handle));
checkSuccess(result.getStatus());
mNeedsPersistToDisk.set(true);
File blobFile = new File(mBlobFilesDir, result.getFileName());
boolean fileExists = blobFile.exists();
boolean digestMatches = false;
if (fileExists) {
// Read the file to check the digest.
byte[] digest;
ParcelFileDescriptor pfd =
ParcelFileDescriptor.open(blobFile, ParcelFileDescriptor.MODE_READ_ONLY);
try (InputStream inputStream = new ParcelFileDescriptor.AutoCloseInputStream(pfd);
DigestInputStream digestInputStream =
new DigestInputStream(
inputStream, MessageDigest.getInstance("SHA-256"))) {
byte[] buffer = new byte[8192];
while (digestInputStream.read(buffer) != -1) {
// pass
}
digest = digestInputStream.getMessageDigest().digest();
} catch (NoSuchAlgorithmException e) {
throw new AppSearchException(
AppSearchResult.RESULT_INTERNAL_ERROR,
"Failed to get MessageDigest for SHA-256.",
e);
}
digestMatches = Arrays.equals(digest, handle.getSha256Digest());
}
// If the file does not exist or the digest is wrong, delete the blob and throw
// an exception.
if (!fileExists || !digestMatches) {
BlobProto removeResult =
mIcingSearchEngineLocked.removeBlob(
BlobHandleToProtoConverter.toBlobHandleProto(handle));
checkSuccess(removeResult.getStatus());
mNeedsPersistToDisk.set(true);
if (!fileExists) {
throw new AppSearchException(
AppSearchResult.RESULT_NOT_FOUND,
"Cannot find the blob for handle: " + handle);
} else {
File blobFileToRemove = new File(mBlobFilesDir, removeResult.getFileName());
if (!blobFileToRemove.delete()) {
throw new AppSearchException(
AppSearchResult.RESULT_IO_ERROR,
"Cannot delete the blob file: " + blobFileToRemove.getName());
}
throw new AppSearchException(
AppSearchResult.RESULT_INVALID_ARGUMENT,
"The blob content doesn't match to the digest.");
}
}
}
/**
* Commits and seals the blob represented by the given {@link AppSearchBlobHandle}.
*
*
After this call, the blob is readable via {@link #openReadBlob}. And any rewrite is not
* allowed.
*
* @param packageName The package name that owns this blob.
* @param databaseName The databaseName this blob resides in.
* @param handle The {@link AppSearchBlobHandle} represent the blob.
*/
public void commitBlob(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull AppSearchBlobHandle handle,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException, IOException {
if (mRevocableFileDescriptorStore == null) {
throw new UnsupportedOperationException(
"BLOB_STORAGE is not available on this AppSearch implementation.");
}
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
mReadWriteLock.writeLock().lock();
long javaLockAcquisitionEndTimeMillis = 0;
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked);
}
verifyCallingBlobHandle(packageName, databaseName, handle);
// If AppSearch manages blob files, it is responsible for verifying the digest of the
// blob file.
if (Flags.enableAppSearchManageBlobFiles()) {
verifyBlobIntegrityLocked(handle);
}
BlobProto result =
mIcingSearchEngineLocked.commitBlob(
BlobHandleToProtoConverter.toBlobHandleProto(handle));
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(result.getGetVmLatencyMs());
}
checkSuccess(result.getStatus());
mNeedsPersistToDisk.set(true);
// The blob is committed and sealed, revoke the sent pfd for writing.
mRevocableFileDescriptorStore.revokeFdForWrite(packageName, handle);
} finally {
logWriteOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_COMMIT_BLOB,
callStatsBuilder);
mReadWriteLock.writeLock().unlock();
}
}
/**
* Gets the {@link ParcelFileDescriptor} for read only purpose of the given {@link
* AppSearchBlobHandle}.
*
*
The target must be committed via {@link #commitBlob};
*
* @param packageName The package name that owns this blob.
* @param databaseName The databaseName this blob resides in.
* @param handle The {@link AppSearchBlobHandle} represent the blob.
*/
public @NonNull ParcelFileDescriptor openReadBlob(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull AppSearchBlobHandle handle,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException, IOException {
if (mRevocableFileDescriptorStore == null) {
throw new UnsupportedOperationException(
"BLOB_STORAGE is not available on this AppSearch implementation.");
}
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.readLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastWriteOperationLatencyMillisLocked);
}
verifyCallingBlobHandle(packageName, databaseName, handle);
mRevocableFileDescriptorStore.checkBlobStoreLimit(packageName);
BlobProto result =
mIcingSearchEngineLocked.openReadBlob(
BlobHandleToProtoConverter.toBlobHandleProto(handle));
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(result.getGetVmLatencyMs());
}
ParcelFileDescriptor pfd =
retrieveFileDescriptorLocked(result, ParcelFileDescriptor.MODE_READ_ONLY);
// We do NOT need to look up the revocable file descriptor for read, skip passing the
// blob handle key.
return mRevocableFileDescriptorStore.wrapToRevocableFileDescriptor(
packageName, /* blobHandle= */ null, pfd, ParcelFileDescriptor.MODE_READ_ONLY);
} finally {
logReadOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_OPEN_READ_BLOB,
callStatsBuilder);
mReadWriteLock.readLock().unlock();
}
}
/**
* Gets the {@link ParcelFileDescriptor} for read only purpose of the given {@link
* AppSearchBlobHandle}.
*
*
The target must be committed via {@link #commitBlob};
*
* @param handle The {@link AppSearchBlobHandle} represent the blob.
*/
public @NonNull ParcelFileDescriptor globalOpenReadBlob(
@NonNull AppSearchBlobHandle handle,
@NonNull CallerAccess access,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException, IOException {
if (mRevocableFileDescriptorStore == null) {
throw new UnsupportedOperationException(
"BLOB_STORAGE is not available on this AppSearch implementation.");
}
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.readLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastWriteOperationLatencyMillisLocked);
}
mRevocableFileDescriptorStore.checkBlobStoreLimit(access.getCallingPackageName());
String prefixedNamespace =
createPrefix(handle.getPackageName(), handle.getDatabaseName())
+ handle.getNamespace();
PropertyProto.BlobHandleProto blobHandleProto =
BlobHandleToProtoConverter.toBlobHandleProto(handle);
// We are using namespace to check blob's visibility.
if (!VisibilityUtil.isSchemaSearchableByCaller(
access,
handle.getPackageName(),
prefixedNamespace,
mBlobVisibilityStoreLocked,
mVisibilityCheckerLocked)) {
// Caller doesn't have access to this namespace.
throw new AppSearchException(
AppSearchResult.RESULT_NOT_FOUND,
"Cannot find the blob for handle: "
+ blobHandleProto.getDigest().toStringUtf8());
}
BlobProto result = mIcingSearchEngineLocked.openReadBlob(blobHandleProto);
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(result.getGetVmLatencyMs());
}
ParcelFileDescriptor pfd =
retrieveFileDescriptorLocked(result, ParcelFileDescriptor.MODE_READ_ONLY);
// We do NOT need to look up the revocable file descriptor for read, skip passing the
// blob handle key.
return mRevocableFileDescriptorStore.wrapToRevocableFileDescriptor(
access.getCallingPackageName(),
/* blobHandle= */ null,
pfd,
ParcelFileDescriptor.MODE_READ_ONLY);
} finally {
logReadOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_GLOBAL_OPEN_READ_BLOB,
callStatsBuilder);
mReadWriteLock.readLock().unlock();
}
}
/**
* Updates the visibility configuration for a specified namespace within a blob storage.
*
*
This method configures the visibility blob namespaces in the given specific database.
*
*
After applying the new visibility configurations, the method identifies and removes any
* existing visibility settings that do not included in the new visibility configurations from
* the visibility store.
*
* @param packageName The package name that owns these blobs.
* @param databaseName The databaseName these blobs resides in.
* @param visibilityConfigs a list of {@link InternalVisibilityConfig} objects representing the
* visibility configurations to be set for the specified namespace.
* @throws AppSearchException if an error occurs while updating the visibility configurations.
* This could happen if the database is closed or in an invalid state.
*/
public void setBlobNamespaceVisibility(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull List visibilityConfigs,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.writeLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastWriteOperationLatencyMillisLocked);
}
if (mBlobVisibilityStoreLocked != null) {
String prefix = createPrefix(packageName, databaseName);
Set removedVisibilityConfigs =
mNamespaceCacheLocked.getPrefixedBlobNamespaces(prefix);
if (removedVisibilityConfigs == null) {
removedVisibilityConfigs = new ArraySet<>();
} else {
// wrap it to allow rewriteVisibilityConfigs modify it.
removedVisibilityConfigs = new ArraySet<>(removedVisibilityConfigs);
}
List prefixedVisibilityConfigs =
rewriteVisibilityConfigs(
prefix, visibilityConfigs, removedVisibilityConfigs);
for (int i = 0; i < prefixedVisibilityConfigs.size(); i++) {
// We are using schema type to represent blob's namespace in
// InternalVisibilityConfig.
mNamespaceCacheLocked.addToBlobNamespaceMap(
prefix, prefixedVisibilityConfigs.get(i).getSchemaType());
}
// Now removedVisibilityConfigs contains those existing schemas that has
// all-default visibility settings, add deleted schemas. That's all we need to
// remove.
mBlobVisibilityStoreLocked.setVisibility(
prefixedVisibilityConfigs, callStatsBuilder);
mBlobVisibilityStoreLocked.removeVisibility(
removedVisibilityConfigs, callStatsBuilder);
} else {
throw new UnsupportedOperationException(
"BLOB_STORAGE is not available on this AppSearch implementation.");
}
} finally {
logWriteOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_SET_BLOB_VISIBILITY,
callStatsBuilder);
mReadWriteLock.writeLock().unlock();
}
}
/**
* Retrieves the {@link ParcelFileDescriptor} from a {@link BlobProto}.
*
* This method handles retrieving the actual file descriptor from the provided {@link
* BlobProto}, taking into account whether AppSearch manages blob files directly. If AppSearch
* manages blob files ({@code Flags.enableAppSearchManageBlobFiles()} is true), it opens the
* file using the file name from the {@link BlobProto}. Otherwise, it retrieves the file
* descriptor directly from the {@link BlobProto}.
*
* @return The {@link ParcelFileDescriptor} for the blob.
* @throws AppSearchException if the {@link BlobProto}'s status indicates an error.
* @throws IOException if there is an error opening the file, such as the file not being found.
*/
@GuardedBy("mReadWriteLock")
private ParcelFileDescriptor retrieveFileDescriptorLocked(BlobProto blobProto, int mode)
throws AppSearchException, IOException {
checkSuccess(blobProto.getStatus());
if (Flags.enableAppSearchManageBlobFiles()) {
File blobFile = new File(mBlobFilesDir, blobProto.getFileName());
return ParcelFileDescriptor.open(blobFile, mode);
} else {
return ParcelFileDescriptor.adoptFd(blobProto.getFileDescriptor());
}
}
/**
* Checks that a new document can be added to the given packageName with the given serialized
* size without violating our {@link LimitConfig}.
*
* @throws AppSearchException with a code of {@link AppSearchResult#RESULT_OUT_OF_SPACE} if the
* limits are violated by the new document.
*/
@GuardedBy("mReadWriteLock")
private void enforceLimitConfigLocked(
String packageName,
String newDocUri,
int newDocSize,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
// Limits check: size of document
if (newDocSize > mConfig.getMaxDocumentSizeBytes()) {
throw new AppSearchException(
AppSearchResult.RESULT_OUT_OF_SPACE,
"Document \""
+ newDocUri
+ "\" for package \""
+ packageName
+ "\" serialized to "
+ newDocSize
+ " bytes, which exceeds "
+ "limit of "
+ mConfig.getMaxDocumentSizeBytes()
+ " bytes");
}
mDocumentLimiterLocked.enforceDocumentCountLimit(
packageName,
() ->
getRawStorageInfoProto(callStatsBuilder)
.getDocumentStorageInfo()
.getNamespaceStorageInfoList());
}
/**
* Retrieves a document from the AppSearch index by namespace and document ID from any
* application the caller is allowed to view
*
*
This method will handle both Icing engine errors as well as permission errors by throwing
* an obfuscated RESULT_NOT_FOUND exception. This is done so the caller doesn't receive
* information on whether or not a file they are not allowed to access exists or not. This is
* different from the behavior of {@link #getDocument}.
*
* @param packageName The package that owns this document.
* @param databaseName The databaseName this document resides in.
* @param namespace The namespace this document resides in.
* @param id The ID of the document to get.
* @param typePropertyPaths A map of schema type to a list of property paths to return in the
* result.
* @param callerAccess Visibility access info of the calling app
* @return The Document contents
* @throws AppSearchException on IcingSearchEngine error or invalid permissions
*/
public @NonNull GenericDocument globalGetDocument(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull String namespace,
@NonNull String id,
@NonNull Map> typePropertyPaths,
@NonNull CallerAccess callerAccess,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.readLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastWriteOperationLatencyMillisLocked);
}
// We retrieve the document before checking for access, as we do not know which
// schema the document is under. Schema is required for checking access
DocumentProto documentProto;
try {
documentProto =
getDocumentProtoByIdLocked(
packageName,
databaseName,
namespace,
id,
typePropertyPaths,
callStatsBuilder);
if (!VisibilityUtil.isSchemaSearchableByCaller(
callerAccess,
packageName,
documentProto.getSchema(),
mDocumentVisibilityStoreLocked,
mVisibilityCheckerLocked)) {
throw new AppSearchException(AppSearchResult.RESULT_NOT_FOUND);
}
} catch (AppSearchException e) {
// Not passing cause in AppSearchException as that violates privacy guarantees as
// user could differentiate between document not existing and not having access.
throw new AppSearchException(
AppSearchResult.RESULT_NOT_FOUND,
"Document (" + namespace + ", " + id + ") not found.");
}
DocumentProto.Builder documentBuilder = documentProto.toBuilder();
removePrefixesFromDocument(documentBuilder);
String prefix = createPrefix(packageName, databaseName);
return GenericDocumentToProtoConverter.toGenericDocument(
documentBuilder.build(), prefix, mSchemaCacheLocked, mConfig);
} finally {
logReadOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_GLOBAL_GET_DOCUMENT_BY_ID,
callStatsBuilder);
mReadWriteLock.readLock().unlock();
}
}
/**
* Retrieves a document from the AppSearch index by namespace and document ID.
*
* This method belongs to query group.
*
* @param packageName The package that owns this document.
* @param databaseName The databaseName this document resides in.
* @param namespace The namespace this document resides in.
* @param id The ID of the document to get.
* @param typePropertyPaths A map of schema type to a list of property paths to return in the
* result.
* @return The Document contents
* @throws AppSearchException on IcingSearchEngine error.
*/
public @NonNull GenericDocument getDocument(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull String namespace,
@NonNull String id,
@NonNull Map> typePropertyPaths,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.readLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastWriteOperationLatencyMillisLocked);
}
DocumentProto documentProto =
getDocumentProtoByIdLocked(
packageName,
databaseName,
namespace,
id,
typePropertyPaths,
callStatsBuilder);
DocumentProto.Builder documentBuilder = documentProto.toBuilder();
removePrefixesFromDocument(documentBuilder);
String prefix = createPrefix(packageName, databaseName);
// The schema type map cannot be null at this point. It could only be null if no
// schema had ever been set for that prefix. Given we have retrieved a document from
// the index, we know a schema had to have been set.
return GenericDocumentToProtoConverter.toGenericDocument(
documentBuilder.build(), prefix, mSchemaCacheLocked, mConfig);
} finally {
logReadOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_GET_DOCUMENT,
callStatsBuilder);
mReadWriteLock.readLock().unlock();
}
}
/**
* Retrieves a list document from the AppSearch index by namespace and document ID.
*
* This method belongs to query group.
*
* @param packageName The package that owns this document.
* @param databaseName The databaseName this document resides in.
* @param request The request configuration for BatchGet.
* @param callerAccess The information about caller. Visibility will be checked if it is not
* NULL. This indicates it is a global get call.
* @return The Document contents in a {@link AppSearchBatchResult}.
*/
public @NonNull AppSearchBatchResult batchGetDocuments(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull GetByDocumentIdRequest request,
@Nullable CallerAccess callerAccess,
CallStats.@Nullable Builder callStatsBuilder) {
AppSearchBatchResult.Builder resultBuilder =
new AppSearchBatchResult.Builder<>();
// If the id list is empty, we can just return directly.
if (request.getIds().isEmpty()) {
return resultBuilder.build();
}
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.readLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastWriteOperationLatencyMillisLocked);
}
BatchGetResultProto batchGetResultProto =
batchGetDocumentProtoByIdLocked(
packageName, databaseName, request, callStatsBuilder);
for (int i = 0; i < batchGetResultProto.getGetResultProtosCount(); ++i) {
GetResultProto getResultProto = batchGetResultProto.getGetResultProtos(i);
String id = getResultProto.getUri();
try {
checkSuccess(getResultProto.getStatus());
// Check if the schema is visible to the caller. This is only done if
// callerAccess is not null.
// TODO(b/404643381) We can cache the results and use those if we have seen
// the same schema before.
if (callerAccess != null
&& !VisibilityUtil.isSchemaSearchableByCaller(
callerAccess,
packageName,
getResultProto.getDocument().getSchema(),
mDocumentVisibilityStoreLocked,
mVisibilityCheckerLocked)) {
throw new AppSearchException(AppSearchResult.RESULT_NOT_FOUND);
}
DocumentProto.Builder documentBuilder =
getResultProto.getDocument().toBuilder();
removePrefixesFromDocument(documentBuilder);
String prefix = createPrefix(packageName, databaseName);
// The schema type map cannot be null at this point. It could only be null if no
// schema had ever been set for that prefix. Given we have retrieved a document
// from the index, we know a schema had to have been set.
GenericDocument doc =
GenericDocumentToProtoConverter.toGenericDocument(
documentBuilder.build(), prefix, mSchemaCacheLocked, mConfig);
resultBuilder.setSuccess(id, doc);
} catch (Throwable t) {
// Global get
if (callerAccess != null) {
// Not passing cause in AppSearchException as that violates privacy
// guarantees as user could differentiate between document not existing
// and not having access.
resultBuilder.setResult(
id,
new AppSearchException(
AppSearchResult.RESULT_NOT_FOUND,
"Document ("
+ request.getNamespace()
+ ", "
+ id
+ ") not found.")
.toAppSearchResult());
} else {
resultBuilder.setResult(id, throwableToFailedResult(t));
}
}
}
return resultBuilder.build();
} finally {
logReadOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_GET_DOCUMENTS,
callStatsBuilder);
mReadWriteLock.readLock().unlock();
}
}
private GetResultSpecProto createGetResultSpecProto(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull String namespace,
@NonNull Map> typePropertyPaths,
@Nullable Set ids) {
String prefix = createPrefix(packageName, databaseName);
List nonPrefixedPropertyMaskBuilders =
TypePropertyPathToProtoConverter.toTypePropertyMaskBuilderList(typePropertyPaths);
List prefixedPropertyMasks =
new ArrayList<>(nonPrefixedPropertyMaskBuilders.size());
for (int i = 0; i < nonPrefixedPropertyMaskBuilders.size(); ++i) {
String nonPrefixedType = nonPrefixedPropertyMaskBuilders.get(i).getSchemaType();
String prefixedType = nonPrefixedType;
if (!nonPrefixedType.equals(GetByDocumentIdRequest.PROJECTION_SCHEMA_TYPE_WILDCARD)) {
// Append prefix if it is not a wildcard.
prefixedType = prefix + nonPrefixedType;
}
prefixedPropertyMasks.add(
nonPrefixedPropertyMaskBuilders.get(i).setSchemaType(prefixedType).build());
}
GetResultSpecProto.Builder resultSpecProtoBuilder =
GetResultSpecProto.newBuilder()
.setNamespaceRequested(namespace)
.addAllTypePropertyMasks(prefixedPropertyMasks);
// For old getDocumentProtoByIdLocked, we don't need to set the ids in the request.
// So we don't pass the ids in from there.
if (ids != null && !ids.isEmpty()) {
resultSpecProtoBuilder.addAllIds(ids);
}
return resultSpecProtoBuilder.build();
}
/**
* Returns a DocumentProto from Icing.
*
* @param packageName The package that owns this document.
* @param databaseName The databaseName this document resides in.
* @param namespace The namespace this document resides in.
* @param id The ID of the document to get.
* @param typePropertyPaths A map of schema type to a list of property paths to return in the
* result.
* @return the DocumentProto object
* @throws AppSearchException on IcingSearchEngine error
*/
@GuardedBy("mReadWriteLock")
// We only log getResultProto.toString() in fullPii trace for debugging.
@SuppressWarnings("LiteProtoToString")
private @NonNull DocumentProto getDocumentProtoByIdLocked(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull String namespace,
@NonNull String id,
@NonNull Map> typePropertyPaths,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
String finalNamespace = createPrefix(packageName, databaseName) + namespace;
GetResultSpecProto getResultSpec =
createGetResultSpecProto(
packageName,
databaseName,
finalNamespace,
typePropertyPaths,
/* ids= */ null);
if (LogUtil.isPiiTraceEnabled()) {
LogUtil.piiTrace(
TAG, "getDocument, request", finalNamespace + ", " + id + "," + getResultSpec);
}
GetResultProto getResultProto =
mIcingSearchEngineLocked.get(finalNamespace, id, getResultSpec);
LogUtil.piiTrace(TAG, "getDocument, response", getResultProto.getStatus(), getResultProto);
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(getResultProto.getGetVmLatencyMs());
}
checkSuccess(getResultProto.getStatus());
return getResultProto.getDocument();
}
/*
* Returns a BatchGetResultProto from Icing. It contains GetResultProto for each id.
*/
@GuardedBy("mReadWriteLock")
@SuppressWarnings("LiteProtoToString")
private @NonNull BatchGetResultProto batchGetDocumentProtoByIdLocked(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull GetByDocumentIdRequest request,
CallStats.@Nullable Builder callStatsBuilder) {
String finalNamespace = createPrefix(packageName, databaseName) + request.getNamespace();
GetResultSpecProto getResultSpec =
createGetResultSpecProto(
packageName,
databaseName,
finalNamespace,
request.getProjections(),
request.getIds());
LogUtil.piiTrace(TAG, "getDocument, request", getResultSpec);
BatchGetResultProto batchGetResultProto = mIcingSearchEngineLocked.batchGet(getResultSpec);
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(batchGetResultProto.getGetVmLatencyMs());
}
LogUtil.piiTrace(
TAG, "getDocument, response", batchGetResultProto.getStatus(), batchGetResultProto);
return batchGetResultProto;
}
/**
* Executes a query against the AppSearch index and returns results.
*
* This method belongs to query group.
*
* @param packageName The package name that is performing the query.
* @param databaseName The databaseName this query for.
* @param queryExpression Query String to search.
* @param searchSpec Spec for setting filters, raw query etc.
* @param logger logger to collect query stats
* @return The results of performing this search. It may contain an empty list of results if no
* documents matched the query.
* @throws AppSearchException on IcingSearchEngine error.
*/
public @NonNull SearchResultPage query(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull String queryExpression,
@NonNull SearchSpec searchSpec,
@Nullable AppSearchLogger logger,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
QueryStats.Builder sStatsBuilder = null;
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.readLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
if (logger != null) {
sStatsBuilder =
new QueryStats.Builder(QueryStats.VISIBILITY_SCOPE_LOCAL, packageName)
.setDatabase(databaseName)
.setSearchSourceLogTag(searchSpec.getSearchSourceLogTag())
.setLaunchVMEnabled(mIsVMEnabled)
.setLastBlockingOperation(mLastWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastWriteOperationLatencyMillisLocked)
.setJavaLockAcquisitionLatencyMillis(
(int)
(javaLockAcquisitionEndTimeMillis
- totalLatencyStartMillis));
}
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastWriteOperationLatencyMillisLocked);
}
throwIfClosedLocked();
List filterPackageNames = searchSpec.getFilterPackageNames();
if (!filterPackageNames.isEmpty() && !filterPackageNames.contains(packageName)) {
// Client wanted to query over some packages that weren't its own. This isn't
// allowed through local query so we can return early with no results.
if (sStatsBuilder != null && logger != null) {
sStatsBuilder.setStatusCode(RESULT_SECURITY_ERROR);
}
return new SearchResultPage();
}
String prefix = createPrefix(packageName, databaseName);
SearchSpecToProtoConverter searchSpecToProtoConverter =
new SearchSpecToProtoConverter(
queryExpression,
searchSpec,
Collections.singleton(prefix),
mNamespaceCacheLocked,
mSchemaCacheLocked,
mConfig);
if (searchSpecToProtoConverter.hasNothingToSearch()) {
// there is nothing to search over given their search filters, so we can return an
// empty SearchResult and skip sending request to Icing.
return new SearchResultPage();
}
SearchResultPage searchResultPage =
doQueryLocked(searchSpecToProtoConverter, sStatsBuilder, callStatsBuilder);
addNextPageToken(packageName, searchResultPage.getNextPageToken());
return searchResultPage;
} finally {
logReadOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_SEARCH,
callStatsBuilder);
mReadWriteLock.readLock().unlock();
if (sStatsBuilder != null && logger != null) {
sStatsBuilder.setTotalLatencyMillis(
(int) (SystemClock.elapsedRealtime() - totalLatencyStartMillis));
logger.logStats(sStatsBuilder.build());
}
}
}
/**
* Executes a global query, i.e. over all permitted prefixes, against the AppSearch index and
* returns results.
*
* This method belongs to query group.
*
* @param queryExpression Query String to search.
* @param searchSpec Spec for setting filters, raw query etc.
* @param callerAccess Visibility access info of the calling app
* @param logger logger to collect globalQuery stats
* @return The results of performing this search. It may contain an empty list of results if no
* documents matched the query.
* @throws AppSearchException on IcingSearchEngine error.
*/
public @NonNull SearchResultPage globalQuery(
@NonNull String queryExpression,
@NonNull SearchSpec searchSpec,
@NonNull CallerAccess callerAccess,
@Nullable AppSearchLogger logger,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
QueryStats.Builder sStatsBuilder = null;
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.readLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
if (logger != null) {
sStatsBuilder =
new QueryStats.Builder(
QueryStats.VISIBILITY_SCOPE_GLOBAL,
callerAccess.getCallingPackageName())
.setSearchSourceLogTag(searchSpec.getSearchSourceLogTag())
.setLaunchVMEnabled(mIsVMEnabled)
.setLastBlockingOperation(mLastWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastWriteOperationLatencyMillisLocked)
.setJavaLockAcquisitionLatencyMillis(
(int)
(javaLockAcquisitionEndTimeMillis
- totalLatencyStartMillis));
}
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastWriteOperationLatencyMillisLocked);
}
throwIfClosedLocked();
long aclLatencyStartMillis = SystemClock.elapsedRealtime();
// The two scenarios where we want to limit package filters are if the outer
// SearchSpec has package filters and there is no JoinSpec, or if both outer and
// nested SearchSpecs have package filters. If outer SearchSpec has no package
// filters or the nested SearchSpec has no package filters, then we pass the key set of
// documentNamespace map of mNamespaceCachedLocked to the SearchSpecToProtoConverter,
// signifying that there is a SearchSpec that wants to query every visible package.
Set packageFilters = new ArraySet<>();
if (!searchSpec.getFilterPackageNames().isEmpty()) {
JoinSpec joinSpec = searchSpec.getJoinSpec();
if (joinSpec == null) {
packageFilters.addAll(searchSpec.getFilterPackageNames());
} else if (!joinSpec.getNestedSearchSpec().getFilterPackageNames().isEmpty()) {
packageFilters.addAll(searchSpec.getFilterPackageNames());
packageFilters.addAll(joinSpec.getNestedSearchSpec().getFilterPackageNames());
}
}
// Convert package filters to prefix filters
Set prefixFilters = new ArraySet<>();
if (packageFilters.isEmpty()) {
// Client didn't restrict their search over packages. Try to query over all
// packages/prefixes
prefixFilters = mNamespaceCacheLocked.getAllDocumentPrefixes();
} else {
// Client did restrict their search over packages. Only include the prefixes that
// belong to the specified packages.
for (String prefix : mNamespaceCacheLocked.getAllDocumentPrefixes()) {
String packageName = getPackageName(prefix);
if (packageFilters.contains(packageName)) {
prefixFilters.add(prefix);
}
}
}
SearchSpecToProtoConverter searchSpecToProtoConverter =
new SearchSpecToProtoConverter(
queryExpression,
searchSpec,
prefixFilters,
mNamespaceCacheLocked,
mSchemaCacheLocked,
mConfig);
// Remove those inaccessible schemas.
searchSpecToProtoConverter.removeInaccessibleSchemaFilter(
callerAccess, mDocumentVisibilityStoreLocked, mVisibilityCheckerLocked);
if (searchSpecToProtoConverter.hasNothingToSearch()) {
// there is nothing to search over given their search filters, so we can return an
// empty SearchResult and skip sending request to Icing.
return new SearchResultPage();
}
if (sStatsBuilder != null) {
sStatsBuilder.setAclCheckLatencyMillis(
(int) (SystemClock.elapsedRealtime() - aclLatencyStartMillis));
}
SearchResultPage searchResultPage =
doQueryLocked(searchSpecToProtoConverter, sStatsBuilder, callStatsBuilder);
addNextPageToken(
callerAccess.getCallingPackageName(), searchResultPage.getNextPageToken());
return searchResultPage;
} finally {
logReadOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_GLOBAL_SEARCH,
callStatsBuilder);
mReadWriteLock.readLock().unlock();
if (sStatsBuilder != null && logger != null) {
sStatsBuilder.setTotalLatencyMillis(
(int) (SystemClock.elapsedRealtime() - totalLatencyStartMillis));
logger.logStats(sStatsBuilder.build());
}
}
}
@GuardedBy("mReadWriteLock")
private SearchResultPage doQueryLocked(
@NonNull SearchSpecToProtoConverter searchSpecToProtoConverter,
QueryStats.@Nullable Builder sStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
// Rewrite the given SearchSpec into SearchSpecProto, ResultSpecProto and ScoringSpecProto.
// All processes are counted in rewriteSearchSpecLatencyMillis
long rewriteSearchSpecLatencyStartMillis = SystemClock.elapsedRealtime();
SearchSpecProto finalSearchSpec =
searchSpecToProtoConverter.toSearchSpecProto(mIsVMEnabled);
ResultSpecProto finalResultSpec =
searchSpecToProtoConverter.toResultSpecProto(
mNamespaceCacheLocked, mSchemaCacheLocked, mIsVMEnabled);
ScoringSpecProto scoringSpec = searchSpecToProtoConverter.toScoringSpecProto();
if (sStatsBuilder != null) {
sStatsBuilder.setRewriteSearchSpecLatencyMillis(
(int) (SystemClock.elapsedRealtime() - rewriteSearchSpecLatencyStartMillis));
}
// Send request to Icing.
SearchResultProto searchResultProto =
searchInIcingLocked(
finalSearchSpec,
finalResultSpec,
scoringSpec,
sStatsBuilder,
callStatsBuilder);
long rewriteSearchResultLatencyStartMillis = SystemClock.elapsedRealtime();
// Rewrite search result before we return.
SearchResultPage searchResultPage =
SearchResultToProtoConverter.toSearchResultPage(
searchResultProto, mSchemaCacheLocked, mConfig);
if (sStatsBuilder != null) {
sStatsBuilder
.setRewriteSearchResultLatencyMillis(
(int)
(SystemClock.elapsedRealtime()
- rewriteSearchResultLatencyStartMillis))
.addGetVmLatencyMillis(searchResultProto.getGetVmLatencyMs());
}
return searchResultPage;
}
@GuardedBy("mReadWriteLock")
// We only log searchSpec, scoringSpec and resultSpec in fullPii trace for debugging.
@SuppressWarnings("LiteProtoToString")
private SearchResultProto searchInIcingLocked(
@NonNull SearchSpecProto searchSpec,
@NonNull ResultSpecProto resultSpec,
@NonNull ScoringSpecProto scoringSpec,
QueryStats.@Nullable Builder queryStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
if (LogUtil.isPiiTraceEnabled()) {
LogUtil.piiTrace(
TAG,
"search, request",
searchSpec.getQuery(),
searchSpec + ", " + scoringSpec + ", " + resultSpec);
}
SearchResultProto searchResultProto =
mIcingSearchEngineLocked.search(searchSpec, scoringSpec, resultSpec);
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(searchResultProto.getGetVmLatencyMs());
}
LogUtil.piiTrace(
TAG, "search, response", searchResultProto.getResultsCount(), searchResultProto);
checkSuccess(searchResultProto.getStatus());
if (queryStatsBuilder != null) {
queryStatsBuilder.setFirstNativeCallLatency(
searchResultProto.getQueryStats().getLatencyMs());
}
long nextPageToken = searchResultProto.getNextPageToken();
if (nextPageToken != SearchResultPage.EMPTY_PAGE_TOKEN
&& searchResultProto.getResultsCount() > 0
&& searchResultProto.getResultsCount() < resultSpec.getNumPerPage()) {
// Did not get a full page of results in the initial search. Do getNextPage until we
// get a full result page or we run out of results.
SearchResultProto.Builder finalSearchResultProtoBuilder =
SearchResultProto.newBuilder(searchResultProto);
retrieveMoreResultsLocked(
nextPageToken,
/* remainingResultCount= */ resultSpec.getNumPerPage()
- searchResultProto.getResultsCount(),
finalSearchResultProtoBuilder,
queryStatsBuilder,
callStatsBuilder);
searchResultProto = finalSearchResultProtoBuilder.build();
}
if (queryStatsBuilder != null) {
queryStatsBuilder.setStatusCode(statusProtoToResultCode(searchResultProto.getStatus()));
if (searchSpec.hasJoinSpec()) {
queryStatsBuilder.setJoinType(
AppSearchSchema.StringPropertyConfig.JOINABLE_VALUE_TYPE_QUALIFIED_ID);
}
AppSearchLoggerHelper.copyNativeStats(
searchResultProto.getQueryStats(), queryStatsBuilder);
}
checkSuccess(searchResultProto.getStatus());
return searchResultProto;
}
/**
* Calls native getNextPage to retrieve more results until remaining result count is 0 or there
* are no more results left for the query. The results retrieved are added into {@code
* searchResultProtoBuilder}.
*/
private void retrieveMoreResultsLocked(
long nextPageToken,
int remainingResultCount,
SearchResultProto.@NonNull Builder searchResultProtoBuilder,
QueryStats.@Nullable Builder queryStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
int numAdditionalPages = 0;
int totalAdditionalResults = 0;
long additionalPageRetrievalLatencyStartMillis = SystemClock.elapsedRealtime();
while (nextPageToken != SearchResultPage.EMPTY_PAGE_TOKEN && remainingResultCount > 0) {
GetNextPageRequestProto getNextPageRequest =
GetNextPageRequestProto.newBuilder()
.setNextPageToken(nextPageToken)
.setMaxResultsToRetrieveFromPage(remainingResultCount)
.build();
LogUtil.piiTrace(TAG, "getNextPage, request", getNextPageRequest);
SearchResultProto nextResultPageProto =
mIcingSearchEngineLocked.getNextPage(getNextPageRequest);
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(nextResultPageProto.getGetVmLatencyMs());
}
LogUtil.piiTrace(
TAG,
"getNextPage, response",
nextResultPageProto.getResultsCount(),
nextResultPageProto);
checkSuccess(nextResultPageProto.getStatus());
++numAdditionalPages;
nextPageToken = nextResultPageProto.getNextPageToken();
mergeSearchResultProtos(nextResultPageProto, searchResultProtoBuilder);
if (nextResultPageProto.getResultsCount() == 0) {
Log.e(
TAG,
"Got additional page with 0 results during search. This should"
+ " never happen normally. GetNextPage status code: "
+ nextResultPageProto.getStatus().getCode());
break;
}
totalAdditionalResults += nextResultPageProto.getResultsCount();
remainingResultCount -= nextResultPageProto.getResultsCount();
}
if (queryStatsBuilder != null) {
// TODO(b/421230879): Restructure QueryStats to record full stats for GetNextPage calls.
queryStatsBuilder.setAdditionalPageCount(numAdditionalPages);
queryStatsBuilder.setAdditionalPagesReturnedResultCount(totalAdditionalResults);
queryStatsBuilder.setAdditionalPageRetrievalLatencyMillis(
(int)
(SystemClock.elapsedRealtime()
- additionalPageRetrievalLatencyStartMillis));
}
}
/**
* Merges the results from {@code searchResultProto} into {@code finalSearchResultProtoBuilder}.
*/
private static void mergeSearchResultProtos(
@NonNull SearchResultProto searchResultProto,
SearchResultProto.@NonNull Builder finalSearchResultProtoBuilder) {
QueryStatsProto previousStats = finalSearchResultProtoBuilder.getQueryStats();
QueryStatsProto newStats = searchResultProto.getQueryStats();
QueryStatsProto mergedStats =
QueryStatsProto.newBuilder(previousStats)
.setNumResultsReturnedCurrentPage(
finalSearchResultProtoBuilder.getResultsCount()
+ searchResultProto.getResultsCount())
.setNumResultsWithSnippets(
previousStats.getNumResultsWithSnippets()
+ newStats.getNumResultsWithSnippets())
.setLatencyMs(previousStats.getLatencyMs() + newStats.getLatencyMs())
.setRankingLatencyMs(
previousStats.getRankingLatencyMs()
+ newStats.getRankingLatencyMs())
.setDocumentRetrievalLatencyMs(
previousStats.getDocumentRetrievalLatencyMs()
+ newStats.getDocumentRetrievalLatencyMs())
.setLockAcquisitionLatencyMs(
previousStats.getLockAcquisitionLatencyMs()
+ newStats.getLockAcquisitionLatencyMs())
.setNativeToJavaJniLatencyMs(
previousStats.getNativeToJavaJniLatencyMs()
+ newStats.getNativeToJavaJniLatencyMs())
.setJavaToNativeJniLatencyMs(
previousStats.getJavaToNativeJniLatencyMs()
+ newStats.getJavaToNativeJniLatencyMs())
.setJoinLatencyMs(
previousStats.getJoinLatencyMs() + newStats.getJoinLatencyMs())
.setNumJoinedResultsReturnedCurrentPage(
previousStats.getNumJoinedResultsReturnedCurrentPage()
+ newStats.getNumJoinedResultsReturnedCurrentPage())
.setPageTokenType(newStats.getPageTokenType())
.setNumResultStatesEvicted(
previousStats.getNumResultStatesEvicted()
+ newStats.getNumResultStatesEvicted())
.build();
finalSearchResultProtoBuilder
.setStatus(searchResultProto.getStatus())
.addAllResults(searchResultProto.getResultsList())
.setNextPageToken(searchResultProto.getNextPageToken())
.setPageTokenNotFound(searchResultProto.getPageTokenNotFound())
.setQueryStats(mergedStats);
}
/**
* Generates suggestions based on the given search prefix.
*
* This method belongs to query group.
*
* @param packageName The package name that is performing the query.
* @param databaseName The databaseName this query for.
* @param suggestionQueryExpression The non-empty query expression used to be completed.
* @param searchSuggestionSpec Spec for setting filters.
* @return a List of {@link SearchSuggestionResult}. The returned {@link SearchSuggestionResult}
* are order by the number of {@link SearchResult} you could get by using that suggestion in
* {@link #query}.
* @throws AppSearchException if the suggestionQueryExpression is empty.
*/
public @NonNull List searchSuggestion(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull String suggestionQueryExpression,
@NonNull SearchSuggestionSpec searchSuggestionSpec,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.readLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastWriteOperationLatencyMillisLocked);
}
if (suggestionQueryExpression.isEmpty()) {
throw new AppSearchException(
AppSearchResult.RESULT_INVALID_ARGUMENT,
"suggestionQueryExpression cannot be empty.");
}
if (searchSuggestionSpec.getMaximumResultCount() > mConfig.getMaxSuggestionCount()) {
throw new AppSearchException(
AppSearchResult.RESULT_INVALID_ARGUMENT,
"Trying to get "
+ searchSuggestionSpec.getMaximumResultCount()
+ " suggestion results, which exceeds limit of "
+ mConfig.getMaxSuggestionCount());
}
String prefix = createPrefix(packageName, databaseName);
SearchSuggestionSpecToProtoConverter searchSuggestionSpecToProtoConverter =
new SearchSuggestionSpecToProtoConverter(
suggestionQueryExpression,
searchSuggestionSpec,
Collections.singleton(prefix),
mNamespaceCacheLocked,
mSchemaCacheLocked);
if (searchSuggestionSpecToProtoConverter.hasNothingToSearch()) {
// there is nothing to search over given their search filters, so we can return an
// empty SearchResult and skip sending request to Icing.
return new ArrayList<>();
}
SuggestionResponse response =
mIcingSearchEngineLocked.searchSuggestions(
searchSuggestionSpecToProtoConverter.toSearchSuggestionSpecProto());
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(response.getGetVmLatencyMs());
}
checkSuccess(response.getStatus());
List suggestions =
new ArrayList<>(response.getSuggestionsCount());
for (int i = 0; i < response.getSuggestionsCount(); i++) {
suggestions.add(
new SearchSuggestionResult.Builder()
.setSuggestedResult(response.getSuggestions(i).getQuery())
.build());
}
return suggestions;
} finally {
logReadOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_SEARCH_SUGGESTION,
callStatsBuilder);
mReadWriteLock.readLock().unlock();
}
}
/**
* Returns a mapping of package names to all the databases owned by that package.
*
* This method is inefficient to call repeatedly.
*/
public @NonNull Map> getPackageToDatabases() {
mReadWriteLock.readLock().lock();
try {
Map> packageToDatabases = new ArrayMap<>();
for (String prefix : mSchemaCacheLocked.getAllPrefixes()) {
String packageName = getPackageName(prefix);
Set databases = packageToDatabases.get(packageName);
if (databases == null) {
databases = new ArraySet<>();
packageToDatabases.put(packageName, databases);
}
String databaseName = getDatabaseName(prefix);
databases.add(databaseName);
}
return packageToDatabases;
} finally {
mReadWriteLock.readLock().unlock();
}
}
/**
* Fetches the next page of results of a previously executed query. Results can be empty if
* next-page token is invalid or all pages have been returned.
*
* This method belongs to query group.
*
* @param packageName Package name of the caller.
* @param nextPageToken The token of pre-loaded results of previously executed query.
* @return The next page of results of previously executed query.
* @throws AppSearchException on IcingSearchEngine error or if can't advance on nextPageToken.
*/
public @NonNull SearchResultPage getNextPage(
@NonNull String packageName,
long nextPageToken,
QueryStats.@Nullable Builder queryStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.readLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastWriteOperationLatencyMillisLocked);
}
if (queryStatsBuilder != null) {
queryStatsBuilder
.setJavaLockAcquisitionLatencyMillis(
(int) (javaLockAcquisitionEndTimeMillis - totalLatencyStartMillis))
.setLaunchVMEnabled(mIsVMEnabled)
.setLastBlockingOperation(mLastWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastWriteOperationLatencyMillisLocked);
}
LogUtil.piiTrace(TAG, "getNextPage, request", nextPageToken);
checkNextPageToken(packageName, nextPageToken);
SearchResultProto searchResultProto;
if (nextPageToken != SearchResultPage.EMPTY_PAGE_TOKEN) {
searchResultProto = mIcingSearchEngineLocked.getNextPage(nextPageToken);
} else {
// If it is an empty page token, then avoid sending it to Icing to save a JNI call.
searchResultProto =
SearchResultProto.newBuilder()
.setStatus(
StatusProto.newBuilder()
.setCode(StatusProto.Code.OK)
.build())
.setNextPageToken(SearchResultPage.EMPTY_PAGE_TOKEN)
.build();
}
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(searchResultProto.getGetVmLatencyMs());
}
LogUtil.piiTrace(
TAG,
"getNextPage, response",
searchResultProto.getResultsCount(),
searchResultProto);
checkSuccess(searchResultProto.getStatus());
int remainingResultCount =
searchResultProto.getQueryStats().getRequestedPageSize()
- searchResultProto.getResultsCount();
if (nextPageToken != SearchResultPage.EMPTY_PAGE_TOKEN
&& searchResultProto.getResultsCount() > 0
&& remainingResultCount > 0) {
SearchResultProto.Builder finalSearchResultsBuilder =
SearchResultProto.newBuilder(searchResultProto);
// Did not get a full page of results during the initial getNextPage. Do more
// getNextPage calls until we get a full result page or we run out of results.
retrieveMoreResultsLocked(
searchResultProto.getNextPageToken(),
remainingResultCount,
finalSearchResultsBuilder,
queryStatsBuilder,
callStatsBuilder);
searchResultProto = finalSearchResultsBuilder.build();
}
if (queryStatsBuilder != null) {
queryStatsBuilder.setStatusCode(
statusProtoToResultCode(searchResultProto.getStatus()));
// Join query stats are handled by SearchResultsImpl, which has access to the
// original SearchSpec.
if (nextPageToken != SearchResultPage.EMPTY_PAGE_TOKEN) {
AppSearchLoggerHelper.copyNativeStats(
searchResultProto.getQueryStats(), queryStatsBuilder);
}
}
if (nextPageToken != SearchResultPage.EMPTY_PAGE_TOKEN
&& searchResultProto.getNextPageToken() == SearchResultPage.EMPTY_PAGE_TOKEN) {
// At this point, we're guaranteed that this nextPageToken exists for this package,
// otherwise checkNextPageToken would've thrown an exception.
// Since the new token is 0, this is the last page. We should remove the old token
// from our cache since it no longer refers to this query.
synchronized (mNextPageTokensLocked) {
Set nextPageTokensForPackage =
Objects.requireNonNull(mNextPageTokensLocked.get(packageName));
nextPageTokensForPackage.remove(nextPageToken);
}
}
// In normal use case, the page token is guaranteed to be valid, so if page token not
// found flag is true, then it is mostly caused by pagination cache eviction. Therefore,
// throw an exception indicating that the search and pagination is aborted.
if (Flags.enableResultAborted()
&& Flags.enableThrowExceptionForNativeNotFoundPageToken()
&& searchResultProto.getPageTokenNotFound()) {
throw new AppSearchException(
AppSearchResult.RESULT_ABORTED,
"Page token not found. It is usually caused by pagination cache eviction.");
}
long rewriteSearchResultLatencyStartMillis = SystemClock.elapsedRealtime();
// Rewrite search result before we return.
SearchResultPage searchResultPage =
SearchResultToProtoConverter.toSearchResultPage(
searchResultProto, mSchemaCacheLocked, mConfig);
if (queryStatsBuilder != null) {
queryStatsBuilder
.setRewriteSearchResultLatencyMillis(
(int)
(SystemClock.elapsedRealtime()
- rewriteSearchResultLatencyStartMillis))
.addGetVmLatencyMillis(searchResultProto.getGetVmLatencyMs());
}
return searchResultPage;
} finally {
logReadOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_GET_NEXT_PAGE,
callStatsBuilder);
mReadWriteLock.readLock().unlock();
if (queryStatsBuilder != null) {
queryStatsBuilder.setTotalLatencyMillis(
(int) (SystemClock.elapsedRealtime() - totalLatencyStartMillis));
}
}
}
/**
* Invalidates the next-page token so that no more results of the related query can be returned.
*
* This method belongs to query group.
*
* @param packageName Package name of the caller.
* @param nextPageToken The token of pre-loaded results of previously executed query to be
* Invalidated.
* @throws AppSearchException if nextPageToken is unusable.
*/
public void invalidateNextPageToken(
@NonNull String packageName,
long nextPageToken,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
if (nextPageToken == SearchResultPage.EMPTY_PAGE_TOKEN) {
// (b/208305352) Directly return here since we are no longer caching EMPTY_PAGE_TOKEN
// in the cached token set. So no need to remove it anymore.
return;
}
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.readLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastWriteOperationLatencyMillisLocked);
}
LogUtil.piiTrace(TAG, "invalidateNextPageToken, request", nextPageToken);
checkNextPageToken(packageName, nextPageToken);
mIcingSearchEngineLocked.invalidateNextPageToken(nextPageToken);
synchronized (mNextPageTokensLocked) {
Set tokens = mNextPageTokensLocked.get(packageName);
if (tokens != null) {
tokens.remove(nextPageToken);
} else {
Log.e(
TAG,
"Failed to invalidate token "
+ nextPageToken
+ ": tokens are not "
+ "cached.");
}
}
} finally {
logReadOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_INVALIDATE_NEXT_PAGE_TOKEN,
callStatsBuilder);
mReadWriteLock.readLock().unlock();
}
}
/** Reports a usage of the given document at the given timestamp. */
public void reportUsage(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull String namespace,
@NonNull String documentId,
long usageTimestampMillis,
boolean systemUsage,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.writeLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked);
}
String prefixedNamespace = createPrefix(packageName, databaseName) + namespace;
UsageReport.UsageType usageType =
systemUsage
? UsageReport.UsageType.USAGE_TYPE2
: UsageReport.UsageType.USAGE_TYPE1;
UsageReport report =
UsageReport.newBuilder()
.setDocumentNamespace(prefixedNamespace)
.setDocumentUri(documentId)
.setUsageTimestampMs(usageTimestampMillis)
.setUsageType(usageType)
.build();
LogUtil.piiTrace(TAG, "reportUsage, request", report.getDocumentUri(), report);
ReportUsageResultProto result = mIcingSearchEngineLocked.reportUsage(report);
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(result.getGetVmLatencyMs());
}
LogUtil.piiTrace(TAG, "reportUsage, response", result.getStatus(), result);
checkSuccess(result.getStatus());
// Report usage changes document store derived files, so persistToDisk is needed.
mNeedsPersistToDisk.set(true);
} finally {
logWriteOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_REPORT_USAGE,
callStatsBuilder);
mReadWriteLock.writeLock().unlock();
}
}
/**
* Removes the given document by id.
*
* This method belongs to mutate group.
*
* @param packageName The package name that owns the document.
* @param databaseName The databaseName the document is in.
* @param namespace Namespace of the document to remove.
* @param documentId ID of the document to remove.
* @param removeStatsBuilder builder for {@link RemoveStats} to hold stats for remove
* @throws AppSearchException on IcingSearchEngine error.
*/
public void remove(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull String namespace,
@NonNull String documentId,
RemoveStats.@Nullable Builder removeStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.writeLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked);
}
String prefixedNamespace = createPrefix(packageName, databaseName) + namespace;
String schemaType = null;
if (mObserverManager.isPackageObserved(packageName)) {
// Someone might be observing the type this document is under, but we have no way to
// know its type without retrieving it. Do so now.
// TODO(b/193494000): If Icing Lib can return information about the deleted
// document's type we can remove this code.
if (LogUtil.isPiiTraceEnabled()) {
LogUtil.piiTrace(
TAG, "removeById, getRequest", prefixedNamespace + ", " + documentId);
}
GetResultProto getResult =
mIcingSearchEngineLocked.get(
prefixedNamespace, documentId, GET_RESULT_SPEC_NO_PROPERTIES);
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(getResult.getGetVmLatencyMs());
}
LogUtil.piiTrace(TAG, "removeById, getResponse", getResult.getStatus(), getResult);
checkSuccess(getResult.getStatus());
schemaType = removePrefix(getResult.getDocument().getSchema());
}
if (LogUtil.isPiiTraceEnabled()) {
LogUtil.piiTrace(TAG, "removeById, request", prefixedNamespace + ", " + documentId);
}
DeleteResultProto deleteResultProto =
mIcingSearchEngineLocked.delete(prefixedNamespace, documentId);
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(deleteResultProto.getGetVmLatencyMs());
}
LogUtil.piiTrace(
TAG, "removeById, response", deleteResultProto.getStatus(), deleteResultProto);
if (removeStatsBuilder != null) {
removeStatsBuilder
.setStatusCode(statusProtoToResultCode(deleteResultProto.getStatus()))
.setLaunchVMEnabled(mIsVMEnabled)
.setJavaLockAcquisitionLatencyMillis(
(int) (javaLockAcquisitionEndTimeMillis - totalLatencyStartMillis))
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked);
AppSearchLoggerHelper.copyNativeStats(
deleteResultProto.getDeleteStats(), removeStatsBuilder);
}
checkSuccess(deleteResultProto.getStatus());
mNeedsPersistToDisk.set(true);
// Update derived maps
mDocumentLimiterLocked.reportDocumentsRemoved(
packageName, /* numDocumentsDeleted= */ 1);
// Prepare notifications
if (schemaType != null) {
mObserverManager.onDocumentChange(
packageName,
databaseName,
namespace,
schemaType,
documentId,
mDocumentVisibilityStoreLocked,
mVisibilityCheckerLocked);
}
} finally {
logWriteOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_REMOVE_DOCUMENT_BY_ID,
callStatsBuilder);
mReadWriteLock.writeLock().unlock();
if (removeStatsBuilder != null) {
removeStatsBuilder.setTotalLatencyMillis(
(int) (SystemClock.elapsedRealtime() - totalLatencyStartMillis));
}
}
}
/**
* Removes documents by given query.
*
*
This method belongs to mutate group.
*
*
{@link SearchSpec} objects containing a {@link JoinSpec} are not allowed here.
*
* @param packageName The package name that owns the documents.
* @param databaseName The databaseName the document is in.
* @param queryExpression Query String to search.
* @param searchSpec Defines what and how to remove
* @param deletedIds collection to populate with ids of successfully deleted docs.
* @param removeStatsBuilder builder for {@link RemoveStats} to hold stats for remove
* @throws AppSearchException on IcingSearchEngine error.
* @throws IllegalArgumentException if the {@link SearchSpec} contains a {@link JoinSpec}.
*/
public void removeByQuery(
@NonNull String packageName,
@NonNull String databaseName,
@NonNull String queryExpression,
@NonNull SearchSpec searchSpec,
@Nullable Map> deletedIds,
RemoveStats.@Nullable Builder removeStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
if (searchSpec.getJoinSpec() != null) {
throw new IllegalArgumentException(
"JoinSpec not allowed in removeByQuery, but " + "JoinSpec was provided");
}
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.writeLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked);
}
if (removeStatsBuilder != null) {
removeStatsBuilder
.setJavaLockAcquisitionLatencyMillis(
(int) (javaLockAcquisitionEndTimeMillis - totalLatencyStartMillis))
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked);
}
List filterPackageNames = searchSpec.getFilterPackageNames();
if (!filterPackageNames.isEmpty() && !filterPackageNames.contains(packageName)) {
// We're only removing documents within the parameter `packageName`. If we're not
// restricting our remove-query to this package name, then there's nothing for us to
// remove.
return;
}
String prefix = createPrefix(packageName, databaseName);
if (!mNamespaceCacheLocked.getAllDocumentPrefixes().contains(prefix)) {
// The target database is empty so we can return early and skip sending request to
// Icing.
return;
}
SearchSpecToProtoConverter searchSpecToProtoConverter =
new SearchSpecToProtoConverter(
queryExpression,
searchSpec,
Collections.singleton(prefix),
mNamespaceCacheLocked,
mSchemaCacheLocked,
mConfig);
if (searchSpecToProtoConverter.hasNothingToSearch()) {
// there is nothing to search over given their search filters, so we can return
// early and skip sending request to Icing.
return;
}
SearchSpecProto finalSearchSpec =
searchSpecToProtoConverter.toSearchSpecProto(mIsVMEnabled);
Set prefixedObservedSchemas = null;
if (mObserverManager.isPackageObserved(packageName)) {
prefixedObservedSchemas = new ArraySet<>();
List prefixedTargetSchemaTypes = finalSearchSpec.getSchemaTypeFiltersList();
for (int i = 0; i < prefixedTargetSchemaTypes.size(); i++) {
String prefixedType = prefixedTargetSchemaTypes.get(i);
String shortTypeName = removePrefix(prefixedType);
if (mObserverManager.isSchemaTypeObserved(packageName, shortTypeName)) {
prefixedObservedSchemas.add(prefixedType);
}
}
}
doRemoveByQueryLocked(
packageName,
finalSearchSpec,
prefixedObservedSchemas,
deletedIds,
removeStatsBuilder,
callStatsBuilder);
} finally {
logWriteOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_REMOVE_DOCUMENTS_BY_SEARCH,
callStatsBuilder);
mReadWriteLock.writeLock().unlock();
if (removeStatsBuilder != null) {
removeStatsBuilder
.setTotalLatencyMillis(
(int) (SystemClock.elapsedRealtime() - totalLatencyStartMillis))
.setLaunchVMEnabled(mIsVMEnabled);
}
}
}
/**
* Executes removeByQuery.
*
* Change notifications will be created if prefixedObservedSchemas is not null.
*
* @param packageName The package name that owns the documents.
* @param finalSearchSpec The final search spec that has been written through {@link
* SearchSpecToProtoConverter}.
* @param prefixedObservedSchemas The set of prefixed schemas that have valid registered
* observers. Only changes to schemas in this set will be queued.
*/
@GuardedBy("mReadWriteLock")
private void doRemoveByQueryLocked(
@NonNull String packageName,
@NonNull SearchSpecProto finalSearchSpec,
@Nullable Set prefixedObservedSchemas,
@Nullable Map> deletedIds,
RemoveStats.@Nullable Builder removeStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
LogUtil.piiTrace(TAG, "removeByQuery, request", finalSearchSpec);
boolean returnDeletedDocumentInfo =
(prefixedObservedSchemas != null && !prefixedObservedSchemas.isEmpty())
|| deletedIds != null;
DeleteByQueryResultProto deleteResultProto =
mIcingSearchEngineLocked.deleteByQuery(finalSearchSpec, returnDeletedDocumentInfo);
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(deleteResultProto.getGetVmLatencyMs());
}
LogUtil.piiTrace(
TAG, "removeByQuery, response", deleteResultProto.getStatus(), deleteResultProto);
if (removeStatsBuilder != null) {
removeStatsBuilder.setStatusCode(
statusProtoToResultCode(deleteResultProto.getStatus()));
// TODO(b/187206766) also log query stats here once IcingLib returns it
AppSearchLoggerHelper.copyNativeStats(
deleteResultProto.getDeleteByQueryStats(), removeStatsBuilder);
}
// It seems that the caller wants to get success if the data matching the query is
// not in the DB because it was not there or was successfully deleted.
checkCodeOneOf(
deleteResultProto.getStatus(), StatusProto.Code.OK, StatusProto.Code.NOT_FOUND);
mNeedsPersistToDisk.set(true);
// Update derived maps
int numDocumentsDeleted =
deleteResultProto.getDeleteByQueryStats().getNumDocumentsDeleted();
mDocumentLimiterLocked.reportDocumentsRemoved(packageName, numDocumentsDeleted);
if (returnDeletedDocumentInfo) {
processDeletedDocumentInfo(
packageName, deleteResultProto, prefixedObservedSchemas, deletedIds);
}
}
@GuardedBy("mReadWriteLock")
private void processDeletedDocumentInfo(
@NonNull String packageName,
@NonNull DeleteByQueryResultProto deleteResultProto,
@Nullable Set prefixedObservedSchemas,
@Nullable Map> deletedIds)
throws AppSearchException {
for (int i = 0; i < deleteResultProto.getDeletedDocumentsCount(); ++i) {
DeleteByQueryResultProto.DocumentGroupInfo group =
deleteResultProto.getDeletedDocuments(i);
// 1. Populate deletedIds if it is provided.
String namespace = null;
if (deletedIds != null) {
namespace = removePrefix(group.getNamespace());
deletedIds.put(namespace, new ArraySet<>(group.getUrisList()));
}
// 2. If this schema type is observed, then notify the observer
if (prefixedObservedSchemas != null
&& prefixedObservedSchemas.contains(group.getSchema())) {
if (namespace == null) {
namespace = removePrefix(group.getNamespace());
}
String databaseName = getDatabaseName(group.getNamespace());
String schemaType = removePrefix(group.getSchema());
for (int j = 0; j < group.getUrisCount(); ++j) {
String uri = group.getUris(j);
mObserverManager.onDocumentChange(
packageName,
databaseName,
namespace,
schemaType,
uri,
mDocumentVisibilityStoreLocked,
mVisibilityCheckerLocked);
}
}
}
}
/** Estimates the total storage usage info data size for a specific set of packages. */
public @NonNull StorageInfo getStorageInfoForPackages(
@NonNull Set packageNames, CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.readLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked);
}
StorageInfo.Builder storageInfoBuilder = new StorageInfo.Builder();
// read document storage info and set to storageInfoBuilder
Set wantedPrefixedDocumentNamespaces =
mNamespaceCacheLocked.getAllPrefixedDocumentNamespaceForPackages(packageNames);
Set wantedPrefixedBlobNamespaces =
mNamespaceCacheLocked.getAllPrefixedBlobNamespaceForPackages(packageNames);
if (wantedPrefixedDocumentNamespaces.isEmpty()
&& wantedPrefixedBlobNamespaces.isEmpty()) {
return storageInfoBuilder.build();
}
StorageInfoProto storageInfoProto = getRawStorageInfoProto(callStatsBuilder);
if (Flags.enableBlobStore() && !wantedPrefixedBlobNamespaces.isEmpty()) {
getBlobStorageInfoForNamespaces(
storageInfoProto, wantedPrefixedBlobNamespaces, storageInfoBuilder);
}
if (!wantedPrefixedDocumentNamespaces.isEmpty()) {
getDocumentStorageInfoForNamespaces(
storageInfoProto, wantedPrefixedDocumentNamespaces, storageInfoBuilder);
}
return storageInfoBuilder.build();
} finally {
logReadOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_GET_STORAGE_INFO,
callStatsBuilder);
mReadWriteLock.readLock().unlock();
}
}
/** Estimates the storage usage info for a specific database in a package. */
public @NonNull StorageInfo getStorageInfoForDatabase(
@NonNull String packageName,
@NonNull String databaseName,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.readLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked);
}
StorageInfo.Builder storageInfoBuilder = new StorageInfo.Builder();
String prefix = createPrefix(packageName, databaseName);
if (Flags.enableBlobStore()) {
// read blob storage info and set to storageInfoBuilder
StorageInfoProto storageInfoProto = getRawStorageInfoProto(callStatsBuilder);
getBlobStorageInfoForPrefix(storageInfoProto, prefix, storageInfoBuilder);
// read document storage info and set to storageInfoBuilder
Set wantedPrefixedDocumentNamespaces =
mNamespaceCacheLocked.getPrefixedDocumentNamespaces(prefix);
if (wantedPrefixedDocumentNamespaces == null
|| wantedPrefixedDocumentNamespaces.isEmpty()) {
return storageInfoBuilder.build();
}
getDocumentStorageInfoForNamespaces(
storageInfoProto, wantedPrefixedDocumentNamespaces, storageInfoBuilder);
} else {
Map> packageToDatabases = getPackageToDatabases();
Set databases = packageToDatabases.get(packageName);
if (databases == null) {
// Package doesn't exist, no storage info to report
return storageInfoBuilder.build();
}
if (!databases.contains(databaseName)) {
// Database doesn't exist, no storage info to report
return storageInfoBuilder.build();
}
Set wantedPrefixedDocumentNamespaces =
mNamespaceCacheLocked.getPrefixedDocumentNamespaces(prefix);
if (wantedPrefixedDocumentNamespaces == null
|| wantedPrefixedDocumentNamespaces.isEmpty()) {
return storageInfoBuilder.build();
}
getDocumentStorageInfoForNamespaces(
getRawStorageInfoProto(callStatsBuilder),
wantedPrefixedDocumentNamespaces,
storageInfoBuilder);
}
return storageInfoBuilder.build();
} finally {
logReadOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_GET_STORAGE_INFO,
callStatsBuilder);
mReadWriteLock.readLock().unlock();
}
}
/**
* Returns the native storage info capsuled in {@link StorageInfoResultProto} directly from
* IcingSearchEngine.
*/
public @NonNull StorageInfoProto getRawStorageInfoProto(
CallStats.@Nullable Builder callStatsBuilder) throws AppSearchException {
mReadWriteLock.readLock().lock();
try {
throwIfClosedLocked();
LogUtil.piiTrace(TAG, "getStorageInfo, request");
StorageInfoResultProto storageInfoResult = mIcingSearchEngineLocked.getStorageInfo();
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(storageInfoResult.getGetVmLatencyMs());
}
LogUtil.piiTrace(
TAG,
"getStorageInfo, response",
storageInfoResult.getStatus(),
storageInfoResult);
checkSuccess(storageInfoResult.getStatus());
return storageInfoResult.getStorageInfo();
} finally {
mReadWriteLock.readLock().unlock();
}
}
/** Get {@link GetSchemaResponse} for a given visibility database. */
@NonNull
public GetSchemaResponse getVisibilitySchema(@NonNull String prefix) throws AppSearchException {
mReadWriteLock.readLock().lock();
try {
GetSchemaResponse.Builder responseBuilder = new GetSchemaResponse.Builder();
Map visibilitySchemaProto =
mSchemaCacheLocked.getSchemaMapForPrefix(prefix);
for (SchemaTypeConfigProto typeConfig : visibilitySchemaProto.values()) {
SchemaTypeConfigProto.Builder typeConfigBuilder = typeConfig.toBuilder();
removePrefixesFromSchemaType(typeConfigBuilder);
responseBuilder.setVersion(typeConfig.getVersion());
responseBuilder.addSchema(
SchemaToProtoConverter.toAppSearchSchema(typeConfigBuilder));
}
return responseBuilder.build();
} finally {
mReadWriteLock.readLock().unlock();
}
}
/**
* Extracts and returns {@link StorageInfo} from {@link StorageInfoProto} based on prefixed
* namespaces.
*
* @param storageInfoProto The source {@link StorageInfoProto} containing storage information to
* be analyzed.
* @param prefixedNamespaces A set of prefixed namespaces that the storage information will be
* filtered against. Only namespaces in this set will be included in the analysis.
* @param storageInfoBuilder The {@link StorageInfo.Builder} used to and build the resulting
* {@link StorageInfo}. This builder will be modified with calculated values.
*/
private static void getDocumentStorageInfoForNamespaces(
@NonNull StorageInfoProto storageInfoProto,
@NonNull Set prefixedNamespaces,
StorageInfo.@NonNull Builder storageInfoBuilder) {
if (!storageInfoProto.hasDocumentStorageInfo()) {
return;
}
long totalStorageSize = storageInfoProto.getTotalStorageSize();
DocumentStorageInfoProto documentStorageInfo = storageInfoProto.getDocumentStorageInfo();
int totalDocuments =
documentStorageInfo.getNumAliveDocuments()
+ documentStorageInfo.getNumExpiredDocuments();
if (totalStorageSize == 0 || totalDocuments == 0) {
// Maybe we can exit early and also avoid a divide by 0 error.
return;
}
// Accumulate stats across the package's namespaces.
int aliveDocuments = 0;
int expiredDocuments = 0;
int aliveNamespaces = 0;
List namespaceStorageInfos =
documentStorageInfo.getNamespaceStorageInfoList();
for (int i = 0; i < namespaceStorageInfos.size(); i++) {
NamespaceStorageInfoProto namespaceStorageInfo = namespaceStorageInfos.get(i);
// The namespace from icing lib is already the prefixed format
if (prefixedNamespaces.contains(namespaceStorageInfo.getNamespace())) {
if (namespaceStorageInfo.getNumAliveDocuments() > 0) {
aliveNamespaces++;
aliveDocuments += namespaceStorageInfo.getNumAliveDocuments();
}
expiredDocuments += namespaceStorageInfo.getNumExpiredDocuments();
}
}
int namespaceDocuments = aliveDocuments + expiredDocuments;
// Since we don't have the exact size of all the documents, we do an estimation. Note
// that while the total storage takes into account schema, index, etc. in addition to
// documents, we'll only calculate the percentage based on number of documents a
// client has.
storageInfoBuilder
.setSizeBytes((long) (namespaceDocuments * 1.0 / totalDocuments * totalStorageSize))
.setAliveDocumentsCount(aliveDocuments)
.setAliveNamespacesCount(aliveNamespaces);
}
/**
* Extracts and returns blob storage information from {@link StorageInfoProto} based on a
* namespace prefix.
*
* @param storageInfoProto The source {@link StorageInfoProto} containing blob storage
* information to be analyzed.
* @param prefix The prefix to match namespaces against. Only blob storage for namespaces
* starting with this prefix will be included.
* @param storageInfoBuilder The {@link StorageInfo.Builder} used to and build the resulting
* {@link StorageInfo}. This builder will be modified with calculated values.
*/
private void getBlobStorageInfoForPrefix(
@NonNull StorageInfoProto storageInfoProto,
@NonNull String prefix,
StorageInfo.@NonNull Builder storageInfoBuilder) {
Set prefixedNamespaces = new ArraySet<>();
List blobStorageInfoProtos =
storageInfoProto.getNamespaceBlobStorageInfoList();
for (int i = 0; i < blobStorageInfoProtos.size(); i++) {
String prefixedNamespace = blobStorageInfoProtos.get(i).getNamespace();
if (prefixedNamespace.startsWith(prefix)) {
prefixedNamespaces.add(prefixedNamespace);
}
}
getBlobStorageInfoForNamespaces(storageInfoProto, prefixedNamespaces, storageInfoBuilder);
}
/**
* Extracts and returns blob storage information from {@link StorageInfoProto} based on prefixed
* namespaces.
*
* @param storageInfoProto The source {@link StorageInfoProto} containing blob storage
* information to be analyzed.
* @param prefixedNamespaces A set of prefixed namespaces that the blob storage information will
* be filtered against. Only namespaces in this set will be included in the analysis.
* @param storageInfoBuilder The {@link StorageInfo.Builder} used to and build the resulting
* {@link StorageInfo}. This builder will be modified with calculated values.
*/
private void getBlobStorageInfoForNamespaces(
@NonNull StorageInfoProto storageInfoProto,
@NonNull Set prefixedNamespaces,
StorageInfo.@NonNull Builder storageInfoBuilder) {
if (storageInfoProto.getNamespaceBlobStorageInfoCount() == 0) {
return;
}
List blobStorageInfoProtos =
storageInfoProto.getNamespaceBlobStorageInfoList();
long blobSizeBytes = 0;
int blobCount = 0;
for (int i = 0; i < blobStorageInfoProtos.size(); i++) {
NamespaceBlobStorageInfoProto blobStorageInfoProto = blobStorageInfoProtos.get(i);
if (prefixedNamespaces.contains(blobStorageInfoProto.getNamespace())) {
if (Flags.enableAppSearchManageBlobFiles()) {
List blobFileNames = blobStorageInfoProto.getBlobFileNamesList();
for (int j = 0; j < blobFileNames.size(); j++) {
File blobFile = new File(mBlobFilesDir, blobFileNames.get(j));
blobSizeBytes += blobFile.length();
}
blobCount += blobFileNames.size();
} else {
blobSizeBytes += blobStorageInfoProto.getBlobSize();
blobCount += blobStorageInfoProto.getNumBlobs();
}
}
}
storageInfoBuilder.setBlobsCount(blobCount).setBlobsSizeBytes(blobSizeBytes);
}
/**
* Returns the native debug info capsuled in {@link DebugInfoResultProto} directly from
* IcingSearchEngine.
*
* @param verbosity The verbosity of the debug info. {@link DebugInfoVerbosity.Code#BASIC} will
* return the simplest debug information. {@link DebugInfoVerbosity.Code#DETAILED} will
* return more detailed debug information as indicated in the comments in debug.proto
*/
public @NonNull DebugInfoProto getRawDebugInfoProto(DebugInfoVerbosity.@NonNull Code verbosity)
throws AppSearchException {
mReadWriteLock.readLock().lock();
try {
throwIfClosedLocked();
LogUtil.piiTrace(TAG, "getDebugInfo, request");
DebugInfoResultProto debugInfoResult = mIcingSearchEngineLocked.getDebugInfo(verbosity);
LogUtil.piiTrace(
TAG, "getDebugInfo, response", debugInfoResult.getStatus(), debugInfoResult);
checkSuccess(debugInfoResult.getStatus());
return debugInfoResult.getDebugInfo();
} finally {
mReadWriteLock.readLock().unlock();
}
}
/**
* Persists all update/delete requests to the disk.
*
* If the app crashes after a call to PersistToDisk with {@link PersistType.Code#FULL}, Icing
* would be able to fully recover all data written up to this point without a costly recovery
* process.
*
*
If the app crashes after a call to PersistToDisk with {@link PersistType.Code#LITE}, Icing
* would trigger a costly recovery process in next initialization. After that, Icing would still
* be able to recover all written data - excepting Usage data. Usage data is only guaranteed to
* be safe after a call to PersistToDisk with {@link PersistType.Code#FULL}
*
*
If the app crashes after an update/delete request has been made, but before any call to
* PersistToDisk, then all data in Icing will be lost.
*
* @param persistType the amount of data to persist. {@link PersistType.Code#LITE} will only
* persist the minimal amount of data to ensure all data can be recovered. {@link
* PersistType.Code#FULL} will persist all data necessary to prevent data loss without
* needing data recovery.
* @throws AppSearchException on any error that AppSearch persist data to disk.
*/
public void persistToDisk(
@Nullable String callingPackageName,
@BaseStats.CallType int triggerCallType,
PersistType.@NonNull Code persistType,
@Nullable AppSearchLogger logger,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
PersistToDiskStats.Builder statsBuilder = null;
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.writeLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (callStatsBuilder != null) {
callStatsBuilder
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked);
}
if (logger != null) {
statsBuilder =
new PersistToDiskStats.Builder(callingPackageName, triggerCallType)
.setJavaLockAcquisitionLatencyMillis(
(int)
(javaLockAcquisitionEndTimeMillis
- totalLatencyStartMillis))
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked)
.setLaunchVMEnabled(mIsVMEnabled);
}
LogUtil.piiTrace(TAG, "persistToDisk, request", persistType);
PersistToDiskResultProto persistToDiskResultProto =
mIcingSearchEngineLocked.persistToDisk(persistType);
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(
persistToDiskResultProto.getGetVmLatencyMs());
}
LogUtil.piiTrace(
TAG,
"persistToDisk, response",
persistToDiskResultProto.getStatus(),
persistToDiskResultProto);
if (statsBuilder != null) {
AppSearchLoggerHelper.copyNativeStats(
persistToDiskResultProto.getPersistStats(), statsBuilder);
statsBuilder
.setStatusCode(
statusProtoToResultCode(persistToDiskResultProto.getStatus()))
.setTotalLatencyMillis(
(int) (SystemClock.elapsedRealtime() - totalLatencyStartMillis));
}
checkSuccess(persistToDiskResultProto.getStatus());
mNeedsPersistToDisk.set(false);
} finally {
logWriteOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_FLUSH,
callStatsBuilder);
mReadWriteLock.writeLock().unlock();
if (logger != null) {
logger.logStats(statsBuilder.build());
}
}
}
/**
* Remove all {@link AppSearchSchema}s and {@link GenericDocument}s under the given package.
*
* @param packageName The name of package to be removed.
* @throws AppSearchException if we cannot remove the data.
*/
public void clearPackageData(@NonNull String packageName)
throws AppSearchException, IOException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.writeLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
if (LogUtil.DEBUG) {
Log.d(TAG, "Clear data for package: " + packageName);
}
// TODO(b/193494000): We are calling getPackageToDatabases here and in several other
// places within AppSearchImpl. This method is not efficient and does a lot of string
// manipulation. We should find a way to cache the package to database map so it can
// just be obtained from a local variable instead of being parsed out of the prefixed
// map.
Set existingPackages = getPackageToDatabases().keySet();
if (existingPackages.contains(packageName)) {
existingPackages.remove(packageName);
prunePackageData(existingPackages);
}
if (mRevocableFileDescriptorStore != null) {
mRevocableFileDescriptorStore.revokeForPackage(packageName);
}
} finally {
logWriteOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.INTERNAL_CALL_TYPE_PRUNE_PACKAGE_DATA,
/* callStatsBuilder= */ null);
mReadWriteLock.writeLock().unlock();
}
}
/**
* Remove all {@link AppSearchSchema}s and {@link GenericDocument}s that doesn't belong to any
* of the given installed packages
*
* @param installedPackages The name of all installed package.
* @throws AppSearchException if we cannot remove the data.
*/
public void prunePackageData(@NonNull Set installedPackages) throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.writeLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
throwIfClosedLocked();
Map> packageToDatabases = getPackageToDatabases();
if (installedPackages.containsAll(packageToDatabases.keySet())) {
// No package got removed. We are good.
return;
}
// Prune schema proto and delete documents
boolean successfullyDeletedData = false;
if (useDatabaseScopedSchemaOperations()) {
Set databasesToDelete = new ArraySet<>();
Set allSchemaPrefixes = mSchemaCacheLocked.getAllPrefixes();
for (String prefix : allSchemaPrefixes) {
String packageName = getPackageName(prefix);
if (!installedPackages.contains(packageName)) {
databasesToDelete.add(prefix);
}
}
if (databasesToDelete.size() < PRUNE_PACKAGE_USING_FULL_SET_SCHEMA_THRESHOLD) {
// Use database-scoped set schema request to prune the schemas and documents
// a single database at a time.
for (String database : databasesToDelete) {
// Apply an empty schema and set force override to true to remove all
// schemas and documents that don't belong to any of the installed packages.
SetSchemaRequestProto emptySetSchemaRequestProto =
SetSchemaRequestProto.newBuilder()
.setSchema(SchemaProto.newBuilder().build())
.setDatabase(database)
.setIgnoreErrorsAndDeleteDocuments(true)
.build();
LogUtil.piiTrace(
TAG,
"clearPackageData.setSchema for database, request",
emptySetSchemaRequestProto);
SetSchemaResultProto setSchemaResultProto =
mIcingSearchEngineLocked.setSchemaWithRequestProto(
emptySetSchemaRequestProto);
LogUtil.piiTrace(
TAG,
"clearPackageData.setSchema, response",
setSchemaResultProto.getStatus(),
setSchemaResultProto);
// Determine whether it succeeded.
checkSuccess(setSchemaResultProto.getStatus());
mNeedsPersistToDisk.set(true);
}
successfullyDeletedData = true;
}
}
if (!successfullyDeletedData) {
prunePackageDataUsingFullSetSchemaLocked(installedPackages);
}
// Prune cached maps once schema and documents have been successfully deleted
for (Map.Entry> entry : packageToDatabases.entrySet()) {
String packageName = entry.getKey();
Set databaseNames = entry.getValue();
if (!installedPackages.contains(packageName) && databaseNames != null) {
mDocumentLimiterLocked.reportPackageRemoved(packageName);
synchronized (mNextPageTokensLocked) {
mNextPageTokensLocked.remove(packageName);
}
for (String databaseName : databaseNames) {
String removedPrefix = createPrefix(packageName, databaseName);
Set removedSchemas = mSchemaCacheLocked.removePrefix(removedPrefix);
if (mDocumentVisibilityStoreLocked != null) {
mDocumentVisibilityStoreLocked.removeVisibility(
removedSchemas, /* callStatsBuilder= */ null);
}
mNamespaceCacheLocked.removeDocumentNamespaces(removedPrefix);
}
}
}
} finally {
logWriteOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.INTERNAL_CALL_TYPE_PRUNE_PACKAGE_DATA,
/* callStatsBuilder= */ null);
mReadWriteLock.writeLock().unlock();
}
}
/**
* Remove all {@link AppSearchSchema}s and {@link GenericDocument}s that doesn't belong to any
* of the given installed packages by resetting the full schema for the remaining installed
* packages.
*
* @param installedPackages The name of all installed package.
* @throws AppSearchException if we cannot remove the data.
*/
private void prunePackageDataUsingFullSetSchemaLocked(@NonNull Set installedPackages)
throws AppSearchException {
SchemaProto existingSchema = getSchemaProtoLocked(/* callStatsBuilder= */ null);
SchemaProto.Builder newSchemaBuilder = SchemaProto.newBuilder();
for (int i = 0; i < existingSchema.getTypesCount(); i++) {
String packageName = getPackageName(existingSchema.getTypes(i).getSchemaType());
if (installedPackages.contains(packageName)) {
newSchemaBuilder.addTypes(existingSchema.getTypes(i));
}
}
SchemaProto finalSchema = newSchemaBuilder.build();
// Apply schema, set force override to true to remove all schemas and documents that
// doesn't belong to any of these installed packages.
LogUtil.piiTrace(
TAG,
"clearPackageData.setSchema, request",
finalSchema.getTypesCount(),
finalSchema);
SetSchemaResultProto setSchemaResultProto =
mIcingSearchEngineLocked.setSchema(
finalSchema, /* ignoreErrorsAndDeleteDocuments= */ true);
LogUtil.piiTrace(
TAG,
"clearPackageData.setSchema, response",
setSchemaResultProto.getStatus(),
setSchemaResultProto);
// Determine whether it succeeded.
checkSuccess(setSchemaResultProto.getStatus());
mNeedsPersistToDisk.set(true);
}
/**
* Deletes all blob files managed by AppSearch.
*
* @throws AppSearchException if an I/O error occurs.
*/
@GuardedBy("mReadWriteLock")
private void deleteBlobFilesLocked() throws AppSearchException {
if (!Flags.enableAppSearchManageBlobFiles()) {
return;
}
if (mBlobFilesDir.isFile() && !mBlobFilesDir.delete()) {
throw new AppSearchException(
AppSearchResult.RESULT_IO_ERROR,
"The blob file directory is a file and cannot delete it.");
}
if (!mBlobFilesDir.exists() && !mBlobFilesDir.mkdirs()) {
throw new AppSearchException(
AppSearchResult.RESULT_IO_ERROR,
"The blob file directory does not exist and cannot create a new one.");
}
File[] blobFiles = mBlobFilesDir.listFiles();
if (blobFiles == null) {
throw new AppSearchException(
AppSearchResult.RESULT_IO_ERROR, "Cannot list the blob files.");
}
for (int i = 0; i < blobFiles.length; i++) {
File blobFile = blobFiles[i];
if (!blobFile.delete()) {
Log.e(TAG, "Cannot delete the blob file: " + blobFile.getName());
}
}
}
/**
* Clears documents and schema across all packages and databaseNames.
*
* This method belongs to mutate group.
*
* @throws AppSearchException on IcingSearchEngine error.
*/
@GuardedBy("mReadWriteLock")
private void resetLocked(
InitializeStats.@Nullable Builder initStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
LogUtil.piiTrace(TAG, "icingSearchEngine.reset, request");
ResetResultProto resetResultProto = mIcingSearchEngineLocked.reset();
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(resetResultProto.getGetVmLatencyMs());
}
LogUtil.piiTrace(
TAG,
"icingSearchEngine.reset, response",
resetResultProto.getStatus(),
resetResultProto);
mOptimizeIntervalCountLocked = 0;
mSchemaCacheLocked.clear();
mNamespaceCacheLocked.clear();
// We just reset the index. So there is no need to retrieve the actual storage info. We know
// that there are no actual namespaces.
List emptyNamespaceInfos = Collections.emptyList();
mDocumentLimiterLocked =
new DocumentLimiter(
mConfig.getDocumentCountLimitStartThreshold(),
mConfig.getPerPackageDocumentCountLimit(),
emptyNamespaceInfos);
synchronized (mNextPageTokensLocked) {
mNextPageTokensLocked.clear();
}
if (initStatsBuilder != null) {
initStatsBuilder
.setHasReset(true)
.setResetStatusCode(statusProtoToResultCode(resetResultProto.getStatus()));
Log.i(
TAG,
"IcingSearchEngine Reset returned with status: "
+ resetResultProto.getStatus());
}
checkSuccess(resetResultProto.getStatus());
mNeedsPersistToDisk.set(true);
// Delete all blob files if AppSearch manages them.
deleteBlobFilesLocked();
}
/** Wrapper around schema changes */
@VisibleForTesting
static class RewrittenSchemaResults {
// Any prefixed types that used to exist in the schema, but are deleted in the new one.
// This set will be empty if rewrittenSchema is called with database-scoped schema
// operations enabled.
final Set mDeletedPrefixedTypes = new ArraySet<>();
// Map of prefixed schema types to SchemaTypeConfigProtos that were part of the new schema.
final Map mRewrittenPrefixedTypes = new ArrayMap<>();
}
/**
* Rewrites all types mentioned in the given {@code newSchema}.
*
* Rewritten types will be added to the {@code existingSchema}.
*
* @param prefix The full prefix to prepend to the schema.
* @param existingSchema A schema that may contain existing types from across all prefixes (only
* if database-scoped schema operations is disabled). Will be mutated to contain the
* properly rewritten schema types from {@code newSchema}.
* @param newSchema Schema with types to add to the {@code existingSchema}.
* @param populateDatabase Whether to populate the database field in the rewritten schema.
* @return a RewrittenSchemaResults that contains all prefixed schema type names in the given
* prefix as well as a set of schema types that were deleted.
*/
@VisibleForTesting
static RewrittenSchemaResults rewriteSchema(
@NonNull String prefix,
SchemaProto.@NonNull Builder existingSchema,
@NonNull SchemaProto newSchema,
boolean populateDatabase)
throws AppSearchException {
Map newTypesToProto =
getRewrittenPrefixedTypes(prefix, newSchema, populateDatabase);
// newTypesToProto is modified below, so we need a copy first
RewrittenSchemaResults rewrittenSchemaResults = new RewrittenSchemaResults();
rewrittenSchemaResults.mRewrittenPrefixedTypes.putAll(newTypesToProto);
// Combine the existing schema (which may have types from other prefixes if
// database-scoped schema operations is disabled) with this prefix's new schema. Modifies
// the existingSchemaBuilder.
// Check if we need to replace any old schema types with the new ones.
for (int i = 0; i < existingSchema.getTypesCount(); i++) {
String schemaType = existingSchema.getTypes(i).getSchemaType();
SchemaTypeConfigProto newProto = newTypesToProto.remove(schemaType);
if (newProto != null) {
// Replacement
existingSchema.setTypes(i, newProto);
} else if (prefix.equals(getPrefix(schemaType))) {
// All types existing before but not in newSchema should be removed.
existingSchema.removeTypes(i);
--i;
rewrittenSchemaResults.mDeletedPrefixedTypes.add(schemaType);
}
}
// We've been removing existing types from newTypesToProto, so everything that remains is
// new.
existingSchema.addAllTypes(newTypesToProto.values());
return rewrittenSchemaResults;
}
/**
* Rewrites all types in the given {@code schema}. The rewrite prepends {@code prefix} to the
* schema types, and also populates the schema's database field accordingly.
*
* @param prefix The full prefix to prepend to the schema.
* @param newSchema Schema with types to rewrite.
* @param populateDatabase Whether to populate the database field in the rewritten schema
* @return a map containing the rewritten schema type names and their corresponding rewritten
* protos.
*/
static Map getRewrittenPrefixedTypes(
@NonNull String prefix, @NonNull SchemaProto newSchema, boolean populateDatabase)
throws AppSearchException {
Map newTypesToProto = new ArrayMap<>();
// Rewrite the schema type to include the typePrefix.
for (int typeIdx = 0; typeIdx < newSchema.getTypesCount(); typeIdx++) {
SchemaTypeConfigProto.Builder typeConfigBuilder =
newSchema.getTypes(typeIdx).toBuilder();
// Rewrite SchemaProto.types.schema_type and populate SchemaProto.types.database
String newSchemaType = prefix + typeConfigBuilder.getSchemaType();
typeConfigBuilder.setSchemaType(newSchemaType);
if (populateDatabase) {
typeConfigBuilder.setDatabase(prefix);
}
// Rewrite SchemaProto.types.properties.schema_type
for (int propertyIdx = 0;
propertyIdx < typeConfigBuilder.getPropertiesCount();
propertyIdx++) {
PropertyConfigProto.Builder propertyConfigBuilder =
typeConfigBuilder.getProperties(propertyIdx).toBuilder();
if (!propertyConfigBuilder.getSchemaType().isEmpty()) {
String newPropertySchemaType = prefix + propertyConfigBuilder.getSchemaType();
propertyConfigBuilder.setSchemaType(newPropertySchemaType);
typeConfigBuilder.setProperties(propertyIdx, propertyConfigBuilder);
}
}
// Rewrite SchemaProto.types.parent_types
for (int parentTypeIdx = 0;
parentTypeIdx < typeConfigBuilder.getParentTypesCount();
parentTypeIdx++) {
String newParentType = prefix + typeConfigBuilder.getParentTypes(parentTypeIdx);
typeConfigBuilder.setParentTypes(parentTypeIdx, newParentType);
}
newTypesToProto.put(newSchemaType, typeConfigBuilder.build());
}
return newTypesToProto;
}
/**
* Rewrite the {@link InternalVisibilityConfig} to add given prefix in the schemaType of the
* given List of {@link InternalVisibilityConfig}
*
* @param prefix The full prefix to prepend to the visibilityConfigs.
* @param visibilityConfigs The visibility configs that need to add prefix
* @param removedVisibilityConfigs The removed configs that is not included in the given
* visibilityConfigs.
* @return The List of {@link InternalVisibilityConfig} that contains prefixed in its schema
* types.
*/
private List rewriteVisibilityConfigs(
@NonNull String prefix,
@NonNull List visibilityConfigs,
@NonNull Set removedVisibilityConfigs) {
List prefixedVisibilityConfigs =
new ArrayList<>(visibilityConfigs.size());
for (int i = 0; i < visibilityConfigs.size(); i++) {
InternalVisibilityConfig visibilityConfig = visibilityConfigs.get(i);
// The VisibilityConfig is controlled by the client and it's untrusted but we
// make it safe by appending a prefix.
// We must control the package-database prefix. Therefore even if the client
// fake the id, they can only mess their own app. That's totally allowed and
// they can do this via the public API too.
// TODO(b/275592563): Move prefixing into VisibilityConfig
// .createVisibilityDocument and createVisibilityOverlay
String namespace = visibilityConfig.getSchemaType();
String prefixedNamespace = prefix + namespace;
prefixedVisibilityConfigs.add(
new InternalVisibilityConfig.Builder(visibilityConfig)
.setSchemaType(prefixedNamespace)
.build());
// This schema has visibility settings. We should keep it from the removal list.
removedVisibilityConfigs.remove(prefixedNamespace);
}
return prefixedVisibilityConfigs;
}
/**
* Retrieves the full SchemaProto stored in IcingLib. The returned SchemaProto contains types
* across all prefixes.
*/
@VisibleForTesting
@GuardedBy("mReadWriteLock")
SchemaProto getSchemaProtoLocked(CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
LogUtil.piiTrace(TAG, "getSchema, request");
GetSchemaResultProto schemaProto = mIcingSearchEngineLocked.getSchema();
LogUtil.piiTrace(TAG, "getSchema, response", schemaProto.getStatus(), schemaProto);
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(schemaProto.getGetVmLatencyMs());
}
// TODO(b/161935693) check GetSchemaResultProto is success or not. Call reset() if it's not.
// TODO(b/161935693) only allow GetSchemaResultProto NOT_FOUND on first run
checkCodeOneOf(schemaProto.getStatus(), StatusProto.Code.OK, StatusProto.Code.NOT_FOUND);
return schemaProto.getSchema();
}
/**
* Retrieves the SchemaProto from IcingLib for the specified prefix. The returned SchemaProto
* will only contain types for the matching the schema database prefix.
*
* Requires {@link #useDatabaseScopedSchemaOperations()} to be true. {@link
* #getSchemaProtoLocked()} should be used instead when {@link
* #useDatabaseScopedSchemaOperations()} is false or when the entire schema is needed.
*
* @param prefix The full prefix for which to retrieve the schema.
*/
@VisibleForTesting
@GuardedBy("mReadWriteLock")
SchemaProto getSchemaProtoForPrefixLocked(@NonNull String prefix) throws AppSearchException {
LogUtil.piiTrace(TAG, "getSchemaForDatabase, request", prefix);
GetSchemaResultProto schemaProto = mIcingSearchEngineLocked.getSchemaForDatabase(prefix);
LogUtil.piiTrace(
TAG, "getSchemaForDatabase, response", schemaProto.getStatus(), schemaProto);
checkCodeOneOf(schemaProto.getStatus(), StatusProto.Code.OK, StatusProto.Code.NOT_FOUND);
return schemaProto.getSchema();
}
private void addNextPageToken(String packageName, long nextPageToken) {
if (nextPageToken == SearchResultPage.EMPTY_PAGE_TOKEN) {
// There is no more pages. No need to add it.
return;
}
synchronized (mNextPageTokensLocked) {
Set tokens = mNextPageTokensLocked.get(packageName);
if (tokens == null) {
tokens = new ArraySet<>();
mNextPageTokensLocked.put(packageName, tokens);
}
tokens.add(nextPageToken);
}
}
private void checkNextPageToken(String packageName, long nextPageToken)
throws AppSearchException {
if (nextPageToken == SearchResultPage.EMPTY_PAGE_TOKEN) {
// Swallow the check for empty page token, token = 0 means there is no more page and it
// won't return anything from Icing.
return;
}
synchronized (mNextPageTokensLocked) {
Set nextPageTokens = mNextPageTokensLocked.get(packageName);
if (nextPageTokens == null || !nextPageTokens.contains(nextPageToken)) {
throw new AppSearchException(
RESULT_SECURITY_ERROR,
"Package \""
+ packageName
+ "\" cannot use nextPageToken: "
+ nextPageToken);
}
}
}
/**
* Adds an {@link ObserverCallback} to monitor changes within the databases owned by {@code
* targetPackageName} if they match the given {@link
* android.app.appsearch.observer.ObserverSpec}.
*
* If the data owned by {@code targetPackageName} is not visible to you, the registration
* call will succeed but no notifications will be dispatched. Notifications could start flowing
* later if {@code targetPackageName} changes its schema visibility settings.
*
*
If no package matching {@code targetPackageName} exists on the system, the registration
* call will succeed but no notifications will be dispatched. Notifications could start flowing
* later if {@code targetPackageName} is installed and starts indexing data.
*
*
Note that this method does not take the standard read/write lock that guards I/O, so it
* will not queue behind I/O. Therefore it is safe to call from any thread including UI or
* binder threads.
*
* @param listeningPackageAccess Visibility information about the app that wants to receive
* notifications.
* @param targetPackageName The package that owns the data the observer wants to be notified
* for.
* @param spec Describes the kind of data changes the observer should trigger for.
* @param executor The executor on which to trigger the observer callback to deliver
* notifications.
* @param observer The callback to trigger on notifications.
*/
public void registerObserverCallback(
@NonNull CallerAccess listeningPackageAccess,
@NonNull String targetPackageName,
@NonNull ObserverSpec spec,
@NonNull Executor executor,
@NonNull ObserverCallback observer) {
// This method doesn't consult mSchemaMap or mNamespaceMap, and it will register
// observers for types that don't exist. This is intentional because we notify for types
// being created or removed. If we only registered observer for existing types, it would
// be impossible to ever dispatch a notification of a type being added.
mObserverManager.registerObserverCallback(
listeningPackageAccess, targetPackageName, spec, executor, observer);
}
/**
* Removes an {@link ObserverCallback} from watching the databases owned by {@code
* targetPackageName}.
*
*
All observers which compare equal to the given observer via {@link
* ObserverCallback#equals} are removed. This may be 0, 1, or many observers.
*
*
Note that this method does not take the standard read/write lock that guards I/O, so it
* will not queue behind I/O. Therefore it is safe to call from any thread including UI or
* binder threads.
*/
public void unregisterObserverCallback(
@NonNull String targetPackageName, @NonNull ObserverCallback observer) {
mObserverManager.unregisterObserverCallback(targetPackageName, observer);
}
/**
* Dispatches the pending change notifications one at a time.
*
*
The notifications are dispatched on the respective executors that were provided at the
* time of observer registration. This method does not take the standard read/write lock that
* guards I/O, so it is safe to call from any thread including UI or binder threads.
*
*
Exceptions thrown from notification dispatch are logged but otherwise suppressed.
*/
public void dispatchAndClearChangeNotifications() {
mObserverManager.dispatchAndClearPendingNotifications();
}
/**
* Checks the given status code and throws an {@link AppSearchException} if code is an error.
*
* @throws AppSearchException on error codes.
*/
private static void checkSuccess(StatusProto statusProto) throws AppSearchException {
checkCodeOneOf(statusProto, StatusProto.Code.OK);
}
/**
* Checks the given status code is one of the provided codes, and throws an {@link
* AppSearchException} if it is not.
*/
private static void checkCodeOneOf(StatusProto statusProto, StatusProto.Code... codes)
throws AppSearchException {
if (!isCodeOneOf(statusProto, codes)) {
throw new AppSearchException(
ResultCodeToProtoConverter.toResultCode(statusProto.getCode()),
statusProto.getMessage());
}
}
/** Returns true if the status is OK or WARNING_DATA_LOSS, false otherwise. */
private static boolean isSuccess(StatusProto statusProto) {
return isCodeOneOf(statusProto, StatusProto.Code.OK);
}
/** Returns true if the status is one of codes or WARNING_DATA_LOSS, false otherwise. */
private static boolean isCodeOneOf(StatusProto statusProto, StatusProto.Code... codes) {
for (int i = 0; i < codes.length; i++) {
if (codes[i] == statusProto.getCode()) {
return true;
}
}
if (statusProto.getCode() == StatusProto.Code.WARNING_DATA_LOSS) {
// TODO: May want to propagate WARNING_DATA_LOSS up to AppSearchSession so they can
// choose to log the error or potentially pass it on to clients.
Log.w(TAG, "Encountered WARNING_DATA_LOSS: " + statusProto.getMessage());
return true;
}
return false;
}
/**
* Checks whether {@link IcingSearchEngine#optimize()} should be called to release resources.
*
*
This method should be only called after a mutation to local storage backend which deletes
* a mass of data and could release lots resources after {@link IcingSearchEngine#optimize()}.
*
*
This method will trigger {@link IcingSearchEngine#getOptimizeInfo()} to check resources
* that could be released for every {@link #CHECK_OPTIMIZE_INTERVAL} mutations.
*
*
{@link IcingSearchEngine#optimize()} should be called only if {@link
* GetOptimizeInfoResultProto} shows there is enough resources could be released.
*
* @param mutationSize The number of how many mutations have been executed for current request.
* An inside counter will accumulates it. Once the counter reaches {@link
* #CHECK_OPTIMIZE_INTERVAL}, {@link IcingSearchEngine#getOptimizeInfo()} will be triggered
* and the counter will be reset.
*/
public void checkForOptimize(
int mutationSize,
OptimizeStats.@Nullable Builder optimizeStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.writeLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
if (optimizeStatsBuilder != null) {
optimizeStatsBuilder.setJavaLockAcquisitionLatencyMillis(
(int) (javaLockAcquisitionEndTimeMillis - totalLatencyStartMillis));
}
mOptimizeIntervalCountLocked += mutationSize;
if (mOptimizeIntervalCountLocked >= CHECK_OPTIMIZE_INTERVAL) {
checkForOptimize(optimizeStatsBuilder, callStatsBuilder);
}
} finally {
logWriteOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_OPTIMIZE,
/* callStatsBuilder= */ null);
mReadWriteLock.writeLock().unlock();
}
}
/**
* Checks whether {@link IcingSearchEngine#optimize()} should be called to release resources.
*
*
This method will directly trigger {@link IcingSearchEngine#getOptimizeInfo()} to check
* resources that could be released.
*
*
{@link IcingSearchEngine#optimize()} should be called only if {@link
* OptimizeStrategy#shouldOptimize(GetOptimizeInfoResultProto)} return true.
*/
public void checkForOptimize(
OptimizeStats.@Nullable Builder optimizeStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.writeLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
if (optimizeStatsBuilder != null) {
optimizeStatsBuilder.setJavaLockAcquisitionLatencyMillis(
(int) (javaLockAcquisitionEndTimeMillis - totalLatencyStartMillis));
}
GetOptimizeInfoResultProto optimizeInfo = getOptimizeInfoResultLocked(callStatsBuilder);
checkSuccess(optimizeInfo.getStatus());
mOptimizeIntervalCountLocked = 0;
if (mOptimizeStrategy.shouldOptimize(optimizeInfo)) {
optimize(optimizeStatsBuilder, callStatsBuilder);
}
} finally {
logWriteOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_OPTIMIZE,
/* callStatsBuilder= */ null);
mReadWriteLock.writeLock().unlock();
}
// TODO(b/147699081): Return OptimizeResultProto & log lost data detail once we add
// a field to indicate lost_schema and lost_documents in OptimizeResultProto.
// go/icing-library-apis.
}
/** Triggers {@link IcingSearchEngine#optimize()} directly. */
public void optimize(
OptimizeStats.@Nullable Builder optimizeStatsBuilder,
CallStats.@Nullable Builder callStatsBuilder)
throws AppSearchException {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
long javaLockAcquisitionEndTimeMillis = 0;
mReadWriteLock.writeLock().lock();
try {
javaLockAcquisitionEndTimeMillis = SystemClock.elapsedRealtime();
int javaLockAcquisitionLatencyMillis =
(int) (javaLockAcquisitionEndTimeMillis - totalLatencyStartMillis);
LogUtil.piiTrace(TAG, "optimize, request");
OptimizeResultProto optimizeResultProto = mIcingSearchEngineLocked.optimize();
// Optimize flushes data internally, so persistToDisk is not needed for the caller.
LogUtil.piiTrace(
TAG,
"optimize, response",
optimizeResultProto.getStatus(),
optimizeResultProto);
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(optimizeResultProto.getGetVmLatencyMs());
}
if (optimizeStatsBuilder != null) {
optimizeStatsBuilder
.setStatusCode(statusProtoToResultCode(optimizeResultProto.getStatus()))
.setJavaLockAcquisitionLatencyMillis(javaLockAcquisitionLatencyMillis)
.setLastBlockingOperation(mLastReadOrWriteOperationLocked)
.setLastBlockingOperationLatencyMillis(
mLastReadOrWriteOperationLatencyMillisLocked)
.setLaunchVMEnabled(mIsVMEnabled)
.addGetVmLatencyMillis(optimizeResultProto.getGetVmLatencyMs());
AppSearchLoggerHelper.copyNativeStats(
optimizeResultProto.getOptimizeStats(), optimizeStatsBuilder);
}
checkSuccess(optimizeResultProto.getStatus());
// If AppSearch manages blob files, remove the optimized blob files.
if (Flags.enableAppSearchManageBlobFiles()) {
List blobFileNamesToRemove =
optimizeResultProto.getBlobFileNamesToRemoveList();
for (int i = 0; i < blobFileNamesToRemove.size(); i++) {
File blobFileToRemove = new File(mBlobFilesDir, blobFileNamesToRemove.get(i));
if (!blobFileToRemove.delete()) {
Log.e(
TAG,
"Cannot delete the optimized blob file: "
+ blobFileToRemove.getName());
}
}
}
} finally {
logWriteOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
/* totalLatencyEndMillis= */ SystemClock.elapsedRealtime(),
BaseStats.CALL_TYPE_OPTIMIZE,
/* callStatsBuilder= */ null);
mReadWriteLock.writeLock().unlock();
}
}
/** Sync the current Android logging level to Icing for the entire process. No lock required. */
public static void syncLoggingLevelToIcing() {
String icingTag = IcingSearchEngine.getLoggingTag();
if (icingTag == null) {
Log.e(TAG, "Received null logging tag from Icing");
return;
}
if (LogUtil.DEBUG) {
if (Log.isLoggable(icingTag, Log.VERBOSE)) {
boolean unused =
IcingSearchEngine.setLoggingLevel(
LogSeverity.Code.VERBOSE, /* verbosity= */ (short) 1);
return;
} else if (Log.isLoggable(icingTag, Log.DEBUG)) {
boolean unused = IcingSearchEngine.setLoggingLevel(LogSeverity.Code.DBG);
return;
}
}
if (LogUtil.INFO) {
if (Log.isLoggable(icingTag, Log.INFO)) {
boolean unused = IcingSearchEngine.setLoggingLevel(LogSeverity.Code.INFO);
return;
}
}
if (Log.isLoggable(icingTag, Log.WARN)) {
boolean unused = IcingSearchEngine.setLoggingLevel(LogSeverity.Code.WARNING);
} else if (Log.isLoggable(icingTag, Log.ERROR)) {
boolean unused = IcingSearchEngine.setLoggingLevel(LogSeverity.Code.ERROR);
} else {
boolean unused = IcingSearchEngine.setLoggingLevel(LogSeverity.Code.FATAL);
}
}
@GuardedBy("mReadWriteLock")
@VisibleForTesting
GetOptimizeInfoResultProto getOptimizeInfoResultLocked(
CallStats.@Nullable Builder callStatsBuilder) {
LogUtil.piiTrace(TAG, "getOptimizeInfo, request");
GetOptimizeInfoResultProto result = mIcingSearchEngineLocked.getOptimizeInfo();
LogUtil.piiTrace(TAG, "getOptimizeInfo, response", result.getStatus(), result);
if (callStatsBuilder != null) {
callStatsBuilder.addGetVmLatencyMillis(result.getGetVmLatencyMs());
}
return result;
}
/**
* Returns all prefixed schema types saved in AppSearch.
*
* This method is inefficient to call repeatedly.
*/
public @NonNull List getAllPrefixedSchemaTypes() {
mReadWriteLock.readLock().lock();
try {
return mSchemaCacheLocked.getAllPrefixedSchemaTypes();
} finally {
mReadWriteLock.readLock().unlock();
}
}
/**
* Returns all prefixed blob namespaces saved in AppSearch.
*
* This method is inefficient to call repeatedly.
*/
public @NonNull List getAllPrefixedBlobNamespaces() {
mReadWriteLock.readLock().lock();
try {
return mNamespaceCacheLocked.getAllPrefixedBlobNamespaces();
} finally {
mReadWriteLock.readLock().unlock();
}
}
/**
* Converts an erroneous status code from the Icing status enums to the AppSearchResult enums.
*
* Callers should ensure that the status code is not OK or WARNING_DATA_LOSS.
*
* @param statusProto StatusProto with error code to translate into an {@link AppSearchResult}
* code.
* @return {@link AppSearchResult} error code
*/
@AppSearchResult.ResultCode
private static int statusProtoToResultCode(@NonNull StatusProto statusProto) {
return ResultCodeToProtoConverter.toResultCode(statusProto.getCode());
}
private static void verifyCallingBlobHandle(
@NonNull String callingPackageName,
@NonNull String callingDatabaseName,
@NonNull AppSearchBlobHandle blobHandle)
throws AppSearchException {
if (!blobHandle.getPackageName().equals(callingPackageName)) {
throw new AppSearchException(
AppSearchResult.RESULT_INVALID_ARGUMENT,
"Blob package doesn't match calling package, calling package: "
+ callingPackageName
+ ", blob package: "
+ blobHandle.getPackageName());
}
if (!blobHandle.getDatabaseName().equals(callingDatabaseName)) {
throw new AppSearchException(
AppSearchResult.RESULT_INVALID_ARGUMENT,
"Blob database doesn't match calling database, calling database: "
+ callingDatabaseName
+ ", blob database: "
+ blobHandle.getDatabaseName());
}
}
/**
* Returns whether the database is in a stable state after initialization. If not, then we may
* need flushing.
*/
private boolean hasDatabaseStateChangedAfterInit(
@NonNull InitializeResultProto initializeResultProto) {
// TODO(b/417463182): add boolean field(s) into InitializeResultProto indicating whether
// ground truths and derived files have changed or not, and we can have a simpler way here
// to decide if persistToDisk is needed or not.
InitializeStatsProto initializeStatsProto = initializeResultProto.getInitializeStats();
return initializeStatsProto.getDocumentStoreDataStatus()
!= InitializeStatsProto.DocumentStoreDataStatus.NO_DATA_LOSS
|| initializeStatsProto.getSchemaStoreRecoveryCause()
!= InitializeStatsProto.RecoveryCause.NONE
|| initializeStatsProto.getDocumentStoreRecoveryCause()
!= InitializeStatsProto.RecoveryCause.NONE
|| initializeStatsProto.getIndexRestorationCause()
!= InitializeStatsProto.RecoveryCause.NONE
|| initializeStatsProto.getIntegerIndexRestorationCause()
!= InitializeStatsProto.RecoveryCause.NONE
|| initializeStatsProto.getQualifiedIdJoinIndexRestorationCause()
!= InitializeStatsProto.RecoveryCause.NONE
|| initializeStatsProto.getEmbeddingIndexRestorationCause()
!= InitializeStatsProto.RecoveryCause.NONE;
}
/** Calls getSchema in a thread safe manner. */
public @NonNull SchemaProto rawGetSchema() {
mReadWriteLock.readLock().lock();
try {
return mIcingSearchEngineLocked.getSchema().getSchema();
} finally {
mReadWriteLock.readLock().unlock();
}
}
/** Calls search in a thread safe manner. */
public @NonNull SearchResultProto rawSearch(
@NonNull SearchSpecProto spec,
@NonNull ScoringSpecProto scoringSpec,
@NonNull ResultSpecProto resultSpec) {
mReadWriteLock.readLock().lock();
try {
return mIcingSearchEngineLocked.search(spec, scoringSpec, resultSpec);
} finally {
mReadWriteLock.readLock().unlock();
}
}
/** Calls getSchema in a thread safe manner. */
public @NonNull SearchResultProto rawGetNextPage(long nextPageToken) {
mReadWriteLock.readLock().lock();
try {
return mIcingSearchEngineLocked.getNextPage(nextPageToken);
} finally {
mReadWriteLock.readLock().unlock();
}
}
/**
* Calls getAllBlobInfos in a thread safe manner.
*
*
Currently this is only used and tested in PlatformStorage.
*/
public @NonNull BlobProto rawGetAllBlobInfos() {
mReadWriteLock.readLock().lock();
try {
return mIcingSearchEngineLocked.getAllBlobInfos();
} finally {
mReadWriteLock.readLock().unlock();
}
}
/**
* Logs the operation latency of a write operation. This method is guarded by {@code
* mReadWriteLock}.
*/
@GuardedBy("mReadWriteLock")
private void logWriteOperationLatencyLocked(
long totalLatencyStartMillis,
long javaLockAcquisitionEndTimeMillis,
long totalLatencyEndMillis,
@BaseStats.CallType int callType,
CallStats.@Nullable Builder callStatsBuilder) {
int executeTime = (int) (totalLatencyEndMillis - javaLockAcquisitionEndTimeMillis);
mLastWriteOperationLocked = callType;
mLastWriteOperationLatencyMillisLocked = executeTime;
// The write operation will also block read operation, calling logReadOperationLatencyLocked
logReadOperationLatencyLocked(
totalLatencyStartMillis,
javaLockAcquisitionEndTimeMillis,
totalLatencyEndMillis,
callType,
callStatsBuilder);
}
/**
* Logs the operation latency of a read operation. This method is guarded by {@code
* mReadWriteLock}.
*/
@GuardedBy("mReadWriteLock")
private void logReadOperationLatencyLocked(
long totalLatencyStartMillis,
long javaLockAcquisitionEndTimeMillis,
long totalLatencyEndMillis,
@BaseStats.CallType int callType,
CallStats.@Nullable Builder callStatsBuilder) {
int executeTime = (int) (totalLatencyEndMillis - javaLockAcquisitionEndTimeMillis);
if (callStatsBuilder != null) {
// This is a read operation, only write operation could be a blocker.
callStatsBuilder
.setJavaLockAcquisitionLatencyMillis(
(int) (totalLatencyStartMillis - javaLockAcquisitionEndTimeMillis))
.setUnblockedAppSearchLatencyMillis(executeTime);
}
mLastReadOrWriteOperationLocked = callType;
mLastReadOrWriteOperationLatencyMillisLocked = executeTime;
}
}