/*
 * Copyright 2022 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
syntax = "proto2";

package fcp.dictionary;

// Describes a mapping of strings to (usually consecutive) integer ids.
message DictionaryDescription {
  // Vocabulary ids with special meaning.
  message SpecialIds {
    // If set and non-negative, id used for an unknown token.
    optional int32 unk = 1 [default = -1];

    // If set and non-negative, id used for the beginning of a sequence.
    optional int32 bos = 2 [default = -1];

    // If set and non-negative, id used for the end of a sequence.
    optional int32 eos = 3 [default = -1];
  }

  // Vocabulary ids that should be filtered from the predictions (e.g.,
  // punctuation, bad words etc.).
  message OutputBlocklistIds {
    repeated int32 id = 1 [packed = true];
  }

  // Optional persistent storage format for the token to id map.
  message Vocabulary {
    message TokenIndex {
      repeated string token = 1;
    }

    reserved 1;

    oneof vocabulary {
      // Repeated strings stored in-order (index begins at 0).
      TokenIndex index = 2;
    }
  }

  optional SpecialIds special_ids = 1;

  optional Vocabulary vocabulary = 2;

  optional OutputBlocklistIds output_blocklist_ids = 3;

  reserved 4;
}