/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*      http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

syntax = "proto3";

package luci.resultdb.v1;

import "google/api/field_behavior.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/struct.proto";
import "google/protobuf/timestamp.proto";
import public "tools/tradefederation/core/proto/resultdb/common.proto";
import public "tools/tradefederation/core/proto/resultdb/test_metadata.proto";
import public "tools/tradefederation/core/proto/resultdb/failure_reason.proto";

option go_package = "go.chromium.org/luci/resultdb/proto/v1;resultpb";
option java_package = "com.android.resultdb.proto";
option java_multiple_files = true;

// A result of a functional test case, representing a single execution attempt.
// Often a single test case is executed multiple times (attempts or runs) and
// has multiple results. Test results are associated with a RootInvocation and
// a WorkUnit that represents the execution context (like a swarming task
// or build step).
//
// This message includes details about the test identity, its execution status/result,
// timing, and potentially failure details.
// This message does not specify the test id.
// It should be available in the message that embeds this message.
//
// Next id: 23.
message TestResult {
  reserved 11;  // test_location

  // This is the resource name of the test result. You can use it to refer
  // to this test result in RPCs like ResultDB.GetTestResult.
  // The format is:
  // "rootinvocations/{ROOT_INVOCATION_ID}/workunits/{WORK_UNIT_ID}/tests/{URL_ESCAPED_TEST_ID}/results/{RESULT_ID}".
  // Note that URL_ESCAPED_TEST_ID uses the flat test ID format.
  //
  // This field is output only, meaning it's assigned by the server.
  // Note: while ResultDB is being updated, a legacy name format that is different to the above may be returned.  This
  // legacy format will also be accepted anywhere the updated format is accepted so they are functionally equivalent.
  string name = 1 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.field_behavior) = IMMUTABLE
  ];

  // The structured test identifier.
  //
  // Uniquely identifies the test that was run, including the specific way of running that
  // test, e.g. build configuration, CPU architecture, OS.
  //
  // When uploading results, this should be specified in preference
  // to test_id and variant. (Uploading test_id and variant is supported only for
  // legacy uploaders).
  TestIdentifier test_id_structured = 19 [(google.api.field_behavior) = IMMUTABLE];

  // A unique identifier of the test in a LUCI project, excluding variant.
  // Regex: ^[[::print::]]{1,512}$
  //
  // This is the flat-form encoding of the structured test variant ID above,
  // excluding information about the specific way of running test (e.g. build configuration,
  // CPU architecture). Such information is captured separately in the `variant` field below.
  //
  // See TestIdentifier for details how a structured test identifier is converted
  // to flat test ID.
  //
  // Output only. (Except for legacy uploaders.)
  string test_id = 2 [(google.api.field_behavior) = OUTPUT_ONLY, (google.api.field_behavior) = IMMUTABLE];

  // This identifies the specific test result within a given invocation and test ID.
  // It's needed because a single test might run multiple times (e.g., retries).
  // This ID should be unique for a given test within the invocation.
  // Test runners or result uploaders typically generate this ID.
  // This forms part of the name (or primary key) of the test result:
  // (root_invocation_id, test_id, result_id)
  // The format follows the regex: ^[a-z0-9\-_.]{1,32}$
  string result_id = 3 [
    (google.api.field_behavior) = IMMUTABLE,
    (google.api.field_behavior) = REQUIRED
  ];

  // Description of one specific way of running the test,
  // e.g. build configuration, CPU architecture, OS.
  //
  // Output only. (Except for legacy uploaders.)
  Variant variant = 4 [(google.api.field_behavior) = OUTPUT_ONLY, (google.api.field_behavior) = IMMUTABLE];

  // DEPRECATED: Replaced by `status_v2` enum.
  //
  // Whether the result of test case execution is expected.
  // In a typical Chromium CL, 99%+ of test results are expected.
  // Users are typically interested only in the unexpected results.
  //
  // An unexpected result != test case failure. There are test cases that are
  // expected to fail/skip/crash. The test harness compares the actual status
  // with the expected one(s) and this field is the result of the comparison.
  bool expected = 5 [(google.api.field_behavior) = IMMUTABLE, deprecated = true];

  // DEPRECATED: Replaced by `status_v2` enum.
  //
  // Machine-readable status of the test case.
  // MUST NOT be STATUS_UNSPECIFIED.
  TestStatus status = 6 [(google.api.field_behavior) = IMMUTABLE, deprecated = true];

  // The machine-readable test result statuses (v2).
  //
  // These high-level categories are further refined:
  // - for failing tests, by the failure_reason.kind.
  // - for skipped tests, by the skipped_reason.kind.
  //
  // Web test users: Refer to framework_extensions.web_tests for the original
  // test status reported by the web test harness. The status in this
  // field is the logical test status, not the original status. E.g. for
  // an expected failure, we record PASSED, not FAILED.
  enum Status {
    // Status was not specified. Do not use.
    STATUS_UNSPECIFIED = 0;

    // The test case has passed.
    PASSED = 1;

    // The test case has failed.
    // Suggests that the code under test is incorrect, but it is also possible
    // that the test is incorrect or it is a flake.
    //
    // If a test failed to complete due to an error that is not the fault of
    // this test's content, use the status EXECUTION_ERRORED (for errors specific
    // to this test) or PRECLUDED (for errors at a higher-level) instead.
    //
    // If you specify this status, you must also populate the failure_reason.kind field.
    FAILED = 2;

    // The test case did not, *and should not*, run to completion in this
    // configuration.
    //
    // For example:
    // - The test is disabled in code
    // - The test assumptions are not met (e.g. JUnit assumption failure
    //   or Tast test hardware dependency unmet)
    // - The test was not stable enough to in presubmit right now.
    //
    // If a test was not run or not run to completion due to an error, use the
    // status EXECUTION_ERRORED (for test-level errors) or PRECLUDED
    // (for higher-level errors) instead.
    //
    // If you specify this status, you must also populate the skipped_reason field.
    SKIPPED = 3;

    // The test did not run to completion, because an infrastructure error
    // precluded it from doing so.
    //
    // Infrastructure here is broadly defined, to mean "not the content
    // of this test".
    //
    // For example:
    // - The test ran, but the result file could not be parsed.
    // - A file this test depends on could not be downloaded.
    //
    // Sometimes it is ambiguous whether test content is at fault or not.
    // For example, loss of SSH connection during the test could be because
    // the test caused a kernel panic or because of a flaky ethernet adapter.
    // Judgement is required. If unsure, use EXECUTION_ERRORED status instead
    // of FAIL to avoid falsely inflating the flakiness rate of a test.
    //
    // Results with this status should be ignored when calculating the flake
    // and failure rates of the test.
    //
    // Currently, there is no dedicated 'reason' field for this status;
    // please just include a suitable description in the result `summary_html`.
    EXECUTION_ERRORED = 4;

    // The test did not run to completion, because its execution is precluded
    // by an error at a higher-level. For example, a work unit-level timeout.
    //
    // If you report this status, you must report an error on the containing
    // work unit. If this restriction is changed in future to allow preclusion
    // by other sources (e.g. a class fixture failed to setup so the tests in
    // using it could not run), a preclusion reason field will be added to
    // capture this.
    //
    // Results with this status should be ignored when calculating the flake
    // and failure rates of the test.
    //
    // Currently, there is no dedicated 'reason' field for this status; please
    // include a suitable description in the result `summary_html`.
    PRECLUDED = 5;
  }

  // Machine-readable status of the test case (v2).
  // MUST NOT be STATUS_UNSPECIFIED.
  Status status_v2 = 20;

  // Human-readable explanation of the result, in HTML.
  // MUST be sanitized before rendering in the browser.
  //
  // The size of the summary must be equal to or smaller than 4096 bytes in
  // UTF-8.
  //
  // Supports artifact embedding using custom tags:
  // * <text-artifact> renders contents of an artifact as text.
  //   Usage:
  //   * To embed result level artifact: <text-artifact
  //   artifact-id="<artifact_id>">
  //   * To embed invocation level artifact: <text-artifact
  //   artifact-id="<artifact_id>" inv-level>
  string summary_html = 7 [(google.api.field_behavior) = IMMUTABLE];

  // The point in time when the test case started to execute.
  google.protobuf.Timestamp start_time = 8
      [(google.api.field_behavior) = IMMUTABLE];

  // Duration of the test case execution.
  // MUST be equal to or greater than 0.
  google.protobuf.Duration duration = 9
      [(google.api.field_behavior) = IMMUTABLE];

  // Metadata for this test result.
  // It might describe this particular execution or the test case.
  // A key can be repeated.
  repeated StringPair tags = 10 [(google.api.field_behavior) = IMMUTABLE];

  // This is a SHA256 hash of the `test_id_structured.module_variant` key-value pairs,
  // encoded as hex.
  //
  // It provides a consistent way to identify unique variants for grouping and
  // analysis.
  //
  // hex(sha256(sorted(''.join('%s:%s\n' for k, v in variant.items()))))
  //
  // For legacy test results which do not upload the test_id_structured field, the
  // module_variant field is equivalent to the legacy `variant` field, and thus
  // this field is effectively a hash of the `variant` field.
  //
  // This field is output only.
  string variant_hash = 12 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.field_behavior) = IMMUTABLE
  ];

  // This holds metadata about the test code itself, like its source code location,
  // captured at the time the test was executed.
  TestMetadata test_metadata = 13;

  // This provides structured information about why a test failed.
  // MUST be set (and only set) when status_v2 is FAILED.
  FailureReason failure_reason = 14;

  // Arbitrary JSON object that contains structured, domain-specific properties
  // of the test result.
  //
  // As well as allowing more structure than tags, it will be exported to BigQuery
  // as a JSON field allowing faster querying than tags.
  //
  // The serialized size must be <= 8 KB.
  google.protobuf.Struct properties = 15;

  // Whether the test result has been masked so that it includes only metadata.
  // The metadata fields for a TestResult are:
  // * name
  // * test_id_structured
  // * test_id
  // * variant
  // * result_id
  // * status_v2
  // * start_time
  // * duration
  // * variant_hash
  // * failure_reason.kind
  // * summary_html
  // * failure_reason.primary_error_message (truncated to 140 characters)
  // * skipped_reason
  // * skip_reason (deprecated)
  // * expected (deprecated)
  // * status (deprecated)
  //
  // This field is output only.
  bool is_masked = 16 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Deprecated: Replaced by skipped_reason message.
  // Reasoning behind a test skip, in machine-readable form.
  // Used to assist downstream analyses, such as automatic bug-filing.
  // MUST not be set unless status is SKIP.
  SkipReason skip_reason = 18 [ deprecated = true ];

  // Reason why a test was skipped.
  // The contents of this message should help the user answer:
  // Why was my test skipped? What can I do to stop it being skipped?
  //
  // MUST be set (and only set) when status_v2 is SKIPPED.
  SkippedReason skipped_reason = 21;

  // Test result framework-specific data model elements.
  FrameworkExtensions framework_extensions = 22;
}

// SkippedReason helps a developer answer the questions:
// - Why was this test skipped?
// - How do I stop it being skipped?
message SkippedReason {
  // Provides structured explanation for why a test was skipped.
  // It must be populated when the status is SKIP.
  // Open to extension.
  enum Kind {
    // Skipped reason was not specified. Do not use this value.
    KIND_UNSPECIFIED = 0;

    // Skipped by an annotation at the test declaration site.
    //
    // This includes:
    // - JUnit4 @Ignore,
    // - JUnit5 @Disabled, @DisabledIf/For/On... @EnabledIf/For/On...
    //   annotations.
    // - GoogleTest DISABLED_ text prefix.
    // - Tast test skips from the declared software or hardware
    //   dependencies being unsatisfied at runtime.
    // and similar statically defined annotations at the test
    // declaration site.
    DISABLED_AT_DECLARATION = 1;

    // Test body began to run but determined the test should
    // not finish. For example, the test did not detect a valid
    // environment meeting the test assumptions.
    //
    // This covers:
    // - a JUnit4 or JUnit5 assumption failure in the body of a test,
    //   from Assume.assumeThat(...) and similar methods.
    // - Use of the GTEST_SKIP() macro within the body of a test.
    // - Golang's t.Skip(...) in the body of a test.
    SKIPPED_BY_TEST_BODY = 2;

    // Used to indicate the test was skipped due to presubmit demotion
    // (e.g. because the test was too flaky).
    //
    // You must specify a human-readable detail message if you use this status.
    // Please explain the criteria that was met and how the user should get the
    // test undemoted or reference a link that explains more.
    DEMOTED = 3;

    // The test was skipped by some other reason.
    //
    // This includes:
    // - Blink web tests disabled in an expectation file.
    //
    // You must specify a human-readable detail message if you use this status.
    OTHER = 4;
  }
  Kind kind = 1;

  // A human-readable message providing more context or details about why the
  // test was skipped. Limited to 1024 bytes of printable UTF-8.
  //
  // Required for skip kinds DEMOTED and OTHER, optional otherwise.
  //
  // For DISABLED_AT_DECLARATION, useful information would be:
  // - For Tast, the Tast-provided skip reason (detailing the declared
  //   hardware/software dependencies that were unmet).
  //
  // For SKIPPED_BY_TEST_BODY, useful information would be:
  // - For JUnit, the JUnit assumption failure that led to the skip.
  // - For GoogleTest, the file and line where the GTEST_SKIP() macro was
  //   invoked and the message passed to it.
  //
  // For DEMOTED and OTHER, the criteria that led to the skip
  // and how the user might get the test running again.
  string reason_message = 2;
}

// Additional information about the test result in source test
// framework-specific format. Data fields may be added here
// before there is sufficient examples to create a standardised
// representation, or to support bespoke harness-specific features.
message FrameworkExtensions {
  // The test status produced by the blink web test harness.
  // This field supports web test rebaselining.
  WebTest web_test = 1;
}

// Web Platform Test specific result data.
message WebTest {
  // An enumeration of web test status values.
  enum Status {
    // Status was not specified.
    // Not to be used in actual test results; serves as a default value for an
    // unset field.
    STATUS_UNSPECIFIED = 0;

    // The test case has passed.
    PASS = 1;

    // The test case has failed.
    FAIL = 2;

    // The test case has crashed during execution.
    CRASH = 3;

    // The test timed out.
    TIMEOUT = 4;

    // The test case did not execute.
    SKIP = 5;
  }

  // Whether the test produced the expected result. Failed results may be
  // expected and passed results unexpected, depending on if we expected
  // the implementation (e.g. Blink) to conform with the web platform feature
  // at this time.
  // See https://chromium.googlesource.com/chromium/src/+/HEAD/docs/testing/web_test_expectations.md.
  // See https://web-platform-tests.org/tools/wptrunner/docs/expectation.html.
  bool is_expected = 1;

  // The status of the test result.
  Status status = 2;
}

// DEPRECATED: Use TestResult.Status instead.
// Machine-readable status of a test result.
enum TestStatus {
  option deprecated = true;

  // Status was not specified.
  // Not to be used in actual test results; serves as a default value for an
  // unset field.
  STATUS_UNSPECIFIED = 0;

  // The test case has passed.
  PASS = 1;

  // The test case has failed.
  // Suggests that the code under test is incorrect, but it is also possible
  // that the test is incorrect or it is a flake.
  FAIL = 2;

  // The test case has crashed during execution.
  // The outcome is inconclusive: the code under test might or might not be
  // correct, but the test+code is incorrect.
  CRASH = 3;

  // The test case has started, but was aborted before finishing.
  // A common reason: timeout.
  ABORT = 4;

  // The test case did not execute.
  // Examples:
  // - The execution of the collection of test cases, such as a test
  //   binary, was aborted prematurely and execution of some test cases was
  //   skipped.
  // - The test harness configuration specified that the test case MUST be
  //   skipped.
  SKIP = 5;
}

// DEPRECATED: Use SkippedReason.Kind instead.
// Machine-readable reason that a test execution was skipped.
// Only reasons actually used are listed here, if you need a new reason
// please add it here and send a CL to the OWNERS.
enum SkipReason {
    option deprecated = true;

  // Skip reason was not specified.
  // This represents an unset field which should be used for non-skip test
  // result statuses.  It can also be used if none of the other statuses
  // apply.
  SKIP_REASON_UNSPECIFIED = 0;

  // Disabled automatically in response to a test skipping policy that skips
  // flaky tests.
  // Used for ChromeOS CQ test filtering.
  AUTOMATICALLY_DISABLED_FOR_FLAKINESS = 1;
}


// Indicates that the test subject (e.g., a ChangeList in Gerrit) should not
// be considered "at fault" for an failed result of a particular test variant.
// This is often used in presubmit checks (like CQ) to prevent blaming a CL
// for pre-existing or unrelated issues.
// Next ID: 10.
message TestExoneration {
  // Can be used to refer to this test exoneration, e.g. in
  // ResultDB.GetTestExoneration RPC.
  // Format:
  // invocations/{INVOCATION_ID}/tests/{URL_ESCAPED_TEST_ID}/exonerations/{EXONERATION_ID}.
  // URL_ESCAPED_TEST_ID uses the flat test ID format.
  //
  // Output only.
  string name = 1 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.field_behavior) = IMMUTABLE
  ];

  // The structured identifier of the test that is being exonerated.
  TestIdentifier test_id_structured = 9 [(google.api.field_behavior) = IMMUTABLE];

  // The flat test id of the test that is being exonerated.
  // Output only. (Except for legacy uploaders.)
  string test_id = 2 [(google.api.field_behavior) = OUTPUT_ONLY, (google.api.field_behavior) = IMMUTABLE];

  // The variant of the test that is being exonerated.
  // Output only. (Except for legacy uploaders.)
  Variant variant = 3 [(google.api.field_behavior) = OUTPUT_ONLY, (google.api.field_behavior) = IMMUTABLE];

  // Identifies an exoneration in a given invocation and test id.
  // It is server-generated.
  string exoneration_id = 4 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.field_behavior) = IMMUTABLE
  ];

  // Humnan readable reasoning behind the exoneration, in HTML.
  // MUST be sanitized before rendering in the browser.
  string explanation_html = 5 [(google.api.field_behavior) = IMMUTABLE];

  // SHA256 hash of the module variant key-value pairs, hex-encoded.
  // See variant_hash on TestResult for more details.
  // Output only.
  string variant_hash = 6 [(google.api.field_behavior) = IMMUTABLE];

  // Reasoning behind the exoneration, in machine-readable form.
  // Used to assist downstream analyses, such as automatic bug-filing.
  // This allow detection of e.g. critical tests failing in presubmit,
  // even if they are being exonerated because they fail on other CLs.
  ExonerationReason reason = 7 [(google.api.field_behavior) = IMMUTABLE];

  // Whether the test exoneration has been masked so that it includes only
  // metadata. The metadata fields for a TestExoneration are:
  // * name
  // * test_id_structured
  // * exoneration_id
  // * variant_hash
  // * explanation_html
  // * reason
  // * test_id (deprecated)
  // * variant (deprecated)
  //
  // This field is output only.
  bool is_masked = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Reason why a test variant was exonerated.
enum ExonerationReason {
  // Reason was not specified.
  // Not to be used in actual test exonerations; serves as a default value for
  // an unset field.
  EXONERATION_REASON_UNSPECIFIED = 0;

  // Similar unexpected results were observed on a mainline branch
  // (i.e. against a build without unsubmitted changes applied).
  // (For avoidance of doubt, this includes both flakily and
  // deterministically occurring unexpected results.)
  // Applies to unexpected results in presubmit/CQ runs only.
  OCCURS_ON_MAINLINE = 1;

  // Similar unexpected results were observed in presubmit run(s) for other,
  // unrelated CL(s). (This is suggestive of the issue being present
  // on mainline but is not confirmed as there are possible confounding
  // factors, like how tests are run on CLs vs how tests are run on
  // mainline branches.)
  // Applies to unexpected results in presubmit/CQ runs only.
  OCCURS_ON_OTHER_CLS = 2;

  // The tests are not critical to the test subject (e.g. CL) passing.
  // This could be because more data is being collected to determine if
  // the tests are stable enough to be made critical (as is often the
  // case for experimental test suites).
  // If information exists indicating the tests are producing unexpected
  // results, and the tests are not critical for that reason,
  // prefer more specific reasons OCCURS_ON_MAINLINE or OCCURS_ON_OTHER_CLS.
  NOT_CRITICAL = 3;

  // The test result was an unexpected pass. (Note that such an exoneration is
  // not automatically created for unexpected passes, unless the option is
  // specified to ResultSink or the project manually creates one).
  UNEXPECTED_PASS = 4;
}