fix: Yamato jobs not testing built player (Backport of: #4194) (#4196)

* fix: Yamato jobs not testing built player (Backport of: #4194)

* fix: Yamato jobs not testing built player

* Next round of fixes

* Set addressables to build on players

* dotnet-fix

* Fix console tests

* Try again to remove graphics from desktop players

* Fix DA NetworkObjectDeserializationFailure test

* Remove the temporarily added N4E settings

* Fix the test failures

* Remove fix that was more a symptom than a cause (keep it simple)

* Match the deregistration to the registration logic

* Don't finish cleaning up if any scene is still loading

* Fix the U-PR config

* Fix IL2CPP player tests

* Remove CoreCLR stuff

* feat: Improve error message on connection failure [2.X] (#4208)

feat: Improve error message on connection failure

* Disabled automatic release preparation

Disabled recurring trigger as in our current situation it would be more messy to handle automated release step. I will leave it manual for now

* chore: Ensure the rerun strategy is baked into the built players (#4218)

* fix: generate canonical bools in NetworkVariable serialization tests (#4198)

* test: generate canonical bools in NetworkVariable serialization tests

RandGenBytes<bool> filled each bool with a random byte, so most values were
neither 0 nor 1. On mac arm64 IL2CPP the List<bool> and NativeHashMap<K, bool>
cases then failed with both sides printing identical all-True contents.
RandGenBool returns only false or true.

* test: use RandGenBool for the NativeArray<bool> case

* test: make changed2 always differ from original2 in collection generators

With RandGenBool an overwrite keeps the existing value half the time, so when
no elements were added or removed, changed2 could equal original2 and the
"do not match" assertions failed. Each overwrite now redraws until it differs
from the original element.

---------

Co-authored-by: Emma <emma.mcmillan@unity3d.com>
(cherry picked from commit b5c186c)

---------

Co-authored-by: Noel Stephens <noel.stephens@unity3d.com>
Co-authored-by: Simon Lemay <simon.lemay@unity3d.com>
Co-authored-by: Michał Chrobot <michal.chrobot@unity3d.com>

Author: 4 people

.pr_agent.toml

@@ -4,6 +4,5 @@
 # https://developer.portal.internal.unity.com/catalog/default/component/u-pr/docs/configuration/automatic_runs/
 
 [github_app]
-handle_pr_actions = ['opened', 'ready_for_review']  # PR events that auto-run pr_commands
 pr_commands       = ["/harness-review"]             # what runs on those events
 reviewer_commands = ["/harness-review"]             # what runs when @u-pr is added as reviewer

.yamato/cmb-service-standalone-tests.yml

@@ -56,8 +56,7 @@ cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}
     # run_cmb_service.sh builds and starts a release version of the full CMB service (along with the limited echo server)
     - ./Tools/CI/service.cmb/run_cmb_service.sh -e $ECHO_SERVER_PORT -s $CMB_SERVICE_PORT -l artifacts
 
-    - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models
-    - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }}
+    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --scripting-backend={{ backend }} --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --extra-player-arg=-batchmode --extra-player-arg=-nographics --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --timeout={{ test_timeout }}
   artifacts:
     logs:
       paths:

.yamato/console-standalone-test.yml

@@ -50,7 +50,7 @@ console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}:
 {% endif %}
   commands:
     - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }}
-    - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics{% if platform.name == "switch" %} --extra-editor-arg=-overrideTextureCompression --extra-editor-arg=ForceUncompressed{% endif %} --reruncount=1 --clean-library-on-rerun --build-only --timeout={{ test_timeout}}
+    - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics{% if platform.name == "switch" %} --extra-editor-arg=-overrideTextureCompression --extra-editor-arg=ForceUncompressed{% endif %} --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --build-only --timeout={{ test_timeout}}
   variables:
 # PS4 related
     SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00'
@@ -88,8 +88,17 @@ console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}:
     model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile)
 {% endif %}
   commands:
-    - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }}
-    - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }}
+    # Console players connect back to UTR's player connection listener on this VM, so the firewall must allow it
+    - command: gsudo NetSh Advfirewall set allprofiles state off
+    # Register the Bokken devkit with the platform's target tools so UTR can deploy and run the prebuilt player
+{% if platform.name == "ps5" %}
+    - command: gsudo prospero-ctrl target add %BOKKEN_DEVICE_IP%
+{% elsif platform.name == "ps4" %}
+    - command: orbis-ctrl add %BOKKEN_DEVICE_IP%
+{% else %}
+    - command: '"%GameDK%/bin/xbconnect" /WS %BOKKEN_DEVICE_IP%'
+{% endif %}
+    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --scripting-backend=il2cpp --testproject={{ project.path }} --artifacts-path=test-results --player-load-path=build/players --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --timeout={{ test_timeout }}
   variables:
 # PS4 related
     SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00'

.yamato/desktop-standalone-tests.yml

@@ -44,7 +44,7 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{
 {% endif %}
   commands:
     - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %}
-    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout={{ test_timeout }}
+    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*;TestProject.RuntimeTests.*" --playergraphicsapi=Null --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --build-only --timeout={{ test_timeout }}
   artifacts:
     players:
       paths:
@@ -79,8 +79,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{
 {% endif %}
 
   commands:
-    - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models
-    - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }}
+    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --scripting-backend={{ backend }} --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --extra-player-arg=-batchmode --extra-player-arg=-nographics --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --timeout={{ test_timeout }}
   artifacts:
     logs:
       paths:

.yamato/mobile-standalone-test.yml

@@ -55,7 +55,7 @@ mobile_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}:
     UNITY_HANDLEUIINTERRUPTIONS: 1
   commands:
     - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %}
-    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout={{ test_timeout }}
+    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --build-only --timeout={{ test_timeout }}
   artifacts:
     players:
       paths:
@@ -82,9 +82,6 @@ mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}:
     model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile)
 {% endif %}
   commands:
-    # Installing editor. We still need the editor to run tests on standalone build and for that the Editor is required
-    - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %}
-
 {% if platform.standalone == "Android" %}
     # Download standalone UnityTestRunner and ADB setup
     - command: wget http://artifactory-slo.bf.unity3d.com/artifactory/mobile-generic/android/ADBKeys.zip!/adbkey.pub -O %USERPROFILE%/.android/adbkey.pub
@@ -97,11 +94,11 @@ mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}:
     # Run tests for Android devices
     - |
       set ANDROID_DEVICE_CONNECTION=%BOKKEN_DEVICE_IP%
-      UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }}
+      UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --timeout={{ test_timeout }}
 {% else %}
 
     # Run tests for non-Android devices
-    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }} --device-id=%BOKKEN_DEVICE_ID%
+    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --timeout={{ test_timeout }} --device-id=%BOKKEN_DEVICE_ID%
 {% endif %}
   artifacts:
     logs:

.yamato/webgl-build.yml

@@ -45,7 +45,7 @@ webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }}:
     # Engine is initialized in ‘nographics’ mode since we don't need any graphics for this case (--extra-editor-arg=-nographics)
     # In case of failure the job will be rerunned once (--reruncount=1) with clean library (--clean-library-on-rerun). Note that this is build only step so no tests are being executed
     # This will perform only building phase (--build-only) with a timeout set by the test_timeout variable in the project metafile.
-    - UnifiedTestRunner --suite=playmode --platform=WebGL --scripting-backend=il2cpp --testproject={{ project.path }} --editor-location=.Editor --artifacts_path=artifacts --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout={{ test_timeout }}
+    - UnifiedTestRunner --suite=playmode --platform=WebGL --scripting-backend=il2cpp --testproject={{ project.path }} --editor-location=.Editor --artifacts_path=artifacts --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --build-only --timeout={{ test_timeout }}
   artifacts:
     logs:
       paths:

Tools/CI/service.cmb/run_cmb_service.sh

@@ -171,8 +171,8 @@ cargo build --release --locked
 # The infinite loop is required as the service will exit each time all connected clients disconnect.
 # This means the service will exit after each test. The infinite loop will immediately restart the service each time it exits.
 logMessage "Running service integration tests..."
-echo "comb-server -l error --metrics-port 5000 standalone --port $service_port -t 60m --unity-version $unity_version"
+echo "comb-server -l warn -l ngo::session=info -l comb_server::session=info --metrics-port 5000 standalone --port $service_port -t 60m --unity-version $unity_version"
 
 while :; do
-  ./target/release/comb-server -l error --metrics-port 5000 standalone --port $service_port -t 60m --unity-version $unity_version;
+  ./target/release/comb-server -l warn -l ngo::session=info -l comb_server::session=info --metrics-port 5000 standalone --port $service_port -t 60m --unity-version $unity_version;
 done & # <- use & to run the entire loop in the background

com.unity.netcode.gameobjects/Tests/Runtime/NetworkTransform/NetworkTransformOrderOfOperations.cs

@@ -24,6 +24,7 @@ namespace Unity.Netcode.RuntimeTests
     [TestFixture(HostOrServer.DAHost)]
     [TestFixture(HostOrServer.Host)]
     [TestFixture(HostOrServer.Server)]
+    [UnityPlatform(exclude = new[] { RuntimePlatform.PS4 })] // Ignored test tracked in MTT-16494
     internal class NetworkTransformOrderOfOperations : IntegrationTestWithApproximation
     {
         #region Configuration, Properties, and Overrides