ladybird/.github/workflows/libjs-test262.yml

name: Run test262 and test-wasm

on: [push, pull_request]

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || format('{0}-{1}', github.ref, github.run_number) }}
  cancel-in-progress: true

env:
  LADYBIRD_SOURCE_DIR: ${{ github.workspace }}
  CCACHE_DIR: ${{ github.workspace }}/.ccache
  VCPKG_ROOT: ${{ github.workspace }}/Build/vcpkg

jobs:
  run_and_update_results:
    runs-on: 'blacksmith-16vcpu-ubuntu-2404'
    if: github.repository == 'LadybirdBrowser/ladybird'

    steps:
      - name: Checkout LadybirdBrowser/ladybird
        uses: actions/checkout@v6.0.2
        with:
          persist-credentials: false

      - name: Checkout LadybirdBrowser/libjs-test262
        uses: actions/checkout@v6.0.2
        with:
          repository: LadybirdBrowser/libjs-test262
          path: libjs-test262

      - name: Checkout LadybirdBrowser/libjs-data
        uses: actions/checkout@v6.0.2
        with:
          repository: LadybirdBrowser/libjs-data
          path: libjs-data

      - name: Checkout tc39/test262
        uses: actions/checkout@v6.0.2
        with:
          repository: tc39/test262
          path: test262

      - name: Checkout tc39/test262-parser-tests
        uses: actions/checkout@v6.0.2
        with:
          repository: tc39/test262-parser-tests
          path: test262-parser-tests

      - name: Set up environment
        uses: ./.github/actions/setup
        id: 'setup'
        with:
          os: 'Linux'
          arch: 'x86_64'
          toolchain: 'Clang'

      - name: Install Python dependencies
        run: |
          pip install -r libjs-test262/requirements.txt

      - name: Restore Caches
        uses: ./.github/actions/cache-restore
        id: 'cache-restore'
        with:
          runner_labels: '["blacksmith-16vcpu-ubuntu-2404"]'
          os: 'Linux'
          arch: 'x86_64'
          toolchain: 'Clang'
          cache_key_extra: 'test262'
          ccache_path: ${{ env.CCACHE_DIR }}
          vcpkg_cache_path: ${{ github.workspace }}/Build/caches/vcpkg-binary-cache

      - name: Get previous results
        run: |
          mkdir -p old-libjs-data
          cp -R libjs-data/test262 libjs-data/wasm old-libjs-data

      - name: Build test262-runner, test-js, test-wasm and test-js-bytecode
        run: |
          cmake --preset Release -B libjs-test262/Build \
            -DCMAKE_C_COMPILER=clang-${{ steps.setup.outputs.llvm_version }} \
            -DCMAKE_CXX_COMPILER=clang++-${{ steps.setup.outputs.llvm_version }} \
            -DWASM_SPEC_TEST_SKIP_FORMATTING=ON \
            -DINCLUDE_WASM_SPEC_TESTS=ON \
            -DENABLE_GUI_TARGETS=OFF
          ninja -C libjs-test262/Build test262-runner test-js test-wasm test-js-bytecode

      - name: Save Caches
        uses: ./.github/actions/cache-save
        with:
          runner_labels: '["blacksmith-16vcpu-ubuntu-2404"]'
          arch: 'x86_64'
          ccache_path: ${{ env.CCACHE_DIR }}
          ccache_primary_key: ${{ steps.cache-restore.outputs.ccache_primary_key }}
          vcpkg_cache_path: ${{ github.workspace }}/Build/caches/vcpkg-binary-cache
          vcpkg_cache_primary_key: ${{ steps.cache-restore.outputs.vcpkg_cache_primary_key }}

      - name: Run test262 and test262-parser-tests
        working-directory: libjs-test262
        run: |
          python3 run_all_and_update_results.py \
            --serenity .. \
            --test262 ../test262 \
            --test262-parser-tests ../test262-parser-tests \
            --results-json ../libjs-data/test262/results.json \
            --per-file-output ../libjs-data/test262/per-file-master.json

      - name: Run test-wasm
        working-directory: libjs-test262
        run: |
          Build/bin/test-wasm --per-file Build/Lagom/Libraries/LibWasm/Tests > ../libjs-data/wasm/per-file-master.json || true
          jq -nc -f /dev/stdin <<-EOF --slurpfile previous ../libjs-data/wasm/results.json --slurpfile details ../libjs-data/wasm/per-file-master.json > wasm-new-results.json
            \$details[0] as \$details | \$previous[0] + [{
              "commit_timestamp": $(git -C .. log -1 --format=%ct),
              "run_timestamp": $(date +%s),
              "versions": {
                "serenity": "$(git -C .. rev-parse HEAD)"
              },
              "tests": {
                "spectest": {
                  "duration": (\$details.duration),
                  "results": {
                    "total": (\$details.results | keys | length),
                    "passed": ([\$details.results | values[] | select(. == "PASSED")] | length),
                    "failed": ([\$details.results | values[] | select(. == "FAILED")] | length),
                    "skipped": ([\$details.results | values[] | select(. == "SKIPPED")] | length),
                    "process_error": ([\$details.results | values[] | select(. == "PROCESS_ERROR")] | length)
                  }
                }
              }
            }]
          EOF
          mv wasm-new-results.json ../libjs-data/wasm/results.json

      - name: Compare test262 results
        run: ./libjs-test262/per_file_result_diff.py -o old-libjs-data/test262/per-file-master.json -n libjs-data/test262/per-file-master.json

      - name: Compare Wasm results
        run: ./libjs-test262/per_file_result_diff.py -o old-libjs-data/wasm/per-file-master.json -n libjs-data/wasm/per-file-master.json

      - name: Deploy to GitHub
        if: ${{ github.ref == 'refs/heads/master' }}
        uses: JamesIves/github-pages-deploy-action@v4.8.0
        with:
          git-config-name: LadybirdBot
          git-config-email: ladybirdbot@ladybird.org
          branch: master
          repository-name: LadybirdBrowser/libjs-data
          token: ${{ secrets.LADYBIRD_BOT_TOKEN }}
          folder: libjs-data