--stockfish-- evaluation

.github/workflows/games.yml

@@ -37,7 +37,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           path: new
           persist-credentials: false
@@ -64,7 +64,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           ref: ${{ github.event.inputs.base_ref }}
           path: base
@@ -86,60 +86,117 @@ jobs:
         with:
           name: base-engine
           path: base/build/${{ github.event.inputs.output_exec }}
-
-
-  test:
+  prepare-tools:
     runs-on: ubuntu-latest
-    needs: [build-new, build-base]
 
     steps:
-      - name: Download engines
-        uses: actions/download-artifact@v4
+      - run: |
+          mkdir tools
+          cd tools
 
-      - name: Install tools
-        env:
-          OUTPUT_EXEC: ${{ github.event.inputs.output_exec }}
-        run: |
           sudo apt update
           sudo apt install -y wget unzip
-
           wget https://github.com/Disservin/fastchess/releases/download/v1.8.0-alpha/fastchess-linux-x86-64.tar
           tar -xf fastchess-linux-x86-64.tar
-          chmod +x fastchess-linux-x86-64/fastchess
           mv fastchess-linux-x86-64/fastchess fastchess
-
           wget https://github.com/official-stockfish/books/raw/refs/heads/master/UHO_Lichess_4852_v1.epd.zip
           unzip UHO_Lichess_4852_v1.epd.zip
-
           wget https://github.com/michiguel/Ordo/releases/download/1.0/ordo-linux64
-          chmod +x ordo-linux64
 
+      - uses: actions/upload-artifact@v4
+        with:
+          name: tools
+          path: tools/
+  test:
+    needs: [build-new, build-base, prepare-tools]
+    strategy:
+      fail-fast: true
+      matrix:
+        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,20,21,22,23,24,25,26,27,28,29]
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/download-artifact@v4
+      - name: Play games
+        env:
+          OUTPUT_EXEC: ${{ github.event.inputs.output_exec }}
+          TC: ${{ github.event.inputs.tc }}
+          ELO0: ${{ github.event.inputs.elo0 }}
+          ELO1: ${{ github.event.inputs.elo1 }}
+        run: |
+          set -eo pipefail
           chmod +x new-engine/engine
           chmod +x base-engine/$OUTPUT_EXEC
+          chmod +x tools/fastchess
+          chmod +x tools/ordo-linux64
+          tools/fastchess \
+            -engine cmd=new-engine/engine name=new \
+            -engine cmd=base-engine/$OUTPUT_EXEC name=base \
+            -openings file=tools/UHO_Lichess_4852_v1.epd format=epd order=random \
+            -each tc=$TC -startup-ms 600000 -ucinewgame-ms 600000 -ping-ms 600000 \
+          ROUNDS: ${{ github.event.inputs.rounds }}
+        run: |
+          set -eo pipefail
+          if [[ ! "$ROUNDS" =~ ^[0-9]+$ ]]; then
+            echo "rounds must be a decimal integer" >&2
+            exit 1
+          fi
+          rounds_value=$((10#$ROUNDS))
+          shard_rounds=$((rounds_value / 30))
+            -rounds "$shard_rounds" -repeat -log level=err \
+            -concurrency $(nproc) -testEnv -sprt elo0=$ELO0 elo1=$ELO1 alpha=0.05 beta=0.05 \
+            -pgnout notation=san nodes=true \
+                    file=games_${{ matrix.shard }}.pgn \
+            | tee results_${{ matrix.shard }}.txt
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: shard-${{ matrix.shard }}
+          path: |
+            games_${{ matrix.shard }}.pgn
+            results_${{ matrix.shard }}.txt
 
-      - name: Run fastchess
+  evaluate:
+    needs: test
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: actions/download-artifact@v4
+
+      - name: Run Ordo
+        run: |
+          chmod +x tools/ordo-linux64
+          printf '<%s>\n' shard-*/games_*.pgn
+          cat shard-*/games_*.pgn > games.pgn
+          tools/ordo-linux64 -o ratings.txt games.pgn
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.13'
+
+      - name: SPRT
         env:
           ELO0: ${{ github.event.inputs.elo0 }}
           ELO1: ${{ github.event.inputs.elo1 }}
-          TC: ${{ github.event.inputs.tc }}
-          OUTPUT_EXEC: ${{ github.event.inputs.output_exec }}
-          ROUNDS: ${{ github.event.inputs.rounds }}
         run: |
-          ./fastchess -recover \
-             -engine cmd=new-engine/engine name=new \
-             -engine cmd=base-engine/$OUTPUT_EXEC name=base \
-             -openings file=UHO_Lichess_4852_v1.epd format=epd order=random \
-             -each tc=$TC \
-             -rounds $ROUNDS \
-             -concurrency $(nproc) \
-             -pgnout notation=san nodes=true file=games.pgn \
-             -sprt elo0=$ELO0 elo1=$ELO1 alpha=0.05 beta=0.05 | tee results.txt
-          ./ordo-linux64 -o ratings.txt -- games.pgn
-      - name: Upload results
-        uses: actions/upload-artifact@v4
+          pip install chess scipy
+          git clone https://github.com/jdart1/stats.git
+          RESULTS=$(python extract_ptnml.py shard-*/*.pgn | tee results.txt | tail -n1)
+          python -m stats.sprt \
+              --elo0 "$ELO0" \
+              --elo1 "$ELO1" \
+              --alpha 0.05 \
+              --beta 0.05 \
+              --results $RESULTS | tee -a results.txt
+
+      - uses: actions/upload-artifact@v4
         with:
           name: results
           path: |
-            games.pgn
             ratings.txt
+            sprt.txt
             results.txt

.github/workflows/spsa.yml

@@ -2,23 +2,13 @@ name: run SPSA (see gist)
 
 on:
   workflow_dispatch:
-    inputs:
-      base_ref:
-        description: "Tune branch"
-        required: false
-        default: "HandcraftedEngine"
-      output_exec:
-        description: "Executable output file name"
-        required: false
-        default: "engine"
 jobs:
   worker:
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v6
         with:
-          ref: ${{ github.event.inputs.base_ref }}
           persist-credentials: false
 
       - uses: actions/setup-python@v6
@@ -40,13 +30,13 @@ jobs:
           cmake --build . -j$(nproc)
       - name: Tune engine
         run: |
-          python spsa.py --iters 150 --pairs 30 --engine build/engine --workers $(nproc)
+          python spsa.py --iters 40 --pairs 30 --engine build/engine --workers $(nproc)
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
           name: spsa-results
           path: |
             ./Weights.h
-            *.pgn
-            *.epd
-          compression-level: 0
+            games*.pgn
+            games*.epd
+          compression-level: 9

Weights.h

@@ -2,12 +2,17 @@
 #define WEIGHTS_H
 #include "eval.h"
 namespace engine::eval {
-inline Value tempo = 20;
-inline Value PawnValue = 100;
-inline Value KnightValue = 320;
-inline Value BishopValue = 330;
-inline Value RookValue = 500;
-inline Value QueenValue = 900;
+inline Value tempo = 28;
+inline Value PawnValueMG = 124;
+inline Value KnightValueMG = 781;
+inline Value BishopValueMG = 825;
+inline Value RookValueMG = 1276;
+inline Value QueenValueMG = 2538;
+inline Value PawnValueEG = 206;
+inline Value KnightValueEG = 854;
+inline Value BishopValueEG = 915;
+inline Value RookValueEG = 1380;
+inline Value QueenValueEG = 2682;
 inline Value fianchettoBonus = 20;
 inline Value trappedBishopPenalty = 60;
 inline Value centerWeight = 5;
@@ -98,7 +103,6 @@ inline Value eg_pawn_table[56] = { 0,   0,   0,   0,   0,  0,   0,   0,  0,   0,
                                    5,   5,   2,   -2,  -5, -10, -5,  10, 20,  20,  10,  -5,  -10, -20, -10, 30,  55,  55, 30,
                                    -10, -20, -35, -20, 70, 110, 110, 70, -20, -35, -60, -45, 120, 180, 180, 120, -45, -60 };
 inline Value developedMg = 8;
-inline Value developedEg = 4;
 inline Value outpostBonusKnight[2] = { 15, 30 };
 inline Value outpostBonusBishop[2] = { 10, 25 };
 inline Value kingProtector[6][2] = {