Performance and size #266
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance and size | |
| on: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| env: | |
| CARGO_TERM_COLOR: always | |
| jobs: | |
| run-benchmarks: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - uses: brndnmtthws/rust-action-cargo-binstall@v1 | |
| with: | |
| packages: hyperfine ripgrep | |
| - name: Install valgrind | |
| id: install-valgrind | |
| continue-on-error: true | |
| run: sudo apt-get install valgrind | |
| - uses: actions/checkout@v4 | |
| - name: Build Ezno (current branch) | |
| run: cargo build --release | |
| env: | |
| CARGO_PROFILE_RELEASE_DEBUG: true | |
| # TEMPORARY LOCATION, SHOULD BE DONE ON MAIN | |
| # Code generated from specification.md. this is not meant to accurately represent a | |
| # program but instead give an idea for how it scales across all the type checking features | |
| - name: Build Ezno (current branch) | |
| run: | | |
| # Generate a file which contains everything that the checker currently implements | |
| # It is intentional that this will be on main branch if it is comparing, because we | |
| # do not want to include new tests | |
| cargo run -p ezno-checker-specification \ | |
| --example amalgamate ./checker/specification/specification.md \ | |
| --comment-headers \ | |
| --out ./demo.tsx | |
| env: | |
| CARGO_PROFILE_RELEASE_DEBUG: true | |
| - uses: actions/checkout@v4 | |
| if: ${{ github.ref_name != 'main' }} | |
| with: | |
| ref: main | |
| clean: false | |
| - name: Build Ezno (main branch) | |
| run: | | |
| mkdir bin2cmp # = binaries to compare | |
| mv ./target/release/ezno bin2cmp/current | |
| cargo build --release | |
| mv ./target/release/ezno bin2cmp/main | |
| if: ${{ github.ref_name != 'main' }} | |
| env: | |
| CARGO_PROFILE_RELEASE_DEBUG: true | |
| - name: Set compilers | |
| id: compilers | |
| shell: bash | |
| run: | | |
| if [ -d "bin2cmp" ]; then | |
| echo "::notice::Comparing against main branch" | |
| echo "BINARIES=bin2cmp/current,bin2cmp/main" >> "$GITHUB_OUTPUT" | |
| echo "BINARY=bin2cmp/current" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "BINARIES=./target/release/ezno" >> "$GITHUB_OUTPUT" | |
| echo "BINARY=./target/release/ezno" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Run checker performance | |
| shell: bash | |
| run: | | |
| echo "### Checking | |
| \`\`\`shell | |
| $(hyperfine -i -L compiler ${{ steps.compilers.outputs.BINARIES }} '{compiler} check demo.tsx') | |
| \`\`\`" >> $GITHUB_STEP_SUMMARY | |
| echo "<details> | |
| <summary>Input</summary> | |
| \`\`\`tsx | |
| $(cat ./demo.tsx) | |
| \`\`\` | |
| </details> | |
| " >> $GITHUB_STEP_SUMMARY | |
| echo "::info::Wrote code to summary" | |
| command_output=$($BINARY check demo.tsx --timings --max-diagnostics all 2>&1 || true) | |
| diagnostics=""; statistics=""; found_splitter=false; | |
| while IFS= read -r line; do | |
| if [[ "$line" == "---"* ]]; then found_splitter=true; | |
| elif [[ "$found_splitter" == false ]]; then diagnostics+="$line"$'\n'; | |
| else statistics+="$line"$'\n'; fi | |
| done <<< "$command_output" | |
| echo "<details> | |
| <summary>Diagnostics</summary> | |
| \`\`\` | |
| $diagnostics | |
| \`\`\` | |
| </details> | |
| " >> $GITHUB_STEP_SUMMARY | |
| if [ -d "bin2cmp" ]; then | |
| OUT=$(./bin2cmp/main check demo.tsx --timings --max-diagnostics all 2>&1 || true) | |
| $main_statistics=$(echo $OUT | rg "Diagnostics:" -A 100) | |
| echo " | |
| <details> | |
| <summary>Statistics</summary> | |
| \`\`\` | |
| $statistics | |
| \`\`\` | |
| against main | |
| \`\`\` | |
| $main_statistics | |
| \`\`\` | |
| against main | |
| </details> | |
| " >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "<details> | |
| <summary>Statistics</summary> | |
| \`\`\` | |
| $statistics | |
| \`\`\` | |
| </details>" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| # - name: Run checker performance w/staging | |
| # shell: bash | |
| # if: github.ref_name != 'main' | |
| # run: | | |
| # echo "::group::Running with staging" | |
| # cat ./checker/specification/specification.md ./checker/specification/staging.md > with-staging.md | |
| # cargo run -p ezno-checker-specification \ | |
| # --example amalgamate with-staging.md \ | |
| # --comment-headers \ | |
| # --out ./with-staging.tsx | |
| # $BINARY check with-staging.tsx --timings || true | |
| # $BINARY check with-staging.tsx --timings || true | |
| # echo "::endgroup::" | |
| - name: Run checker performance on large file | |
| shell: bash | |
| run: | | |
| echo "::group::Running large" | |
| # Repeat the file | |
| for i in {1..10}; do | |
| cat ./demo.tsx >> large.tsx | |
| done | |
| $BINARY check large.tsx --timings --max-diagnostics 0 || true | |
| hyperfine -i -L compiler ${{ steps.compilers.outputs.BINARIES }} '{compiler} check large.tsx' | |
| echo "::endgroup::" | |
| - name: Valgrind and callgrind | |
| shell: bash | |
| if: ${{ steps.install-valgrind.outcome == 'success' }} | |
| continue-on-error: true | |
| run: | | |
| IFS=',' read -ra ITEMS <<< ${{ steps.compilers.outputs.BINARIES }} | |
| export RUST_BACKTRACE=1 | |
| for compiler in ${ITEMS[@]}; do | |
| echo "::group::Running $compiler" | |
| echo "::group::Callgrind" | |
| { | |
| valgrind --tool=callgrind --callgrind-out-file=./cpu-out $compiler check demo.tsx --max-diagnostics 0 | |
| } || true | |
| echo "::notice::CPU usage:$(rg "summary: (.*)" -or '$1' -N --color never cpu_out)" | |
| cat cpu_out | |
| echo "::endgroup::" | |
| echo "::group::Valgrind" | |
| { | |
| valgrind --log-file=memory-out $compiler check demo.tsx --max-diagnostics 0 | |
| } || true | |
| echo "::notice::Memory usage:$(rg "([0-9,]*) bytes allocated" -or '$1' -N --color never memory-out)" | |
| echo "::endgroup::" | |
| echo "::endgroup::" | |
| done | |
| - name: Run parsing & stringing (minfied) benchmarks | |
| shell: bash | |
| continue-on-error: true | |
| run: | | |
| CORPUS_URL="https://gist.githubusercontent.com/kaleidawave/6708f604bc403021b56cb54ea50cac62/raw/javascript_files.txt" | |
| curl -s "$CORPUS_URL" | while IFS= read -r URL; do | |
| FILENAME="${URL##*/}" | |
| curl -s "$URL" > $FILENAME | |
| echo "::group::Comparison" | |
| $BINARY ast-explorer full $FILENAME --check | |
| # hyperfine -i -L compiler ${{ steps.compilers.outputs.BINARIES }} \ | |
| # '{compiler} ast-explorer full $FILENAME --check' | |
| echo "::endgroup::" | |
| done | |
| # - name: Upload checker | |
| # if: ${{ github.ref_name == 'main' }} | |
| # uses: actions/upload-artifact@v4 | |
| # with: | |
| # name: latest-checker | |
| # path: target/release/ezno | |
| # retention-days: 90 |