Let's just check in the PGO Firefox binaries, one build step less

Originally, the PGO data was a bunch of proprietary blobs which I could not legally redistribute. When switching to Firefox binaries, I wanted to keep things as similar as possible, to keep the repo download size down, and not waste disk space. But in this year, megabytes are cheap, everything else is huge, I doubt many people said no, asking the user is an extra step, and buildbots are often sandboxed and can't download them. Let's just cut out the middlemen.
2026-09-07 01:56:07 -05:00 · 2020-01-26 05:12:36 +01:00
parent e8f2b7ceaf
commit 56f58e57ca
9 changed files with 24 additions and 138 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,3 @@
-profile/choice
-profile/firefox-10.0esr.tar
-profile/firefox-17.0esr.tar
-
 obj/
 flips
 flips.exe
--- a/2
+++ b/2
@@ -1,3 +1,5 @@
+#This script creates a debug-optimized binary by default. If you're on Linux, you'll get a faster binary from make.sh.
+
 CFLAGS_gtk = -DFLIPS_GTK $(GTKFLAGS) $(GTKLIBS)
 CFLAGS_windows := -DFLIPS_WINDOWS -mwindows -lgdi32 -lcomdlg32 -lcomctl32 -luser32 -lkernel32 -lshell32 -ladvapi32
 CFLAGS_cli := -DFLIPS_CLI
--- a/make-maintainer.sh
+++ b/make-maintainer.sh
@@ -10,7 +10,6 @@ fi

 rm floating.zip

-profile/download.sh || exit $?
 # . rather than ./make.sh, so $FLAGS remains set
 . ./make.sh
 mv flips flips-linux
@@ -31,7 +30,8 @@ echo 'Windows (1/3)'
 rm obj/* flips.exe; wine mingw32-make TARGET=windows OPTFLAGS="$FLAGS -fprofile-generate -lgcov"
 [ -e flips.exe ] || exit
 echo 'Windows (2/3)'
-profile/profile.sh 'wine flips.exe'
+/usr/bin/time --verbose wine flips.exe --create --bps-delta         profile/firefox-10.0esr.tar profile/firefox-17.0esr.tar /dev/null
+/usr/bin/time --verbose wine flips.exe --create --bps-delta-moremem profile/firefox-10.0esr.tar profile/firefox-17.0esr.tar /dev/null
 echo 'Windows (3/3)'
 rm flips.exe; wine mingw32-make TARGET=windows OPTFLAGS="$FLAGS -fprofile-use"

--- a/make.bat
+++ b/make.bat
@@ -1,56 +0,0 @@
-::This script creates a heavily optimized binary. For debugging, you're better off using 'mingw32-make'.
-
-::Feel free to point this elsewhere.
-call "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\vcvars32.bat"
-::call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\vcvars32.bat"
-::call "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\vcvars64.bat"
-
-goto q
-:h
-pause
-:q
-cls
-del flips.exe
-if exist flips.exe goto h
-::/c - don't link
-::/Oy - omit frame pointer
-::/Oi - use intrinsics
-::/Os - smaller code
-::/Ox - all optimizations
-::/EHs-c- - no exceptions
-::/Gz - use stdcall
-::/D_CRT_bluh - shut the fuck up about strcpy
-::/WX - enable warnings
-::/wd4700 - uninitialized variable, I know what I'm doing
-::/nologo - I know your version, shut up
-::/TP - force C++ because void foo(int bar = 0); is syntax error in C
-::/MT - use static msvcrt
-::/GL - LTCG
-::/openmp - enable OpenMP (adds an extra DLL, so let's not)
-cl /c /Oy /Oi /Os /Ox /EHs-c- /Gz /D_CRT_SECURE_NO_WARNINGS /WX /wd4700 /nologo /TP /MT /GL /Imsvc ^
-   /DUSE_DIVSUFSORT /Ilibdivsufsort-2.0.1\include /DHAVE_CONFIG_H libdivsufsort-2.0.1\lib\*.c ^
-   *.c *.cpp
-::rc doesn't have a /nologo, just let it spam
-rc flips.rc
-
-link /LTCG /nologo *.obj /subsystem:windows flips.res user32.lib gdi32.lib comctl32.lib shell32.lib comdlg32.lib advapi32.lib /out:flips.exe
-
-::FOR %%G IN (2dland-dec2013,smwcp,nsmw-tll,senate13,kamek) DO (
-::FOR %%G IN (2dland-dec2013) DO (
-::flips --create --ips profile/smw.smc profile/%%G.smc profile/tmp
-::flips --apply  --ips profile/tmp profile/smw.smc NUL.smc
-
-::flips --create --bps-linear profile/smw.smc profile/%%G.smc profile/tmp
-::flips --apply  --bps-linear profile/tmp profile/smw.smc NUL.smc
-
-::flips --create --bps-delta profile/smw.smc profile/%%G.smc profile/tmp
-::flips --apply  --bps-delta profile/tmp profile/smw.smc NUL.smc
-::)
-::del profile\tmp
-
-::link /nologo /ltcg:pgoptimize *.obj /subsystem:windows flips.res user32.lib gdi32.lib comctl32.lib shell32.lib comdlg32.lib advapi32.lib /out:flips.exe
-
-del *.obj flips.res *.pgc *.pgd
-goto h
-
-::kernel32.lib shell32.lib user32.lib gdi32.lib comctl32.lib comdlg32.lib dinput8.lib
--- a/make.sh
+++ b/make.sh
@@ -1,44 +1,33 @@
 #!/bin/sh

-#This script creates a heavily optimized binary. For debugging, you're better off using 'make'.
+#This script creates a heavily optimized Linux binary. For debugging or Windows, you're better off using the Makefile directly.

-FLAGS='-Wall -Werror -O3 -s -flto -fuse-linker-plugin -fweb -fomit-frame-pointer -fmerge-all-constants -fvisibility=hidden'
+FLAGS='-Wall -Werror -O3 -flto -fuse-linker-plugin -fomit-frame-pointer -fmerge-all-constants -fvisibility=hidden'
 FLAGS=$FLAGS' -fno-exceptions -fno-unwind-tables -fno-asynchronous-unwind-tables'
 FLAGS=$FLAGS' -ffunction-sections -fdata-sections -Wl,--gc-sections -fprofile-dir=obj/'
 #Linux flags, they don't make sense on Windows
 #make-release.sh uses this
 LINFLAGS=' -Wl,-z,relro,-z,now,--as-needed,--hash-style=gnu,--relax'

-#if old profile data is present, download the new one instead
-#do it before checking for arguments, so --profile=no with old data existing doesn't do anything silly
-if [ -e profile/firefox-45.0esr.tar ]; then
-  rm profile/firefox-45.0esr.tar profile/firefox-52.0esr.tar
-  touch profile/firefox-10.0esr.tar
-fi
-
 PROFILE=yes
+STRIP="-s"

 for i in "$@"; do
 case "$i" in
-  --harden=no)
-  ;;
  --harden=yes)
    FLAGS=$FLAGS' -fstack-protector-all -Wstack-protector --param ssp-buffer-size=4 -pie -fPIE -D_FORTIFY_SOURCE=2'
    true | gcc -E - -mmitigate-rop > /dev/null 2>&1 &&
      FLAGS=$FLAGS' -mmitigate-rop'
  ;;
-  --profile=no)
-    echo n > profile/choice
-    rm profile/firefox-10.0esr.tar
-    rm profile/firefox-17.0esr.tar
-  ;;
-  --profile=no-once)
+  --profile=no|--profile=no-once)
+    # With PGO, compilation takes about 75 seconds for me, with a peak memory use of 814MB.
+    # Without PGO, it takes 2.5 seconds and 75MB RAM. However, the resulting binary is about 2% slower.
    PROFILE=no
  ;;
-  --profile=yes)
-    if [ ! -e profile/firefox-10.0esr.tar ]; then
-      profile/download.sh || exit $?
-    fi
+  --strip=no)
+    STRIP="-g"
+  ;;
+  --harden=no|--strip=yes|--profile=yes)
  ;;
  *)
  echo "Unknown argument $1; valid arguments are:"
@@ -48,38 +37,23 @@ case "$i" in
 esac
 done

+rm flips obj/*

-if [ $PROFILE = yes -a ! -e profile/choice ]; then
-  while true; do
-    #              1         2         3         4         5         6         7         8
-    #     12345678901234567890123456789012345678901234567890123456789012345678901234567890
-    echo "Do you wish to use profile-guided optimization? This will download 40MB"
-    echo "data from the internet and use 90MB disk space, and use 800MB RAM during"
-    echo "compilation. The result is typically 2% faster."
-    read -p "(y/n)" yn || exit $?
-    case $yn in
-      [Yy]*) profile/download.sh || exit $?; break;;
-      [Nn]*) echo n > profile/choice; break;;
-      *) echo "Please answer yes or no.";;
-    esac
-  done
-fi
+if [ $PROFILE = yes ]; then

-#if download was aborted, resume it
-if [ -e profile/firefox-10.0esr.tar ]; then
-  profile/download.sh || exit $?
-fi
-
-rm flips flips.exe obj/*
-
-if [ $PROFILE = yes -a -e profile/firefox-10.0esr.tar ]; then
 echo 'GTK+ (1/3)'
 rm obj/* flips; TARGET=gtk make OPTFLAGS="$FLAGS$LINFLAGS -fprofile-generate -lgcov" || exit $?
 [ -e flips ] || exit 1
 echo 'GTK+ (2/3)'
-profile/profile.sh ./flips
+
+# These files are downloaded and extracted from
+# https://ftp.mozilla.org/pub/firefox/releases/10.0esr/linux-x86_64/en-US/firefox-10.0esr.tar.bz2
+# https://ftp.mozilla.org/pub/firefox/releases/17.0esr/linux-x86_64/en-US/firefox-17.0esr.tar.bz2
+/usr/bin/time --verbose ./flips --create --bps-delta         profile/firefox-10.0esr.tar profile/firefox-17.0esr.tar /dev/null
+/usr/bin/time --verbose ./flips --create --bps-delta-moremem profile/firefox-10.0esr.tar profile/firefox-17.0esr.tar /dev/null
+
 echo 'GTK+ (3/3)'
-rm flips; TARGET=gtk make OPTFLAGS="$FLAGS$LINFLAGS -fprofile-use"
+rm flips; TARGET=gtk make OPTFLAGS="$FLAGS$LINFLAGS -fprofile-use" || exit $?
 else
 rm flips; TARGET=gtk make OPTFLAGS="$FLAGS$LINFLAGS" || exit $?
 fi
--- a/profile/download.sh
+++ b/profile/download.sh
@@ -1,20 +0,0 @@
-#!/bin/sh
-
-#why firefox? I need two large, freely available, mostly similar, binary files; two versions of an executable would be a good choice
-#why 10/17? older firefoxes are smaller, so profiling needs less RAM; but I also want to skip a few versions, to ensure they're
-# not too similar; and I want something chosen with a minimum of arbitrariness. The oldest ESRs satisfy those constraints.
-SRC=profile/firefox-10.0esr.tar
-DST=profile/firefox-17.0esr.tar
-SRCSIZE=39526400
-DSTSIZE=49192960
-SRCURL=https://ftp.mozilla.org/pub/firefox/releases/10.0esr/linux-x86_64/en-US/firefox-10.0esr.tar.bz2
-DSTURL=https://ftp.mozilla.org/pub/firefox/releases/17.0esr/linux-x86_64/en-US/firefox-17.0esr.tar.bz2
-
-if [ ! -f $SRC ] || [ $(wc -c <$SRC) -lt $SRCSIZE ]; then
-  wget $SRCURL -O- | bzcat > $SRC || curl $SRCURL | bzcat > $SRC || exit $?
-fi
-if [ ! -f $DST ] || [ $(wc -c <$DST) -lt $DSTSIZE ]; then
-  wget $DSTURL -O- | bzcat > $DST || curl $DSTURL | bzcat > $DST || exit $?
-fi
-
-echo y > profile/choice
--- a/profile/firefox-10.0esr.tar
+++ b/profile/firefox-10.0esr.tar
--- a/profile/firefox-17.0esr.tar
+++ b/profile/firefox-17.0esr.tar
--- a/profile/profile.sh
+++ b/profile/profile.sh
@@ -1,10 +0,0 @@
-#!/bin/sh
-
-#profiling plus threading tends to act weirdly, disable it
-export OMP_NUM_THREADS=1
-
-SRC=profile/firefox-10.0esr.tar
-DST=profile/firefox-17.0esr.tar
-
-/usr/bin/time --verbose $1 --create --bps-delta         $SRC $DST /dev/null
-/usr/bin/time --verbose $1 --create --bps-delta-moremem $SRC $DST /dev/null