Changeset 0b5aa1f0 in mainline for tools/toolchain.sh


Ignore:
Timestamp:
2023-10-24T16:38:29Z (13 months ago)
Author:
Jiří Zárevúcky <zarevucky.jiri@…>
Branches:
master, ticket/834-toolchain-update, topic/msim-upgrade, topic/simplify-dev-export
Children:
29ed06d
Parents:
cc73f6d4
git-author:
Jiří Zárevúcky <zarevucky.jiri@…> (2023-10-24 16:25:02)
git-committer:
Jiří Zárevúcky <zarevucky.jiri@…> (2023-10-24 16:38:29)
Message:

toolchain.sh: extensive changes

libgcc may need binutils and GCC to be properly installed to
build correctly. To avoid problems, the —no-install option was
removed and the script reorganized so that system-wide installation
is still as convenient as possible.

Overall, the process occurs in several stages:

  • binutils is built for all architectures
  • all binutils are installed with help of sudo if needed
  • gcc is built for all architectures
  • all GCCs are installed with help of sudo if needed
  • libgcc is built for all architectures
  • GDB is built for all architectures (if enabled)
  • all files are installed to target location (with sudo if needed)

This process ensures that a system-wide install of all platforms
only asks for password three times in total, which is two more times
than ideal, but acceptable nonetheless.

Futher changes include:

  • GDB build is optional and disabled by default
  • non-HelenOS target build removed
  • removed 'essential' option, since we want keep HelenOS riscv64 port buildable regardless of its current low functionality
File:
1 edited

Legend:

Unmodified
Added
Removed

  • tools/toolchain.sh

    rcc73f6d4 r0b5aa1f0  
    4444SRCDIR="$(readlink -f $(dirname "$0"))"
    4545
     46# If we install into a temporary directory and copy from there,
     47# we don't have to trust the upstream makefiles respect the prefix for
     48# all files, and we also have a ready-made package for distribution.
     49INSTALL_DIR="${BASEDIR}/PKG"
     50
    4651SYSTEM_INSTALL=false
    47 REAL_INSTALL=true
    48 USE_HELENOS_TARGET=true
     52
     53BUILD_GDB=false
     54BUILD_BINUTILS=true
     55BUILD_GCC=true
     56
     57if [ -z "$JOBS" ] ; then
     58        JOBS="`nproc`"
     59fi
    4960
    5061check_error() {
    … …  
    6172        echo
    6273        echo "Syntax:"
    63         echo " $0 [--system-wide] [--no-install] [--non-helenos-target] <platform>"
     74        echo " $0 [--system-wide] [--with-gdb|--only-gdb] <platform>"
    6475        echo " $0 [--system-wide] --test-version [<platform>]"
    6576        echo
    … …  
    7687        echo " sparc64    SPARC V9"
    7788        echo " all        build all targets"
    78         echo " essential  build only targets currently needed for HelenOS development"
    7989        echo " parallel   same as 'all', but all in parallel"
    8090        echo " 2-way      same as 'all', but 2-way parallel"
    … …  
    92102        echo "     variable is not set, then the default value of \$HOME/.local/share"
    93103        echo "     is assumed."
    94         echo
    95         echo "If the --no-install option is used, the toolchain still uses the"
    96         echo "target installation directory as determined above, but the files"
    97         echo "are actually copied into the PKG/ subdirectory during the installation"
    98         echo "without affecting the actual target file system. This might be useful"
    99         echo "when preparing a system-wide installation, but avoiding running this"
    100         echo "script under the super user."
    101104        echo
    102105        echo "The --non-helenos-target option will build non-HelenOS-specific"
    … …  
    177180}
    178181
     182ring_bell() {
     183        printf '\a'
     184        sleep 0.1
     185        printf '\a'
     186        sleep 0.1
     187        printf '\a'
     188        sleep 0.1
     189        printf '\a'
     190        sleep 0.1
     191        printf '\a'
     192}
     193
    179194show_countdown() {
    180195        TM="$1"
    … …  
    251266        check_error $? "Unable to change directory to ${ORIGINAL}."
    252267
    253         mkdir -p "${OUTSIDE}"
     268        if $SYSTEM_INSTALL && [ ! -d "${OUTSIDE}" ]; then
     269                ring_bell
     270                ( set -x ; sudo -k mkdir -p "${OUTSIDE}" )
     271        else
     272                mkdir -p "${OUTSIDE}"
     273        fi
     274
    254275        cd "${OUTSIDE}"
    255276        check_error $? "Unable to change directory to ${OUTSIDE}."
    … …  
    280301        change_title "Downloading sources"
    281302        echo ">>> Downloading sources"
    282         git clone --depth 1 -b "$BINUTILS_BRANCH" "$BINUTILS_GDB_GIT" "binutils-$BINUTILS_VERSION"
    283         git clone --depth 1 -b "$GDB_BRANCH" "$BINUTILS_GDB_GIT" "gdb-$GDB_VERSION"
    284         git clone --depth 1 -b "$GCC_BRANCH" "$GCC_GIT" "gcc-$GCC_VERSION"
    285 
    286         # If the directory already existed, pull upstream changes.
    287         git -C "binutils-$BINUTILS_VERSION" pull
    288         git -C "gdb-$GDB_VERSION" pull
    289         git -C "gcc-$GCC_VERSION" pull
    290 
    291         change_title "Downloading GCC prerequisites"
    292         echo ">>> Downloading GCC prerequisites"
    293         cd "gcc-${GCC_VERSION}"
    294         ./contrib/download_prerequisites
    295         cd ..
     303
     304        if $BUILD_BINUTILS ; then
     305                git clone --depth 1 -b "$BINUTILS_BRANCH" "$BINUTILS_GDB_GIT" "binutils-$BINUTILS_VERSION"
     306                # If the directory already existed, pull upstream changes.
     307                git -C "binutils-$BINUTILS_VERSION" pull
     308        fi
     309
     310        if $BUILD_GCC ; then
     311                git clone --depth 1 -b "$GCC_BRANCH" "$GCC_GIT" "gcc-$GCC_VERSION"
     312                git -C "gcc-$GCC_VERSION" pull
     313
     314                change_title "Downloading GCC prerequisites"
     315                echo ">>> Downloading GCC prerequisites"
     316                cd "gcc-${GCC_VERSION}"
     317                ./contrib/download_prerequisites
     318                cd ..
     319        fi
     320
     321        if $BUILD_GDB ; then
     322                git clone --depth 1 -b "$GDB_BRANCH" "$BINUTILS_GDB_GIT" "gdb-$GDB_VERSION"
     323                git -C "gdb-$GDB_VERSION" pull
     324        fi
     325
     326        # This sets the CROSS_PREFIX variable
     327        set_cross_prefix
     328
     329        DESTDIR_SPEC="DESTDIR=${INSTALL_DIR}"
    296330}
    297331
    … …  
    321355        esac
    322356
    323         HELENOS_TARGET="${GNU_ARCH}-helenos"
    324 
    325         case "$1" in
    326                 "arm32")
    327                         LINUX_TARGET="${GNU_ARCH}-linux-gnueabi"
    328                         ;;
    329                 *)
    330                         LINUX_TARGET="${GNU_ARCH}-linux-gnu"
    331                         ;;
    332         esac
    333 }
    334 
    335 build_target() {
    336         PLATFORM="$1"
    337 
    338         # This sets the *_TARGET variables
    339         set_target_from_platform "$PLATFORM"
    340         if $USE_HELENOS_TARGET ; then
    341                 TARGET="$HELENOS_TARGET"
    342         else
    343                 TARGET="$LINUX_TARGET"
    344         fi
     357        TARGET="${GNU_ARCH}-helenos"
     358}
     359
     360build_binutils() {
     361        # This sets the TARGET variable
     362        set_target_from_platform "$1"
    345363
    346364        WORKDIR="${BASEDIR}/${TARGET}"
    347         INSTALL_DIR="${BASEDIR}/PKG"
    348365        BINUTILSDIR="${WORKDIR}/binutils-${BINUTILS_VERSION}"
    349         GCCDIR="${WORKDIR}/gcc-${GCC_VERSION}"
    350         GDBDIR="${WORKDIR}/gdb-${GDB_VERSION}"
    351 
    352         # This sets the CROSS_PREFIX variable
    353         set_cross_prefix
    354 
    355         if [ -z "$JOBS" ] ; then
    356                 JOBS="`nproc`"
    357         fi
    358 
    359         PREFIX="${CROSS_PREFIX}"
    360366
    361367        echo ">>> Removing previous content"
    362368        cleanup_dir "${WORKDIR}"
    363369        mkdir -p "${WORKDIR}"
    364         check_dirs "${PREFIX}" "${WORKDIR}"
    365 
    366         if $USE_HELENOS_TARGET ; then
    367                 change_title "Creating build sysroot"
    368                 echo ">>> Creating build sysroot"
    369                 mkdir -p "${WORKDIR}/sysroot/include"
    370                 mkdir "${WORKDIR}/sysroot/lib"
    371                 ARCH="$PLATFORM"
    372                 if [ "$ARCH" = "mips32eb" ]; then
    373                         ARCH=mips32
    374                 fi
    375 
    376                 cp -r -L -t "${WORKDIR}/sysroot/include" \
    377                         ${SRCDIR}/../abi/include/* \
    378                         ${SRCDIR}/../uspace/lib/c/arch/${ARCH}/include/* \
    379                         ${SRCDIR}/../uspace/lib/c/include/*
    380                 check_error $? "Failed to create build sysroot."
    381         fi
    382 
     370        check_dirs "${CROSS_PREFIX}" "${WORKDIR}"
    383371
    384372        echo ">>> Processing binutils (${PLATFORM})"
    … …  
    390378        CFLAGS="-Wno-error -fcommon" "${BASEDIR}/downloads/binutils-${BINUTILS_VERSION}/configure" \
    391379                "--target=${TARGET}" \
    392                 "--prefix=${PREFIX}" \
     380                "--prefix=${CROSS_PREFIX}" \
    393381                "--program-prefix=${TARGET}-" \
    394382                --disable-nls \
    … …  
    405393
    406394        change_title "binutils: install (${PLATFORM})"
    407         make install "DESTDIR=${INSTALL_DIR}"
     395        make install $DESTDIR_SPEC
    408396        check_error $? "Error installing binutils."
    409 
     397}
     398
     399build_gcc() {
     400        # This sets the TARGET variable
     401        set_target_from_platform "$1"
     402
     403        WORKDIR="${BASEDIR}/${TARGET}"
     404        GCCDIR="${WORKDIR}/gcc-${GCC_VERSION}"
     405
     406        echo ">>> Removing previous content"
     407        cleanup_dir "${WORKDIR}"
     408        mkdir -p "${WORKDIR}"
     409        check_dirs "${CROSS_PREFIX}" "${WORKDIR}"
    410410
    411411        echo ">>> Processing GCC (${PLATFORM})"
    … …  
    414414        check_error $? "Change directory failed."
    415415
    416         if $USE_HELENOS_TARGET ; then
    417                 SYSROOT=--with-sysroot --with-build-sysroot="${WORKDIR}/sysroot"
    418         else
    419                 SYSROOT=--without-headers
    420         fi
     416        BUILDPATH="${CROSS_PREFIX}/bin:${PATH}"
    421417
    422418        change_title "GCC: configure (${PLATFORM})"
    423         PATH="$PATH:${INSTALL_DIR}/${PREFIX}/bin" "${BASEDIR}/downloads/gcc-${GCC_VERSION}/configure" \
     419        PATH="${BUILDPATH}" "${BASEDIR}/downloads/gcc-${GCC_VERSION}/configure" \
    424420                "--target=${TARGET}" \
    425                 "--prefix=${PREFIX}" \
     421                "--prefix=${CROSS_PREFIX}" \
    426422                "--program-prefix=${TARGET}-" \
    427423                --with-gnu-as \
    … …  
    432428                --disable-shared \
    433429                --disable-werror \
    434                 $SYSROOT
     430                --without-headers  # TODO: Replace with proper sysroot so we can build more libs
    435431        check_error $? "Error configuring GCC."
    436432
    437433        change_title "GCC: make (${PLATFORM})"
    438         PATH="${PATH}:${PREFIX}/bin:${INSTALL_DIR}/${PREFIX}/bin" make all-gcc -j$JOBS
     434        PATH="${BUILDPATH}" make all-gcc -j$JOBS
    439435        check_error $? "Error compiling GCC."
    440436
    441         if $USE_HELENOS_TARGET ; then
    442                 PATH="${PATH}:${PREFIX}/bin:${INSTALL_DIR}/${PREFIX}/bin" make all-target-libgcc -j$JOBS
    443                 check_error $? "Error compiling libgcc."
    444                 # TODO: libatomic and libstdc++ need some extra care
    445                 #    PATH="${PATH}:${PREFIX}/bin:${INSTALL_DIR}/${PREFIX}/bin" make all-target-libatomic -j$JOBS
    446                 #    check_error $? "Error compiling libatomic."
    447                 #    PATH="${PATH}:${PREFIX}/bin:${INSTALL_DIR}/${PREFIX}/bin" make all-target-libstdc++-v3 -j$JOBS
    448                 #    check_error $? "Error compiling libstdc++."
    449         fi
    450 
    451437        change_title "GCC: install (${PLATFORM})"
    452         PATH="${PATH}:${INSTALL_DIR}/${PREFIX}/bin" make install-gcc "DESTDIR=${INSTALL_DIR}"
    453         if $USE_HELENOS_TARGET ; then
    454                 PATH="${PATH}:${INSTALL_DIR}/${PREFIX}/bin" make install-target-libgcc "DESTDIR=${INSTALL_DIR}"
    455                 #    PATH="${PATH}:${INSTALL_DIR}/${PREFIX}/bin" make install-target-libatomic "DESTDIR=${INSTALL_DIR}"
    456                 #    PATH="${PATH}:${INSTALL_DIR}/${PREFIX}/bin" make install-target-libstdc++-v3 "DESTDIR=${INSTALL_DIR}"
    457         fi
     438        PATH="${BUILDPATH}" make install-gcc $DESTDIR_SPEC
    458439        check_error $? "Error installing GCC."
    459 
     440}
     441
     442build_libgcc() {
     443        # This sets the TARGET variable
     444        set_target_from_platform "$1"
     445
     446        WORKDIR="${BASEDIR}/${TARGET}"
     447        GCCDIR="${WORKDIR}/gcc-${GCC_VERSION}"
     448
     449        # No removing previous content here, we need the previous GCC build
     450
     451        cd "${GCCDIR}"
     452        check_error $? "Change directory failed."
     453
     454        BUILDPATH="${CROSS_PREFIX}/bin:${PATH}"
     455
     456        change_title "libgcc: make (${PLATFORM})"
     457
     458        PATH="${BUILDPATH}" make all-target-libgcc -j$JOBS
     459        check_error $? "Error compiling libgcc."
     460        # TODO: libatomic and libstdc++ need some extra care
     461        #    PATH="${BUILDPATH}" make all-target-libatomic -j$JOBS
     462        #    check_error $? "Error compiling libatomic."
     463        #    PATH="${BUILDPATH}" make all-target-libstdc++-v3 -j$JOBS
     464        #    check_error $? "Error compiling libstdc++."
     465
     466        change_title "libgcc: install (${PLATFORM})"
     467
     468        PATH="${BUILDPATH}" make install-target-libgcc $DESTDIR_SPEC
     469        #    PATH="${BUILDPATH}" make install-target-libatomic $DESTDIR_SPEC
     470        #    PATH="${BUILDPATH}" make install-target-libstdc++-v3 $DESTDIR_SPEC
     471        check_error $? "Error installing libgcc."
     472}
     473
     474build_gdb() {
     475        # This sets the TARGET variable
     476        set_target_from_platform "$1"
     477
     478        WORKDIR="${BASEDIR}/${TARGET}"
     479        GDBDIR="${WORKDIR}/gdb-${GDB_VERSION}"
     480
     481        echo ">>> Removing previous content"
     482        cleanup_dir "${WORKDIR}"
     483        mkdir -p "${WORKDIR}"
     484        check_dirs "${CROSS_PREFIX}" "${WORKDIR}"
    460485
    461486        echo ">>> Processing GDB (${PLATFORM})"
    … …  
    465490
    466491        change_title "GDB: configure (${PLATFORM})"
    467         CFLAGS="-fcommon" PATH="$PATH:${INSTALL_DIR}/${PREFIX}/bin" "${BASEDIR}/downloads/gdb-${GDB_VERSION}/configure" \
     492        CFLAGS="-fcommon" "${BASEDIR}/downloads/gdb-${GDB_VERSION}/configure" \
    468493                "--target=${TARGET}" \
    469                 "--prefix=${PREFIX}" \
     494                "--prefix=${CROSS_PREFIX}" \
    470495                "--program-prefix=${TARGET}-" \
    471496                --enable-werror=no
    … …  
    473498
    474499        change_title "GDB: make (${PLATFORM})"
    475         PATH="${PATH}:${PREFIX}/bin:${INSTALL_DIR}/${PREFIX}/bin" make all-gdb -j$JOBS
     500        make all-gdb -j$JOBS
    476501        check_error $? "Error compiling GDB."
    477502
    478         change_title "GDB: make (${PLATFORM})"
    479         PATH="${PATH}:${INSTALL_DIR}/${PREFIX}/bin" make install-gdb "DESTDIR=${INSTALL_DIR}"
     503        change_title "GDB: install (${PLATFORM})"
     504        make install-gdb $DESTDIR_SPEC
    480505        check_error $? "Error installing GDB."
    481 
     506}
     507
     508install_pkg() {
     509        echo ">>> Moving to the destination directory."
     510        if $SYSTEM_INSTALL ; then
     511                ring_bell
     512                ( set -x ; sudo -k cp -r -t "${CROSS_PREFIX}" "${INSTALL_DIR}${CROSS_PREFIX}/"* )
     513        else
     514                cp -r -t "${CROSS_PREFIX}" "${INSTALL_DIR}${CROSS_PREFIX}/"*
     515        fi
     516}
     517
     518link_clang() {
    482519        # Symlink clang and lld to the install path.
    483520        CLANG="`which clang 2> /dev/null || echo "/usr/bin/clang"`"
    484521        LLD="`which ld.lld 2> /dev/null || echo "/usr/bin/ld.lld"`"
    485522
    486         ln -s $CLANG "${INSTALL_DIR}/${PREFIX}/bin/${TARGET}-clang"
    487         ln -s $LLD "${INSTALL_DIR}/${PREFIX}/bin/${TARGET}-ld.lld"
    488 
    489         if $REAL_INSTALL ; then
    490                 echo ">>> Moving to the destination directory."
    491                 cp -r -t "${PREFIX}" "${INSTALL_DIR}/${PREFIX}/"*
    492         fi
    493 
    494         cd "${BASEDIR}"
    495         check_error $? "Change directory failed."
    496 
    497         echo ">>> Cleaning up"
    498         cleanup_dir "${WORKDIR}"
    499 
    500         echo
    501         echo ">>> Cross-compiler for ${TARGET} installed."
     523        ln -s $CLANG "${INSTALL_DIR}${CROSS_PREFIX}/bin/${TARGET}-clang"
     524        ln -s $LLD "${INSTALL_DIR}${CROSS_PREFIX}/bin/${TARGET}-ld.lld"
    502525}
    503526
    … …  
    512535                        exit
    513536                        ;;
    514                 --no-install)
    515                         REAL_INSTALL=false
     537                --with-gdb)
     538                        BUILD_GDB=true
    516539                        shift
    517540                        ;;
    518                 --non-helenos-target)
    519                         USE_HELENOS_TARGET=false
     541                --only-gdb)
     542                        BUILD_GDB=true
     543                        BUILD_BINUTILS=false
     544                        BUILD_GCC=false
    520545                        shift
    521546                        ;;
    … …  
    529554        show_usage
    530555fi
     556
     557PLATFORMS="amd64 arm32 arm64 ia32 ia64 mips32 mips32eb ppc32 riscv64 sparc64"
     558
     559run_one() {
     560        $1 $PLATFORM
     561}
     562
     563run_all() {
     564        for x in $PLATFORMS ; do
     565                $1 $x
     566        done
     567}
     568
     569run_parallel() {
     570        for x in $PLATFORMS ; do
     571                $1 $x &
     572        done
     573        wait
     574}
     575
     576run_2way() {
     577        $1 amd64 &
     578        $1 arm32 &
     579        wait
     580
     581        $1 arm64 &
     582        $1 ia32 &
     583        wait
     584
     585        $1 ia64 &
     586        $1 mips32 &
     587        wait
     588
     589        $1 mips32eb &
     590        $1 ppc32 &
     591        wait
     592
     593        $1 riscv64 &
     594        $1 sparc64 &
     595        wait
     596}
     597
     598everything() {
     599        RUNNER="$1"
     600
     601        prepare
     602
     603        if $BUILD_BINUTILS ; then
     604                $RUNNER build_binutils
     605
     606                if $BUILD_GCC ; then
     607                        # gcc/libgcc may fail to build correctly if binutils is not installed first
     608                        echo ">>> Installing binutils"
     609                        install_pkg
     610                fi
     611        fi
     612
     613        if $BUILD_GCC ; then
     614                $RUNNER build_gcc
     615
     616                # libgcc may fail to build correctly if gcc is not installed first
     617                echo ">>> Installing GCC"
     618                install_pkg
     619
     620                for x in $PLATFORMS ; do
     621                        build_libgcc $x
     622                done
     623        fi
     624
     625        if $BUILD_GDB ; then
     626                $RUNNER build_gdb
     627        fi
     628
     629        echo ">>> Installing all files"
     630        install_pkg
     631
     632        link_clang
     633}
    531634
    532635case "$1" in
    … …  
    536639                ;;
    537640        amd64|arm32|arm64|ia32|ia64|mips32|mips32eb|ppc32|riscv64|sparc64)
    538                 prepare
    539                 build_target "$1"
     641                PLATFORM="$1"
     642                everything run_one
    540643                ;;
    541644        "all")
    542                 prepare
    543                 build_target "amd64"
    544                 build_target "arm32"
    545                 build_target "arm64"
    546                 build_target "ia32"
    547                 build_target "ia64"
    548                 build_target "mips32"
    549                 build_target "mips32eb"
    550                 build_target "ppc32"
    551                 build_target "riscv64"
    552                 build_target "sparc64"
    553                 ;;
    554         "essential")
    555                 prepare
    556                 build_target "amd64"
    557                 build_target "arm32"
    558                 build_target "arm64"
    559                 build_target "ia32"
    560                 build_target "ia64"
    561                 build_target "mips32"
    562                 build_target "mips32eb"
    563                 build_target "ppc32"
    564                 build_target "sparc64"
     645                everything run_all
    565646                ;;
    566647        "parallel")
    567                 prepare
    568                 build_target "amd64" &
    569                 build_target "arm32" &
    570                 build_target "arm64" &
    571                 build_target "ia32" &
    572                 build_target "ia64" &
    573                 build_target "mips32" &
    574                 build_target "mips32eb" &
    575                 build_target "ppc32" &
    576                 build_target "riscv64" &
    577                 build_target "sparc64" &
    578                 wait
     648                everything run_parallel
    579649                ;;
    580650        "2-way")
    581                 prepare
    582                 build_target "amd64" &
    583                 build_target "arm32" &
    584                 wait
    585 
    586                 build_target "arm64" &
    587                 build_target "ia32" &
    588                 wait
    589 
    590                 build_target "ia64" &
    591                 build_target "mips32" &
    592                 wait
    593 
    594                 build_target "mips32eb" &
    595                 build_target "ppc32" &
    596                 wait
    597 
    598                 build_target "riscv64" &
    599                 build_target "sparc64" &
    600                 wait
     651                everything run_2way
    601652                ;;
    602653        *)
Note: See TracChangeset for help on using the changeset viewer.