From 7b781d88909214120ae2f5655e12d68cbc08ec2c Mon Sep 17 00:00:00 2001 From: "Alexander \"Ananace\" Olofsson" Date: Sun, 17 May 2020 01:12:04 +0200 Subject: [PATCH] Windows CI dependency upgrade (#2847) * Windows CI: Use OSG 3.4-experimental for 0.46 * Update compiled Windows CI dependencies Only built and pushed so far, still need to try making full OpenMW builds with them as well. * Update missed Bullet version number * MyGUI uses RelWithDebInfo for Release builds now * Update Windows CI dependencies, switch Qt install * Fix aqt retrieval and setup * Make aqt install output slightly nicer * Bump to Qt 5.15 for VS2019 support * Fix FFmpeg and Qt install parts * Fix OSG plugin DLL copying * Add CMake flag for double-precision bullet * Roll back 2019 to Boost 1.71 for CI * Move aqt into unpack step, to allow manual install --- CI/before_script.msvc.sh | 211 ++++++++++++++++++++++++--------------- 1 file changed, 132 insertions(+), 79 deletions(-) diff --git a/CI/before_script.msvc.sh b/CI/before_script.msvc.sh index 3a0b062e6e..c1786889ae 100644 --- a/CI/before_script.msvc.sh +++ b/CI/before_script.msvc.sh @@ -5,6 +5,7 @@ MISSINGTOOLS=0 command -v 7z >/dev/null 2>&1 || { echo "Error: 7z (7zip) is not on the path."; MISSINGTOOLS=1; } command -v cmake >/dev/null 2>&1 || { echo "Error: cmake (CMake) is not on the path."; MISSINGTOOLS=1; } +command -v python >/dev/null 2>&1 || { echo "Warning: Python is not on the path, automatic Qt installation impossible."; } if [ $MISSINGTOOLS -ne 0 ]; then exit 1 @@ -32,11 +33,15 @@ KEEP="" UNITY_BUILD="" VS_VERSION="" NMAKE="" +PDBS="" PLATFORM="" CONFIGURATION="" TEST_FRAMEWORK="" GOOGLE_INSTALL_ROOT="" INSTALL_PREFIX="." +BULLET_DOUBLE="" +BULLET_DBL="" +BULLET_DBL_DISPLAY="Single precision" while [ $# -gt 0 ]; do ARGSTR=$1 @@ -57,6 +62,9 @@ while [ $# -gt 0 ]; do d ) SKIP_DOWNLOAD=true ;; + D ) + BULLET_DOUBLE=true ;; + e ) SKIP_EXTRACT=true ;; @@ -77,6 +85,9 @@ while [ $# -gt 0 ]; do PLATFORM=$1 shift ;; + P ) + PDBS=true ;; + c ) CONFIGURATION=$1 shift ;; @@ -96,6 +107,8 @@ Options: Set the configuration, can also be set with environment variable CONFIGURATION. -d Skip checking the downloads. + -D + Use double-precision Bullet -e Skip extracting dependencies. -h @@ -112,6 +125,8 @@ Options: Choose the Visual Studio version to use. -n Produce NMake makefiles instead of a Visual Studio solution. + -P + Download debug symbols where available -V Run verbosely -i @@ -264,6 +279,7 @@ case $VS_VERSION in MSVC_REAL_VER="16" MSVC_VER="14.2" MSVC_YEAR="2015" + MSVC_REAL_YEAR="2019" MSVC_DISPLAY_YEAR="2019" BOOST_VER="1.71.0" BOOST_VER_URL="1_71_0" @@ -276,6 +292,7 @@ case $VS_VERSION in MSVC_REAL_VER="15" MSVC_VER="14.1" MSVC_YEAR="2015" + MSVC_REAL_YEAR="2017" MSVC_DISPLAY_YEAR="2017" BOOST_VER="1.67.0" BOOST_VER_URL="1_67_0" @@ -288,6 +305,7 @@ case $VS_VERSION in MSVC_REAL_VER="14" MSVC_VER="14.0" MSVC_YEAR="2015" + MSVC_REAL_YEAR="2015" MSVC_DISPLAY_YEAR="2015" BOOST_VER="1.67.0" BOOST_VER_URL="1_67_0" @@ -295,15 +313,8 @@ case $VS_VERSION in ;; 12|12.0|2013 ) - GENERATOR="Visual Studio 12 2013" - TOOLSET="vc120" - MSVC_REAL_VER="12" - MSVC_VER="12.0" - MSVC_YEAR="2013" - MSVC_DISPLAY_YEAR="2013" - BOOST_VER="1.58.0" - BOOST_VER_URL="1_58_0" - BOOST_VER_SDK="105800" + echo "Visual Studio 2013 is no longer supported" + exit 1 ;; esac @@ -369,6 +380,12 @@ if ! [ -z $UNITY_BUILD ]; then add_cmake_opts "-DOPENMW_UNITY_BUILD=True" fi +if [ -n "$BULLET_DOUBLE" ]; then + BULLET_DBL="-double" + BULLET_DBL_DISPLAY="Double precision" + add_cmake_opts "-DBULLET_USE_DOUBLES=True" +fi + echo echo "===================================" echo "Starting prebuild on MSVC${MSVC_DISPLAY_YEAR} WIN${BITS}" @@ -393,45 +410,54 @@ if [ -z $SKIP_DOWNLOAD ]; then fi # Bullet - download "Bullet 2.87" \ - "https://www.lysator.liu.se/~ace/OpenMW/deps/Bullet-2.87-msvc${MSVC_YEAR}-win${BITS}.7z" \ - "Bullet-2.87-msvc${MSVC_YEAR}-win${BITS}.7z" + download "Bullet 2.89 (${BULLET_DBL_DISPLAY})" \ + "https://rgw.ctrl-c.liu.se/openmw/Deps/Bullet-2.89-msvc${MSVC_YEAR}-win${BITS}${BULLET_DBL}.7z" \ + "Bullet-2.89-msvc${MSVC_YEAR}-win${BITS}${BULLET_DBL}.7z" # FFmpeg - download "FFmpeg 3.2.4" \ - "https://ffmpeg.zeranoe.com/builds/win${BITS}/shared/ffmpeg-3.2.4-win${BITS}-shared.zip" \ - "ffmpeg-3.2.4-win${BITS}.zip" \ - "https://ffmpeg.zeranoe.com/builds/win${BITS}/dev/ffmpeg-3.2.4-win${BITS}-dev.zip" \ - "ffmpeg-3.2.4-dev-win${BITS}.zip" + download "FFmpeg 4.2.2" \ + "https://ffmpeg.zeranoe.com/builds/win${BITS}/shared/ffmpeg-4.2.2-win${BITS}-shared.zip" \ + "ffmpeg-4.2.2-win${BITS}.zip" \ + "https://ffmpeg.zeranoe.com/builds/win${BITS}/dev/ffmpeg-4.2.2-win${BITS}-dev.zip" \ + "ffmpeg-4.2.2-dev-win${BITS}.zip" # MyGUI - download "MyGUI 3.2.2" \ - "https://www.lysator.liu.se/~ace/OpenMW/deps/MyGUI-3.2.2-msvc${MSVC_YEAR}-win${BITS}.7z" \ - "MyGUI-3.2.2-msvc${MSVC_YEAR}-win${BITS}.7z" + download "MyGUI 3.4.0" \ + "https://rgw.ctrl-c.liu.se/openmw/Deps/MyGUI-3.4.0-msvc${MSVC_REAL_YEAR}-win${BITS}.7z" \ + "MyGUI-3.4.0-msvc${MSVC_REAL_YEAR}-win${BITS}.7z" + + if [ -n "$PDBS" ]; then + download "MyGUI symbols" \ + "https://rgw.ctrl-c.liu.se/openmw/Deps/MyGUI-3.4.0-msvc${MSVC_REAL_YEAR}-win${BITS}-sym.7z" \ + "MyGUI-3.4.0-msvc${MSVC_REAL_YEAR}-win${BITS}-sym.7z" + fi # OpenAL - download "OpenAL-Soft 1.19.1" \ - "http://openal-soft.org/openal-binaries/openal-soft-1.19.1-bin.zip" \ - "OpenAL-Soft-1.19.1.zip" + download "OpenAL-Soft 1.20.1" \ + "http://openal-soft.org/openal-binaries/openal-soft-1.20.1-bin.zip" \ + "OpenAL-Soft-1.20.1.zip" # OSG - download "OpenSceneGraph 3.4.1-scrawl" \ - "https://www.lysator.liu.se/~ace/OpenMW/deps/OSG-3.4.1-scrawl-msvc${MSVC_YEAR}-win${BITS}.7z" \ - "OSG-3.4.1-scrawl-msvc${MSVC_YEAR}-win${BITS}.7z" + download "OpenSceneGraph 3.6.5" \ + "https://rgw.ctrl-c.liu.se/openmw/Deps/OSG-3.6.5-msvc${MSVC_REAL_YEAR}-win${BITS}.7z" \ + "OSG-3.6.5-msvc${MSVC_REAL_YEAR}-win${BITS}.7z" + + if [ -n "$PDBS" ]; then + download "OpenSceneGraph symbols" \ + "https://rgw.ctrl-c.liu.se/openmw/Deps/OSG-3.6.5-msvc${MSVC_REAL_YEAR}-win${BITS}-sym.7z" \ + "OSG-3.6.5-msvc${MSVC_REAL_YEAR}-win${BITS}-sym.7z" + fi # Qt if [ -z $APPVEYOR ]; then - if [ $BITS == "64" ]; then - QT_SUFFIX="_64" - else - QT_SUFFIX="" + if [ "${MSVC_REAL_YEAR}" = "2015" ] && [ "${BITS}" = "32" ]; then + echo "Qt no longer provides MSVC2015 Win32 packages, switch to 64-bit or a newer Visual Studio. Sorry." + exit 1 fi - download "Qt 5.7.0" \ - "https://download.qt.io/new_archive/qt/5.7/5.7.0/qt-opensource-windows-x86-msvc${MSVC_YEAR}${QT_SUFFIX}-5.7.0.exe" \ - "qt-5.7.0-msvc${MSVC_YEAR}-win${BITS}.exe" \ - "https://www.lysator.liu.se/~ace/OpenMW/deps/qt-5-install.qs" \ - "qt-5-install.qs" + download "AQt installer" \ + "https://files.pythonhosted.org/packages/f3/bb/aee972f08deecca31bfc46b5aedfad1ce6c7f3aaf1288d685e4a914b53ac/aqtinstall-0.8-py2.py3-none-any.whl" \ + "aqtinstall-0.8-py2.py3-none-any.whl" fi # SDL2 @@ -533,15 +559,15 @@ fi cd $DEPS echo # Bullet -printf "Bullet 2.87... " +printf "Bullet 2.89 (${BULLET_DBL_DISPLAY})... " { cd $DEPS_INSTALL if [ -d Bullet ]; then printf -- "Exists. (No version checking) " elif [ -z $SKIP_EXTRACT ]; then rm -rf Bullet - eval 7z x -y "${DEPS}/Bullet-2.87-msvc${MSVC_YEAR}-win${BITS}.7z" $STRIP - mv "Bullet-2.87-msvc${MSVC_YEAR}-win${BITS}" Bullet + eval 7z x -y "${DEPS}/Bullet-2.89-msvc${MSVC_YEAR}-win${BITS}${BULLET_DBL}.7z" $STRIP + mv "Bullet-2.89-msvc${MSVC_YEAR}-win${BITS}${BULLET_DBL}" Bullet fi export BULLET_ROOT="$(real_pwd)/Bullet" echo Done. @@ -549,21 +575,21 @@ printf "Bullet 2.87... " cd $DEPS echo # FFmpeg -printf "FFmpeg 3.2.4... " +printf "FFmpeg 4.2.2... " { cd $DEPS_INSTALL - if [ -d FFmpeg ] && grep "FFmpeg version: 3.2.4" FFmpeg/README.txt > /dev/null; then + if [ -d FFmpeg ] && grep "4.2.2" FFmpeg/README.txt > /dev/null; then printf "Exists. " elif [ -z $SKIP_EXTRACT ]; then rm -rf FFmpeg - eval 7z x -y "${DEPS}/ffmpeg-3.2.4-win${BITS}.zip" $STRIP - eval 7z x -y "${DEPS}/ffmpeg-3.2.4-dev-win${BITS}.zip" $STRIP - mv "ffmpeg-3.2.4-win${BITS}-shared" FFmpeg - cp -r "ffmpeg-3.2.4-win${BITS}-dev/"* FFmpeg/ - rm -rf "ffmpeg-3.2.4-win${BITS}-dev" + eval 7z x -y "${DEPS}/ffmpeg-4.2.2-win${BITS}.zip" $STRIP + eval 7z x -y "${DEPS}/ffmpeg-4.2.2-dev-win${BITS}.zip" $STRIP + mv "ffmpeg-4.2.2-win${BITS}-shared" FFmpeg + cp -r "ffmpeg-4.2.2-win${BITS}-dev/"* FFmpeg/ + rm -rf "ffmpeg-4.2.2-win${BITS}-dev" fi export FFMPEG_HOME="$(real_pwd)/FFmpeg" - add_runtime_dlls "$(pwd)/FFmpeg/bin/"{avcodec-57,avformat-57,avutil-55,swresample-2,swscale-4}.dll + add_runtime_dlls "$(pwd)/FFmpeg/bin/"{avcodec-58,avformat-58,avutil-56,swresample-3,swscale-5}.dll if [ $BITS -eq 32 ]; then add_cmake_opts "-DCMAKE_EXE_LINKER_FLAGS=\"/machine:X86 /safeseh:no\"" fi @@ -572,62 +598,66 @@ printf "FFmpeg 3.2.4... " cd $DEPS echo # MyGUI -printf "MyGUI 3.2.2... " +printf "MyGUI 3.4.0... " { cd $DEPS_INSTALL if [ -d MyGUI ] && \ grep "MYGUI_VERSION_MAJOR 3" MyGUI/include/MYGUI/MyGUI_Prerequest.h > /dev/null && \ - grep "MYGUI_VERSION_MINOR 2" MyGUI/include/MYGUI/MyGUI_Prerequest.h > /dev/null && \ - grep "MYGUI_VERSION_PATCH 2" MyGUI/include/MYGUI/MyGUI_Prerequest.h > /dev/null + grep "MYGUI_VERSION_MINOR 4" MyGUI/include/MYGUI/MyGUI_Prerequest.h > /dev/null && \ + grep "MYGUI_VERSION_PATCH 0" MyGUI/include/MYGUI/MyGUI_Prerequest.h > /dev/null then printf "Exists. " elif [ -z $SKIP_EXTRACT ]; then rm -rf MyGUI - eval 7z x -y "${DEPS}/MyGUI-3.2.2-msvc${MSVC_YEAR}-win${BITS}.7z" $STRIP - mv "MyGUI-3.2.2-msvc${MSVC_YEAR}-win${BITS}" MyGUI + eval 7z x -y "${DEPS}/MyGUI-3.4.0-msvc${MSVC_REAL_YEAR}-win${BITS}.7z" $STRIP + [ -n "$PDBS" ] && eval 7z x -y "${DEPS}/MyGUI-3.4.0-msvc${MSVC_REAL_YEAR}-win${BITS}-sym.7z" $STRIP + mv "MyGUI-3.4.0-msvc${MSVC_REAL_YEAR}-win${BITS}" MyGUI fi export MYGUI_HOME="$(real_pwd)/MyGUI" if [ $CONFIGURATION == "Debug" ]; then SUFFIX="_d" + MYGUI_CONFIGURATION="Debug" else SUFFIX="" + MYGUI_CONFIGURATION="RelWithDebInfo" fi - add_runtime_dlls "$(pwd)/MyGUI/bin/${CONFIGURATION}/MyGUIEngine${SUFFIX}.dll" + add_runtime_dlls "$(pwd)/MyGUI/bin/${MYGUI_CONFIGURATION}/MyGUIEngine${SUFFIX}.dll" echo Done. } cd $DEPS echo # OpenAL -printf "OpenAL-Soft 1.19.1... " +printf "OpenAL-Soft 1.20.1... " { - if [ -d openal-soft-1.19.1-bin ]; then + if [ -d openal-soft-1.20.1-bin ]; then printf "Exists. " elif [ -z $SKIP_EXTRACT ]; then - rm -rf openal-soft-1.19.1-bin - eval 7z x -y OpenAL-Soft-1.19.1.zip $STRIP + rm -rf openal-soft-1.20.1-bin + eval 7z x -y OpenAL-Soft-1.20.1.zip $STRIP fi - OPENAL_SDK="$(real_pwd)/openal-soft-1.19.1-bin" + OPENAL_SDK="$(real_pwd)/openal-soft-1.20.1-bin" add_cmake_opts -DOPENAL_INCLUDE_DIR="${OPENAL_SDK}/include/AL" \ -DOPENAL_LIBRARY="${OPENAL_SDK}/libs/Win${BITS}/OpenAL32.lib" - add_runtime_dlls "$(pwd)/openal-soft-1.19.1-bin/bin/WIN${BITS}/soft_oal.dll:OpenAL32.dll" + add_runtime_dlls "$(pwd)/openal-soft-1.20.1-bin/bin/WIN${BITS}/soft_oal.dll:OpenAL32.dll" echo Done. } cd $DEPS echo # OSG -printf "OSG 3.4.1-scrawl... " +printf "OSG 3.6.5... " { cd $DEPS_INSTALL if [ -d OSG ] && \ grep "OPENSCENEGRAPH_MAJOR_VERSION 3" OSG/include/osg/Version > /dev/null && \ - grep "OPENSCENEGRAPH_MINOR_VERSION 4" OSG/include/osg/Version > /dev/null && \ - grep "OPENSCENEGRAPH_PATCH_VERSION 1" OSG/include/osg/Version > /dev/null + grep "OPENSCENEGRAPH_MINOR_VERSION 6" OSG/include/osg/Version > /dev/null && \ + grep "OPENSCENEGRAPH_PATCH_VERSION 5" OSG/include/osg/Version > /dev/null then printf "Exists. " elif [ -z $SKIP_EXTRACT ]; then rm -rf OSG - eval 7z x -y "${DEPS}/OSG-3.4.1-scrawl-msvc${MSVC_YEAR}-win${BITS}.7z" $STRIP - mv "OSG-3.4.1-scrawl-msvc${MSVC_YEAR}-win${BITS}" OSG + eval 7z x -y "${DEPS}/OSG-3.6.5-msvc${MSVC_REAL_YEAR}-win${BITS}.7z" $STRIP + [ -n "$PDBS" ] && eval 7z x -y "${DEPS}/OSG-3.6.5-msvc${MSVC_REAL_YEAR}-win${BITS}-sym.7z" $STRIP + mv "OSG-3.6.5-msvc${MSVC_REAL_YEAR}-win${BITS}" OSG fi OSG_SDK="$(real_pwd)/OSG" add_cmake_opts -DOSG_DIR="$OSG_SDK" @@ -636,17 +666,17 @@ printf "OSG 3.4.1-scrawl... " else SUFFIX="" fi - add_runtime_dlls "$(pwd)/OSG/bin/"{OpenThreads,zlib,libpng*}${SUFFIX}.dll \ + add_runtime_dlls "$(pwd)/OSG/bin/"{OpenThreads,zlib,libpng}${SUFFIX}.dll \ "$(pwd)/OSG/bin/osg"{,Animation,DB,FX,GA,Particle,Text,Util,Viewer,Shadow}${SUFFIX}.dll - add_osg_dlls "$(pwd)/OSG/bin/osgPlugins-3.4.1/osgdb_"{bmp,dds,freetype,jpeg,osg,png,tga}${SUFFIX}.dll - add_osg_dlls "$(pwd)/OSG/bin/osgPlugins-3.4.1/osgdb_serializers_osg"{,animation,fx,ga,particle,text,util,viewer,shadow}${SUFFIX}.dll + add_osg_dlls "$(pwd)/OSG/bin/osgPlugins-3.6.5/osgdb_"{bmp,dds,freetype,jpeg,osg,png,tga}${SUFFIX}.dll + add_osg_dlls "$(pwd)/OSG/bin/osgPlugins-3.6.5/osgdb_serializers_osg"{,animation,fx,ga,particle,text,util,viewer,shadow}${SUFFIX}.dll echo Done. } cd $DEPS echo # Qt if [ -z $APPVEYOR ]; then - printf "Qt 5.7.0... " + printf "Qt 5.15.0... " else printf "Qt 5.13 AppVeyor... " fi @@ -658,21 +688,44 @@ fi fi if [ -z $APPVEYOR ]; then cd $DEPS_INSTALL - QT_SDK="$(real_pwd)/Qt/5.7/msvc${MSVC_YEAR}${SUFFIX}" - if [ -d Qt ] && head -n2 Qt/InstallationLog.txt | grep "5.7.0" > /dev/null; then + QT_SDK="$(real_pwd)/Qt/5.15.0/msvc${MSVC_YEAR}${SUFFIX}" + + if [ -d 'Qt/5.15.0' ]; then printf "Exists. " elif [ -z $SKIP_EXTRACT ]; then + pushd "$DEPS" > /dev/null + if ! [ -d 'aqt-venv' ]; then + echo " Creating Virtualenv for aqt..." + eval python -m venv aqt-venv $STRIP + fi + if [ -d 'aqt-venv/bin' ]; then + VENV_BIN_DIR='bin' + elif [ -d 'aqt-venv/Scripts' ]; then + VENV_BIN_DIR='Scripts' + else + echo "Error: Failed to create virtualenv." + exit 1 + fi + + if ! [ -e "aqt-venv/${VENV_BIN_DIR}/aqt" ]; then + echo " Installing aqt wheel into virtualenv..." + eval "aqt-venv/${VENV_BIN_DIR}/pip" install aqtinstall-0.8-py2.py3-none-any.whl $STRIP + fi + popd > /dev/null + rm -rf Qt - cp "${DEPS}/qt-5-install.qs" qt-install.qs - sed -i "s|INSTALL_DIR|$(real_pwd)/Qt|" qt-install.qs - sed -i "s/qt.VERSION.winBITS_msvcYEAR/qt.57.win${BITS}_msvc${MSVC_YEAR}${SUFFIX}/" qt-install.qs - printf -- "(Installation might take a while) " - "${DEPS}/qt-5.7.0-msvc${MSVC_YEAR}-win${BITS}.exe" --script qt-install.qs --silent - mv qt-install.qs Qt/ - echo Done. + + mkdir Qt + cd Qt + + eval "${DEPS}/aqt-venv/${VENV_BIN_DIR}/aqt" install 5.15.0 windows desktop "win${BITS}_msvc${MSVC_REAL_YEAR}${SUFFIX}" $STRIP + printf " Cleaning up extraneous data... " - rm -r "$(real_pwd)/Qt/"{dist,Docs,Examples,Tools,vcredist,components.xml,MaintenanceTool.dat,MaintenanceTool.exe,MaintenanceTool.ini,network.xml,qt-install.qs} + rm -rf Qt/{aqtinstall.log,Tools} + + echo Done. fi + cd $QT_SDK add_cmake_opts -DDESIRED_QT_VERSION=5 \ -DQT_QMAKE_EXECUTABLE="${QT_SDK}/bin/qmake.exe" \ @@ -822,10 +875,10 @@ fi done echo echo "- OSG Plugin DLLs..." - mkdir -p ${DLL_PREFIX}osgPlugins-3.4.1 + mkdir -p ${DLL_PREFIX}osgPlugins-3.6.5 for DLL in $OSG_PLUGINS; do echo " $(basename $DLL)." - cp "$DLL" ${DLL_PREFIX}osgPlugins-3.4.1 + cp "$DLL" ${DLL_PREFIX}osgPlugins-3.6.5 done echo echo "- Qt Platform DLLs..."