From 5d7ce3cf7188fdaaaad3b9f439d7fa62a8b40274 Mon Sep 17 00:00:00 2001 From: "Lennart J. Kurzweg (Nx2)" Date: Mon, 26 Aug 2024 21:20:47 +0200 Subject: [PATCH] mf1 --- .direnv/bin/nix-direnv-reload | 19 - .direnv/nix-profile-24.05-19l2iplgd8kw8b1i | 1 - .direnv/nix-profile-24.05-19l2iplgd8kw8b1i.rc | 1895 ----------------- libs/ollama_functions.py | 199 +- libs/run_tests.py | 22 +- libs/runnables.py | 37 +- libs/tools.py | 3 + print_saved_results.py | 96 - saved_results-bak.json | 1651 ++++++++++++++ suite_settings/models.py | 168 +- suite_settings/techniques.py | 14 +- suite_settings/tests.py | 300 ++- 12 files changed, 2055 insertions(+), 2350 deletions(-) delete mode 100755 .direnv/bin/nix-direnv-reload delete mode 120000 .direnv/nix-profile-24.05-19l2iplgd8kw8b1i delete mode 100644 .direnv/nix-profile-24.05-19l2iplgd8kw8b1i.rc delete mode 100644 print_saved_results.py create mode 100644 saved_results-bak.json diff --git a/.direnv/bin/nix-direnv-reload b/.direnv/bin/nix-direnv-reload deleted file mode 100755 index b1042d4..0000000 --- a/.direnv/bin/nix-direnv-reload +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -set -e -if [[ ! -d "/home/nx2/test-small-llms" ]]; then - echo "Cannot find source directory; Did you move it?" - echo "(Looking for "/home/nx2/test-small-llms")" - echo 'Cannot force reload with this script - use "direnv reload" manually and then try again' - exit 1 -fi - -# rebuild the cache forcefully -_nix_direnv_force_reload=1 direnv exec "/home/nx2/test-small-llms" true - -# Update the mtime for .envrc. -# This will cause direnv to reload again - but without re-building. -touch "/home/nx2/test-small-llms/.envrc" - -# Also update the timestamp of whatever profile_rc we have. -# This makes sure that we know we are up to date. -touch -r "/home/nx2/test-small-llms/.envrc" "/home/nx2/test-small-llms/.direnv"/*.rc diff --git a/.direnv/nix-profile-24.05-19l2iplgd8kw8b1i b/.direnv/nix-profile-24.05-19l2iplgd8kw8b1i deleted file mode 120000 index 8d63904..0000000 --- a/.direnv/nix-profile-24.05-19l2iplgd8kw8b1i +++ /dev/null @@ -1 +0,0 @@ -/nix/store/j2vf461mp9h2y9awkklbfawf3dz7cs1p-nix-shell-env \ No newline at end of file diff --git a/.direnv/nix-profile-24.05-19l2iplgd8kw8b1i.rc b/.direnv/nix-profile-24.05-19l2iplgd8kw8b1i.rc deleted file mode 100644 index 30cbf24..0000000 --- a/.direnv/nix-profile-24.05-19l2iplgd8kw8b1i.rc +++ /dev/null @@ -1,1895 +0,0 @@ -unset shellHook -PATH=${PATH:-} -nix_saved_PATH="$PATH" -XDG_DATA_DIRS=${XDG_DATA_DIRS:-} -nix_saved_XDG_DATA_DIRS="$XDG_DATA_DIRS" -AR='ar' -export AR -AS='as' -export AS -BASH='/nix/store/5jw69mbaj5dg4l2bj58acg3gxywfszpj-bash-5.2p26/bin/bash' -CC='gcc' -export CC -CONFIG_SHELL='/nix/store/5jw69mbaj5dg4l2bj58acg3gxywfszpj-bash-5.2p26/bin/bash' -export CONFIG_SHELL -CXX='g++' -export CXX -HOSTTYPE='x86_64' -HOST_PATH='/nix/store/m1850izd0rq01zj1yqrmzqzjzas0x42h-python3-3.12.4-env/bin:/nix/store/ysqx2xfzygv2rxl7nxnw48276z5ckppn-coreutils-9.5/bin:/nix/store/36rvynxwln7iz0qq3k1v3r1mna8bma8s-findutils-4.9.0/bin:/nix/store/0fw4a3z849azkhyjxnpxbygj4g5qhd0v-diffutils-3.10/bin:/nix/store/7xwbkzfrs6flyvjyvd23m8r2mlnycinq-gnused-4.9/bin:/nix/store/d9xr7s3z0r8rf0ba22q6ilqv68agymdb-gnugrep-3.11/bin:/nix/store/9fklixgxv2a9xxpbn504mpifnbz94rb5-gawk-5.2.2/bin:/nix/store/iz0gv4jb2b8bkb9krmha68b00b24p3rl-gnutar-1.35/bin:/nix/store/g14bdsvp1lqqhiyd9g88nyjhp16hs9wj-gzip-1.13/bin:/nix/store/fmk8lz57yy64hvz9rp58krlzb3q9df70-bzip2-1.0.8-bin/bin:/nix/store/3hnf34qxi3h6c62dw95crgxdxvibasml-gnumake-4.4.1/bin:/nix/store/5jw69mbaj5dg4l2bj58acg3gxywfszpj-bash-5.2p26/bin:/nix/store/f46k9pgiq88v6yh76wxb3dv1ggpn6fml-patch-2.7.6/bin:/nix/store/5srqwq17md7w5ln001iaxhxpd0839d8r-xz-5.4.7-bin/bin:/nix/store/j4aja22d2ngbpmi4vdpdd2dvp1hm7dvr-file-5.45/bin' -export HOST_PATH -IFS=' -' -IN_NIX_SHELL='impure' -export IN_NIX_SHELL -LD='ld' -export LD -LINENO='76' -MACHTYPE='x86_64-pc-linux-gnu' -NIX_BINTOOLS='/nix/store/bl7gf4crmr480jfm8raswv9xn2v1qwiw-binutils-wrapper-2.41' -export NIX_BINTOOLS -NIX_BINTOOLS_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu='1' -export NIX_BINTOOLS_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu -NIX_BUILD_CORES='16' -export NIX_BUILD_CORES -NIX_CC='/nix/store/r73z9i18vbjcph7k2f3isrysxzx6sqjx-gcc-wrapper-13.2.0' -export NIX_CC -NIX_CC_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu='1' -export NIX_CC_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu -NIX_CFLAGS_COMPILE=' -frandom-seed=j2vf461mp9 -isystem /nix/store/m1850izd0rq01zj1yqrmzqzjzas0x42h-python3-3.12.4-env/include -isystem /nix/store/m1850izd0rq01zj1yqrmzqzjzas0x42h-python3-3.12.4-env/include' -export NIX_CFLAGS_COMPILE -NIX_ENFORCE_NO_NATIVE='1' -export NIX_ENFORCE_NO_NATIVE -NIX_HARDENING_ENABLE='bindnow format fortify fortify3 pic relro stackprotector strictoverflow' -export NIX_HARDENING_ENABLE -NIX_LDFLAGS='-rpath /home/nx2/test-small-llms/outputs/out/lib -L/nix/store/m1850izd0rq01zj1yqrmzqzjzas0x42h-python3-3.12.4-env/lib -L/nix/store/m1850izd0rq01zj1yqrmzqzjzas0x42h-python3-3.12.4-env/lib' -export NIX_LDFLAGS -NIX_NO_SELF_RPATH='1' -NIX_STORE='/nix/store' -export NIX_STORE -NM='nm' -export NM -OBJCOPY='objcopy' -export OBJCOPY -OBJDUMP='objdump' -export OBJDUMP -OLDPWD='' -export OLDPWD -OPTERR='1' -OSTYPE='linux-gnu' -PATH='/nix/store/cy6j2yii4y9jlbqr10k190v0b3jdbwl3-patchelf-0.15.0/bin:/nix/store/r73z9i18vbjcph7k2f3isrysxzx6sqjx-gcc-wrapper-13.2.0/bin:/nix/store/llmjvk4i2yncv8xqdvs4382wr3kgdmvp-gcc-13.2.0/bin:/nix/store/z5gvbj7p7dr89k42ncrl2mlv5v3ymjlp-glibc-2.39-52-bin/bin:/nix/store/ysqx2xfzygv2rxl7nxnw48276z5ckppn-coreutils-9.5/bin:/nix/store/bl7gf4crmr480jfm8raswv9xn2v1qwiw-binutils-wrapper-2.41/bin:/nix/store/7v7g86ml0ri171gfcrs1d442px5bi1p3-binutils-2.41/bin:/nix/store/m1850izd0rq01zj1yqrmzqzjzas0x42h-python3-3.12.4-env/bin:/nix/store/ysqx2xfzygv2rxl7nxnw48276z5ckppn-coreutils-9.5/bin:/nix/store/36rvynxwln7iz0qq3k1v3r1mna8bma8s-findutils-4.9.0/bin:/nix/store/0fw4a3z849azkhyjxnpxbygj4g5qhd0v-diffutils-3.10/bin:/nix/store/7xwbkzfrs6flyvjyvd23m8r2mlnycinq-gnused-4.9/bin:/nix/store/d9xr7s3z0r8rf0ba22q6ilqv68agymdb-gnugrep-3.11/bin:/nix/store/9fklixgxv2a9xxpbn504mpifnbz94rb5-gawk-5.2.2/bin:/nix/store/iz0gv4jb2b8bkb9krmha68b00b24p3rl-gnutar-1.35/bin:/nix/store/g14bdsvp1lqqhiyd9g88nyjhp16hs9wj-gzip-1.13/bin:/nix/store/fmk8lz57yy64hvz9rp58krlzb3q9df70-bzip2-1.0.8-bin/bin:/nix/store/3hnf34qxi3h6c62dw95crgxdxvibasml-gnumake-4.4.1/bin:/nix/store/5jw69mbaj5dg4l2bj58acg3gxywfszpj-bash-5.2p26/bin:/nix/store/f46k9pgiq88v6yh76wxb3dv1ggpn6fml-patch-2.7.6/bin:/nix/store/5srqwq17md7w5ln001iaxhxpd0839d8r-xz-5.4.7-bin/bin:/nix/store/j4aja22d2ngbpmi4vdpdd2dvp1hm7dvr-file-5.45/bin' -export PATH -PS4='+ ' -RANLIB='ranlib' -export RANLIB -READELF='readelf' -export READELF -SIZE='size' -export SIZE -SOURCE_DATE_EPOCH='315532800' -export SOURCE_DATE_EPOCH -STRINGS='strings' -export STRINGS -STRIP='strip' -export STRIP -XDG_DATA_DIRS='/nix/store/cy6j2yii4y9jlbqr10k190v0b3jdbwl3-patchelf-0.15.0/share' -export XDG_DATA_DIRS -__structuredAttrs='' -export __structuredAttrs -_substituteStream_has_warned_replace_deprecation='false' -buildInputs='/nix/store/m1850izd0rq01zj1yqrmzqzjzas0x42h-python3-3.12.4-env' -export buildInputs -buildPhase='{ echo "------------------------------------------------------------"; - echo " WARNING: the existence of this path is not guaranteed."; - echo " It is an internal implementation detail for pkgs.mkShell."; - echo "------------------------------------------------------------"; - echo; - # Record all build inputs as runtime dependencies - export; -} >> "$out" -' -export buildPhase -builder='/nix/store/5jw69mbaj5dg4l2bj58acg3gxywfszpj-bash-5.2p26/bin/bash' -export builder -cmakeFlags='' -export cmakeFlags -configureFlags='' -export configureFlags -defaultBuildInputs='' -defaultNativeBuildInputs='/nix/store/cy6j2yii4y9jlbqr10k190v0b3jdbwl3-patchelf-0.15.0 /nix/store/l4sqcq84ibmissf3427f34mjlzx28h20-update-autotools-gnu-config-scripts-hook /nix/store/h9lc1dpi14z7is86ffhl3ld569138595-audit-tmpdir.sh /nix/store/m54bmrhj6fqz8nds5zcj97w9s9bckc9v-compress-man-pages.sh /nix/store/wgrbkkaldkrlrni33ccvm3b6vbxzb656-make-symlinks-relative.sh /nix/store/5yzw0vhkyszf2d179m0qfkgxmp5wjjx4-move-docs.sh /nix/store/fyaryjvghbkpfnsyw97hb3lyb37s1pd6-move-lib64.sh /nix/store/kd4xwxjpjxi71jkm6ka0np72if9rm3y0-move-sbin.sh /nix/store/pag6l61paj1dc9sv15l7bm5c17xn5kyk-move-systemd-user-units.sh /nix/store/jivxp510zxakaaic7qkrb7v1dd2rdbw9-multiple-outputs.sh /nix/store/ilaf1w22bxi6jsi45alhmvvdgy4ly3zs-patch-shebangs.sh /nix/store/cickvswrvann041nqxb0rxilc46svw1n-prune-libtool-files.sh /nix/store/xyff06pkhki3qy1ls77w10s0v79c9il0-reproducible-builds.sh /nix/store/ngg1cv31c8c7bcm2n8ww4g06nq7s4zhm-set-source-date-epoch-to-latest.sh /nix/store/gps9qrh99j7g02840wv5x78ykmz30byp-strip.sh /nix/store/r73z9i18vbjcph7k2f3isrysxzx6sqjx-gcc-wrapper-13.2.0' -depsBuildBuild='' -export depsBuildBuild -depsBuildBuildPropagated='' -export depsBuildBuildPropagated -depsBuildTarget='' -export depsBuildTarget -depsBuildTargetPropagated='' -export depsBuildTargetPropagated -depsHostHost='' -export depsHostHost -depsHostHostPropagated='' -export depsHostHostPropagated -depsTargetTarget='' -export depsTargetTarget -depsTargetTargetPropagated='' -export depsTargetTargetPropagated -doCheck='' -export doCheck -doInstallCheck='' -export doInstallCheck -dontAddDisableDepTrack='1' -export dontAddDisableDepTrack -declare -a envBuildBuildHooks=() -declare -a envBuildHostHooks=() -declare -a envBuildTargetHooks=() -declare -a envHostHostHooks=('ccWrapper_addCVars' 'bintoolsWrapper_addLDVars' ) -declare -a envHostTargetHooks=('ccWrapper_addCVars' 'bintoolsWrapper_addLDVars' ) -declare -a envTargetTargetHooks=() -declare -a fixupOutputHooks=('if [ -z "${dontPatchELF-}" ]; then patchELF "$prefix"; fi' 'if [[ -z "${noAuditTmpdir-}" && -e "$prefix" ]]; then auditTmpdir "$prefix"; fi' 'if [ -z "${dontGzipMan-}" ]; then compressManPages "$prefix"; fi' '_moveLib64' '_moveSbin' '_moveSystemdUserUnits' 'patchShebangsAuto' '_pruneLibtoolFiles' '_doStrip' ) -guess='16' -initialPath='/nix/store/ysqx2xfzygv2rxl7nxnw48276z5ckppn-coreutils-9.5 /nix/store/36rvynxwln7iz0qq3k1v3r1mna8bma8s-findutils-4.9.0 /nix/store/0fw4a3z849azkhyjxnpxbygj4g5qhd0v-diffutils-3.10 /nix/store/7xwbkzfrs6flyvjyvd23m8r2mlnycinq-gnused-4.9 /nix/store/d9xr7s3z0r8rf0ba22q6ilqv68agymdb-gnugrep-3.11 /nix/store/9fklixgxv2a9xxpbn504mpifnbz94rb5-gawk-5.2.2 /nix/store/iz0gv4jb2b8bkb9krmha68b00b24p3rl-gnutar-1.35 /nix/store/g14bdsvp1lqqhiyd9g88nyjhp16hs9wj-gzip-1.13 /nix/store/fmk8lz57yy64hvz9rp58krlzb3q9df70-bzip2-1.0.8-bin /nix/store/3hnf34qxi3h6c62dw95crgxdxvibasml-gnumake-4.4.1 /nix/store/5jw69mbaj5dg4l2bj58acg3gxywfszpj-bash-5.2p26 /nix/store/f46k9pgiq88v6yh76wxb3dv1ggpn6fml-patch-2.7.6 /nix/store/5srqwq17md7w5ln001iaxhxpd0839d8r-xz-5.4.7-bin /nix/store/j4aja22d2ngbpmi4vdpdd2dvp1hm7dvr-file-5.45' -mesonFlags='' -export mesonFlags -name='nix-shell-env' -export name -nativeBuildInputs='' -export nativeBuildInputs -out='/home/nx2/test-small-llms/outputs/out' -export out -outputBin='out' -outputDev='out' -outputDevdoc='REMOVE' -outputDevman='out' -outputDoc='out' -outputInclude='out' -outputInfo='out' -outputLib='out' -outputMan='out' -outputs='out' -export outputs -patches='' -export patches -phases='buildPhase' -export phases -pkg='/nix/store/r73z9i18vbjcph7k2f3isrysxzx6sqjx-gcc-wrapper-13.2.0' -declare -a pkgsBuildBuild=() -declare -a pkgsBuildHost=('/nix/store/cy6j2yii4y9jlbqr10k190v0b3jdbwl3-patchelf-0.15.0' '/nix/store/l4sqcq84ibmissf3427f34mjlzx28h20-update-autotools-gnu-config-scripts-hook' '/nix/store/h9lc1dpi14z7is86ffhl3ld569138595-audit-tmpdir.sh' '/nix/store/m54bmrhj6fqz8nds5zcj97w9s9bckc9v-compress-man-pages.sh' '/nix/store/wgrbkkaldkrlrni33ccvm3b6vbxzb656-make-symlinks-relative.sh' '/nix/store/5yzw0vhkyszf2d179m0qfkgxmp5wjjx4-move-docs.sh' '/nix/store/fyaryjvghbkpfnsyw97hb3lyb37s1pd6-move-lib64.sh' '/nix/store/kd4xwxjpjxi71jkm6ka0np72if9rm3y0-move-sbin.sh' '/nix/store/pag6l61paj1dc9sv15l7bm5c17xn5kyk-move-systemd-user-units.sh' '/nix/store/jivxp510zxakaaic7qkrb7v1dd2rdbw9-multiple-outputs.sh' '/nix/store/ilaf1w22bxi6jsi45alhmvvdgy4ly3zs-patch-shebangs.sh' '/nix/store/cickvswrvann041nqxb0rxilc46svw1n-prune-libtool-files.sh' '/nix/store/xyff06pkhki3qy1ls77w10s0v79c9il0-reproducible-builds.sh' '/nix/store/ngg1cv31c8c7bcm2n8ww4g06nq7s4zhm-set-source-date-epoch-to-latest.sh' '/nix/store/gps9qrh99j7g02840wv5x78ykmz30byp-strip.sh' '/nix/store/r73z9i18vbjcph7k2f3isrysxzx6sqjx-gcc-wrapper-13.2.0' '/nix/store/bl7gf4crmr480jfm8raswv9xn2v1qwiw-binutils-wrapper-2.41' ) -declare -a pkgsBuildTarget=() -declare -a pkgsHostHost=() -declare -a pkgsHostTarget=('/nix/store/m1850izd0rq01zj1yqrmzqzjzas0x42h-python3-3.12.4-env' ) -declare -a pkgsTargetTarget=() -declare -a postFixupHooks=('_makeSymlinksRelativeInAllOutputs' '_multioutPropagateDev' ) -postShellHook='ln -sf lib/python3.12/site-packages/* ./.venv/lib/python3.12/site-packages -' -export postShellHook -declare -a postUnpackHooks=('_updateSourceDateEpochFromSourceRoot' ) -declare -a preConfigureHooks=('_multioutConfig' ) -preConfigurePhases=' updateAutotoolsGnuConfigScriptsPhase' -declare -a preFixupHooks=('_moveToShare' '_multioutDocs' '_multioutDevs' ) -preferLocalBuild='1' -export preferLocalBuild -prefix='/home/nx2/test-small-llms/outputs/out' -declare -a propagatedBuildDepFiles=('propagated-build-build-deps' 'propagated-native-build-inputs' 'propagated-build-target-deps' ) -propagatedBuildInputs='' -export propagatedBuildInputs -declare -a propagatedHostDepFiles=('propagated-host-host-deps' 'propagated-build-inputs' ) -propagatedNativeBuildInputs='' -export propagatedNativeBuildInputs -declare -a propagatedTargetDepFiles=('propagated-target-target-deps' ) -shell='/nix/store/5jw69mbaj5dg4l2bj58acg3gxywfszpj-bash-5.2p26/bin/bash' -export shell -shellHook='SOURCE_DATE_EPOCH=$(date +%s) -export "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/nix/store/4g5w7fpyn7f1iw1n6l9lcvciz21icmb5-libffi-3.4.6/lib:/nix/store/anqmzdm5aqw5dvqcjivhhnpbb2y99a36-openssl-3.0.14/lib:/nix/store/p44qan69linp3ii0xrviypsw2j4qdcp2-gcc-13.2.0-lib/lib" -VENV=.venv - -if test ! -d $VENV; then - python3 -m venv $VENV -fi -source ./$VENV/bin/activate -export PYTHONPATH=`pwd`/$VENV/lib/python3.12/site-packages/ -# export PYTHONPATH=`pwd`/$VENV/lib/python3.12/site-packages/:$PYTHONPATH -pip install -r requirements.txt -' -export shellHook -stdenv='/nix/store/dd7nxjnni7nzm0846fq5xrm89ais5lwz-stdenv-linux' -export stdenv -strictDeps='' -export strictDeps -system='x86_64-linux' -export system -declare -a unpackCmdHooks=('_defaultUnpack' ) -_accumFlagsArray () -{ - - local name; - if [ -n "$__structuredAttrs" ]; then - for name in "$@"; - do - local -n nameref="$name"; - flagsArray+=(${nameref+"${nameref[@]}"}); - done; - else - for name in "$@"; - do - local -n nameref="$name"; - case "$name" in - *Array) - flagsArray+=(${nameref+"${nameref[@]}"}) - ;; - *) - flagsArray+=(${nameref-}) - ;; - esac; - done; - fi -} -_activatePkgs () -{ - - local hostOffset targetOffset; - local pkg; - for hostOffset in "${allPlatOffsets[@]}"; - do - local pkgsVar="${pkgAccumVarVars[hostOffset + 1]}"; - for targetOffset in "${allPlatOffsets[@]}"; - do - (( hostOffset <= targetOffset )) || continue; - local pkgsRef="${pkgsVar}[$targetOffset - $hostOffset]"; - local pkgsSlice="${!pkgsRef}[@]"; - for pkg in ${!pkgsSlice+"${!pkgsSlice}"}; - do - activatePackage "$pkg" "$hostOffset" "$targetOffset"; - done; - done; - done -} -_addRpathPrefix () -{ - - if [ "${NIX_NO_SELF_RPATH:-0}" != 1 ]; then - export NIX_LDFLAGS="-rpath $1/lib ${NIX_LDFLAGS-}"; - fi -} -_addToEnv () -{ - - local depHostOffset depTargetOffset; - local pkg; - for depHostOffset in "${allPlatOffsets[@]}"; - do - local hookVar="${pkgHookVarVars[depHostOffset + 1]}"; - local pkgsVar="${pkgAccumVarVars[depHostOffset + 1]}"; - for depTargetOffset in "${allPlatOffsets[@]}"; - do - (( depHostOffset <= depTargetOffset )) || continue; - local hookRef="${hookVar}[$depTargetOffset - $depHostOffset]"; - if [[ -z "${strictDeps-}" ]]; then - local visitedPkgs=""; - for pkg in "${pkgsBuildBuild[@]}" "${pkgsBuildHost[@]}" "${pkgsBuildTarget[@]}" "${pkgsHostHost[@]}" "${pkgsHostTarget[@]}" "${pkgsTargetTarget[@]}"; - do - if [[ "$visitedPkgs" = *"$pkg"* ]]; then - continue; - fi; - runHook "${!hookRef}" "$pkg"; - visitedPkgs+=" $pkg"; - done; - else - local pkgsRef="${pkgsVar}[$depTargetOffset - $depHostOffset]"; - local pkgsSlice="${!pkgsRef}[@]"; - for pkg in ${!pkgsSlice+"${!pkgsSlice}"}; - do - runHook "${!hookRef}" "$pkg"; - done; - fi; - done; - done -} -_allFlags () -{ - - export system pname name version; - for varName in $(awk 'BEGIN { for (v in ENVIRON) if (v ~ /^[a-z][a-zA-Z0-9_]*$/) print v }'); - do - if (( "${NIX_DEBUG:-0}" >= 1 )); then - printf "@%s@ -> %q\n" "${varName}" "${!varName}" 1>&2; - fi; - args+=("--subst-var" "$varName"); - done -} -_assignFirst () -{ - - local varName="$1"; - local _var; - local REMOVE=REMOVE; - shift; - for _var in "$@"; - do - if [ -n "${!_var-}" ]; then - eval "${varName}"="${_var}"; - return; - fi; - done; - echo; - echo "error: _assignFirst: could not find a non-empty variable whose name to assign to ${varName}."; - echo " The following variables were all unset or empty:"; - echo " $*"; - if [ -z "${out:-}" ]; then - echo ' If you do not want an "out" output in your derivation, make sure to define'; - echo ' the other specific required outputs. This can be achieved by picking one'; - echo " of the above as an output."; - echo ' You do not have to remove "out" if you want to have a different default'; - echo ' output, because the first output is taken as a default.'; - echo; - fi; - return 1 -} -_callImplicitHook () -{ - - local def="$1"; - local hookName="$2"; - if declare -F "$hookName" > /dev/null; then - "$hookName"; - else - if type -p "$hookName" > /dev/null; then - source "$hookName"; - else - if [ -n "${!hookName:-}" ]; then - eval "${!hookName}"; - else - return "$def"; - fi; - fi; - fi -} -_defaultUnpack () -{ - - local fn="$1"; - local destination; - if [ -d "$fn" ]; then - destination="$(stripHash "$fn")"; - if [ -e "$destination" ]; then - echo "Cannot copy $fn to $destination: destination already exists!"; - echo "Did you specify two \"srcs\" with the same \"name\"?"; - return 1; - fi; - cp -pr --reflink=auto -- "$fn" "$destination"; - else - case "$fn" in - *.tar.xz | *.tar.lzma | *.txz) - ( XZ_OPT="--threads=$NIX_BUILD_CORES" xz -d < "$fn"; - true ) | tar xf - --mode=+w --warning=no-timestamp - ;; - *.tar | *.tar.* | *.tgz | *.tbz2 | *.tbz) - tar xf "$fn" --mode=+w --warning=no-timestamp - ;; - *) - return 1 - ;; - esac; - fi -} -_doStrip () -{ - - local -ra flags=(dontStripHost dontStripTarget); - local -ra debugDirs=(stripDebugList stripDebugListTarget); - local -ra allDirs=(stripAllList stripAllListTarget); - local -ra stripCmds=(STRIP STRIP_FOR_TARGET); - local -ra ranlibCmds=(RANLIB RANLIB_FOR_TARGET); - stripDebugList=${stripDebugList[*]:-lib lib32 lib64 libexec bin sbin}; - stripDebugListTarget=${stripDebugListTarget[*]:-}; - stripAllList=${stripAllList[*]:-}; - stripAllListTarget=${stripAllListTarget[*]:-}; - local i; - for i in ${!stripCmds[@]}; - do - local -n flag="${flags[$i]}"; - local -n debugDirList="${debugDirs[$i]}"; - local -n allDirList="${allDirs[$i]}"; - local -n stripCmd="${stripCmds[$i]}"; - local -n ranlibCmd="${ranlibCmds[$i]}"; - if [[ -n "${dontStrip-}" || -n "${flag-}" ]] || ! type -f "${stripCmd-}" 2> /dev/null 1>&2; then - continue; - fi; - stripDirs "$stripCmd" "$ranlibCmd" "$debugDirList" "${stripDebugFlags[*]:--S -p}"; - stripDirs "$stripCmd" "$ranlibCmd" "$allDirList" "${stripAllFlags[*]:--s -p}"; - done -} -_eval () -{ - - if declare -F "$1" > /dev/null 2>&1; then - "$@"; - else - eval "$1"; - fi -} -_makeSymlinksRelative () -{ - - local symlinkTarget; - if [ "${dontRewriteSymlinks-}" ] || [ ! -e "$prefix" ]; then - return; - fi; - while IFS= read -r -d '' f; do - symlinkTarget=$(readlink "$f"); - if [[ "$symlinkTarget"/ != "$prefix"/* ]]; then - continue; - fi; - if [ ! -e "$symlinkTarget" ]; then - echo "the symlink $f is broken, it points to $symlinkTarget (which is missing)"; - fi; - echo "rewriting symlink $f to be relative to $prefix"; - ln -snrf "$symlinkTarget" "$f"; - done < <(find $prefix -type l -print0) -} -_makeSymlinksRelativeInAllOutputs () -{ - - local output; - for output in $(getAllOutputNames); - do - prefix="${!output}" _makeSymlinksRelative; - done -} -_moveLib64 () -{ - - if [ "${dontMoveLib64-}" = 1 ]; then - return; - fi; - if [ ! -e "$prefix/lib64" -o -L "$prefix/lib64" ]; then - return; - fi; - echo "moving $prefix/lib64/* to $prefix/lib"; - mkdir -p $prefix/lib; - shopt -s dotglob; - for i in $prefix/lib64/*; - do - mv --no-clobber "$i" $prefix/lib; - done; - shopt -u dotglob; - rmdir $prefix/lib64; - ln -s lib $prefix/lib64 -} -_moveSbin () -{ - - if [ "${dontMoveSbin-}" = 1 ]; then - return; - fi; - if [ ! -e "$prefix/sbin" -o -L "$prefix/sbin" ]; then - return; - fi; - echo "moving $prefix/sbin/* to $prefix/bin"; - mkdir -p $prefix/bin; - shopt -s dotglob; - for i in $prefix/sbin/*; - do - mv "$i" $prefix/bin; - done; - shopt -u dotglob; - rmdir $prefix/sbin; - ln -s bin $prefix/sbin -} -_moveSystemdUserUnits () -{ - - if [ "${dontMoveSystemdUserUnits:-0}" = 1 ]; then - return; - fi; - if [ ! -e "${prefix:?}/lib/systemd/user" ]; then - return; - fi; - local source="$prefix/lib/systemd/user"; - local target="$prefix/share/systemd/user"; - echo "moving $source/* to $target"; - mkdir -p "$target"; - ( shopt -s dotglob; - for i in "$source"/*; - do - mv "$i" "$target"; - done ); - rmdir "$source"; - ln -s "$target" "$source" -} -_moveToShare () -{ - - if [ -n "$__structuredAttrs" ]; then - if [ -z "${forceShare-}" ]; then - forceShare=(man doc info); - fi; - else - forceShare=(${forceShare:-man doc info}); - fi; - if [[ -z "$out" ]]; then - return; - fi; - for d in "${forceShare[@]}"; - do - if [ -d "$out/$d" ]; then - if [ -d "$out/share/$d" ]; then - echo "both $d/ and share/$d/ exist!"; - else - echo "moving $out/$d to $out/share/$d"; - mkdir -p $out/share; - mv $out/$d $out/share/; - fi; - fi; - done -} -_multioutConfig () -{ - - if [ "$(getAllOutputNames)" = "out" ] || [ -z "${setOutputFlags-1}" ]; then - return; - fi; - if [ -z "${shareDocName:-}" ]; then - local confScript="${configureScript:-}"; - if [ -z "$confScript" ] && [ -x ./configure ]; then - confScript=./configure; - fi; - if [ -f "$confScript" ]; then - local shareDocName="$(sed -n "s/^PACKAGE_TARNAME='\(.*\)'$/\1/p" < "$confScript")"; - fi; - if [ -z "$shareDocName" ] || echo "$shareDocName" | grep -q '[^a-zA-Z0-9_-]'; then - shareDocName="$(echo "$name" | sed 's/-[^a-zA-Z].*//')"; - fi; - fi; - prependToVar configureFlags --bindir="${!outputBin}"/bin --sbindir="${!outputBin}"/sbin --includedir="${!outputInclude}"/include --oldincludedir="${!outputInclude}"/include --mandir="${!outputMan}"/share/man --infodir="${!outputInfo}"/share/info --docdir="${!outputDoc}"/share/doc/"${shareDocName}" --libdir="${!outputLib}"/lib --libexecdir="${!outputLib}"/libexec --localedir="${!outputLib}"/share/locale; - prependToVar installFlags pkgconfigdir="${!outputDev}"/lib/pkgconfig m4datadir="${!outputDev}"/share/aclocal aclocaldir="${!outputDev}"/share/aclocal -} -_multioutDevs () -{ - - if [ "$(getAllOutputNames)" = "out" ] || [ -z "${moveToDev-1}" ]; then - return; - fi; - moveToOutput include "${!outputInclude}"; - moveToOutput lib/pkgconfig "${!outputDev}"; - moveToOutput share/pkgconfig "${!outputDev}"; - moveToOutput lib/cmake "${!outputDev}"; - moveToOutput share/aclocal "${!outputDev}"; - for f in "${!outputDev}"/{lib,share}/pkgconfig/*.pc; - do - echo "Patching '$f' includedir to output ${!outputInclude}"; - sed -i "/^includedir=/s,=\${prefix},=${!outputInclude}," "$f"; - done -} -_multioutDocs () -{ - - local REMOVE=REMOVE; - moveToOutput share/info "${!outputInfo}"; - moveToOutput share/doc "${!outputDoc}"; - moveToOutput share/gtk-doc "${!outputDevdoc}"; - moveToOutput share/devhelp/books "${!outputDevdoc}"; - moveToOutput share/man "${!outputMan}"; - moveToOutput share/man/man3 "${!outputDevman}" -} -_multioutPropagateDev () -{ - - if [ "$(getAllOutputNames)" = "out" ]; then - return; - fi; - local outputFirst; - for outputFirst in $(getAllOutputNames); - do - break; - done; - local propagaterOutput="$outputDev"; - if [ -z "$propagaterOutput" ]; then - propagaterOutput="$outputFirst"; - fi; - if [ -z "${propagatedBuildOutputs+1}" ]; then - local po_dirty="$outputBin $outputInclude $outputLib"; - set +o pipefail; - propagatedBuildOutputs=`echo "$po_dirty" | tr -s ' ' '\n' | grep -v -F "$propagaterOutput" | sort -u | tr '\n' ' ' `; - set -o pipefail; - fi; - if [ -z "$propagatedBuildOutputs" ]; then - return; - fi; - mkdir -p "${!propagaterOutput}"/nix-support; - for output in $propagatedBuildOutputs; - do - echo -n " ${!output}" >> "${!propagaterOutput}"/nix-support/propagated-build-inputs; - done -} -_overrideFirst () -{ - - if [ -z "${!1-}" ]; then - _assignFirst "$@"; - fi -} -_pruneLibtoolFiles () -{ - - if [ "${dontPruneLibtoolFiles-}" ] || [ ! -e "$prefix" ]; then - return; - fi; - find "$prefix" -type f -name '*.la' -exec grep -q '^# Generated by .*libtool' {} \; -exec grep -q "^old_library=''" {} \; -exec sed -i {} -e "/^dependency_libs='[^']/ c dependency_libs='' #pruned" \; -} -_updateSourceDateEpochFromSourceRoot () -{ - - if [ -n "$sourceRoot" ]; then - updateSourceDateEpoch "$sourceRoot"; - fi -} -activatePackage () -{ - - local pkg="$1"; - local -r hostOffset="$2"; - local -r targetOffset="$3"; - (( hostOffset <= targetOffset )) || exit 1; - if [ -f "$pkg" ]; then - source "$pkg"; - fi; - if [[ -z "${strictDeps-}" || "$hostOffset" -le -1 ]]; then - addToSearchPath _PATH "$pkg/bin"; - fi; - if (( hostOffset <= -1 )); then - addToSearchPath _XDG_DATA_DIRS "$pkg/share"; - fi; - if [[ "$hostOffset" -eq 0 && -d "$pkg/bin" ]]; then - addToSearchPath _HOST_PATH "$pkg/bin"; - fi; - if [[ -f "$pkg/nix-support/setup-hook" ]]; then - source "$pkg/nix-support/setup-hook"; - fi -} -addEnvHooks () -{ - - local depHostOffset="$1"; - shift; - local pkgHookVarsSlice="${pkgHookVarVars[$depHostOffset + 1]}[@]"; - local pkgHookVar; - for pkgHookVar in "${!pkgHookVarsSlice}"; - do - eval "${pkgHookVar}s"'+=("$@")'; - done -} -addToSearchPath () -{ - - addToSearchPathWithCustomDelimiter ":" "$@" -} -addToSearchPathWithCustomDelimiter () -{ - - local delimiter="$1"; - local varName="$2"; - local dir="$3"; - if [[ -d "$dir" && "${!varName:+${delimiter}${!varName}${delimiter}}" != *"${delimiter}${dir}${delimiter}"* ]]; then - export "${varName}=${!varName:+${!varName}${delimiter}}${dir}"; - fi -} -appendToVar () -{ - - local -n nameref="$1"; - local useArray type; - if [ -n "$__structuredAttrs" ]; then - useArray=true; - else - useArray=false; - fi; - if declare -p "$1" 2> /dev/null | grep -q '^'; then - type="$(declare -p "$1")"; - if [[ "$type" =~ "declare -A" ]]; then - echo "appendToVar(): ERROR: trying to use appendToVar on an associative array, use variable+=([\"X\"]=\"Y\") instead." 1>&2; - return 1; - else - if [[ "$type" =~ "declare -a" ]]; then - useArray=true; - else - useArray=false; - fi; - fi; - fi; - shift; - if $useArray; then - nameref=(${nameref+"${nameref[@]}"} "$@"); - else - nameref="${nameref-} $*"; - fi -} -auditTmpdir () -{ - - local dir="$1"; - [ -e "$dir" ] || return 0; - echo "checking for references to $TMPDIR/ in $dir..."; - local i; - find "$dir" -type f -print0 | while IFS= read -r -d '' i; do - if [[ "$i" =~ .build-id ]]; then - continue; - fi; - if isELF "$i"; then - if { - printf :; - patchelf --print-rpath "$i" - } | grep -q -F ":$TMPDIR/"; then - echo "RPATH of binary $i contains a forbidden reference to $TMPDIR/"; - exit 1; - fi; - fi; - if isScript "$i"; then - if [ -e "$(dirname "$i")/.$(basename "$i")-wrapped" ]; then - if grep -q -F "$TMPDIR/" "$i"; then - echo "wrapper script $i contains a forbidden reference to $TMPDIR/"; - exit 1; - fi; - fi; - fi; - done -} -bintoolsWrapper_addLDVars () -{ - - local role_post; - getHostRoleEnvHook; - if [[ -d "$1/lib64" && ! -L "$1/lib64" ]]; then - export NIX_LDFLAGS${role_post}+=" -L$1/lib64"; - fi; - if [[ -d "$1/lib" ]]; then - local -a glob=($1/lib/lib*); - if [ "${#glob[*]}" -gt 0 ]; then - export NIX_LDFLAGS${role_post}+=" -L$1/lib"; - fi; - fi -} -buildPhase () -{ - - runHook preBuild; - if [[ -z "${makeFlags-}" && -z "${makefile:-}" && ! ( -e Makefile || -e makefile || -e GNUmakefile ) ]]; then - echo "no Makefile or custom buildPhase, doing nothing"; - else - foundMakefile=1; - local flagsArray=(${enableParallelBuilding:+-j${NIX_BUILD_CORES}} SHELL=$SHELL); - _accumFlagsArray makeFlags makeFlagsArray buildFlags buildFlagsArray; - echoCmd 'build flags' "${flagsArray[@]}"; - make ${makefile:+-f $makefile} "${flagsArray[@]}"; - unset flagsArray; - fi; - runHook postBuild -} -ccWrapper_addCVars () -{ - - local role_post; - getHostRoleEnvHook; - if [ -d "$1/include" ]; then - export NIX_CFLAGS_COMPILE${role_post}+=" -isystem $1/include"; - fi; - if [ -d "$1/Library/Frameworks" ]; then - export NIX_CFLAGS_COMPILE${role_post}+=" -iframework $1/Library/Frameworks"; - fi -} -checkPhase () -{ - - runHook preCheck; - if [[ -z "${foundMakefile:-}" ]]; then - echo "no Makefile or custom checkPhase, doing nothing"; - runHook postCheck; - return; - fi; - if [[ -z "${checkTarget:-}" ]]; then - if make -n ${makefile:+-f $makefile} check > /dev/null 2>&1; then - checkTarget=check; - else - if make -n ${makefile:+-f $makefile} test > /dev/null 2>&1; then - checkTarget=test; - fi; - fi; - fi; - if [[ -z "${checkTarget:-}" ]]; then - echo "no check/test target in ${makefile:-Makefile}, doing nothing"; - else - local flagsArray=(${enableParallelChecking:+-j${NIX_BUILD_CORES}} SHELL=$SHELL); - _accumFlagsArray makeFlags makeFlagsArray; - if [ -n "$__structuredAttrs" ]; then - flagsArray+=("${checkFlags[@]:-VERBOSE=y}"); - else - flagsArray+=(${checkFlags:-VERBOSE=y}); - fi; - _accumFlagsArray checkFlagsArray; - flagsArray+=(${checkTarget}); - echoCmd 'check flags' "${flagsArray[@]}"; - make ${makefile:+-f $makefile} "${flagsArray[@]}"; - unset flagsArray; - fi; - runHook postCheck -} -compressManPages () -{ - - local dir="$1"; - if [ -L "$dir"/share ] || [ -L "$dir"/share/man ] || [ ! -d "$dir/share/man" ]; then - return; - fi; - echo "gzipping man pages under $dir/share/man/"; - find "$dir"/share/man/ -type f -a '!' -regex '.*\.\(bz2\|gz\|xz\)$' -print0 | while IFS= read -r -d '' f; do - if gzip -c -n "$f" > "$f".gz; then - rm "$f"; - else - rm "$f".gz; - fi; - done; - find "$dir"/share/man/ -type l -a '!' -regex '.*\.\(bz2\|gz\|xz\)$' -print0 | sort -z | while IFS= read -r -d '' f; do - local target; - target="$(readlink -f "$f")"; - if [ -f "$target".gz ]; then - ln -sf "$target".gz "$f".gz && rm "$f"; - fi; - done -} -configurePhase () -{ - - runHook preConfigure; - : "${configureScript=}"; - if [[ -z "$configureScript" && -x ./configure ]]; then - configureScript=./configure; - fi; - if [ -z "${dontFixLibtool:-}" ]; then - export lt_cv_deplibs_check_method="${lt_cv_deplibs_check_method-pass_all}"; - local i; - find . -iname "ltmain.sh" -print0 | while IFS='' read -r -d '' i; do - echo "fixing libtool script $i"; - fixLibtool "$i"; - done; - CONFIGURE_MTIME_REFERENCE=$(mktemp configure.mtime.reference.XXXXXX); - find . -executable -type f -name configure -exec grep -l 'GNU Libtool is free software; you can redistribute it and/or modify' {} \; -exec touch -r {} "$CONFIGURE_MTIME_REFERENCE" \; -exec sed -i s_/usr/bin/file_file_g {} \; -exec touch -r "$CONFIGURE_MTIME_REFERENCE" {} \;; - rm -f "$CONFIGURE_MTIME_REFERENCE"; - fi; - if [[ -z "${dontAddPrefix:-}" && -n "$prefix" ]]; then - prependToVar configureFlags "${prefixKey:---prefix=}$prefix"; - fi; - if [[ -f "$configureScript" ]]; then - if [ -z "${dontAddDisableDepTrack:-}" ]; then - if grep -q dependency-tracking "$configureScript"; then - prependToVar configureFlags --disable-dependency-tracking; - fi; - fi; - if [ -z "${dontDisableStatic:-}" ]; then - if grep -q enable-static "$configureScript"; then - prependToVar configureFlags --disable-static; - fi; - fi; - if [ -z "${dontPatchShebangsInConfigure:-}" ]; then - patchShebangs --build "$configureScript"; - fi; - fi; - if [ -n "$configureScript" ]; then - local -a flagsArray; - _accumFlagsArray configureFlags configureFlagsArray; - echoCmd 'configure flags' "${flagsArray[@]}"; - $configureScript "${flagsArray[@]}"; - unset flagsArray; - else - echo "no configure script, doing nothing"; - fi; - runHook postConfigure -} -consumeEntire () -{ - - if IFS='' read -r -d '' "$1"; then - echo "consumeEntire(): ERROR: Input null bytes, won't process" 1>&2; - return 1; - fi -} -distPhase () -{ - - runHook preDist; - local flagsArray=(); - _accumFlagsArray distFlags distFlagsArray; - flagsArray+=(${distTarget:-dist}); - echo 'dist flags: %q' "${flagsArray[@]}"; - make ${makefile:+-f $makefile} "${flagsArray[@]}"; - if [ "${dontCopyDist:-0}" != 1 ]; then - mkdir -p "$out/tarballs"; - cp -pvd ${tarballs[*]:-*.tar.gz} "$out/tarballs"; - fi; - runHook postDist -} -dumpVars () -{ - - if [ "${noDumpEnvVars:-0}" != 1 ]; then - export 2> /dev/null >| "$NIX_BUILD_TOP/env-vars" || true; - fi -} -echoCmd () -{ - - printf "%s:" "$1"; - shift; - printf ' %q' "$@"; - echo -} -exitHandler () -{ - - exitCode="$?"; - set +e; - if [ -n "${showBuildStats:-}" ]; then - read -r -d '' -a buildTimes < <(times); - echo "build times:"; - echo "user time for the shell ${buildTimes[0]}"; - echo "system time for the shell ${buildTimes[1]}"; - echo "user time for all child processes ${buildTimes[2]}"; - echo "system time for all child processes ${buildTimes[3]}"; - fi; - if (( "$exitCode" != 0 )); then - runHook failureHook; - if [ -n "${succeedOnFailure:-}" ]; then - echo "build failed with exit code $exitCode (ignored)"; - mkdir -p "$out/nix-support"; - printf "%s" "$exitCode" > "$out/nix-support/failed"; - exit 0; - fi; - else - runHook exitHook; - fi; - return "$exitCode" -} -findInputs () -{ - - local -r pkg="$1"; - local -r hostOffset="$2"; - local -r targetOffset="$3"; - (( hostOffset <= targetOffset )) || exit 1; - local varVar="${pkgAccumVarVars[hostOffset + 1]}"; - local varRef="$varVar[$((targetOffset - hostOffset))]"; - local var="${!varRef}"; - unset -v varVar varRef; - local varSlice="$var[*]"; - case "${!varSlice-}" in - *" $pkg "*) - return 0 - ;; - esac; - unset -v varSlice; - eval "$var"'+=("$pkg")'; - if ! [ -e "$pkg" ]; then - echo "build input $pkg does not exist" 1>&2; - exit 1; - fi; - function mapOffset () - { - local -r inputOffset="$1"; - local -n outputOffset="$2"; - if (( inputOffset <= 0 )); then - outputOffset=$((inputOffset + hostOffset)); - else - outputOffset=$((inputOffset - 1 + targetOffset)); - fi - }; - local relHostOffset; - for relHostOffset in "${allPlatOffsets[@]}"; - do - local files="${propagatedDepFilesVars[relHostOffset + 1]}"; - local hostOffsetNext; - mapOffset "$relHostOffset" hostOffsetNext; - (( -1 <= hostOffsetNext && hostOffsetNext <= 1 )) || continue; - local relTargetOffset; - for relTargetOffset in "${allPlatOffsets[@]}"; - do - (( "$relHostOffset" <= "$relTargetOffset" )) || continue; - local fileRef="${files}[$relTargetOffset - $relHostOffset]"; - local file="${!fileRef}"; - unset -v fileRef; - local targetOffsetNext; - mapOffset "$relTargetOffset" targetOffsetNext; - (( -1 <= hostOffsetNext && hostOffsetNext <= 1 )) || continue; - [[ -f "$pkg/nix-support/$file" ]] || continue; - local pkgNext; - read -r -d '' pkgNext < "$pkg/nix-support/$file" || true; - for pkgNext in $pkgNext; - do - findInputs "$pkgNext" "$hostOffsetNext" "$targetOffsetNext"; - done; - done; - done -} -fixLibtool () -{ - - local search_path; - for flag in $NIX_LDFLAGS; - do - case $flag in - -L*) - search_path+=" ${flag#-L}" - ;; - esac; - done; - sed -i "$1" -e "s^eval \(sys_lib_search_path=\).*^\1'${search_path:-}'^" -e 's^eval sys_lib_.+search_path=.*^^' -} -fixupPhase () -{ - - local output; - for output in $(getAllOutputNames); - do - if [ -e "${!output}" ]; then - chmod -R u+w,u-s,g-s "${!output}"; - fi; - done; - runHook preFixup; - local output; - for output in $(getAllOutputNames); - do - prefix="${!output}" runHook fixupOutput; - done; - recordPropagatedDependencies; - if [ -n "${setupHook:-}" ]; then - mkdir -p "${!outputDev}/nix-support"; - substituteAll "$setupHook" "${!outputDev}/nix-support/setup-hook"; - fi; - if [ -n "${setupHooks:-}" ]; then - mkdir -p "${!outputDev}/nix-support"; - local hook; - for hook in ${setupHooks[@]}; - do - local content; - consumeEntire content < "$hook"; - substituteAllStream content "file '$hook'" >> "${!outputDev}/nix-support/setup-hook"; - unset -v content; - done; - unset -v hook; - fi; - if [ -n "${propagatedUserEnvPkgs:-}" ]; then - mkdir -p "${!outputBin}/nix-support"; - printWords $propagatedUserEnvPkgs > "${!outputBin}/nix-support/propagated-user-env-packages"; - fi; - runHook postFixup -} -genericBuild () -{ - - export GZIP_NO_TIMESTAMPS=1; - if [ -f "${buildCommandPath:-}" ]; then - source "$buildCommandPath"; - return; - fi; - if [ -n "${buildCommand:-}" ]; then - eval "$buildCommand"; - return; - fi; - if [ -z "${phases[*]:-}" ]; then - phases="${prePhases[*]:-} unpackPhase patchPhase ${preConfigurePhases[*]:-} configurePhase ${preBuildPhases[*]:-} buildPhase checkPhase ${preInstallPhases[*]:-} installPhase ${preFixupPhases[*]:-} fixupPhase installCheckPhase ${preDistPhases[*]:-} distPhase ${postPhases[*]:-}"; - fi; - for curPhase in ${phases[*]}; - do - runPhase "$curPhase"; - done -} -getAllOutputNames () -{ - - if [ -n "$__structuredAttrs" ]; then - echo "${!outputs[*]}"; - else - echo "$outputs"; - fi -} -getHostRole () -{ - - getRole "$hostOffset" -} -getHostRoleEnvHook () -{ - - getRole "$depHostOffset" -} -getRole () -{ - - case $1 in - -1) - role_post='_FOR_BUILD' - ;; - 0) - role_post='' - ;; - 1) - role_post='_FOR_TARGET' - ;; - *) - echo "binutils-wrapper-2.41: used as improper sort of dependency" 1>&2; - return 1 - ;; - esac -} -getTargetRole () -{ - - getRole "$targetOffset" -} -getTargetRoleEnvHook () -{ - - getRole "$depTargetOffset" -} -getTargetRoleWrapper () -{ - - case $targetOffset in - -1) - export NIX_BINTOOLS_WRAPPER_TARGET_BUILD_x86_64_unknown_linux_gnu=1 - ;; - 0) - export NIX_BINTOOLS_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu=1 - ;; - 1) - export NIX_BINTOOLS_WRAPPER_TARGET_TARGET_x86_64_unknown_linux_gnu=1 - ;; - *) - echo "binutils-wrapper-2.41: used as improper sort of dependency" 1>&2; - return 1 - ;; - esac -} -installCheckPhase () -{ - - runHook preInstallCheck; - if [[ -z "${foundMakefile:-}" ]]; then - echo "no Makefile or custom installCheckPhase, doing nothing"; - else - if [[ -z "${installCheckTarget:-}" ]] && ! make -n ${makefile:+-f $makefile} "${installCheckTarget:-installcheck}" > /dev/null 2>&1; then - echo "no installcheck target in ${makefile:-Makefile}, doing nothing"; - else - local flagsArray=(${enableParallelChecking:+-j${NIX_BUILD_CORES}} SHELL=$SHELL); - _accumFlagsArray makeFlags makeFlagsArray installCheckFlags installCheckFlagsArray; - flagsArray+=(${installCheckTarget:-installcheck}); - echoCmd 'installcheck flags' "${flagsArray[@]}"; - make ${makefile:+-f $makefile} "${flagsArray[@]}"; - unset flagsArray; - fi; - fi; - runHook postInstallCheck -} -installPhase () -{ - - runHook preInstall; - if [[ -z "${makeFlags-}" && -z "${makefile:-}" && ! ( -e Makefile || -e makefile || -e GNUmakefile ) ]]; then - echo "no Makefile or custom installPhase, doing nothing"; - runHook postInstall; - return; - else - foundMakefile=1; - fi; - if [ -n "$prefix" ]; then - mkdir -p "$prefix"; - fi; - local flagsArray=(${enableParallelInstalling:+-j${NIX_BUILD_CORES}} SHELL=$SHELL); - _accumFlagsArray makeFlags makeFlagsArray installFlags installFlagsArray; - if [ -n "$__structuredAttrs" ]; then - flagsArray+=("${installTargets[@]:-install}"); - else - flagsArray+=(${installTargets:-install}); - fi; - echoCmd 'install flags' "${flagsArray[@]}"; - make ${makefile:+-f $makefile} "${flagsArray[@]}"; - unset flagsArray; - runHook postInstall -} -isELF () -{ - - local fn="$1"; - local fd; - local magic; - exec {fd}< "$fn"; - read -r -n 4 -u "$fd" magic; - exec {fd}>&-; - if [ "$magic" = 'ELF' ]; then - return 0; - else - return 1; - fi -} -isMachO () -{ - - local fn="$1"; - local fd; - local magic; - exec {fd}< "$fn"; - read -r -n 4 -u "$fd" magic; - exec {fd}>&-; - if [[ "$magic" = $(echo -ne "\xfe\xed\xfa\xcf") || "$magic" = $(echo -ne "\xcf\xfa\xed\xfe") ]]; then - return 0; - else - if [[ "$magic" = $(echo -ne "\xfe\xed\xfa\xce") || "$magic" = $(echo -ne "\xce\xfa\xed\xfe") ]]; then - return 0; - else - if [[ "$magic" = $(echo -ne "\xca\xfe\xba\xbe") || "$magic" = $(echo -ne "\xbe\xba\xfe\xca") ]]; then - return 0; - else - return 1; - fi; - fi; - fi -} -isScript () -{ - - local fn="$1"; - local fd; - local magic; - exec {fd}< "$fn"; - read -r -n 2 -u "$fd" magic; - exec {fd}>&-; - if [[ "$magic" =~ \#! ]]; then - return 0; - else - return 1; - fi -} -mapOffset () -{ - - local -r inputOffset="$1"; - local -n outputOffset="$2"; - if (( inputOffset <= 0 )); then - outputOffset=$((inputOffset + hostOffset)); - else - outputOffset=$((inputOffset - 1 + targetOffset)); - fi -} -moveToOutput () -{ - - local patt="$1"; - local dstOut="$2"; - local output; - for output in $(getAllOutputNames); - do - if [ "${!output}" = "$dstOut" ]; then - continue; - fi; - local srcPath; - for srcPath in "${!output}"/$patt; - do - if [ ! -e "$srcPath" ] && [ ! -L "$srcPath" ]; then - continue; - fi; - if [ "$dstOut" = REMOVE ]; then - echo "Removing $srcPath"; - rm -r "$srcPath"; - else - local dstPath="$dstOut${srcPath#${!output}}"; - echo "Moving $srcPath to $dstPath"; - if [ -d "$dstPath" ] && [ -d "$srcPath" ]; then - rmdir "$srcPath" --ignore-fail-on-non-empty; - if [ -d "$srcPath" ]; then - mv -t "$dstPath" "$srcPath"/*; - rmdir "$srcPath"; - fi; - else - mkdir -p "$(readlink -m "$dstPath/..")"; - mv "$srcPath" "$dstPath"; - fi; - fi; - local srcParent="$(readlink -m "$srcPath/..")"; - if [ -n "$(find "$srcParent" -maxdepth 0 -type d -empty 2> /dev/null)" ]; then - echo "Removing empty $srcParent/ and (possibly) its parents"; - rmdir -p --ignore-fail-on-non-empty "$srcParent" 2> /dev/null || true; - fi; - done; - done -} -patchELF () -{ - - local dir="$1"; - [ -e "$dir" ] || return 0; - echo "shrinking RPATHs of ELF executables and libraries in $dir"; - local i; - while IFS= read -r -d '' i; do - if [[ "$i" =~ .build-id ]]; then - continue; - fi; - if ! isELF "$i"; then - continue; - fi; - echo "shrinking $i"; - patchelf --shrink-rpath "$i" || true; - done < <(find "$dir" -type f -print0) -} -patchPhase () -{ - - runHook prePatch; - local -a patchesArray; - if [ -n "$__structuredAttrs" ]; then - patchesArray=(${patches:+"${patches[@]}"}); - else - patchesArray=(${patches:-}); - fi; - for i in "${patchesArray[@]}"; - do - echo "applying patch $i"; - local uncompress=cat; - case "$i" in - *.gz) - uncompress="gzip -d" - ;; - *.bz2) - uncompress="bzip2 -d" - ;; - *.xz) - uncompress="xz -d" - ;; - *.lzma) - uncompress="lzma -d" - ;; - esac; - local -a flagsArray; - if [ -n "$__structuredAttrs" ]; then - flagsArray=("${patchFlags[@]:--p1}"); - else - flagsArray=(${patchFlags:--p1}); - fi; - $uncompress < "$i" 2>&1 | patch "${flagsArray[@]}"; - done; - runHook postPatch -} -patchShebangs () -{ - - local pathName; - local update; - while [[ $# -gt 0 ]]; do - case "$1" in - --host) - pathName=HOST_PATH; - shift - ;; - --build) - pathName=PATH; - shift - ;; - --update) - update=true; - shift - ;; - --) - shift; - break - ;; - -* | --*) - echo "Unknown option $1 supplied to patchShebangs" 1>&2; - return 1 - ;; - *) - break - ;; - esac; - done; - echo "patching script interpreter paths in $@"; - local f; - local oldPath; - local newPath; - local arg0; - local args; - local oldInterpreterLine; - local newInterpreterLine; - if [[ $# -eq 0 ]]; then - echo "No arguments supplied to patchShebangs" 1>&2; - return 0; - fi; - local f; - while IFS= read -r -d '' f; do - isScript "$f" || continue; - read -r oldInterpreterLine < "$f" || [ "$oldInterpreterLine" ]; - read -r oldPath arg0 args <<< "${oldInterpreterLine:2}"; - if [[ -z "${pathName:-}" ]]; then - if [[ -n $strictDeps && $f == "$NIX_STORE"* ]]; then - pathName=HOST_PATH; - else - pathName=PATH; - fi; - fi; - if [[ "$oldPath" == *"/bin/env" ]]; then - if [[ $arg0 == "-S" ]]; then - arg0=${args%% *}; - args=${args#* }; - newPath="$(PATH="${!pathName}" command -v "env" || true)"; - args="-S $(PATH="${!pathName}" command -v "$arg0" || true) $args"; - else - if [[ $arg0 == "-"* || $arg0 == *"="* ]]; then - echo "$f: unsupported interpreter directive \"$oldInterpreterLine\" (set dontPatchShebangs=1 and handle shebang patching yourself)" 1>&2; - exit 1; - else - newPath="$(PATH="${!pathName}" command -v "$arg0" || true)"; - fi; - fi; - else - if [[ -z $oldPath ]]; then - oldPath="/bin/sh"; - fi; - newPath="$(PATH="${!pathName}" command -v "$(basename "$oldPath")" || true)"; - args="$arg0 $args"; - fi; - newInterpreterLine="$newPath $args"; - newInterpreterLine=${newInterpreterLine%${newInterpreterLine##*[![:space:]]}}; - if [[ -n "$oldPath" && ( "$update" == true || "${oldPath:0:${#NIX_STORE}}" != "$NIX_STORE" ) ]]; then - if [[ -n "$newPath" && "$newPath" != "$oldPath" ]]; then - echo "$f: interpreter directive changed from \"$oldInterpreterLine\" to \"$newInterpreterLine\""; - escapedInterpreterLine=${newInterpreterLine//\\/\\\\}; - timestamp=$(stat --printf "%y" "$f"); - sed -i -e "1 s|.*|#\!$escapedInterpreterLine|" "$f"; - touch --date "$timestamp" "$f"; - fi; - fi; - done < <(find "$@" -type f -perm -0100 -print0) -} -patchShebangsAuto () -{ - - if [[ -z "${dontPatchShebangs-}" && -e "$prefix" ]]; then - if [[ "$output" != out && "$output" = "$outputDev" ]]; then - patchShebangs --build "$prefix"; - else - patchShebangs --host "$prefix"; - fi; - fi -} -prependToVar () -{ - - local -n nameref="$1"; - local useArray type; - if [ -n "$__structuredAttrs" ]; then - useArray=true; - else - useArray=false; - fi; - if declare -p "$1" 2> /dev/null | grep -q '^'; then - type="$(declare -p "$1")"; - if [[ "$type" =~ "declare -A" ]]; then - echo "prependToVar(): ERROR: trying to use prependToVar on an associative array." 1>&2; - return 1; - else - if [[ "$type" =~ "declare -a" ]]; then - useArray=true; - else - useArray=false; - fi; - fi; - fi; - shift; - if $useArray; then - nameref=("$@" ${nameref+"${nameref[@]}"}); - else - nameref="$* ${nameref-}"; - fi -} -printLines () -{ - - (( "$#" > 0 )) || return 0; - printf '%s\n' "$@" -} -printWords () -{ - - (( "$#" > 0 )) || return 0; - printf '%s ' "$@" -} -recordPropagatedDependencies () -{ - - declare -ra flatVars=(depsBuildBuildPropagated propagatedNativeBuildInputs depsBuildTargetPropagated depsHostHostPropagated propagatedBuildInputs depsTargetTargetPropagated); - declare -ra flatFiles=("${propagatedBuildDepFiles[@]}" "${propagatedHostDepFiles[@]}" "${propagatedTargetDepFiles[@]}"); - local propagatedInputsIndex; - for propagatedInputsIndex in "${!flatVars[@]}"; - do - local propagatedInputsSlice="${flatVars[$propagatedInputsIndex]}[@]"; - local propagatedInputsFile="${flatFiles[$propagatedInputsIndex]}"; - [[ -n "${!propagatedInputsSlice}" ]] || continue; - mkdir -p "${!outputDev}/nix-support"; - printWords ${!propagatedInputsSlice} > "${!outputDev}/nix-support/$propagatedInputsFile"; - done -} -runHook () -{ - - local hookName="$1"; - shift; - local hooksSlice="${hookName%Hook}Hooks[@]"; - local hook; - for hook in "_callImplicitHook 0 $hookName" ${!hooksSlice+"${!hooksSlice}"}; - do - _eval "$hook" "$@"; - done; - return 0 -} -runOneHook () -{ - - local hookName="$1"; - shift; - local hooksSlice="${hookName%Hook}Hooks[@]"; - local hook ret=1; - for hook in "_callImplicitHook 1 $hookName" ${!hooksSlice+"${!hooksSlice}"}; - do - if _eval "$hook" "$@"; then - ret=0; - break; - fi; - done; - return "$ret" -} -runPhase () -{ - - local curPhase="$*"; - if [[ "$curPhase" = unpackPhase && -n "${dontUnpack:-}" ]]; then - return; - fi; - if [[ "$curPhase" = patchPhase && -n "${dontPatch:-}" ]]; then - return; - fi; - if [[ "$curPhase" = configurePhase && -n "${dontConfigure:-}" ]]; then - return; - fi; - if [[ "$curPhase" = buildPhase && -n "${dontBuild:-}" ]]; then - return; - fi; - if [[ "$curPhase" = checkPhase && -z "${doCheck:-}" ]]; then - return; - fi; - if [[ "$curPhase" = installPhase && -n "${dontInstall:-}" ]]; then - return; - fi; - if [[ "$curPhase" = fixupPhase && -n "${dontFixup:-}" ]]; then - return; - fi; - if [[ "$curPhase" = installCheckPhase && -z "${doInstallCheck:-}" ]]; then - return; - fi; - if [[ "$curPhase" = distPhase && -z "${doDist:-}" ]]; then - return; - fi; - if [[ -n $NIX_LOG_FD ]]; then - echo "@nix { \"action\": \"setPhase\", \"phase\": \"$curPhase\" }" >&"$NIX_LOG_FD"; - fi; - showPhaseHeader "$curPhase"; - dumpVars; - local startTime=$(date +"%s"); - eval "${!curPhase:-$curPhase}"; - local endTime=$(date +"%s"); - showPhaseFooter "$curPhase" "$startTime" "$endTime"; - if [ "$curPhase" = unpackPhase ]; then - [ -n "${sourceRoot:-}" ] && chmod +x "${sourceRoot}"; - cd "${sourceRoot:-.}"; - fi -} -showPhaseFooter () -{ - - local phase="$1"; - local startTime="$2"; - local endTime="$3"; - local delta=$(( endTime - startTime )); - (( delta < 30 )) && return; - local H=$((delta/3600)); - local M=$((delta%3600/60)); - local S=$((delta%60)); - echo -n "$phase completed in "; - (( H > 0 )) && echo -n "$H hours "; - (( M > 0 )) && echo -n "$M minutes "; - echo "$S seconds" -} -showPhaseHeader () -{ - - local phase="$1"; - echo "Running phase: $phase" -} -stripDirs () -{ - - local cmd="$1"; - local ranlibCmd="$2"; - local paths="$3"; - local stripFlags="$4"; - local excludeFlags=(); - local pathsNew=; - [ -z "$cmd" ] && echo "stripDirs: Strip command is empty" 1>&2 && exit 1; - [ -z "$ranlibCmd" ] && echo "stripDirs: Ranlib command is empty" 1>&2 && exit 1; - local pattern; - if [ -n "${stripExclude:-}" ]; then - for pattern in "${stripExclude[@]}"; - do - excludeFlags+=(-a '!' '(' -name "$pattern" -o -wholename "$prefix/$pattern" ')'); - done; - fi; - local p; - for p in ${paths}; - do - if [ -e "$prefix/$p" ]; then - pathsNew="${pathsNew} $prefix/$p"; - fi; - done; - paths=${pathsNew}; - if [ -n "${paths}" ]; then - echo "stripping (with command $cmd and flags $stripFlags) in $paths"; - local striperr; - striperr="$(mktemp --tmpdir="$TMPDIR" 'striperr.XXXXXX')"; - find $paths -type f "${excludeFlags[@]}" -a '!' -path "$prefix/lib/debug/*" -printf '%D-%i,%p\0' | sort -t, -k1,1 -u -z | cut -d, -f2- -z | xargs -r -0 -n1 -P "$NIX_BUILD_CORES" -- $cmd $stripFlags 2> "$striperr" || exit_code=$?; - [[ "$exit_code" = 123 || -z "$exit_code" ]] || ( cat "$striperr" 1>&2 && exit 1 ); - rm "$striperr"; - find $paths -name '*.a' -type f -exec $ranlibCmd '{}' \; 2> /dev/null; - fi -} -stripHash () -{ - - local strippedName casematchOpt=0; - strippedName="$(basename -- "$1")"; - shopt -q nocasematch && casematchOpt=1; - shopt -u nocasematch; - if [[ "$strippedName" =~ ^[a-z0-9]{32}- ]]; then - echo "${strippedName:33}"; - else - echo "$strippedName"; - fi; - if (( casematchOpt )); then - shopt -s nocasematch; - fi -} -substitute () -{ - - local input="$1"; - local output="$2"; - shift 2; - if [ ! -f "$input" ]; then - echo "substitute(): ERROR: file '$input' does not exist" 1>&2; - return 1; - fi; - local content; - consumeEntire content < "$input"; - if [ -e "$output" ]; then - chmod +w "$output"; - fi; - substituteStream content "file '$input'" "$@" > "$output" -} -substituteAll () -{ - - local input="$1"; - local output="$2"; - local -a args=(); - _allFlags; - substitute "$input" "$output" "${args[@]}" -} -substituteAllInPlace () -{ - - local fileName="$1"; - shift; - substituteAll "$fileName" "$fileName" "$@" -} -substituteAllStream () -{ - - local -a args=(); - _allFlags; - substituteStream "$1" "$2" "${args[@]}" -} -substituteInPlace () -{ - - local -a fileNames=(); - for arg in "$@"; - do - if [[ "$arg" = "--"* ]]; then - break; - fi; - fileNames+=("$arg"); - shift; - done; - for file in "${fileNames[@]}"; - do - substitute "$file" "$file" "$@"; - done -} -substituteStream () -{ - - local var=$1; - local description=$2; - shift 2; - while (( "$#" )); do - local replace_mode="$1"; - case "$1" in - --replace) - if ! "$_substituteStream_has_warned_replace_deprecation"; then - echo "substituteStream() in derivation $name: WARNING: '--replace' is deprecated, use --replace-{fail,warn,quiet}. ($description)" 1>&2; - _substituteStream_has_warned_replace_deprecation=true; - fi; - replace_mode='--replace-warn' - ;& - --replace-quiet | --replace-warn | --replace-fail) - pattern="$2"; - replacement="$3"; - shift 3; - local savedvar; - savedvar="${!var}"; - eval "$var"'=${'"$var"'//"$pattern"/"$replacement"}'; - if [ "$pattern" != "$replacement" ]; then - if [ "${!var}" == "$savedvar" ]; then - if [ "$replace_mode" == --replace-warn ]; then - printf "substituteStream() in derivation $name: WARNING: pattern %q doesn't match anything in %s\n" "$pattern" "$description" 1>&2; - else - if [ "$replace_mode" == --replace-fail ]; then - printf "substituteStream() in derivation $name: ERROR: pattern %q doesn't match anything in %s\n" "$pattern" "$description" 1>&2; - return 1; - fi; - fi; - fi; - fi - ;; - --subst-var) - local varName="$2"; - shift 2; - if ! [[ "$varName" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then - echo "substituteStream() in derivation $name: ERROR: substitution variables must be valid Bash names, \"$varName\" isn't." 1>&2; - return 1; - fi; - if [ -z ${!varName+x} ]; then - echo "substituteStream() in derivation $name: ERROR: variable \$$varName is unset" 1>&2; - return 1; - fi; - pattern="@$varName@"; - replacement="${!varName}"; - eval "$var"'=${'"$var"'//"$pattern"/"$replacement"}' - ;; - --subst-var-by) - pattern="@$2@"; - replacement="$3"; - eval "$var"'=${'"$var"'//"$pattern"/"$replacement"}'; - shift 3 - ;; - *) - echo "substituteStream() in derivation $name: ERROR: Invalid command line argument: $1" 1>&2; - return 1 - ;; - esac; - done; - printf "%s" "${!var}" -} -unpackFile () -{ - - curSrc="$1"; - echo "unpacking source archive $curSrc"; - if ! runOneHook unpackCmd "$curSrc"; then - echo "do not know how to unpack source archive $curSrc"; - exit 1; - fi -} -unpackPhase () -{ - - runHook preUnpack; - if [ -z "${srcs:-}" ]; then - if [ -z "${src:-}" ]; then - echo 'variable $src or $srcs should point to the source'; - exit 1; - fi; - srcs="$src"; - fi; - local -a srcsArray; - if [ -n "$__structuredAttrs" ]; then - srcsArray=("${srcs[@]}"); - else - srcsArray=($srcs); - fi; - local dirsBefore=""; - for i in *; - do - if [ -d "$i" ]; then - dirsBefore="$dirsBefore $i "; - fi; - done; - for i in "${srcsArray[@]}"; - do - unpackFile "$i"; - done; - : "${sourceRoot=}"; - if [ -n "${setSourceRoot:-}" ]; then - runOneHook setSourceRoot; - else - if [ -z "$sourceRoot" ]; then - for i in *; - do - if [ -d "$i" ]; then - case $dirsBefore in - *\ $i\ *) - - ;; - *) - if [ -n "$sourceRoot" ]; then - echo "unpacker produced multiple directories"; - exit 1; - fi; - sourceRoot="$i" - ;; - esac; - fi; - done; - fi; - fi; - if [ -z "$sourceRoot" ]; then - echo "unpacker appears to have produced no directories"; - exit 1; - fi; - echo "source root is $sourceRoot"; - if [ "${dontMakeSourcesWritable:-0}" != 1 ]; then - chmod -R u+w -- "$sourceRoot"; - fi; - runHook postUnpack -} -updateAutotoolsGnuConfigScriptsPhase () -{ - - if [ -n "${dontUpdateAutotoolsGnuConfigScripts-}" ]; then - return; - fi; - for script in config.sub config.guess; - do - for f in $(find . -type f -name "$script"); - do - echo "Updating Autotools / GNU config script to a newer upstream version: $f"; - cp -f "/nix/store/pqjr4bpaykb34bp13m1gc5sin1gzb1a8-gnu-config-2024-01-01/$script" "$f"; - done; - done -} -updateSourceDateEpoch () -{ - - local path="$1"; - local -a res=($(find "$path" -type f -not -newer "$NIX_BUILD_TOP/.." -printf '%T@ %p\0' | sort -n --zero-terminated | tail -n1 --zero-terminated | head -c -1)); - local time="${res[0]//\.[0-9]*/}"; - local newestFile="${res[1]}"; - if [ "${time:-0}" -gt "$SOURCE_DATE_EPOCH" ]; then - echo "setting SOURCE_DATE_EPOCH to timestamp $time of file $newestFile"; - export SOURCE_DATE_EPOCH="$time"; - local now="$(date +%s)"; - if [ "$time" -gt $((now - 60)) ]; then - echo "warning: file $newestFile may be generated; SOURCE_DATE_EPOCH may be non-deterministic"; - fi; - fi -} -PATH="$PATH${nix_saved_PATH:+:$nix_saved_PATH}" -XDG_DATA_DIRS="$XDG_DATA_DIRS${nix_saved_XDG_DATA_DIRS:+:$nix_saved_XDG_DATA_DIRS}" -export NIX_BUILD_TOP="$(mktemp -d -t nix-shell.XXXXXX)" -export TMP="$NIX_BUILD_TOP" -export TMPDIR="$NIX_BUILD_TOP" -export TEMP="$NIX_BUILD_TOP" -export TEMPDIR="$NIX_BUILD_TOP" -eval "$shellHook" diff --git a/libs/ollama_functions.py b/libs/ollama_functions.py index 71d2679..673daf6 100644 --- a/libs/ollama_functions.py +++ b/libs/ollama_functions.py @@ -20,11 +20,10 @@ from langchain_core.language_models import LanguageModelInput from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage, BaseMessage, ToolCall from langchain_core.outputs import ChatGeneration, ChatResult from langchain_core.prompts import SystemMessagePromptTemplate -from langchain_core.pydantic_v1 import BaseModel +from langchain_core.pydantic_v1 import BaseModel from langchain_core.runnables import Runnable from langchain_core.tools import BaseTool, Tool from langchain_core.utils.pydantic import is_basemodel_instance, is_basemodel_subclass -from textwrap import dedent from libs.functions import nxhash @@ -98,14 +97,15 @@ def _is_pydantic_class(obj: Any) -> bool: is_basemodel_subclass(obj) or BaseModel in obj.__bases__ ) -class OllamaFunctions(ChatOllama): +class OllamaFunctionsBase(ChatOllama): """Function chat model that uses Ollama API.""" tool_system_prompt_template: str = DEFAULT_SYTEM_PROMPT tool_system_prompt_template_with_history: str = DEFAULT_SYTEM_PROMPT_WITH_HISTORY max_tool_call_fails: int = 5 - def __init__(self, max_tool_call_fails, **kwargs: Any) -> None: + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) def bind_tools( @@ -115,6 +115,8 @@ class OllamaFunctions(ChatOllama): ) -> Runnable[LanguageModelInput, BaseMessage]: return self.bind(functions=tools, **kwargs) + def _get_final_message(self, messages: list, functions_str: str) -> list: + raise NotImplementedError def _generate(self, messages: List[BaseMessage], stop: Optional[List[str]] = None, run_manager: Optional[CallbackManagerForLLMRun] = None, **kwargs: Any) -> ChatResult: def _convert_to_ollama_tool(self, tool: Any) -> Dict: @@ -177,11 +179,11 @@ class OllamaFunctions(ChatOllama): return called_tool def _extract_conversaional_response(self, d: dict) -> str: - if ("tool_input" in d and "response" in d["tool_input"]): + if ("tool_input" in d and d["tool_input"] and "response" in d["tool_input"]): response = d["tool_input"]["response"] - elif ("input" in d and "response" in d["input"]): + elif ("input" in d and d["input"] and "response" in d["input"]): response = d["input"]["response"] - elif ("args" in d and "response" in d["args"]): + elif ("args" in d and d["args"] and "response" in d["args"]): response = d["args"]["response"] elif "response" in d: response = d["response"] @@ -220,66 +222,6 @@ class OllamaFunctions(ChatOllama): called_tool_args = {} return called_tool_args - def _get_final_message(self, messages: list, functions_str: str) -> list: - def _get_system_msg_and_formatted_history(self, messages: list) -> Tuple[str, str]: - def _format_tools_for_history(tool_calls: list[ToolCall]) -> str: - call_list = [] - for c in tool_calls: - call_list.append({ - "id": nxhash(c['id'])[-4:], - "tool": c['name'], - "args": c['args'] - }) - if len(call_list) == 1: - return json.dumps(obj=call_list[0], ensure_ascii=False, indent=2) - else: - return json.dumps(obj=call_list, ensure_ascii=False, indent=2) - formated_history = "" - system_msg = messages[0] - for m in messages[1:]: - - if formated_history != "": - formated_history += "\n\n" - - if isinstance(m, SystemMessage): - formated_history += "The system provided the info:\n" + str(m.content) - elif isinstance(m, HumanMessage): - formated_history += "The Human said:\n" + str(m.content) - elif isinstance(m, AIMessage) and m.tool_calls: - formated_history += "So you called the tool" + (":\n" if len(m.tool_calls) == 1 else "s:\n") + _format_tools_for_history(m.tool_calls) - elif isinstance(m, ToolMessage): - formated_history += "To which the tool (" + nxhash(m.tool_call_id)[-4:] + ") replied with:\n" + str(m.content) - elif isinstance(m, AIMessage) and not m.tool_calls: - formated_history += "You said:\n" + str(m.content) - else: - raise TypeError("OllamaFunctions only supports SystemMessage HumanMessage ToolMessage AIMessage but got " + str(type(m))) - - return system_msg, formated_history - - # prepare generation with history - if True in [ isinstance(m, ToolMessage) for m in messages ]: - system_msg, formated_history = _get_system_msg_and_formatted_history(self, messages=messages) - - system_message_prompt_template = SystemMessagePromptTemplate.from_template(self.tool_system_prompt_template_with_history) - system_message = system_message_prompt_template.format( - tools=functions_str, - history=formated_history, - system_msg=system_msg - ) - final_messages = [ system_message ] - - # prepare generation without history - else: - system_message_prompt_template = SystemMessagePromptTemplate.from_template(self.tool_system_prompt_template) - system_message = system_message_prompt_template.format( - tools=functions_str - ) - final_messages = [ system_message ] + messages - - return final_messages - - - def gen(self, failed_tool_calls: int, messages: list) -> ChatResult: @@ -289,7 +231,7 @@ class OllamaFunctions(ChatOllama): functions_str = json.dumps(functions_list, indent=2) # get messages to prompt with - final_messages = _get_final_message(self, messages=messages, functions_str=functions_str) + final_messages = self._get_final_message(messages=messages, functions_str=functions_str) # genrerate chat result response_message = super()._generate(final_messages, stop=stop, run_manager=run_manager, **kwargs) @@ -329,6 +271,125 @@ class OllamaFunctions(ChatOllama): return gen(self, failed_tool_calls=0, messages=messages) + + + + + +class OllamaFunctionsLSM(OllamaFunctionsBase): + """Function chat model that uses Ollama API.""" + + def _get_final_message(self, messages: list, functions_str: str) -> list: + def _get_system_msg_and_formatted_history(self, messages: list) -> Tuple[str, str]: + def _format_tools_for_history(tool_calls: list[ToolCall]) -> str: + call_list = [] + for c in tool_calls: + call_list.append({ + "id": nxhash(c['id'])[-4:], + "tool": c['name'], + "args": c['args'] + }) + if len(call_list) == 1: + return json.dumps(obj=call_list[0], ensure_ascii=False, indent=2) + else: + return json.dumps(obj=call_list, ensure_ascii=False, indent=2) + formated_history = "" + system_msg = messages[0] + for m in messages[1:]: + + if formated_history != "": + formated_history += "\n\n" + + if isinstance(m, SystemMessage): + formated_history += "The system provided the info:\n" + str(m.content) + elif isinstance(m, HumanMessage): + formated_history += "The Human said:\n" + str(m.content) + elif isinstance(m, AIMessage) and m.tool_calls: + formated_history += "So you called the tool" + (":\n" if len(m.tool_calls) == 1 else "s:\n") + _format_tools_for_history(m.tool_calls) + elif isinstance(m, ToolMessage): + formated_history += "To which the tool (" + nxhash(m.tool_call_id)[-4:] + ") replied with:\n" + str(m.content) + elif isinstance(m, AIMessage) and not m.tool_calls: + formated_history += "You said:\n" + str(m.content) + else: + try: + raise TypeError("OllamaFunctions only supports SystemMessage HumanMessage ToolMessage AIMessage but got " + str(type(m))) + except NameError: + raise TypeError("OllamaFunctions only supports SystemMessage HumanMessage ToolMessage AIMessage.") + + + return system_msg, formated_history + + # prepare generation with history + if True in [ isinstance(m, ToolMessage) for m in messages ]: + system_msg, formated_history = _get_system_msg_and_formatted_history(self, messages=messages) + + system_message_prompt_template = SystemMessagePromptTemplate.from_template(self.tool_system_prompt_template_with_history) + system_message = system_message_prompt_template.format( + tools=functions_str, + history=formated_history, + system_msg=system_msg + ) + final_messages = [ system_message ] + + # prepare generation without history + else: + system_message_prompt_template = SystemMessagePromptTemplate.from_template(self.tool_system_prompt_template) + system_message = system_message_prompt_template.format( + tools=functions_str + ) + final_messages = [ system_message ] + messages + + return final_messages + @property def _llm_type(self) -> str: - return "ollama_functions" + return "ollama_functions_lsm" + + + +class OllamaFunctionsT2S(OllamaFunctionsBase): + """Function chat model that uses Ollama API.""" + + def _get_final_message(self, messages: list, functions_str: str) -> list: + # prepare generation with history + if True in [ isinstance(m, ToolMessage) for m in messages ]: + + transformed_messages = [] + for m in messages: + if isinstance(m, ToolMessage): + transformed_messages.append(SystemMessage(content=( + f"The Tool '{m.name}' replied with:" + "\n" + str(m.content) + ))) + elif isinstance(m, AIMessage): + if m.tool_calls: + l = [] + for call in m.tool_calls: + l.append({ + "tool": call['name'], + "tool_input": call['args'] + }) + if len(l) == 1: + transformed_messages.append(AIMessage(content=json.dumps(l[0]))) + else: + transformed_messages.append(AIMessage(content=json.dumps(l))) + else: + transformed_messages.append(m) + + system_message_prompt_template = SystemMessagePromptTemplate.from_template(self.tool_system_prompt_template) + system_message = system_message_prompt_template.format(tools=functions_str) + + final_messages = [ system_message ] + transformed_messages + + # prepare generation without history + else: + system_message_prompt_template = SystemMessagePromptTemplate.from_template(self.tool_system_prompt_template) + system_message = system_message_prompt_template.format( + tools=functions_str + ) + final_messages = [ system_message ] + messages + + return final_messages + + @property + def _llm_type(self) -> str: + return "ollama_functions_t2s" diff --git a/libs/run_tests.py b/libs/run_tests.py index 8f1faa6..05d1a79 100644 --- a/libs/run_tests.py +++ b/libs/run_tests.py @@ -79,8 +79,8 @@ def run_tests(models: dict[int, Model], seeds: list[int], tests: dict[int, Test] 'technique_name': technique.name, }) - # if hash_key == "DE3D137E": - # pass + if hash_key == "0DEB2030": + pass if hash_key not in saved_results.keys(): try: @@ -105,7 +105,7 @@ def run_tests(models: dict[int, Model], seeds: list[int], tests: dict[int, Test] "\033[0;35m)\033[0m", end="" ) - answer = test.runnable(model=model, seed=seed, test=test, base_url=base_url) + answer = test.runnable(model=model, seed=seed, test=test, technique=technique, base_url=base_url) if isinstance(answer, str): combination['answer'] = answer # combination['tool_calls'] = [] # no entry @@ -172,13 +172,15 @@ def run_tests(models: dict[int, Model], seeds: list[int], tests: dict[int, Test] try: entry = { - 'test_name': result['test_name'], - 'test_id': result['test_id'], - 'model_name': result['model_name'], - 'model_id': result['model_id'], - 'seed': result['seed'], - 'answer': result['answer'], - 'validation': result['test'].validator(test=result['test'], answer=result['answer'], base_url=base_url), + 'test_name': result['test_name'], + 'test_id': result['test_id'], + 'model_name': result['model_name'], + 'model_id': result['model_id'], + 'technique_name': result['technique_name'], + 'technique_id': result['technique_id'], + 'seed': result['seed'], + 'answer': result['answer'], + 'validation': result['test'].validator(test=result['test'], answer=result['answer'], base_url=base_url), } except Exception as e: print("\033[0;31mError validating entry (\033[0m" + hash_key + "\033[0;31m). <\033[0m" + str(e) + "\033[0;31m> Continuing...\033[0m ") diff --git a/libs/runnables.py b/libs/runnables.py index a3cf00f..b6cec9d 100644 --- a/libs/runnables.py +++ b/libs/runnables.py @@ -1,8 +1,8 @@ from types import NoneType from langchain_ollama.chat_models import ChatOllama -from libs.ollama_functions import OllamaFunctions +from libs.ollama_functions import OllamaFunctionsLSM, OllamaFunctionsT2S from langchain_core.messages import AIMessage, SystemMessage, HumanMessage, ToolMessage -from libs.classes import Test, Model +from libs.classes import Technique, Test, Model from langchain.tools import Tool from typing import Literal @@ -10,22 +10,31 @@ from langgraph.graph import StateGraph, MessagesState import json from pydantic import ValidationError -def _get_llm(model: Model, base_url: str, seed: int, tools: list[Tool]|NoneType = None): - if model.supports_tools: +from suite_settings.techniques import techniques + +def _get_llm(model: Model, base_url: str, seed: int, technique: Technique, tools: list[Tool]|NoneType = None): + if technique == techniques[1]: # Native llm = ChatOllama( model=model.identifier, seed=seed, base_url=base_url ) - else: - llm = OllamaFunctions( + elif technique == techniques[903]: # Long System Message + llm = OllamaFunctionsLSM( model=model.identifier, seed=seed, base_url=base_url, format="json", - max_tool_call_fails=3, - temperature=0.0 ) + elif technique == techniques[572]: # ToolMessages to SystemMessages + llm = OllamaFunctionsT2S( + model=model.identifier, + seed=seed, + base_url=base_url, + format="json", + ) + else: + raise ValueError("Unkown Technique in _get_llm()") if tools: llm = llm.bind_tools(tools=tools) @@ -33,7 +42,7 @@ def _get_llm(model: Model, base_url: str, seed: int, tools: list[Tool]|NoneType return llm -def basic_prompt(model: Model, seed: int, test: Test, base_url: str) -> str: +def basic_prompt(model: Model, seed: int, test: Test, technique: Technique, base_url: str) -> str: messages = [SystemMessage(test.runnable_input['system_msg'])] try: @@ -42,20 +51,20 @@ def basic_prompt(model: Model, seed: int, test: Test, base_url: str) -> str: pass messages += [ HumanMessage(test.runnable_input['human_msg']) ] - llm = _get_llm(model=model, base_url=base_url, seed=seed) + llm = _get_llm(model=model, base_url=base_url, technique=technique, seed=seed) ai_msg = llm.invoke(messages) assert isinstance(ai_msg.content, str) return ai_msg.content -def one_tool_call_answer(model: Model, seed: int, test: Test, base_url: str) -> dict: +def one_tool_call_answer(model: Model, seed: int, test: Test, technique: Technique, base_url: str) -> dict: tools_dict = test.runnable_input['tools'] tools = [] for key in tools_dict: tools.append(tools_dict[key]) - llm = _get_llm(model=model, base_url=base_url, seed=seed, tools=tools) + llm = _get_llm(model=model, base_url=base_url, seed=seed, technique=technique, tools=tools) messages = [SystemMessage(test.runnable_input['system_msg'])] try: @@ -108,7 +117,7 @@ def one_tool_call_answer(model: Model, seed: int, test: Test, base_url: str) -> "tool_calls": tool_calls, } -def agent_with_tools(model: Model, seed: int, test: Test, base_url: str) -> dict[str, str|list]: +def agent_with_tools(model: Model, seed: int, test: Test, technique: Technique, base_url: str) -> dict[str, str|list]: tool_calls = [] index = -1 @@ -173,7 +182,7 @@ def agent_with_tools(model: Model, seed: int, test: Test, base_url: str) -> dict for key in tools_dict: tools.append(tools_dict[key]) tool_node = NxToolNode(tools) - llm = _get_llm(model=model, base_url=base_url, seed=seed, tools=tools) + llm = _get_llm(model=model, base_url=base_url, seed=seed, technique=technique, tools=tools) workflow = StateGraph(MessagesState) diff --git a/libs/tools.py b/libs/tools.py index b8f848a..78ea391 100644 --- a/libs/tools.py +++ b/libs/tools.py @@ -141,6 +141,9 @@ def get_notes_containing(patterns: Union[list[str], str]) -> str: ret += f"{datetime.strftime(entry.time, '%Y/%m/%d %H:%M')} {entry.content}" is_first = False + if ret == "": + ret = "No matching notes were found. Try diffrent patterns." + return ret @tool diff --git a/print_saved_results.py b/print_saved_results.py deleted file mode 100644 index 22cd804..0000000 --- a/print_saved_results.py +++ /dev/null @@ -1,96 +0,0 @@ -import json -import os -import sys - -def print_help(): - print("""Example usages: - -python print_saved_results.py -python print_saved_results.py -m llama3.1 -python print_saved_results.py -m llama3.1,mixtral-nemo:12b -python print_saved_results.py -m llama3.1 -s 2222,2 -t "Healthy Vegetables in Chinese" - -Note: If one of the "fileters" does not exist, no error is thrown.""") - -def main(argv: list[str]) -> None: - try: - with open("./saved_results.json", "r") as f: - saved_results = json.load(fp=f) - except: - print("saved_results.json not found. Try running test_suite.py first.") - exit(1) - - if "-h" in argv: - print_help() - exit(0) - - try: - if "-m" in argv: - test_str = argv[argv.index("-m")+1] - assert test_str[0] != "-" - models = test_str.split(",") - argv.pop(argv.index("-m")+1) - argv.pop(argv.index("-m")) - else: - models = None - - if "-s" in argv: - test_str = argv[argv.index("-s")+1] - assert test_str[0] != "-" - seeds = test_str.split(",") - argv.pop(argv.index("-s")+1) - argv.pop(argv.index("-s")) - else: - seeds = None - - if "-t" in argv: - test_str = argv[argv.index("-t")+1] - assert test_str[0] != "-" - tests = test_str.split(",") - argv.pop(argv.index("-t")+1) - argv.pop(argv.index("-t")) - else: - tests = None - except: - print("Syntax error. Run `python print_saved_results.py -h` for help.") - print_help() - exit(1) - - argv.pop(0) # remove filename entry - if argv != []: - print("Syntax error. Run `python print_saved_results.py -h` for help.") - print(f"Got unkown argument{'s' if len(argv) != 1 else ''}: {argv}") - print_help() - exit(1) - - - first_print = True - term_size = os.get_terminal_size() - - for hash_key in saved_results: - result = saved_results[hash_key] - if models == None or result['model'] in models: - if seeds == None or str(result['seed']) in seeds: - if tests == None or result['test_name'] in tests: - if not first_print: print('-' * term_size.columns) - - print( - "\n" + - "\033[0;36mTest name:\033[0m " + - result['test_name'] + - "\n\033[0;36mModel:\033[0m " + - result['model'] + - "\n\033[0;36mSeed:\033[0m " + - str(result['seed']) + - "\n\033[0;36mValidation result:\033[0m " + - str(result['validation']) + - "\n\033[0;36mAnswer: »\033[0m" + - result['answer'] + - "\033[0;36m«\033[0m" + - "\n" - ) - - first_print = False - -if __name__ == "__main__": - main(argv=sys.argv) diff --git a/saved_results-bak.json b/saved_results-bak.json new file mode 100644 index 0000000..4b20e8a --- /dev/null +++ b/saved_results-bak.json @@ -0,0 +1,1651 @@ +{ + "0381CD21": { + "answer": "The product of 234,215 and 143,243 is 3,354,965,924.5.", + "model_id": 229, + "model_name": "Phi3 3.8b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 693, + "test_name": "Simple Multiplication", + "tool_calls": [ + { + "args": { + "a": 234215, + "b": 143243 + }, + "times_failed": 0, + "tool": "multiply" + } + ], + "validation": false + }, + "0D6A8004": { + "answer": "Could you please double-check the dates for Grandma's request?", + "model_id": 626, + "model_name": "Gemma2 27b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 260, + "test_name": "Notes from last Saturday TSO", + "tool_calls": [ + { + "args": { + "begin": "2023-08-26", + "to": "2023-08-27" + }, + "index": 0, + "tool": "get_notes_in_timespan" + } + ], + "validation": false + }, + "0DEB2030": { + "answer": ">>Model did not come to a conclusion (Recusion Error)<<", + "model_id": 704, + "model_name": "Yi 9b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 283, + "test_name": "Notes from last Saturday", + "tool_calls": [ + { + "args": { + "patterns": [ + "grandma want me to call" + ] + }, + "index": 0, + "tool": "get_notes_containing" + }, + { + "args": { + "patterns": [ + "grandma want me to call", + "last saturday" + ] + }, + "index": 1, + "tool": "get_notes_containing" + }, + { + "args": { + "patterns": [ + "grandma want me to call", + "last saturday" + ] + }, + "index": 2, + "tool": "get_notes_containing" + }, + { + "args": { + "patterns": [ + "grandma want me to call", + "last saturday" + ] + }, + "index": 3, + "tool": "get_notes_containing" + }, + { + "args": { + "patterns": [ + "last Saturday", + "grandma want me to call" + ] + }, + "index": 4, + "tool": "get_notes_containing" + } + ], + "validation": false + }, + "100971A0": { + "answer": "各种蔬菜都有自己的营养特点，但从健康的角度来看，以下几种蔬菜通常被认为是非常有益的：\n1. 菠菜 - 富含铁质、维生素K和纤维。\n2. 西兰花 - 富含维生素C和多种抗氧化剂。\n3. 番茄 - 富含番茄红素和维生素C。\n4. 黄瓜 - 低热量，高水分，有助于保持身体水合。\n5. 甜薯 - 富含纤维和维生素A。\n6. 胡萝卜 - 富含β-胡罗卜素和维生素K。\n7. 花椰菜 - 富含维生素C和抗癌化合物。\n\n记得食物多样性是健康饮食的关键，因此应该尝试食用各种蔬菜以获取不同的营养素。", + "model_id": 724, + "model_name": "Yi 34b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 607, + "test_name": "Healthy Vegetables in Chinese", + "validation": true + }, + "11342E8A": { + "answer": "全球范围内的最健康食物通常是新鲜、有色的各种蔬菜。例如，西班牙海蘇是螿黄素肥而丰富的蔬菜，非洲可乐椒也为我们提供了天然的电解能和粮食保护作用。美国的夏娃根是高酶乌龙，富含无麸质。即使是我这个AI助手，也应当认为食品质量和多样性都非常重要。", + "model_id": 229, + "model_name": "Phi3 3.8b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 607, + "test_name": "Healthy Vegetables in Chinese", + "validation": true + }, + "1180F980": { + "answer": "各种蔬菜都有其独特的营养价值，但一般来说，深绿色蔬菜（如菠菜、羽衣甘蓝等）和浆果类蔬菜（如甜豌豆、南瓜等）含有较高的维生素 A 和 C。此外，任何种类的新鲜蔬菜都是健康的选择，因为它们富含纤维、抗氧化剂和其他对健康有益的化合物。所以可以说，没有哪种蔬菜是最健康的，而是应该均衡摄取各种不同的蔬菜来获取更全面的健康益处。", + "model_id": 704, + "model_name": "Yi 9b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 607, + "test_name": "Healthy Vegetables in Chinese", + "validation": true + }, + "1789EA85": { + "answer": "Grandma likely wanted you to call someone for a specific reason! Do you recall what that might have been?", + "model_id": 626, + "model_name": "Gemma2 27b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 283, + "test_name": "Notes from last Saturday", + "tool_calls": [ + { + "args": { + "patterns": [ + "Grandma called" + ] + }, + "index": 0, + "tool": "get_notes_containing" + } + ], + "validation": false + }, + "17A83FF1": { + "answer": ">>Model did not come to a conclusion (Recusion Error)<<", + "model_id": 704, + "model_name": "Yi 9b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 260, + "test_name": "Notes from last Saturday TSO", + "tool_calls": [ + { + "args": { + "begin": "2019/04/13", + "to": "2019/04/15" + }, + "index": 0, + "tool": "get_notes_in_timespan" + }, + { + "args": { + "begin": "2019/04/13", + "to": "2019/04/15" + }, + "index": 1, + "tool": "get_notes_in_timespan" + }, + { + "args": { + "begin": "2019-04-13", + "to": "2019-04-15" + }, + "index": 2, + "tool": "get_notes_in_timespan" + }, + { + "args": { + "begin": "2019-04-13", + "to": "2019-04-15" + }, + "index": 3, + "tool": "get_notes_in_timespan" + }, + { + "args": { + "begin": "2019-04-13", + "to": "2019-04-15" + }, + "index": 4, + "tool": "get_notes_in_timespan" + } + ], + "validation": false + }, + "18DEEF2A": { + "answer": "I apologize for the confusion earlier. Let me try again. Who did grandma want you to call on Saturday?", + "model_id": 724, + "model_name": "Yi 34b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 283, + "test_name": "Notes from last Saturday", + "tool_calls": [ + { + "args": { + "patterns": [ + "Saturday(?i).*grandma", + "call" + ] + }, + "index": 0, + "tool": "get_notes_containing" + } + ], + "validation": false + }, + "1AC5BD63": { + "answer": ">>Model failed<<", + "model_id": 853, + "model_name": "Qwen2 0.5b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 607, + "test_name": "Healthy Vegetables in Chinese", + "validation": false + }, + "20DFA39A": { + "answer": "I remember Grandma mentioning that she wanted me to call Aunt Sarah last Saturday. She was feeling homesick and missed our family gatherings.", + "model_id": 229, + "model_name": "Phi3 3.8b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 856, + "test_name": "Notes from last Saturday TSO FSP", + "tool_calls": [ + { + "args": {}, + "index": 0, + "tool": "get_current_date_and_time" + } + ], + "validation": false + }, + "224DA2FB": { + "answer": "On Thursday, grandma wanted you to call someone to pick up the dry cleaning. They close early on Fridays, so make sure to get it before then!", + "model_id": 724, + "model_name": "Yi 34b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 260, + "test_name": "Notes from last Saturday TSO", + "tool_calls": [ + { + "args": {}, + "index": 0, + "tool": "get_current_date_and_time" + }, + { + "args": { + "begin": "2024/08/06", + "to": "2024/08/06" + }, + "index": 1, + "tool": "get_notes_in_timespan" + }, + { + "args": { + "begin": "2024/08/06", + "to": "2024/08/06" + }, + "index": 2, + "tool": "get_notes_in_timespan" + } + ], + "validation": false + }, + "24FF2149": { + "answer": "Oh no! I apologize, it seems like my programming needs a bit of help with date formatting for that request. Could you please try to rephrase your question?", + "model_id": 475, + "model_name": "Gemma2 9b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 856, + "test_name": "Notes from last Saturday TSO FSP", + "tool_calls": [ + { + "args": { + "begin": "2024-08-04", + "to": "2024-08-04" + }, + "index": 0, + "tool": "get_notes_in_timespan" + } + ], + "validation": false + }, + "2C175709": { + "answer": "蔬菜有很多种，比如菠菜、胡萝卜、芹菜等都是健康蔬菜。", + "model_id": 853, + "model_name": "Qwen2 0.5b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 607, + "test_name": "Healthy Vegetables in Chinese", + "validation": true + }, + "2C4F8792": { + "answer": ">>Model did not come to a conclusion (Recusion Error)<<", + "model_id": 704, + "model_name": "Yi 9b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 120, + "test_name": "Complex Multiplication", + "tool_calls": [ + { + "args": { + "a": 235, + "b": 1243 + }, + "index": 0, + "tool": "multiply" + }, + { + "args": { + "a": 292105.0, + "b": 21 + }, + "index": 1, + "tool": "multiply" + }, + { + "args": { + "a": 6134205.0, + "b": 21 + }, + "index": 2, + "tool": "multiply" + }, + { + "args": { + "a": 128818305.0, + "b": 1 + }, + "index": 3, + "tool": "multiply" + }, + { + "args": { + "a": 128818305.0, + "b": 1.0 + }, + "index": 4, + "tool": "multiply" + } + ], + "validation": false + }, + "2EDBC730": { + "answer": "全球范围内的一些营养丰富的蔬菜包括西红色甜菜、黄瓜和冷却后的花生。它们都富含钾，维生素C，镁等益物，并且肥力大.", + "model_id": 229, + "model_name": "Phi3 3.8b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 607, + "test_name": "Healthy Vegetables in Chinese", + "validation": true + }, + "2F04B7E1": { + "answer": ">>Model failed<<", + "model_id": 229, + "model_name": "Phi3 3.8b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 260, + "test_name": "Notes from last Saturday TSO", + "tool_calls": [], + "validation": false + }, + "312BEDBF": { + "answer": "I am sorry, I do not have access to the notes. Can you provide me with more context on what notes you are referring to?", + "model_id": 626, + "model_name": "Gemma2 27b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 283, + "test_name": "Notes from last Saturday", + "tool_calls": [ + { + "args": { + "patterns": [ + "grandma" + ] + }, + "index": 0, + "tool": "get_notes_containing" + } + ], + "validation": false + }, + "388DFAEA": { + "answer": "很多蔬菜都含有丰富的维生素和矿物质，对健康很有好处。但是要说哪一种蔬菜最健康，很难有一个绝对的答案，因为这取决于你的整体饮食和生活方式。不过，一般来说，深绿色、黄色的蔬菜如菠菜、花椰菜和胡萝卜等都比较受推崇，因为它们富含营养物质并且有助于增强免疫力。", + "model_id": 704, + "model_name": "Yi 9b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 607, + "test_name": "Healthy Vegetables in Chinese", + "validation": true + }, + "39629BC0": { + "answer": "I'm sorry, but I can't recall any specifics about your interactions. However, if you have a notebook or record of such events it might be helpful to revisit that.", + "model_id": 229, + "model_name": "Phi3 3.8b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 283, + "test_name": "Notes from last Saturday", + "tool_calls": [], + "validation": false + }, + "3B563F6E": { + "answer": "I do not have access to personal information like past conversations with your grandma.", + "model_id": 626, + "model_name": "Gemma2 27b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 260, + "test_name": "Notes from last Saturday TSO", + "tool_calls": [], + "validation": false + }, + "3CC0DBE8": { + "answer": "That's quite a large number! ", + "model_id": 475, + "model_name": "Gemma2 9b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 693, + "test_name": "Simple Multiplication", + "tool_calls": [ + { + "args": { + "a": 234215, + "b": 143243 + }, + "times_failed": 0, + "tool": "multiply" + } + ], + "validation": false + }, + "4012D925": { + "answer": "Oh no! I apologize, it seems like my programming needs a bit of help with date formatting for that request. Could you please try to rephrase your question?", + "model_id": 626, + "model_name": "Gemma2 27b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 856, + "test_name": "Notes from last Saturday TSO FSP", + "tool_calls": [ + { + "args": { + "begin": "2024-08-04", + "to": "2024-08-04" + }, + "index": 0, + "tool": "get_notes_in_timespan" + } + ], + "validation": false + }, + "440796EA": { + "answer": "Based on your notes from last Saturday, Grandma wanted you to call Uncle George regarding the birthday card and also suggested that you consider signing up for a local yoga class.", + "model_id": 724, + "model_name": "Yi 34b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 856, + "test_name": "Notes from last Saturday TSO FSP", + "tool_calls": [ + { + "args": { + "begin": "2024/07/29", + "to": "2024/07/30" + }, + "index": 0, + "tool": "get_notes_in_timespan" + } + ], + "validation": false + }, + "48A4CBEB": { + "answer": "I'm sorry, but I can only provide factual information and answers. If you have a question about events or schedules from last Saturday that we could find records of in the system database, let me know!", + "model_id": 229, + "model_name": "Phi3 3.8b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 260, + "test_name": "Notes from last Saturday TSO", + "tool_calls": [], + "validation": false + }, + "4DD53B70": { + "answer": "Last Saturday, Grandma wanted me to call Dr. Miller at 9 am.", + "model_id": 704, + "model_name": "Yi 9b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 856, + "test_name": "Notes from last Saturday TSO FSP", + "tool_calls": [ + { + "args": {}, + "index": 0, + "tool": "get_current_date_and_time" + }, + { + "args": { + "content": "Last Saturday, Grandma wanted me to call Dr. Miller at 9 am." + }, + "index": 1, + "tool": "write_note" + } + ], + "validation": false + }, + "53321A3A": { + "answer": "I'm sorry I couldn't help you find your grandma last Saturday, but it seems like there might have been a misunderstanding or perhaps she was too busy. Let me know if there's anything specific about her memories that we could look into.", + "model_id": 229, + "model_name": "Phi3 3.8b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 283, + "test_name": "Notes from last Saturday", + "tool_calls": [ + { + "args": { + "patterns": [ + "grandmother" + ] + }, + "index": 0, + "tool": "get_notes_containing" + }, + { + "args": { + "content": "I'm sorry I couldn't help you find your grandma last Saturday, but it seems like there might have been a misunderstanding or perhaps she was too busy. Let me know if there's anything specific about her memories that we could look into." + }, + "index": 1, + "tool": "write_note" + } + ], + "validation": false + }, + "57DFC15F": { + "answer": "It's awesome you're thinking about healthy eating! There isn't one 'healthiest' vegetable, as they all bring unique benefits. For example: \n\n* **Leafy greens** like spinach and kale are packed with vitamins A & C and fiber.\n* **Cruciferous veggies** such as broccoli and cauliflower are known for their cancer-fighting properties.\n* **Beets and carrots** provide potassium and vitamin K, boosting your heart health and bone strength.\n\nWhat kind of vegetables are you interested in trying more of? ", + "model_id": 626, + "model_name": "Gemma2 27b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 607, + "test_name": "Healthy Vegetables in Chinese", + "validation": false + }, + "5C8EF609": { + "answer": "It's tough to say definitively which vegetable is *the* healthiest, as they all bring unique benefits! 🥦🥬🥕 But, some standouts include leafy greens like spinach and kale for their vitamin boost, broccoli for its fiber and vitamin C, and carrots for their vibrant color and beta-carotene. What kind of vegetables do you enjoy eating?", + "model_id": 475, + "model_name": "Gemma2 9b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 607, + "test_name": "Healthy Vegetables in Chinese", + "validation": false + }, + "62ED480A": { + "answer": ">>Model failed<<", + "model_id": 704, + "model_name": "Yi 9b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 856, + "test_name": "Notes from last Saturday TSO FSP", + "tool_calls": [ + { + "args": { + "begin": "2024/08/05", + "to": "2024/08/06" + }, + "index": 0, + "tool": "get_notes_in_timespan" + }, + { + "args": { + "begin": "2024/08/01", + "to": "2024/08/05" + }, + "index": 1, + "tool": "get_notes_in_timespan" + } + ], + "validation": false + }, + "63846F62": { + "answer": "That's a big number! Let me know if you want to try something else.", + "model_id": 475, + "model_name": "Gemma2 9b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 693, + "test_name": "Simple Multiplication", + "tool_calls": [ + { + "args": { + "a": 234215, + "b": 143243 + }, + "times_failed": 0, + "tool": "multiply" + } + ], + "validation": false + }, + "63F0FD5B": { + "answer": "The product of 234215 and 143243 is 33549659245.", + "model_id": 704, + "model_name": "Yi 9b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 693, + "test_name": "Simple Multiplication", + "tool_calls": [ + { + "args": { + "a": 234215, + "b": 143243 + }, + "times_failed": 0, + "tool": "multiply" + } + ], + "validation": true + }, + "6B7D8418": { + "answer": "The product you're looking for is quite large! I found that multiplying 235 by 1243 first gives us a pretty big number, and then when we multiply it with 21, the result becomes even more substantial. The exact calculation would be taking this initial massive figure from our multiplication of 235 and 1243, and afterwards multiplying that result by 21 to get your final answer.", + "model_id": 229, + "model_name": "Phi3 3.8b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 120, + "test_name": "Complex Multiplication", + "tool_calls": [ + { + "args": { + "a": 235, + "b": 1243 + }, + "index": 0, + "tool": "multiply" + }, + { + "args": { + "a": 292105, + "b": 21 + }, + "index": 1, + "tool": "multiply" + } + ], + "validation": false + }, + "6E623449": { + "answer": "Error: Invalid input. Date format is %Y/%m/%d", + "model_id": 853, + "model_name": "Qwen2 0.5b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 856, + "test_name": "Notes from last Saturday TSO FSP", + "tool_calls": [ + { + "args": {}, + "index": 0, + "tool": "get_current_date_and_time" + }, + { + "args": { + "begin": "Saturday the 7th of August 2024 19:59", + "to": "Monday the 8th of August 2024 20:45" + }, + "index": 1, + "tool": "get_notes_in_timespan" + } + ], + "validation": false + }, + "7981570B": { + "answer": "That's a pretty big number! What else can I help you with?", + "model_id": 475, + "model_name": "Gemma2 9b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 120, + "test_name": "Complex Multiplication", + "tool_calls": [ + { + "args": { + "a": 235, + "b": 1243, + "c": 21 + }, + "index": 0, + "tool": "multiply" + } + ], + "validation": false + }, + "7BF1D261": { + "answer": "That is the product of 235 times 1243 times 21.", + "model_id": 475, + "model_name": "Gemma2 9b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 120, + "test_name": "Complex Multiplication", + "tool_calls": [ + { + "args": { + "a": 235, + "b": 1243, + "c": 21 + }, + "index": 0, + "tool": "multiply" + } + ], + "validation": false + }, + "80B0EE0B": { + "answer": ">>LLM did not respond conversationally<<", + "model_id": 724, + "model_name": "Yi 34b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 693, + "test_name": "Simple Multiplication", + "tool_calls": [ + { + "args": { + "a": 234215, + "b": 143243 + }, + "times_failed": 0, + "tool": "multiply" + }, + { + "args": { + "a": 10, + "b": 20 + }, + "tool": "add" + } + ], + "validation": false + }, + "8235F5CF": { + "answer": "Grandma wanted you to call her niece, Lisa, on Saturday. ", + "model_id": 626, + "model_name": "Gemma2 27b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 856, + "test_name": "Notes from last Saturday TSO FSP", + "tool_calls": [], + "validation": false + }, + "87024585": { + "answer": "It's tricky to say definitively which vegetable is *the* healthiest, as they all bring unique benefits! 🥦🥕🥬 But some consistently rank high: leafy greens like spinach and kale are packed with vitamins, broccoli offers fiber and antioxidants, and sweet potatoes give you a boost of vitamin A. 😊 What kind of veggies do you enjoy? I might have some recipe ideas for you!", + "model_id": 626, + "model_name": "Gemma2 27b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 607, + "test_name": "Healthy Vegetables in Chinese", + "validation": false + }, + "88613663": { + "answer": "It's awesome you're thinking about healthy veggies! There isn't one 'best' vegetable, as they all offer unique nutrients. But some standouts include leafy greens like spinach and kale for vitamins, colorful peppers for antioxidants, or broccoli for its fiber content.", + "model_id": 475, + "model_name": "Gemma2 9b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 607, + "test_name": "Healthy Vegetables in Chinese", + "validation": false + }, + "92F08F91": { + "answer": "There doesn't seem to be any note from last Saturday regarding grandma wanting you to call someone.", + "model_id": 724, + "model_name": "Yi 34b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 260, + "test_name": "Notes from last Saturday TSO", + "tool_calls": [ + { + "args": { + "begin": "2022/08/27", + "to": "2022/08/27" + }, + "index": 0, + "tool": "get_notes_in_timespan" + } + ], + "validation": false + }, + "96477B91": { + "answer": "The result of multiplying 234215 by 143243 is 3,354,965,924.5.", + "model_id": 724, + "model_name": "Yi 34b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 693, + "test_name": "Simple Multiplication", + "tool_calls": [ + { + "args": { + "a": 234215, + "b": 143243 + }, + "times_failed": 0, + "tool": "multiply" + } + ], + "validation": false + }, + "98115D53": { + "answer": ">>Model did not come to a conclusion (Recusion Error)<<", + "model_id": 853, + "model_name": "Qwen2 0.5b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 856, + "test_name": "Notes from last Saturday TSO FSP", + "tool_calls": [ + { + "args": { + "content": "Happy anniversary Charlotte! Here are some notes for you." + }, + "index": 0, + "tool": "write_note" + }, + { + "args": { + "content": "Happy anniversary Charlotte! Here are some notes for you." + }, + "index": 1, + "tool": "write_note" + }, + { + "args": { + "content": "Happy anniversary Charlotte! Here are some notes for you." + }, + "index": 2, + "tool": "write_note" + }, + { + "args": { + "content": "Happy anniversary Charlotte! Here are some notes for you." + }, + "index": 3, + "tool": "write_note" + }, + { + "args": { + "content": "Happy anniversary Charlotte! Here are some notes for you." + }, + "index": 4, + "tool": "write_note" + } + ], + "validation": false + }, + "98188C83": { + "answer": "I am sorry, I do not have access to the notes. Can you provide me with more context on what notes you are referring to?", + "model_id": 475, + "model_name": "Gemma2 9b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 283, + "test_name": "Notes from last Saturday", + "tool_calls": [ + { + "args": { + "patterns": [ + "grandma" + ] + }, + "index": 0, + "tool": "get_notes_containing" + } + ], + "validation": false + }, + "9960F531": { + "answer": "The result of 234215 times 143243 is 33549659245.0. If you have any other questions or need further assistance, feel free to ask!", + "model_id": 704, + "model_name": "Yi 9b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 693, + "test_name": "Simple Multiplication", + "tool_calls": [ + { + "args": { + "a": 234215, + "b": 143243 + }, + "times_failed": 0, + "tool": "multiply" + } + ], + "validation": true + }, + "9D216390": { + "answer": "It seems there are no notes about grandma wanting you to call someone last Saturday. Please check with her again for any missed messages or instructions.", + "model_id": 724, + "model_name": "Yi 34b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 283, + "test_name": "Notes from last Saturday", + "tool_calls": [ + { + "args": { + "patterns": [ + "grandma saturday call" + ] + }, + "index": 0, + "tool": "get_notes_containing" + } + ], + "validation": false + }, + "A0E20580": { + "answer": "On Saturday, Grandma asked you to send a thank-you note to Mrs. Jenkins for the lovely dinner last weekend.", + "model_id": 724, + "model_name": "Yi 34b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 856, + "test_name": "Notes from last Saturday TSO FSP", + "tool_calls": [ + { + "args": { + "begin": "2024/08/01", + "to": "2024/08/02" + }, + "index": 0, + "tool": "get_notes_in_timespan" + } + ], + "validation": false + }, + "A1A11822": { + "answer": "I'm sorry, but it seems there has been an error in the calculation process.", + "model_id": 229, + "model_name": "Phi3 3.8b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 120, + "test_name": "Complex Multiplication", + "tool_calls": [ + { + "args": { + "a": 235, + "b": 1243 + }, + "index": 0, + "tool": "multiply" + }, + { + "args": { + "a": null, + "b": [ + 292105.0, + 21 + ] + }, + "index": 1, + "tool": "add" + }, + { + "args": { + "a": null, + "b": [ + 292105.0, + 21 + ] + }, + "index": 2, + "tool": "multiply" + }, + { + "args": { + "a": null, + "b": [ + 292105.0, + 21 + ] + }, + "index": 3, + "tool": "multiply" + } + ], + "validation": false + }, + "A2E943E2": { + "answer": "I do not have access to personal information like past conversations with your grandma.", + "model_id": 475, + "model_name": "Gemma2 9b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 260, + "test_name": "Notes from last Saturday TSO", + "tool_calls": [], + "validation": false + }, + "A43A7AE8": { + "answer": "The sum of 234215 times 143243 is 377458.", + "model_id": 853, + "model_name": "Qwen2 0.5b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 693, + "test_name": "Simple Multiplication", + "tool_calls": [ + { + "args": { + "a": 234215, + "b": 143243 + }, + "times_failed": 0, + "tool": "add" + } + ], + "validation": false + }, + "A492D5A7": { + "answer": "That's quite a big number! Want to see what happens when we multiply that by a different number?", + "model_id": 626, + "model_name": "Gemma2 27b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 120, + "test_name": "Complex Multiplication", + "tool_calls": [ + { + "args": { + "a": 235, + "b": 1243, + "c": 21 + }, + "index": 0, + "tool": "multiply" + } + ], + "validation": false + }, + "A8FEB462": { + "answer": ">>LLM did not respond conversationally<<", + "model_id": 853, + "model_name": "Qwen2 0.5b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 693, + "test_name": "Simple Multiplication", + "tool_calls": [ + { + "args": { + "a": 234215, + "b": 143243 + }, + "times_failed": 0, + "tool": "add" + }, + { + "args": { + "a": 234215, + "b": 143243 + }, + "tool": "multiply" + } + ], + "validation": false + }, + "B094EC53": { + "answer": "Grandma probably wanted you to call someone about something last Saturday. Who is she thinking of?", + "model_id": 475, + "model_name": "Gemma2 9b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 856, + "test_name": "Notes from last Saturday TSO FSP", + "tool_calls": [ + { + "args": {}, + "index": 0, + "tool": "get_current_date_and_time" + } + ], + "validation": false + }, + "B09DCB84": { + "answer": "That's an impressive number! You could say that result is quite large.", + "model_id": 626, + "model_name": "Gemma2 27b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 693, + "test_name": "Simple Multiplication", + "tool_calls": [ + { + "args": { + "a": 234215, + "b": 143243 + }, + "times_failed": 0, + "tool": "multiply" + } + ], + "validation": false + }, + "B3895FB2": { + "answer": "The result of 235 times 1243 times 21 is 6134205.", + "model_id": 724, + "model_name": "Yi 34b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 120, + "test_name": "Complex Multiplication", + "tool_calls": [ + { + "args": { + "a": 235, + "b": 1243 + }, + "index": 0, + "tool": "multiply" + }, + { + "args": { + "a": 292105, + "b": 21 + }, + "index": 1, + "tool": "multiply" + } + ], + "validation": true + }, + "BC229AE2": { + "answer": "Grandma wanted me to call her on Thursday the 8th of August 2024 at 18:03", + "model_id": 853, + "model_name": "Qwen2 0.5b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 260, + "test_name": "Notes from last Saturday TSO", + "tool_calls": [ + { + "args": { + "begin": "2017/10/31", + "to": "2018/10/31" + }, + "index": 0, + "tool": "get_notes_in_timespan" + }, + { + "args": {}, + "index": 1, + "tool": "get_current_date_and_time" + } + ], + "validation": false + }, + "BE20B7DD": { + "answer": "That is the product of 235 times 1243 times 21.", + "model_id": 626, + "model_name": "Gemma2 27b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 120, + "test_name": "Complex Multiplication", + "tool_calls": [ + { + "args": { + "a": 235, + "b": 1243, + "c": 21 + }, + "index": 0, + "tool": "multiply" + } + ], + "validation": false + }, + "C0BAAC78": { + "answer": "The result of multiplying 235 by 1243 by 21 is 6134205.", + "model_id": 704, + "model_name": "Yi 9b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 120, + "test_name": "Complex Multiplication", + "tool_calls": [ + { + "args": { + "a": 235, + "b": 1243 + }, + "index": 0, + "tool": "multiply" + }, + { + "args": { + "a": 292105.0, + "b": 21 + }, + "index": 1, + "tool": "multiply" + } + ], + "validation": true + }, + "C588983E": { + "answer": "That's a big number! Let me know if you want to try something else.", + "model_id": 626, + "model_name": "Gemma2 27b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 693, + "test_name": "Simple Multiplication", + "tool_calls": [ + { + "args": { + "a": 234215, + "b": 143243 + }, + "times_failed": 0, + "tool": "multiply" + } + ], + "validation": false + }, + "C5D2EB83": { + "answer": "Grandma wanted me to call her on Thursday the 8th of August, 2024 at 18:03.", + "model_id": 853, + "model_name": "Qwen2 0.5b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 283, + "test_name": "Notes from last Saturday", + "tool_calls": [ + { + "args": { + "begin": "2018/10/31", + "to": "2019/04/15" + }, + "index": 0, + "tool": "get_notes_in_timespan" + }, + { + "args": {}, + "index": 1, + "tool": "get_current_date_and_time" + } + ], + "validation": false + }, + "CDCD4AE9": { + "answer": "Grandma likely wanted you to call someone specific, but I don't have her notes on hand. Could you remind me who you need to call?", + "model_id": 475, + "model_name": "Gemma2 9b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 283, + "test_name": "Notes from last Saturday", + "tool_calls": [ + { + "args": { + "patterns": [ + "Grandma called" + ] + }, + "index": 0, + "tool": "get_notes_containing" + } + ], + "validation": false + }, + "D37C735A": { + "answer": "Based on the information I have now, it seems there is no note about grandma wanting you to call last Saturday. Could you double-check or provide more context?", + "model_id": 704, + "model_name": "Yi 9b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 283, + "test_name": "Notes from last Saturday", + "tool_calls": [ + { + "args": { + "patterns": [ + "grandma want me to call" + ] + }, + "index": 0, + "tool": "get_notes_containing" + } + ], + "validation": false + }, + "D5A11058": { + "answer": "Could you tell me what date you are thinking of?", + "model_id": 475, + "model_name": "Gemma2 9b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 260, + "test_name": "Notes from last Saturday TSO", + "tool_calls": [ + { + "args": { + "begin": "2023-08-26", + "to": "2023-08-27" + }, + "index": 0, + "tool": "get_notes_in_timespan" + } + ], + "validation": false + }, + "D8448C0B": { + "answer": ">>Model failed<<", + "model_id": 853, + "model_name": "Qwen2 0.5b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 120, + "test_name": "Complex Multiplication", + "tool_calls": [ + { + "args": { + "a": 235, + "b": 1243 + }, + "index": 0, + "tool": "add" + }, + { + "args": { + "a": 235, + "b": 1243 + }, + "index": 1, + "tool": "add" + } + ], + "validation": false + }, + "D9FE1561": { + "answer": "Error: Failed to parse response.", + "model_id": 853, + "model_name": "Qwen2 0.5b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 120, + "test_name": "Complex Multiplication", + "tool_calls": [ + { + "args": { + "a": 235, + "b": 1243, + "c": 21 + }, + "index": 0, + "tool": "add" + }, + { + "args": { + "a": 1243, + "b": 21 + }, + "index": 1, + "tool": "multiply" + } + ], + "validation": false + }, + "DD8B4308": { + "answer": ">>Model did not come to a conclusion (Recusion Error)<<", + "model_id": 724, + "model_name": "Yi 34b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 120, + "test_name": "Complex Multiplication", + "tool_calls": [ + { + "args": { + "a": 235, + "b": 1243 + }, + "index": 0, + "tool": "multiply" + }, + { + "args": { + "a": 292105, + "b": 21 + }, + "index": 1, + "tool": "multiply" + }, + { + "args": { + "a": 6134205, + "b": 21 + }, + "index": 2, + "tool": "multiply" + }, + { + "args": { + "a": 6134205, + "b": 21 + }, + "index": 3, + "tool": "multiply" + }, + { + "args": { + "a": 128818305, + "b": 21 + }, + "index": 4, + "tool": "multiply" + } + ], + "validation": false + }, + "DD96E74B": { + "answer": "It seems like there are no notes from Grandma about calling anyone on last Saturday. Do you have another question?", + "model_id": 704, + "model_name": "Yi 9b", + "seed": 2, + "technique_id": 572, + "technique_name": "T2S", + "test_id": 260, + "test_name": "Notes from last Saturday TSO", + "tool_calls": [ + { + "args": { + "begin": "2021/08/27", + "to": "2021/08/29" + }, + "index": 0, + "tool": "get_notes_in_timespan" + } + ], + "validation": false + }, + "ECF1749A": { + "answer": "各种蔬菜都有自己的营养特点，但从健康的角度来看，有些蔬菜因其营养价值和益处而备受推崇。例如：菠菜、羽衣甘蓝、西兰花、甜薯、西红柿等都含有丰富的维生素、矿物质和抗氧化物质。然而，最重要的是饮食多样化，均衡摄入多种蔬菜以获取不同的营养素。记得摄入足够的蔬菜同时也要注意品种的搭配哦！", + "model_id": 724, + "model_name": "Yi 34b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 607, + "test_name": "Healthy Vegetables in Chinese", + "validation": true + }, + "ED2DE7E9": { + "answer": "", + "model_id": 853, + "model_name": "Qwen2 0.5b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 283, + "test_name": "Notes from last Saturday", + "tool_calls": [ + { + "args": { + "begin": "2018/10/31", + "to": "2019/04/14" + }, + "index": 0, + "tool": "get_notes_in_timespan" + }, + { + "args": { + "content": "Hello! It seems like you've made a typo in your message. Please try writing it again." + }, + "index": 1, + "tool": "write_note" + }, + { + "args": { + "content": "Hello! It seems like you've made a typo in your message. Please try writing it again." + }, + "index": 2, + "tool": "write_note" + } + ], + "validation": false + }, + "EFE22F3B": { + "answer": "The product you're looking for is approximately 33549659245.", + "model_id": 229, + "model_name": "Phi3 3.8b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 693, + "test_name": "Simple Multiplication", + "tool_calls": [ + { + "args": { + "a": 234215, + "b": 143243 + }, + "times_failed": 0, + "tool": "multiply" + } + ], + "validation": true + }, + "F5CBA558": { + "answer": ">>Model failed<<", + "model_id": 853, + "model_name": "Qwen2 0.5b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 260, + "test_name": "Notes from last Saturday TSO", + "tool_calls": [ + { + "args": { + "begin": "", + "to": "" + }, + "index": 0, + "tool": "get_notes_in_timespan" + } + ], + "validation": false + }, + "FDE24DD0": { + "answer": "I'm sorry, but I don't have that information right now. Maybe you can check with Grandma directly? She might remember who she wanted to call.", + "model_id": 229, + "model_name": "Phi3 3.8b", + "seed": 2, + "technique_id": 903, + "technique_name": "LSM", + "test_id": 856, + "test_name": "Notes from last Saturday TSO FSP", + "tool_calls": [], + "validation": false + } +} \ No newline at end of file diff --git a/suite_settings/models.py b/suite_settings/models.py index 711059a..2a9b61b 100644 --- a/suite_settings/models.py +++ b/suite_settings/models.py @@ -1,90 +1,90 @@ from libs.classes import Model models = { - 245: Model( - display_name="llama3.1 8b", - identifier="llama3.1", - supports_tools=True, - parameter_count_in_b=8 - ), - 238: Model( - display_name="llama3.1 70b", - identifier="llama3.1:70b", - supports_tools=True, - parameter_count_in_b=70 - ), - 120: Model( - display_name="llama3 groq TU 8b", - identifier="llama3-groq-tool-use", - supports_tools=True, - parameter_count_in_b=8 - ), - 890: Model( - display_name="llama3 groq TU 70b", - identifier="llama3-groq-tool-use:70b", - supports_tools=True, - parameter_count_in_b=70 - ), - 348: Model( - display_name="Mixtral MoE 8x7b", - identifier="mixtral:8x7b", - supports_tools=False, - parameter_count_in_b=13, - ), - 789: Model( - display_name="Mixtral MoE 8x22b", - identifier="mixtral:8x22b", - supports_tools=True, - parameter_count_in_b=39 - ), - 445: Model( - display_name="Gemma2 2b", - identifier="gemma2:2b", - supports_tools=False, - parameter_count_in_b=2 - ), - 475: Model( - display_name="Gemma2 9b", - identifier="gemma2:2b", - supports_tools=False, - parameter_count_in_b=9 - ), - 626: Model( - display_name="Gemma2 27b", - identifier="gemma2:2b", - supports_tools=False, - parameter_count_in_b=27 - ), - 229: Model( - display_name="Phi3 3.8b", - identifier="phi3", - supports_tools=False, - parameter_count_in_b=3.8 - ), - 903: Model( - display_name="Tinyllama 1.1b", - identifier="tinyllama:1.1b", - supports_tools=False, - parameter_count_in_b=1.1 - ), - 670: Model( - display_name="Mistral Nemo 12b", - identifier="mistral-nemo:12b", - supports_tools=True, - parameter_count_in_b=12 - ), - 404: Model( - display_name="Command R+ 104b", - identifier="command-r-plus:104b", - supports_tools=True, - parameter_count_in_b=104 - ), - 701: Model( - display_name="Yi 6b", - identifier="yi:6b", - supports_tools=False, - parameter_count_in_b=6 - ), +# 245: Model( +# display_name="llama3.1 8b", +# identifier="llama3.1", +# supports_tools=True, +# parameter_count_in_b=8 +# ), +# 238: Model( +# display_name="llama3.1 70b", +# identifier="llama3.1:70b", +# supports_tools=True, +# parameter_count_in_b=70 +# ), +# 120: Model( +# display_name="llama3 groq TU 8b", +# identifier="llama3-groq-tool-use", +# supports_tools=True, +# parameter_count_in_b=8 +# ), +# 890: Model( +# display_name="llama3 groq TU 70b", +# identifier="llama3-groq-tool-use:70b", +# supports_tools=True, +# parameter_count_in_b=70 +# ), +# 348: Model( +# display_name="Mixtral MoE 8x7b", +# identifier="mixtral:8x7b", +# supports_tools=False, +# parameter_count_in_b=13, +# ), +# 789: Model( +# display_name="Mixtral MoE 8x22b", +# identifier="mixtral:8x22b", +# supports_tools=True, +# parameter_count_in_b=39 +# ), +# 445: Model( +# display_name="Gemma2 2b", +# identifier="gemma2:2b", +# supports_tools=False, +# parameter_count_in_b=2 +# ), +# 475: Model( +# display_name="Gemma2 9b", +# identifier="gemma2:2b", +# supports_tools=False, +# parameter_count_in_b=9 +# ), +# 626: Model( +# display_name="Gemma2 27b", +# identifier="gemma2:2b", +# supports_tools=False, +# parameter_count_in_b=27 +# ), +# 229: Model( +# display_name="Phi3 3.8b", +# identifier="phi3", +# supports_tools=False, +# parameter_count_in_b=3.8 +# ), +# 903: Model( +# display_name="Tinyllama 1.1b", +# identifier="tinyllama:1.1b", +# supports_tools=False, +# parameter_count_in_b=1.1 +# ), +# 670: Model( +# display_name="Mistral Nemo 12b", +# identifier="mistral-nemo:12b", +# supports_tools=True, +# parameter_count_in_b=12 +# ), +# 404: Model( +# display_name="Command R+ 104b", +# identifier="command-r-plus:104b", +# supports_tools=True, +# parameter_count_in_b=104 +# ), +# 701: Model( +# display_name="Yi 6b", +# identifier="yi:6b", +# supports_tools=False, +# parameter_count_in_b=6 +# ), 704: Model( display_name="Yi 9b", identifier="yi:9b", diff --git a/suite_settings/techniques.py b/suite_settings/techniques.py index 67d3cb7..b1f9e9d 100644 --- a/suite_settings/techniques.py +++ b/suite_settings/techniques.py @@ -1,19 +1,19 @@ from libs.classes import Technique techniques = { - 190: Technique( + 1: Technique( name="Native", for_supports_tools=True, for_not_supports_tools=False, ), 903: Technique( - name="Long System Message", + name="LSM", # Long System Message + for_supports_tools=False, + for_not_supports_tools=True, + ), + 572: Technique( + name="T2S", # Tool to System Messsages for_supports_tools=False, for_not_supports_tools=True, ), - # 572: Technique( - # name="Tool to System Messsages", - # for_supports_tools=False, - # for_not_supports_tools=True, - # ), } diff --git a/suite_settings/tests.py b/suite_settings/tests.py index 50bd7e0..41eb68a 100644 --- a/suite_settings/tests.py +++ b/suite_settings/tests.py @@ -1,160 +1,150 @@ -from libs.classes import Test -from libs.runnables import basic_prompt, one_tool_call_answer, agent_with_tools -from libs.validators import regex_match_any, system_human_answer_match -from libs.tools import add, multiply, get_current_date_and_time, get_notes_in_timespan, get_notes_containing, write_note -from textwrap import dedent +from libs.classes import Test +from libs.runnables import basic_prompt, one_tool_call_answer, agent_with_tools +from libs.validators import regex_match_any, system_human_answer_match +from libs.tools import add, multiply, get_current_date_and_time, get_notes_in_timespan, get_notes_containing, write_note +from textwrap import dedent from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage, AIMessage tests = { - 607: Test( - name="Healthy Vegetables in Chinese", - runnable=basic_prompt, - runnable_input={ - "system_msg": "You are a helpful assistant. You serve people across the globe.", - "human_msg": "什么蔬菜最健康？", - }, - validator=system_human_answer_match, - validation_input={ - "criteria": dedent("""- in Mandarin Chinese from front to finnish - - factually correct - - about healthy vegetables - - just one single conversational answer, without any AI fragments (A/B versions, "end of message" parts, unfitting discalimers or notes) - - Again, the message has to be entirely in Manadarin Chineese. - That means If the answer is not in Chinese the answer is NOT correct! Only if the message in in Chinese rate as correct"""), - } - ), - 693: Test( - name="Simple Multiplication", - runnable=one_tool_call_answer, - runnable_input={ - "system_msg": "You are a helpful assistant.", - "human_msg": "What is 234215 times 143243?", - "tools": { - "add": add, - "multiply": multiply - } - }, - validator=regex_match_any, - validation_input={ - "patterns": ["33549659245", "33,549,659,245", "33.549.659.245"] - # "patterns": ["3[,\. ]?3[,\. ]?5[,\. ]?4[,\. ]?9[,\. ]?6[,\. ]?5[,\. ]?9[,\. ]?2[,\. ]?4[,\. ]?5"] # Would accept 3.354.965.9245 - } - ), - 120: Test( - name="Complex Multiplication", - runnable=agent_with_tools, - runnable_input={ - "system_msg": 'You are a helpful assistant. You can use tools to accomplish the task. Once you\'ve called a tool. the resulting tool_message content can be taken into consideration again. With that you can do "multiple rounds" of tool calling.', - "human_msg": "What is 235 times 1243 times 21?", - "tools": { - "add": add, - "multiply": multiply - } - }, - validator=regex_match_any, - validation_input={ - "patterns": [ "6134205", "6.134.205", "6,134,205" ] - } - ), - 283: Test( - name="Notes from last Saturday", - runnable=agent_with_tools, - runnable_input={ - "system_msg": "You are a helpful assistant. You can use tools to accomplish tasks. Once you've called a tool, the resulting tool_message content can be taken into consideration again. With that you can do \"multiple rounds\" of tool calling. To know the date, use the tool get_current_date_and_time.", - "human_msg": "Last Saturday, who did grandma want me to call?", - "tools": { - "get_current_date_and_time": get_current_date_and_time, - "get_notes_in_timespan": get_notes_in_timespan, - "get_notes_containing": get_notes_containing, - "Write note": write_note, - } - }, - validator=system_human_answer_match, - validation_input={ - "criteria": dedent("""- containing the information that the Human should call Wolfgang - - just one single conversational answer, without any AI fragments (A/B versions, "end of message" parts, unfitting discalimers or notes, what specific tool was used to get the answer, etc.)""") - } - ), - 260: Test( - name="Notes from last Saturday TSO", # time span only - runnable=agent_with_tools, - runnable_input={ - "system_msg": "You are a helpful assistant. You can use tools to accomplish tasks. Once you've called a tool, the resulting tool_message content can be taken into consideration again. With that you can do \"multiple rounds\" of tool calling. To know the date, use the tool get_current_date_and_time.", - "human_msg": "Last Saturday, who did grandma want me to call?", - "tools": { - "get_current_date_and_time": get_current_date_and_time, - "get_notes_in_timespan": get_notes_in_timespan, - "Write note": write_note - } - }, - validator=system_human_answer_match, - validation_input={ - "criteria": dedent("""- containing the information that the Human should call Wolfgang - - just one single conversational answer, without any AI fragments (A/B versions, "end of message" parts, unfitting discalimers or notes, what specific tool was used to get the answer, etc.)""") - } - ), - 856: Test( - name="Notes from last Saturday TSO FSP", - runnable=agent_with_tools, - runnable_input={ - "system_msg": "You are a helpful assistant. You can use tools to accomplish tasks. Once you've called a tool, the resulting tool_message content can be taken into consideration again. With that you can do \"multiple rounds\" of tool calling. To know the date, use the tool get_current_date_and_time.", - "fsp_messages": [ - HumanMessage("Tomorrow is the anniversary! Any tips what I should by her?"), # One year ago - AIMessage("", tool_calls=[{"name": "get_current_date_and_time", "args": {}, "id": "11" }]), - ToolMessage("Wednesday the 31st of Juli 2024 09:31", tool_call_id="11" ), - AIMessage("", tool_calls=[{"name": "get_notes_in_timespan", "args": {"begin": "2023/08/01", "to": "2023/08/01"}, "id": "12"}], ), - ToolMessage("2023/08/01 23:10 Went out with Charlotte for our anniversary. Pizza at Cavalinos. She loved the rose necklace!", tool_call_id="12"), - AIMessage("I'm afraid I cannot be of great help, since I obviously know charlotte way less than you, but last year you two went out to Cavalinons and you got her a rose necklace as a present. And she liked it. So maybe a pair of earrings would be something she'd like?", name="example_assistant", ), - - HumanMessage("Did I write down anything yesterday or the day before that?"), - AIMessage("", tool_calls=[{"name": "get_current_date_and_time", "args": {}, "id": "21" }], ), - ToolMessage("Wednesday the 7th of August 2024 16:23", tool_call_id="21" ), - AIMessage("", tool_calls=[{"name": "get_notes_in_timespan", "args": {"begin": "2024/08/05", "to": "2024/08/06"}, "id": "22"}], ), - ToolMessage("2024/08/05 11:45 Ask Dr. Mills about the side effects of the new medication he got me.\n\n2024/08/06 18:30 Pick up the dry cleaning on Thursday; they close early on Fridays.", tool_call_id="22"), - AIMessage("Yes. I found two entries.\n- From yesterday stating that you wanted to pickup the dry cleaning on Thursday, because they close early on Fridays\n- From Monday a note saying that you want to ask Dr. Mills about the side effects of the new medication he got you.", name="example_assistant", ), - ], - "human_msg": "Last Saturday, who did grandma want me to call?", - "tools": { - "get_current_date_and_time": get_current_date_and_time, - "get_notes_in_timespan": get_notes_in_timespan, - "Write note": write_note - } - }, + 607: Test( + name="Healthy Vegetables in Chinese", + runnable=basic_prompt, + runnable_input={ + "system_msg": "You are a helpful assistant. You serve people across the globe.", + "human_msg": "什么蔬菜最健康？", + }, validator=system_human_answer_match, - validation_input={ - "criteria": dedent("""- containing the information that the Human should call Wolfgang - - just one single conversational answer, without any AI fragments (A/B versions, "end of message" parts, unfitting discalimers or notes, what specific tool was used to get the answer, etc.)""") - } - ), - - # 363: Test(), - # 600: Test(), - # 221: Test(), - # 985: Test(), - # 634: Test(), - # 927: Test(), - # 346: Test(), - # 995: Test(), - # 404: Test(), - # 299: Test(), - # 275: Test(), - # 852: Test(), - # 376: Test(), - # 263: Test(), - # 432: Test(), - # 270: Test(), - # 797: Test(), - # 340: Test(), - # 489: Test(), - # 786: Test(), - # 121: Test(), - # 971: Test(), - # 436: Test(), - # 147: Test(), - # 534: Test(), - # 190: Test(), - # 158: Test(), - # 191: Test(), -} + validation_input={ + "criteria": dedent("""- in Mandarin Chinese from front to finnish + - factually correct + - about healthy vegetables + - just one single conversational answer, without any AI fragments (A/B versions, "end of message" parts, unfitting discalimers or notes) + Again, the message has to be entirely in Manadarin Chineese. + That means If the answer is not in Chinese the answer is NOT correct! Only if the message in in Chinese rate as correct"""), + }, + ), + 693: Test( + name="Simple Multiplication", + runnable=one_tool_call_answer, + runnable_input={ + "system_msg": "You are a helpful assistant.", + "human_msg": "What is 234215 times 143243?", + "tools": {"add": add, "multiply": multiply}, + }, + validator=regex_match_any, + validation_input={ + "patterns": ["33549659245", "33,549,659,245", "33.549.659.245"] + # "patterns": ["3[,\. ]?3[,\. ]?5[,\. ]?4[,\. ]?9[,\. ]?6[,\. ]?5[,\. ]?9[,\. ]?2[,\. ]?4[,\. ]?5"] # Would accept 3.354.965.9245 + }, + ), + 120: Test( + name="Complex Multiplication", + runnable=agent_with_tools, + runnable_input={ + "system_msg": 'You are a helpful assistant. You can use tools to accomplish the task. Once you\'ve called a tool. the resulting tool_message content can be taken into consideration again. With that you can do "multiple rounds" of tool calling.', + "human_msg": "What is 235 times 1243 times 21?", + "tools": {"add": add, "multiply": multiply}, + }, + validator=regex_match_any, + validation_input={"patterns": ["6134205", "6.134.205", "6,134,205"]}, + ), + 283: Test( + name="Notes from last Saturday", + runnable=agent_with_tools, + runnable_input={ + "system_msg": 'You are a helpful assistant. You can use tools to accomplish tasks. Once you\'ve called a tool, the resulting tool_message content can be taken into consideration again. With that you can do "multiple rounds" of tool calling. To know the date, use the tool get_current_date_and_time.', + "human_msg": "Last Saturday, who did grandma want me to call?", + "tools": { + "get_current_date_and_time": get_current_date_and_time, + "get_notes_in_timespan": get_notes_in_timespan, + "get_notes_containing": get_notes_containing, + "Write note": write_note, + }, + }, + validator=system_human_answer_match, + validation_input={ + "criteria": dedent("""- containing the information that the Human should call Wolfgang + - just one single conversational answer, without any AI fragments (A/B versions, "end of message" parts, unfitting discalimers or notes, what specific tool was used to get the answer, etc.)""") + }, + ), + 260: Test( + name="Notes from last Saturday TSO", # time span only + runnable=agent_with_tools, + runnable_input={ + "system_msg": 'You are a helpful assistant. You can use tools to accomplish tasks. Once you\'ve called a tool, the resulting tool_message content can be taken into consideration again. With that you can do "multiple rounds" of tool calling. To know the date, use the tool get_current_date_and_time.', + "human_msg": "Last Saturday, who did grandma want me to call?", + "tools": { + "get_current_date_and_time": get_current_date_and_time, + "get_notes_in_timespan": get_notes_in_timespan, + "Write note": write_note, + }, + }, + validator=system_human_answer_match, + validation_input={ + "criteria": dedent("""- containing the information that the Human should call Wolfgang + - just one single conversational answer, without any AI fragments (A/B versions, "end of message" parts, unfitting discalimers or notes, what specific tool was used to get the answer, etc.)""") + }, + ), + 856: Test( + name="Notes from last Saturday TSO FSP", + runnable=agent_with_tools, + runnable_input={ + "system_msg": 'You are a helpful assistant. You can use tools to accomplish tasks. Once you\'ve called a tool, the resulting tool_message content can be taken into consideration again. With that you can do "multiple rounds" of tool calling. To know the date, use the tool get_current_date_and_time.', + "fsp_messages": [ + HumanMessage("Tomorrow is the anniversary! Any tips what I should by her?"), # One year ago + AIMessage("", tool_calls=[{"name": "get_current_date_and_time", "args": {}, "id": "11"}]), + ToolMessage("Wednesday the 31st of Juli 2024 09:31", tool_call_id="11"), + AIMessage("", tool_calls=[ { "name": "get_notes_in_timespan", "args": {"begin": "2023/08/01", "to": "2023/08/01"}, "id": "12", } ]), + ToolMessage("2023/08/01 23:10 Went out with Charlotte for our anniversary. Pizza at Cavalinos. She loved the rose necklace!", tool_call_id="12"), + AIMessage("I'm afraid I cannot be of great help, since I obviously know charlotte way less than you, but last year you two went out to Cavalinons and you got her a rose necklace as a present. And she liked it. So maybe a pair of earrings would be something she'd like?", name="example_assistant"), + + HumanMessage("Did I write down anything yesterday or the day before that?"), + AIMessage( "", tool_calls=[{"name": "get_current_date_and_time", "args": {}, "id": "21"}]), + ToolMessage("Wednesday the 7th of August 2024 16:23", tool_call_id="21"), + AIMessage( "", tool_calls=[ { "name": "get_notes_in_timespan", "args": {"begin": "2024/08/05", "to": "2024/08/06"}, "id": "22"}]), + ToolMessage( "2024/08/05 11:45 Ask Dr. Mills about the side effects of the new medication he got me.\n\n2024/08/06 18:30 Pick up the dry cleaning on Thursday; they close early on Fridays.", tool_call_id="22"), + AIMessage( "Yes. I found two entries.\n- From yesterday stating that you wanted to pickup the dry cleaning on Thursday, because they close early on Fridays\n- From Monday a note saying that you want to ask Dr. Mills about the side effects of the new medication he got you.", name="example_assistant"), + ], + "human_msg": "Last Saturday, who did grandma want me to call?", + "tools": { + "get_current_date_and_time": get_current_date_and_time, + "get_notes_in_timespan": get_notes_in_timespan, + "Write note": write_note, + }, + }, + validator=system_human_answer_match, + validation_input={ + "criteria": dedent("""- containing the information that the Human should call Wolfgang + - just one single conversational answer, without any AI fragments (A/B versions, "end of message" parts, unfitting discalimers or notes, what specific tool was used to get the answer, etc.)""") + }, + ), + # 363: Test(), + # 600: Test(), + # 221: Test(), + # 985: Test(), + # 634: Test(), + # 927: Test(), + # 346: Test(), + # 995: Test(), + # 404: Test(), + # 299: Test(), + # 275: Test(), + # 852: Test(), + # 376: Test(), + # 263: Test(), + # 432: Test(), + # 270: Test(), + # 797: Test(), + # 340: Test(), + # 489: Test(), + # 786: Test(), + # 121: Test(), + # 971: Test(), + # 436: Test(), + # 147: Test(), + # 534: Test(), + # 190: Test(), + # 158: Test(), + # 191: Test(), +}