test/gate-source-under-test.sh

1#!/usr/bin/env bash

2#

3# gate-source-under-test.sh — run this repo's tests against THIS REPO'S SOURCE.

4#

5# WHY THIS EXISTS. The `sigil` binary embeds a copy of `(sigil log)`, and that

6# copy wins over the project's own source, over its built .sgb, and over an

7# explicit from-path redirect. So plain `sigil test` in this repo is green

8# about the binary's bundled copy and never reads src/sigil/log.sgl at all —

9# demonstrated by appending a syntax error to that file and watching the suite

10# produce an identical result. See the folio note

11# `topics/sigil-cli-bundle-shadows-extracted-package-modules`.

12#

13# HOW IT GETS AROUND IT. The one thing the binary cannot shadow is a module

14# name it does not embed. This gate generates a copy of the library with its

15# `define-library` head renamed to `(loguut log)`, drops it in a throwaway

16# project under /tmp, rewrites each test file's import to match, and runs the

17# suite there.

18#

19# WHAT MAKES THE COPY TRUSTWORTHY. Two things, and the gate fails SETUP if

20# either does not hold:

21# * the copy is generated from src/sigil/log.sgl on every run, never committed;

22# * the diff between original and copy is asserted to be EXACTLY the renamed

23# line, so the gate cannot quietly be testing something else.

24#

25# Outcomes, which are four and not two:

26# GREEN every arm passed against this repo's source

27# RED the source is under test and an arm failed

28# SETUP-FAILED the source was never put under test; says nothing about it

29# TIMED-OUT bounded wait elapsed; says which checks did not run

30#

31# Nothing is written inside the repo: all scratch lives under /tmp.

33set -u

35REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"

36SRC="$REPO/src/sigil/log.sgl"

37UUT_NAME="loguut"

38SIGIL="${SIGIL:-sigil}"

39TIMEOUT_SECS="${GATE_TIMEOUT_SECS:-300}"

40WORK="$(mktemp -d /tmp/sigil-log-uut.XXXXXX)"

42setup_failed() {

43 echo

44 echo "SETUP-FAILED: $*"

45 echo " The library was NOT put under test. This result says nothing about"

46 echo " whether the source is correct — do not read it as a pass or a fail."

47 echo " Scratch left for inspection: $WORK"

48 exit 70

49}

51echo "=== gate-source-under-test ==="

52echo "repo: $REPO"

53echo "source: $SRC"

54[ -f "$SRC" ] || setup_failed "no library source at $SRC"

55echo "src-sha: $(sha256sum "$SRC" | cut -d' ' -f1)"

56echo "sigil: $(command -v "$SIGIL") -> $("$SIGIL" --version 2>&1 | tr -d '\033' | sed 's/\[[0-9]*m//g')"

57echo "scratch: $WORK"

58echo

60# --- Establish the shadowing this gate exists to route around -----------------

61# A presence probe whose healthy answer is POSITIVE: the bundled copy must be

62# reachable with no project at all. If it is not, the premise has changed and

63# the rename is unnecessary — which the reader needs told, loudly.

64mkdir -p "$WORK/shadowcheck/test"

65cat > "$WORK/shadowcheck/test/test-s.sgl" <<'EOF'

66(import (sigil test) (sigil log))

67(test-group "g" (test "bundled (sigil log) resolves with no project" (assert-true #t)))

68(run-tests)

69EOF

70( cd "$WORK/shadowcheck" && "$SIGIL" test --sgl >/dev/null 2>&1 )

71if [ $? -ne 0 ]; then

72 echo "NOTE: (sigil log) did NOT resolve in a project with no dependencies."

73 echo " The binary may no longer embed it. This gate still works, but the"

74 echo " plain 'sigil test' in this repo may now be meaningful on its own."

75 echo " Re-read topics/sigil-cli-bundle-shadows-extracted-package-modules."

76 echo

77fi

79# --- Generate the renamed library --------------------------------------------

80mkdir -p "$WORK/proj/src/$UUT_NAME" "$WORK/proj/test"

81COPY="$WORK/proj/src/$UUT_NAME/log.sgl"

82sed "s/^(define-library (sigil log)\$/(define-library ($UUT_NAME log)/" "$SRC" > "$COPY"

84CHANGED=$(diff "$SRC" "$COPY" | grep -c '^[<>]')

85if [ "$CHANGED" -ne 2 ]; then

86 echo "--- diff between source and renamed copy ---"

87 diff "$SRC" "$COPY" | head -40

88 setup_failed "expected exactly one renamed line (2 diff lines), got $CHANGED"

89fi

90grep -q "^(define-library ($UUT_NAME log)\$" "$COPY" \

91 || setup_failed "the rename did not land in the copy"

92echo "rename: 1 line, verified ($(sha256sum "$COPY" | cut -d' ' -f1))"

94cat > "$WORK/proj/package.sgl" <<EOF

95(package

96 name: "$UUT_NAME"

97 version: "0.0.0"

98 sigil: "^0.16"

99 description: "throwaway harness — generated by test/gate-source-under-test.sh"

100 license: "BSD-3-Clause")

101EOF

102

103

104# The throwaway project still needs its dependency closure resolved: the copy's

105# imports name stdlib modules, and `sigil test` refuses to start without them.

106( cd "$WORK/proj" && timeout "$TIMEOUT_SECS" "$SIGIL" deps install > "$WORK/deps.log" 2>&1 )

107DEPS_RC=$?

108if [ "$DEPS_RC" -ne 0 ]; then

109 sed 's/\x1b\[[0-9;]*m//g' "$WORK/deps.log"

110 setup_failed "sigil deps install failed in the harness project (rc=$DEPS_RC)"

111fi

112# --- Copy each test file, rewriting its import -------------------------------

113# Auto-discovery needs the test- prefix, so test/uut/*.sgl are named to stay OUT

114# of a plain `sigil test` run (where they would test the bundled copy and be

115# permanently, uninformatively red) and are renamed back in here.

116#

117# THE EXPECTED SET IS ENUMERATED FROM DISK, not from the list of copy_suite

118# calls below. Deriving it from those calls would make the "did every suite

119# run?" check compare the run against itself: deleting a copy_suite line would

120# shrink the expectation to match, and the gate would stay green over a smaller

121# suite. That mistake was made here once and caught by testing the check.

122declare -a EXPECTED=()

123for f in "$REPO"/test/test-*.sgl "$REPO"/test/uut/*.sgl; do

124 [ -e "$f" ] && EXPECTED+=("$f")

125done

126[ "${#EXPECTED[@]}" -gt 0 ] || setup_failed "found no test sources under $REPO/test"

127

128declare -a SUITES=()

129copy_suite() { # src-path dest-basename

130 local from="$1" to="$WORK/proj/test/$2"

131 [ -f "$from" ] || setup_failed "expected test file missing: $from"

132 local want; want=$(grep -c '(sigil log)' "$from")

133 [ "$want" -gt 0 ] || setup_failed "$from does not import (sigil log); nothing to rewrite"

134 sed "s/(sigil log)/($UUT_NAME log)/g" "$from" > "$to"

135 local got; got=$(diff "$from" "$to" | grep -c '^[<>]')

136 [ "$got" -eq $(( want * 2 )) ] \

137 || setup_failed "rewriting $from changed $got diff lines, expected $(( want * 2 ))"

138 SUITES+=("$2")

139}

140copy_suite "$REPO/test/test-log.sgl" "test-log.sgl"

141copy_suite "$REPO/test/uut/forgery-arms.sgl" "test-forgery.sgl"

142copy_suite "$REPO/test/uut/module-arms.sgl" "test-modules.sgl"

143

144if [ "${#SUITES[@]}" -ne "${#EXPECTED[@]}" ]; then

145 echo "test sources on disk (${#EXPECTED[@]}):"

146 printf ' %s\n' "${EXPECTED[@]}"

147 echo "suites this gate copies (${#SUITES[@]}):"

148 printf ' %s\n' "${SUITES[@]}"

149

  setup_failed "a test source on disk is not wired into this gate; a green would be over a smaller suite than the repo contains"

150fi

151echo "suites: ${SUITES[*]} (${#EXPECTED[@]} of ${#EXPECTED[@]} sources on disk)"

152echo

153# --- Prove the harness is running the COPY, not the bundle -------------------

154# Without this the whole gate could pass for free: if (loguut log) silently

155# fell back to something else, every arm below would be about that something.

156cat > "$WORK/proj/test/test-presence.sgl" <<EOF

157(import (sigil test) ($UUT_NAME log) (sigil io))

158(test-group "harness presence"

159 (test "the renamed library is the one loaded"

160 ;; THIS IS A LOAD CHECK, NOT A BEHAVIOUR CHECK, and saying so matters:

161 ;; it passes against a pre-fix library and against any other loadable

162 ;; copy. Its only job is to prove the renamed module was loaded and

163 ;; can log at all, so that a green from the arms below cannot mean

164 ;; "the harness never ran anything". The forgery and module arms are

165 ;; what test behaviour.

166 (let ((port (open-output-string)))

167 (log-configure! level: 'info format: 'text target: port)

168 (log-info "presence" k: "a b")

169 (assert-true (string? (get-output-string port))))))

170(run-tests)

171EOF

172

173# --- Run ----------------------------------------------------------------------

174LOG="$WORK/run.log"

175( cd "$WORK/proj" && timeout "$TIMEOUT_SECS" "$SIGIL" test --sgl > "$LOG" 2>&1 )

176RC=$?

177

178sed 's/\x1b\[[0-9;]*m//g' "$LOG" > "$WORK/run.txt"

179cat "$WORK/run.txt"

180

181if [ "$RC" -eq 124 ]; then

182 echo

183 echo "TIMED-OUT after ${TIMEOUT_SECS}s. Everything collected is above; the"

184 echo "arms that had not reported by then did not run and are UNKNOWN, not green."

185 echo "Scratch: $WORK"

186 exit 124

187fi

188

189if ! grep -q 'the renamed library is the one loaded' "$WORK/run.txt"; then

190 setup_failed "the presence arm did not report; the harness may not have loaded the copy"

191fi

192

193# Every suite must have RUN. `sigil test` reports "Files: N of M run", and a

194# suite that failed to load is counted but contributes no arms -- so a silent

195# drop would otherwise look like a smaller, still-green suite.

196WANT_FILES=$(( ${#EXPECTED[@]} + 1 )) # +1 for the presence suite

197if ! grep -qE "Files: +$WANT_FILES of $WANT_FILES run" "$WORK/run.txt"; then

198 echo

199 echo "Expected all $WANT_FILES suites to run. Actual:"

200 grep -E 'Files:' "$WORK/run.txt" || echo " (no Files: line at all)"

201 setup_failed "not every suite ran; a green here would be over a smaller suite than intended"

202fi

203

204echo

205echo "--- verdict ---"

206echo "rc: $RC"

207if [ "$RC" -eq 0 ]; then

208 echo "GREEN — every arm passed against $SRC"

209 rm -rf "$WORK"

210else

211 echo "RED — the source IS under test and at least one arm failed (see above)"

212 echo "Scratch KEPT for inspection: $WORK"

213 echo " Re-run just the failing suite with:"

214 echo " cd $WORK/proj && $SIGIL test --sgl"

215fi

216exit "$RC"