0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-03-11 02:12:21 -05:00

Updated tinybird tooling with usability improvements (#21185)

- Added yarn command to update TB CLI, as that needs doing frequently and I can never remember the command
- Improved safety & usability of tinybird test script by ensuring branches are correctly created before running & adding optional delete
- Updated tinybird test to warn only for sanity check as that's not always a valid check (Will prob remove soon)
- Improved output of tinybird test script on failure, so that the diff is readable and closer to what git shows you
- Added tool to convert tinybird ndjson to csv to make it easier to bring the data into google sheets for verifying numbers
This commit is contained in:
Hannah Wolfe 2024-10-02 15:28:39 +01:00 committed by GitHub
parent 0f7b1567a3
commit bce5d9d588
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 85 additions and 18 deletions

View file

@ -0,0 +1,39 @@
#!/usr/bin/env bash
# Check if the correct number of arguments is provided
if [ "$#" -ne 2 ]; then
echo "Usage: $0 <input_ndjson_file> <output_csv_file>"
exit 1
fi
# Assign input and output file paths from arguments
input_file="$1"
output_file="$2"
# Create the header row
header="timestamp,session_id,action,version,site_uuid,member_uuid,member_status,post_uuid,user-agent,locale,location,referrer,pathname,href"
# Write the header to the output file
echo "$header" > "$output_file"
# Convert NDJSON to CSV and append to the output file
jq -r '
[
.timestamp,
.session_id,
.action,
.version,
(.payload | fromjson | .site_uuid),
(.payload | fromjson | .member_uuid),
(.payload | fromjson | .member_status),
(.payload | fromjson | .post_uuid),
(.payload | fromjson | .["user-agent"]),
(.payload | fromjson | .locale),
(.payload | fromjson | .location),
(.payload | fromjson | .referrer),
(.payload | fromjson | .pathname),
(.payload | fromjson | .href)
] | @csv
' "$input_file" >> "$output_file" # Append to the output file
echo "Conversion complete: $output_file"

View file

@ -5,13 +5,38 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
# Create a branch and test it
## Create a variable with a timestamp to use as a random name for the branch
# Create a variable with a timestamp to use as a random name for the branch
BRANCH_NAME="TEST_$(date +%s)"
tb branch create $BRANCH_NAME
# Check for -y flag
force_delete=false
for arg in "$@"; do
if [[ "$arg" == "-y" ]]; then
force_delete=true
break
fi
done
# Attempt to create the branch and check for errors
if ! tb branch create "$BRANCH_NAME"; then
echo "🚨 ERROR: Failed to create branch $BRANCH_NAME. Exiting."
exit 1
fi
# Run the scripts using their full paths
"$SCRIPT_DIR/append_fixtures.sh"
"$SCRIPT_DIR/exec_test.sh"
tb branch rm $BRANCH_NAME --yes
# Conditional deletion based on the -y flag or user prompt
if [ "$force_delete" = true ]; then
tb branch rm "$BRANCH_NAME" --yes
echo "Branch $BRANCH_NAME removed without prompt."
else
read -p "Do you want to delete the branch $BRANCH_NAME? (y/n): " choice
if [[ "$choice" == "y" || "$choice" == "Y" ]]; then
tb branch rm "$BRANCH_NAME" --yes
echo "Branch $BRANCH_NAME removed."
else
echo "Branch $BRANCH_NAME kept."
fi
fi

View file

@ -38,8 +38,8 @@ check_sum() {
echo "✅ Sanity check passed: Sum of $column_name is $sum (matches NDJSON line count)"
return 0
else
echo "🚨 Sanity check failed: Sum of $column_name is $sum, expected $expected_count (NDJSON line count)"
return 1
echo "⚠️ WARNING: Sanity check failed: Sum of $column_name is $sum, expected $expected_count (NDJSON line count)"
return 1 # Return 1 to indicate a warning, but not a failure
fi
}
@ -54,15 +54,15 @@ run_test() {
# When appending fixtures, we need to retry in case of the data is not replicated in time
while [ $retries -lt $TOTAL_RETRIES ]; do
# Run the test and store the output in a temporary file
bash $t >$tmpfile
bash "$t" >"$tmpfile"
exit_code=$?
if [ "$exit_code" -eq 0 ]; then
# If the test passed, break the loop
if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
if diff -B "${t}.result" "$tmpfile" >/dev/null 2>&1; then
break
# If the test failed, increment the retries counter and try again
else
retries=$((retries+1))
retries=$((retries + 1))
fi
# If the bash command failed, print an error message and break the loop
else
@ -70,24 +70,25 @@ run_test() {
fi
done
if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
if diff -B "${t}.result" "$tmpfile" >/dev/null 2>&1; then
echo "✅ Test $t passed"
check_sum ${t}.result $expected_count || return 1
rm $tmpfile
check_sum "${t}.result" "$expected_count" || echo "⚠️ Warning: Sanity check did not pass."
rm "$tmpfile"
return 0
elif [ $retries -eq $TOTAL_RETRIES ]; then
echo "🚨 ERROR: Test $t failed, diff:";
diff -B ${t}.result $tmpfile
rm $tmpfile
echo "🚨 ERROR: Test $t failed, showing differences:"
diff -B -u --color -U3 "${t}.result" "$tmpfile" # Use unified diff format with 3 lines of context
rm "$tmpfile"
return 1
else
echo "🚨 ERROR: Test $t failed with bash command exit code $?"
cat $tmpfile
rm $tmpfile
cat "$tmpfile"
rm "$tmpfile"
return 1
fi
echo ""
}
export -f run_test
export -f check_sum
@ -110,5 +111,6 @@ else
fi
if [ $fail == 1 ]; then
exit 1
echo "🚨 ERROR: Some tests failed"
exit 1
fi

View file

@ -42,7 +42,8 @@
"main:monorepo": "git checkout main && git pull ${GHOST_UPSTREAM:-origin} main && yarn",
"main:submodules": "git submodule sync && git submodule update && git submodule foreach \"git checkout main && git pull ${GHOST_UPSTREAM:-origin} main\"",
"prepare": "husky install .github/hooks",
"tb": "docker run --rm -v $(pwd):/ghost -w /ghost/ghost/tinybird -it tinybirdco/tinybird-cli-docker"
"tb": "docker run --rm -v $(pwd):/ghost -w /ghost/ghost/tinybird -it tinybirdco/tinybird-cli-docker",
"tb:update": "docker pull tinybirdco/tinybird-cli-docker"
},
"resolutions": {
"@tryghost/errors": "1.3.5",