From bce5d9d588e1981e4b468cbb9b7522519af32505 Mon Sep 17 00:00:00 2001 From: Hannah Wolfe Date: Wed, 2 Oct 2024 15:28:39 +0100 Subject: [PATCH] Updated tinybird tooling with usability improvements (#21185) - Added yarn command to update TB CLI, as that needs doing frequently and I can never remember the command - Improved safety & usability of tinybird test script by ensuring branches are correctly created before running & adding optional delete - Updated tinybird test to warn only for sanity check as that's not always a valid check (Will prob remove soon) - Improved output of tinybird test script on failure, so that the diff is readable and closer to what git shows you - Added tool to convert tinybird ndjson to csv to make it easier to bring the data into google sheets for verifying numbers --- .../fixtures/utils/ndjson_to_csv.sh | 39 +++++++++++++++++++ ghost/tinybird/scripts/branch_and_test.sh | 31 +++++++++++++-- ghost/tinybird/scripts/exec_test.sh | 30 +++++++------- package.json | 3 +- 4 files changed, 85 insertions(+), 18 deletions(-) create mode 100755 ghost/tinybird/datasources/fixtures/utils/ndjson_to_csv.sh diff --git a/ghost/tinybird/datasources/fixtures/utils/ndjson_to_csv.sh b/ghost/tinybird/datasources/fixtures/utils/ndjson_to_csv.sh new file mode 100755 index 0000000000..d037bfa9e5 --- /dev/null +++ b/ghost/tinybird/datasources/fixtures/utils/ndjson_to_csv.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Check if the correct number of arguments is provided +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Assign input and output file paths from arguments +input_file="$1" +output_file="$2" + +# Create the header row +header="timestamp,session_id,action,version,site_uuid,member_uuid,member_status,post_uuid,user-agent,locale,location,referrer,pathname,href" + +# Write the header to the output file +echo "$header" > "$output_file" + +# Convert NDJSON to CSV and append to the output file +jq -r ' + [ + .timestamp, + .session_id, + .action, + .version, + (.payload | fromjson | .site_uuid), + (.payload | fromjson | .member_uuid), + (.payload | fromjson | .member_status), + (.payload | fromjson | .post_uuid), + (.payload | fromjson | .["user-agent"]), + (.payload | fromjson | .locale), + (.payload | fromjson | .location), + (.payload | fromjson | .referrer), + (.payload | fromjson | .pathname), + (.payload | fromjson | .href) + ] | @csv +' "$input_file" >> "$output_file" # Append to the output file + +echo "Conversion complete: $output_file" diff --git a/ghost/tinybird/scripts/branch_and_test.sh b/ghost/tinybird/scripts/branch_and_test.sh index 5fc9489867..ee6cae88f6 100755 --- a/ghost/tinybird/scripts/branch_and_test.sh +++ b/ghost/tinybird/scripts/branch_and_test.sh @@ -5,13 +5,38 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # Create a branch and test it -## Create a variable with a timestamp to use as a random name for the branch +# Create a variable with a timestamp to use as a random name for the branch BRANCH_NAME="TEST_$(date +%s)" -tb branch create $BRANCH_NAME +# Check for -y flag +force_delete=false +for arg in "$@"; do + if [[ "$arg" == "-y" ]]; then + force_delete=true + break + fi +done + +# Attempt to create the branch and check for errors +if ! tb branch create "$BRANCH_NAME"; then + echo "🚨 ERROR: Failed to create branch $BRANCH_NAME. Exiting." + exit 1 +fi # Run the scripts using their full paths "$SCRIPT_DIR/append_fixtures.sh" "$SCRIPT_DIR/exec_test.sh" -tb branch rm $BRANCH_NAME --yes +# Conditional deletion based on the -y flag or user prompt +if [ "$force_delete" = true ]; then + tb branch rm "$BRANCH_NAME" --yes + echo "Branch $BRANCH_NAME removed without prompt." +else + read -p "Do you want to delete the branch $BRANCH_NAME? (y/n): " choice + if [[ "$choice" == "y" || "$choice" == "Y" ]]; then + tb branch rm "$BRANCH_NAME" --yes + echo "Branch $BRANCH_NAME removed." + else + echo "Branch $BRANCH_NAME kept." + fi +fi diff --git a/ghost/tinybird/scripts/exec_test.sh b/ghost/tinybird/scripts/exec_test.sh index af6fb20b59..02d52c08d1 100755 --- a/ghost/tinybird/scripts/exec_test.sh +++ b/ghost/tinybird/scripts/exec_test.sh @@ -38,8 +38,8 @@ check_sum() { echo "✅ Sanity check passed: Sum of $column_name is $sum (matches NDJSON line count)" return 0 else - echo "🚨 Sanity check failed: Sum of $column_name is $sum, expected $expected_count (NDJSON line count)" - return 1 + echo "⚠️ WARNING: Sanity check failed: Sum of $column_name is $sum, expected $expected_count (NDJSON line count)" + return 1 # Return 1 to indicate a warning, but not a failure fi } @@ -54,15 +54,15 @@ run_test() { # When appending fixtures, we need to retry in case of the data is not replicated in time while [ $retries -lt $TOTAL_RETRIES ]; do # Run the test and store the output in a temporary file - bash $t >$tmpfile + bash "$t" >"$tmpfile" exit_code=$? if [ "$exit_code" -eq 0 ]; then # If the test passed, break the loop - if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then + if diff -B "${t}.result" "$tmpfile" >/dev/null 2>&1; then break # If the test failed, increment the retries counter and try again else - retries=$((retries+1)) + retries=$((retries + 1)) fi # If the bash command failed, print an error message and break the loop else @@ -70,24 +70,25 @@ run_test() { fi done - if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then + if diff -B "${t}.result" "$tmpfile" >/dev/null 2>&1; then echo "✅ Test $t passed" - check_sum ${t}.result $expected_count || return 1 - rm $tmpfile + check_sum "${t}.result" "$expected_count" || echo "⚠️ Warning: Sanity check did not pass." + rm "$tmpfile" return 0 elif [ $retries -eq $TOTAL_RETRIES ]; then - echo "🚨 ERROR: Test $t failed, diff:"; - diff -B ${t}.result $tmpfile - rm $tmpfile + echo "🚨 ERROR: Test $t failed, showing differences:" + diff -B -u --color -U3 "${t}.result" "$tmpfile" # Use unified diff format with 3 lines of context + rm "$tmpfile" return 1 else echo "🚨 ERROR: Test $t failed with bash command exit code $?" - cat $tmpfile - rm $tmpfile + cat "$tmpfile" + rm "$tmpfile" return 1 fi echo "" } + export -f run_test export -f check_sum @@ -110,5 +111,6 @@ else fi if [ $fail == 1 ]; then - exit 1 + echo "🚨 ERROR: Some tests failed" + exit 1 fi diff --git a/package.json b/package.json index d739563c7a..84aa352c51 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,8 @@ "main:monorepo": "git checkout main && git pull ${GHOST_UPSTREAM:-origin} main && yarn", "main:submodules": "git submodule sync && git submodule update && git submodule foreach \"git checkout main && git pull ${GHOST_UPSTREAM:-origin} main\"", "prepare": "husky install .github/hooks", - "tb": "docker run --rm -v $(pwd):/ghost -w /ghost/ghost/tinybird -it tinybirdco/tinybird-cli-docker" + "tb": "docker run --rm -v $(pwd):/ghost -w /ghost/ghost/tinybird -it tinybirdco/tinybird-cli-docker", + "tb:update": "docker pull tinybirdco/tinybird-cli-docker" }, "resolutions": { "@tryghost/errors": "1.3.5",