Updated tinybird tooling with usability improvements (#21185)

- Added yarn command to update TB CLI, as that needs doing frequently and I can never remember the command
- Improved safety & usability of tinybird test script by ensuring branches are correctly created before running & adding optional delete
- Updated tinybird test to warn only for sanity check as that's not always a valid check (Will prob remove soon)
- Improved output of tinybird test script on failure, so that the diff is readable and closer to what git shows you
- Added tool to convert tinybird ndjson to csv to make it easier to bring the data into google sheets for verifying numbers

ghost/tinybird/datasources/fixtures/utils/ndjson_to_csv.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Check if the correct number of arguments is provided
+if [ "$#" -ne 2 ]; then
+    echo "Usage: $0 <input_ndjson_file> <output_csv_file>"
+    exit 1
+fi
+
+# Assign input and output file paths from arguments
+input_file="$1"
+output_file="$2"
+
+# Create the header row
+header="timestamp,session_id,action,version,site_uuid,member_uuid,member_status,post_uuid,user-agent,locale,location,referrer,pathname,href"
+
+# Write the header to the output file
+echo "$header" > "$output_file"
+
+# Convert NDJSON to CSV and append to the output file
+jq -r '
+    [
+        .timestamp,
+        .session_id,
+        .action,
+        .version,
+        (.payload | fromjson | .site_uuid),
+        (.payload | fromjson | .member_uuid),
+        (.payload | fromjson | .member_status),
+        (.payload | fromjson | .post_uuid),
+        (.payload | fromjson | .["user-agent"]),
+        (.payload | fromjson | .locale),
+        (.payload | fromjson | .location),
+        (.payload | fromjson | .referrer),
+        (.payload | fromjson | .pathname),
+        (.payload | fromjson | .href)
+    ] | @csv
+' "$input_file" >> "$output_file"  # Append to the output file
+
+echo "Conversion complete: $output_file"

ghost/tinybird/scripts/branch_and_test.sh

@@ -5,13 +5,38 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
 # Create a branch and test it
 
-## Create a variable with a timestamp to use as a random name for the branch
+# Create a variable with a timestamp to use as a random name for the branch
 BRANCH_NAME="TEST_$(date +%s)"
 
-tb branch create $BRANCH_NAME
+# Check for -y flag
+force_delete=false
+for arg in "$@"; do
+    if [[ "$arg" == "-y" ]]; then
+        force_delete=true
+        break
+    fi
+done
+
+# Attempt to create the branch and check for errors
+if ! tb branch create "$BRANCH_NAME"; then
+    echo "🚨 ERROR: Failed to create branch $BRANCH_NAME. Exiting."
+    exit 1
+fi
 
 # Run the scripts using their full paths
 "$SCRIPT_DIR/append_fixtures.sh"
 "$SCRIPT_DIR/exec_test.sh"
 
-tb branch rm $BRANCH_NAME --yes
+# Conditional deletion based on the -y flag or user prompt
+if [ "$force_delete" = true ]; then
+    tb branch rm "$BRANCH_NAME" --yes
+    echo "Branch $BRANCH_NAME removed without prompt."
+else
+    read -p "Do you want to delete the branch $BRANCH_NAME? (y/n): " choice
+    if [[ "$choice" == "y" || "$choice" == "Y" ]]; then
+        tb branch rm "$BRANCH_NAME" --yes
+        echo "Branch $BRANCH_NAME removed."
+    else
+        echo "Branch $BRANCH_NAME kept."
+    fi
+fi

ghost/tinybird/scripts/exec_test.sh

@@ -38,8 +38,8 @@ check_sum() {
         echo "✅ Sanity check passed: Sum of $column_name is $sum (matches NDJSON line count)"
         return 0
     else
-        echo "🚨 Sanity check failed: Sum of $column_name is $sum, expected $expected_count (NDJSON line count)"
+        echo "⚠️  WARNING: Sanity check failed: Sum of $column_name is $sum, expected $expected_count (NDJSON line count)"
-        return 1
+        return 1  # Return 1 to indicate a warning, but not a failure
     fi
 }
 
@@ -54,15 +54,15 @@ run_test() {
     # When appending fixtures, we need to retry in case of the data is not replicated in time
     while [ $retries -lt $TOTAL_RETRIES ]; do
         # Run the test and store the output in a temporary file
-        bash $t >$tmpfile
+        bash "$t" >"$tmpfile"
         exit_code=$?
         if [ "$exit_code" -eq 0 ]; then
             # If the test passed, break the loop
-            if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
+            if diff -B "${t}.result" "$tmpfile" >/dev/null 2>&1; then
                 break
             # If the test failed, increment the retries counter and try again
             else
-                retries=$((retries+1))
+                retries=$((retries + 1))
             fi
         # If the bash command failed, print an error message and break the loop
         else
@@ -70,24 +70,25 @@ run_test() {
         fi
     done
 
-    if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
+    if diff -B "${t}.result" "$tmpfile" >/dev/null 2>&1; then
         echo "✅ Test $t passed"
-        check_sum ${t}.result $expected_count || return 1
+        check_sum "${t}.result" "$expected_count" || echo "⚠️  Warning: Sanity check did not pass."
-        rm $tmpfile
+        rm "$tmpfile"
         return 0
     elif [ $retries -eq $TOTAL_RETRIES ]; then
-        echo "🚨 ERROR: Test $t failed, diff:";
+        echo "🚨 ERROR: Test $t failed, showing differences:"
-        diff -B ${t}.result $tmpfile
+        diff -B -u --color -U3 "${t}.result" "$tmpfile"  # Use unified diff format with 3 lines of context
-        rm $tmpfile
+        rm "$tmpfile"
         return 1
     else
         echo "🚨 ERROR: Test $t failed with bash command exit code $?"
-        cat $tmpfile
+        cat "$tmpfile"
-        rm $tmpfile
+        rm "$tmpfile"
         return 1
     fi
     echo ""
 }
+
 export -f run_test
 export -f check_sum
 
@@ -110,5 +111,6 @@ else
 fi
 
 if [ $fail == 1 ]; then
-  exit 1
+    echo "🚨 ERROR: Some tests failed"
+    exit 1
 fi

package.json

@@ -42,7 +42,8 @@
     "main:monorepo": "git checkout main && git pull ${GHOST_UPSTREAM:-origin} main && yarn",
     "main:submodules": "git submodule sync && git submodule update && git submodule foreach \"git checkout main && git pull ${GHOST_UPSTREAM:-origin} main\"",
     "prepare": "husky install .github/hooks",
-    "tb": "docker run --rm -v $(pwd):/ghost -w /ghost/ghost/tinybird -it tinybirdco/tinybird-cli-docker"
+    "tb": "docker run --rm -v $(pwd):/ghost -w /ghost/ghost/tinybird -it tinybirdco/tinybird-cli-docker",
+    "tb:update": "docker pull tinybirdco/tinybird-cli-docker"
   },
   "resolutions": {
     "@tryghost/errors": "1.3.5",