From 08bf49eaecee7af6004da240f4c5e40037fcb77c Mon Sep 17 00:00:00 2001 From: Hannah Wolfe Date: Thu, 29 Aug 2024 22:03:31 +0100 Subject: [PATCH] Added full suite of tinybird datasources and pipes (#20882) ref https://linear.app/tryghost/issue/ANAL-27/setup-tinybird-project-and-cicd ref https://github.com/tinybirdco/web-analytics-starter-kit/blob/main/tinybird/pipes/analytics_sessions.pipe - These datasources and pipes work together to define the main endpoints we need for our stats dashboard - They are based on the web analytics starter kit from tinybird - We've updated them to handle site_uuid - There's more to do to pipe the member-related and post-related data through the system yet --- .../datasources/analytics_pages_mv.datasource | 18 +++ .../analytics_sessions_mv.datasource | 14 ++ .../analytics_sources_mv.datasource | 13 ++ ghost/tinybird/datasources/fixtures/README.md | 15 ++ .../fixtures/mockingbird-schema.json | 72 ++++++++++ ghost/tinybird/pipes/analytics_hits.pipe | 71 ++++++++++ ghost/tinybird/pipes/analytics_pages.pipe | 24 ++++ ghost/tinybird/pipes/analytics_sessions.pipe | 20 +++ ghost/tinybird/pipes/analytics_sources.pipe | 21 +++ ghost/tinybird/pipes/kpis.pipe | 130 ++++++++++++++++++ ghost/tinybird/pipes/top_browsers.pipe | 32 +++++ ghost/tinybird/pipes/top_devices.pipe | 33 +++++ ghost/tinybird/pipes/top_locations.pipe | 32 +++++ ghost/tinybird/pipes/top_pages.pipe | 38 +++++ ghost/tinybird/pipes/top_sources.pipe | 33 +++++ ghost/tinybird/pipes/trend.pipe | 36 +++++ 16 files changed, 602 insertions(+) create mode 100644 ghost/tinybird/datasources/analytics_pages_mv.datasource create mode 100644 ghost/tinybird/datasources/analytics_sessions_mv.datasource create mode 100644 ghost/tinybird/datasources/analytics_sources_mv.datasource create mode 100644 ghost/tinybird/datasources/fixtures/README.md create mode 100644 ghost/tinybird/datasources/fixtures/mockingbird-schema.json create mode 100644 ghost/tinybird/pipes/analytics_hits.pipe create mode 100644 ghost/tinybird/pipes/analytics_pages.pipe create mode 100644 ghost/tinybird/pipes/analytics_sessions.pipe create mode 100644 ghost/tinybird/pipes/analytics_sources.pipe create mode 100644 ghost/tinybird/pipes/kpis.pipe create mode 100644 ghost/tinybird/pipes/top_browsers.pipe create mode 100644 ghost/tinybird/pipes/top_devices.pipe create mode 100644 ghost/tinybird/pipes/top_locations.pipe create mode 100644 ghost/tinybird/pipes/top_pages.pipe create mode 100644 ghost/tinybird/pipes/top_sources.pipe create mode 100644 ghost/tinybird/pipes/trend.pipe diff --git a/ghost/tinybird/datasources/analytics_pages_mv.datasource b/ghost/tinybird/datasources/analytics_pages_mv.datasource new file mode 100644 index 0000000000..c481ec4ef4 --- /dev/null +++ b/ghost/tinybird/datasources/analytics_pages_mv.datasource @@ -0,0 +1,18 @@ +SCHEMA > + `site_uuid` String, + `member_uuid` String, + `member_status` String, + `post_uuid` String, + `date` Date, + `device` String, + `browser` String, + `location` String, + `pathname` String, + `visits` AggregateFunction(uniq, String), + `hits` AggregateFunction(count), + `logged_in_hits` AggregateFunction(count), + `logged_out_hits` AggregateFunction(count) + +ENGINE AggregatingMergeTree +ENGINE_PARTITION_KEY toYYYYMM(date) +ENGINE_SORTING_KEY date, device, browser, location, pathname, member_uuid, member_status, post_uuid, site_uuid diff --git a/ghost/tinybird/datasources/analytics_sessions_mv.datasource b/ghost/tinybird/datasources/analytics_sessions_mv.datasource new file mode 100644 index 0000000000..f5a0fe32de --- /dev/null +++ b/ghost/tinybird/datasources/analytics_sessions_mv.datasource @@ -0,0 +1,14 @@ +SCHEMA > + `site_uuid` String, + `date` Date, + `session_id` String, + `device` SimpleAggregateFunction(any, String), + `browser` SimpleAggregateFunction(any, String), + `location` SimpleAggregateFunction(any, String), + `first_hit` SimpleAggregateFunction(min, DateTime), + `latest_hit` SimpleAggregateFunction(max, DateTime), + `hits` AggregateFunction(count) + +ENGINE AggregatingMergeTree +ENGINE_PARTITION_KEY toYYYYMM(date) +ENGINE_SORTING_KEY date, session_id, site_uuid diff --git a/ghost/tinybird/datasources/analytics_sources_mv.datasource b/ghost/tinybird/datasources/analytics_sources_mv.datasource new file mode 100644 index 0000000000..91f1dfe3d4 --- /dev/null +++ b/ghost/tinybird/datasources/analytics_sources_mv.datasource @@ -0,0 +1,13 @@ +SCHEMA > + `site_uuid` String, + `date` Date, + `device` String, + `browser` String, + `location` String, + `referrer` String, + `visits` AggregateFunction(uniq, String), + `hits` AggregateFunction(count) + +ENGINE AggregatingMergeTree +ENGINE_PARTITION_KEY toYYYYMM(date) +ENGINE_SORTING_KEY date, device, browser, location, referrer, site_uuid diff --git a/ghost/tinybird/datasources/fixtures/README.md b/ghost/tinybird/datasources/fixtures/README.md new file mode 100644 index 0000000000..622c39c9b5 --- /dev/null +++ b/ghost/tinybird/datasources/fixtures/README.md @@ -0,0 +1,15 @@ +# Datasource fixtures + +The file mockingbird-schema.json is a schema for generating fake data using the Mockingbird CLI. + +The CLI is installed via npm: + +``` +npm install -g @tinybirdco/mockingbird +``` + +The command I'm currently using to generate the data is: + +``` +mockingbird-cli tinybird --schema ghost/tinybird/datasources/fixtures/mockingbird-schema.json --endpoint gcp_europe_west3 --token xxxx --datasource analytics_events --eps 50 --limit 5000 +``` diff --git a/ghost/tinybird/datasources/fixtures/mockingbird-schema.json b/ghost/tinybird/datasources/fixtures/mockingbird-schema.json new file mode 100644 index 0000000000..0475ca0e0b --- /dev/null +++ b/ghost/tinybird/datasources/fixtures/mockingbird-schema.json @@ -0,0 +1,72 @@ +{ + "timestamp": { + "type": "mockingbird.datetimeBetween", + "params": [ + { + "start": "2024-07-01T00:00:00.000Z", + "end": "2024-08-20T12:00:00.000Z" + } + ] + }, + "session_id": { + "type": "string.uuid" + }, + "action": { + "type": "mockingbird.pick", + "params": [ + { + "values": [ + "page_hit" + ] + } + ] + }, + "version": { + "type": "mockingbird.pick", + "params": [ + { + "values": [ + "1" + ] + } + ] + }, + "payload": { + "type": "mockingbird.pickWeighted", + "params": [ + { + "values": [ + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\", \"locale\":\"en-US\", \"referrer\":\"https://www.kike.io\", \"pathname\":\"/coming-soon/\", \"href\":\"https://web-analytics.ghost.is/coming-soon/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/104.0.5112.79 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"IT\", \"referrer\":\"https://www.hn.com\", \"pathname\":\"/about/\", \"href\":\"https://web-analytics.ghost.is/about/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-GB\", \"location\":\"ES\", \"referrer\":\"\", \"pathname\":\"/\", \"href\":\"https://web-analytics.ghost.is\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/\", \"href\":\"https://web-analytics.ghost.is\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://web-analytics.ghost.is/\", \"pathname\":\"/coming-soon/\", \"href\":\"https://web-analytics.ghost.is/coming-soon/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/hello-world/\", \"href\":\"https://web-analytics.ghost.is/hello-world/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"IL\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/hello-world/\", \"href\":\"https://web-analytics.ghost.is/hello-world/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1\", \"locale\":\"es-ES\", \"location\":\"ES\", \"referrer\":\"https://www.twitter.com\", \"pathname\":\"/\", \"href\":\"https://web-analytics.ghost.is/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"GB\", \"referrer\":\"https://www.facebook.com\", \"pathname\":\"/\", \"href\":\"https://web-analytics.ghost.is/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"CH\", \"referrer\":\"https://www.qq.ch\", \"pathname\":\"/coming-soon/\", \"href\":\"https://web-analytics.ghost.is/coming-soon/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.yandex.com\", \"pathname\":\"/about/\", \"href\":\"https://web-analytics.ghost.is/about/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 13; SM-A102U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"FR\", \"referrer\":\"https://www.github.com\", \"pathname\":\"/coming-soon/\", \"href\":\"https://web-analytics.ghost.is/coming-soon/\"}", + + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\", \"locale\":\"en-US\", \"referrer\":\"https://www.kike.io\", \"pathname\":\"/products/\", \"href\":\"https://fake-site.ghost.is/products/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/104.0.5112.79 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"IT\", \"referrer\":\"https://www.hn.com\", \"pathname\":\"/blog/\", \"href\":\"https://fake-site.ghost.is/blog/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-GB\", \"location\":\"ES\", \"referrer\":\"\", \"pathname\":\"/contact/\", \"href\":\"https://fake-site.ghost.is/contact/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/faq/\", \"href\":\"https://fake-site.ghost.is/faq/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://fake-site.ghost.is/\", \"pathname\":\"/services/\", \"href\":\"https://fake-site.ghost.is/services/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/team/\", \"href\":\"https://fake-site.ghost.is/team/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"IL\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/pricing/\", \"href\":\"https://fake-site.ghost.is/pricing/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1\", \"locale\":\"es-ES\", \"location\":\"ES\", \"referrer\":\"https://www.twitter.com\", \"pathname\":\"/resources/\", \"href\":\"https://fake-site.ghost.is/resources/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"GB\", \"referrer\":\"https://www.facebook.com\", \"pathname\":\"/careers/\", \"href\":\"https://fake-site.ghost.is/careers/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"CH\", \"referrer\":\"https://www.qq.ch\", \"pathname\":\"/support/\", \"href\":\"https://fake-site.ghost.is/support/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.yandex.com\", \"pathname\":\"/partners/\", \"href\":\"https://fake-site.ghost.is/partners/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 13; SM-A102U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"FR\", \"referrer\":\"https://www.github.com\", \"pathname\":\"/events/\", \"href\":\"https://fake-site.ghost.is/events/\"}" + ], + "weights": [ + 200, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 400, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100 + ] + } + ] + } + } diff --git a/ghost/tinybird/pipes/analytics_hits.pipe b/ghost/tinybird/pipes/analytics_hits.pipe new file mode 100644 index 0000000000..08fbd7023d --- /dev/null +++ b/ghost/tinybird/pipes/analytics_hits.pipe @@ -0,0 +1,71 @@ +DESCRIPTION > + Parsed `page_hit` events, implementing `browser` and `device` detection logic. + +TOKEN "dashboard" READ + +NODE parsed_hits +DESCRIPTION > + Parse raw page_hit events + +SQL > + SELECT + timestamp, + action, + version, + coalesce(session_id, '0') as session_id, + JSONExtractString(payload, 'locale') as locale, + JSONExtractString(payload, 'location') as location, + JSONExtractString(payload, 'referrer') as referrer, + JSONExtractString(payload, 'pathname') as pathname, + JSONExtractString(payload, 'href') as href, + JSONExtractString(payload, 'site_uuid') as site_uuid, + JSONExtractString(payload, 'member_uuid') as member_uuid, + JSONExtractString(payload, 'member_status') as member_status, + JSONExtractString(payload, 'post_uuid') as post_uuid, + lower(JSONExtractString(payload, 'user-agent')) as user_agent + FROM analytics_events + where action = 'page_hit' + +NODE endpoint +SQL > + SELECT + site_uuid, + timestamp, + action, + version, + session_id, + case + when member_uuid REGEXP '^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$' + then true + else false + END as logged_in, + member_uuid, + member_status, + post_uuid, + location, + referrer, + pathname, + href, + case + when match(user_agent, 'wget|ahrefsbot|curl|urllib|bitdiscovery|\+https://|googlebot') + then 'bot' + when match(user_agent, 'android') + then 'mobile-android' + when match(user_agent, 'ipad|iphone|ipod') + then 'mobile-ios' + else 'desktop' + END as device, + case + when match(user_agent, 'firefox') + then 'firefox' + when match(user_agent, 'chrome|crios') + then 'chrome' + when match(user_agent, 'opera') + then 'opera' + when match(user_agent, 'msie|trident') + then 'ie' + when match(user_agent, 'iphone|ipad|safari') + then 'safari' + else 'Unknown' + END as browser + FROM parsed_hits diff --git a/ghost/tinybird/pipes/analytics_pages.pipe b/ghost/tinybird/pipes/analytics_pages.pipe new file mode 100644 index 0000000000..5974e71071 --- /dev/null +++ b/ghost/tinybird/pipes/analytics_pages.pipe @@ -0,0 +1,24 @@ +NODE analytics_pages_1 +DESCRIPTION > + Aggregate by pathname and calculate session and hits + +SQL > + SELECT + site_uuid, + member_uuid, + member_status, + post_uuid, + toDate(timestamp) AS date, + device, + browser, + location, + pathname, + uniqState(session_id) AS visits, + countState() AS hits, + countStateIf(logged_in = true) AS logged_in_hits, + countStateIf(logged_in = false) AS logged_out_hits + FROM analytics_hits + GROUP BY date, device, browser, location, pathname, member_uuid, member_status, post_uuid, site_uuid + +TYPE MATERIALIZED +DATASOURCE analytics_pages_mv diff --git a/ghost/tinybird/pipes/analytics_sessions.pipe b/ghost/tinybird/pipes/analytics_sessions.pipe new file mode 100644 index 0000000000..1eff0c3a7b --- /dev/null +++ b/ghost/tinybird/pipes/analytics_sessions.pipe @@ -0,0 +1,20 @@ +NODE analytics_sessions_1 +DESCRIPTION > + Aggregate by session_id and calculate session metrics + +SQL > + SELECT + site_uuid, + toDate(timestamp) AS date, + session_id, + anySimpleState(device) AS device, + anySimpleState(browser) AS browser, + anySimpleState(location) AS location, + minSimpleState(timestamp) AS first_hit, + maxSimpleState(timestamp) AS latest_hit, + countState() AS hits + FROM analytics_hits + GROUP BY date, session_id, site_uuid + +TYPE MATERIALIZED +DATASOURCE analytics_sessions_mv diff --git a/ghost/tinybird/pipes/analytics_sources.pipe b/ghost/tinybird/pipes/analytics_sources.pipe new file mode 100644 index 0000000000..99a29e2f96 --- /dev/null +++ b/ghost/tinybird/pipes/analytics_sources.pipe @@ -0,0 +1,21 @@ +NODE analytics_sources_1 +DESCRIPTION > + Aggregate by referral and calculate session and hits + +SQL > + WITH (SELECT domainWithoutWWW(href) FROM analytics_hits LIMIT 1) AS current_domain + SELECT + site_uuid, + toDate(timestamp) AS date, + device, + browser, + location, + referrer, + uniqState(session_id) AS visits, + countState() AS hits + FROM analytics_hits + WHERE domainWithoutWWW(referrer) != current_domain + GROUP BY date, device, browser, location, referrer, site_uuid + +TYPE MATERIALIZED +DATASOURCE analytics_sources_mv diff --git a/ghost/tinybird/pipes/kpis.pipe b/ghost/tinybird/pipes/kpis.pipe new file mode 100644 index 0000000000..06a4f8e06a --- /dev/null +++ b/ghost/tinybird/pipes/kpis.pipe @@ -0,0 +1,130 @@ +DESCRIPTION > + Summary with general KPIs per date, including visits, page views, bounce rate and average session duration. + Accepts `date_from` and `date_to` date filter, all historical data if not passed. + Daily granularity, except when filtering one single day (hourly) + +TOKEN "dashboard" READ + +NODE timeseries +DESCRIPTION > + Generate a timeseries for the specified time range, so we call fill empty data points. + Filters "future" data points. + +SQL > + % + {% set _single_day = defined(date_from) and day_diff(date_from, date_to) == 0 %} + with + {% if defined(date_from) %} + toStartOfDay( + toDate( + {{ + Date( + date_from, + description="Starting day for filtering a date range", + required=False, + ) + }} + ) + ) as start, + {% else %} toStartOfDay(timestampAdd(today(), interval -7 day)) as start, + {% end %} + {% if defined(date_to) %} + toStartOfDay( + toDate( + {{ + Date( + date_to, + description="Finishing day for filtering a date range", + required=False, + ) + }} + ) + ) as end + {% else %} toStartOfDay(today()) as end + {% end %} + {% if _single_day %} + select + arrayJoin( + arrayMap( + x -> toDateTime(x), + range( + toUInt32(toDateTime(start)), toUInt32(timestampAdd(end, interval 1 day)), 3600 + ) + ) + ) as date + {% else %} + select + arrayJoin( + arrayMap( + x -> toDate(x), + range(toUInt32(start), toUInt32(timestampAdd(end, interval 1 day)), 24 * 3600) + ) + ) as date + {% end %} + where date <= now() + +NODE hits +DESCRIPTION > + Group by sessions and calculate metrics at that level + +SQL > + % + {% if defined(date_from) and day_diff(date_from, date_to) == 0 %} + select + site_uuid, + toStartOfHour(timestamp) as date, + session_id, + uniq(session_id) as visits, + count() as pageviews, + case when min(timestamp) = max(timestamp) then 1 else 0 end as is_bounce, + max(timestamp) as latest_hit_aux, + min(timestamp) as first_hit_aux + from analytics_hits + where toDate(timestamp) = {{ Date(date_from) }} + group by toStartOfHour(timestamp), session_id, site_uuid + {% else %} + select + site_uuid, + date, + session_id, + uniq(session_id) as visits, + countMerge(hits) as pageviews, + case when min(first_hit) = max(latest_hit) then 1 else 0 end as is_bounce, + max(latest_hit) as latest_hit_aux, + min(first_hit) as first_hit_aux + from analytics_sessions_mv + where + {% if defined(date_from) %} date >= {{ Date(date_from) }} + {% else %} date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} and date <= {{ Date(date_to) }} + {% else %} and date <= today() + {% end %} + group by date, session_id, site_uuid + {% end %} + +NODE data +DESCRIPTION > + General KPIs per date, works for both summary metrics and trends charts. + +SQL > + select + site_uuid, + date, + uniq(session_id) as visits, + sum(pageviews) as pageviews, + sum(case when latest_hit_aux = first_hit_aux then 1 end) / visits as bounce_rate, + avg(latest_hit_aux - first_hit_aux) as avg_session_sec + from hits + group by date, site_uuid + +NODE endpoint +DESCRIPTION > + Join and generate timeseries with metrics + +SQL > +% + select a.date, b.visits, b.pageviews, b.bounce_rate, b.avg_session_sec + from timeseries a + left join data b using date + where site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} diff --git a/ghost/tinybird/pipes/top_browsers.pipe b/ghost/tinybird/pipes/top_browsers.pipe new file mode 100644 index 0000000000..5789296d84 --- /dev/null +++ b/ghost/tinybird/pipes/top_browsers.pipe @@ -0,0 +1,32 @@ +DESCRIPTION > + Top Browsers ordered by most visits. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by browser and calculate hits and visits + +SQL > + % + select browser, uniqMerge(visits) as visits, countMerge(hits) as hits + from analytics_sources_mv + where + site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and + {% if defined(date_from) %} + date + >= + {{ Date(date_from, description="Starting day for filtering a date range", required=False) }} + {% else %} date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date + <= + {{ Date(date_to, description="Finishing day for filtering a date range", required=False) }} + {% else %} and date <= today() + {% end %} + group by browser + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/ghost/tinybird/pipes/top_devices.pipe b/ghost/tinybird/pipes/top_devices.pipe new file mode 100644 index 0000000000..1988bac9ac --- /dev/null +++ b/ghost/tinybird/pipes/top_devices.pipe @@ -0,0 +1,33 @@ + +DESCRIPTION > + Top Device Types ordered by most visits. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by device and calculate hits and visits + +SQL > + % + select device, uniqMerge(visits) as visits, countMerge(hits) as hits + from analytics_sources_mv + where + site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and + {% if defined(date_from) %} + date + >= + {{ Date(date_from, description="Starting day for filtering a date range", required=False) }} + {% else %} date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date + <= + {{ Date(date_to, description="Finishing day for filtering a date range", required=False) }} + {% else %} and date <= today() + {% end %} + group by device + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/ghost/tinybird/pipes/top_locations.pipe b/ghost/tinybird/pipes/top_locations.pipe new file mode 100644 index 0000000000..61ad02aeb8 --- /dev/null +++ b/ghost/tinybird/pipes/top_locations.pipe @@ -0,0 +1,32 @@ +DESCRIPTION > + Top visiting Countries ordered by most visits. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by pagepath and calculate hits and visits + +SQL > + % + select location, uniqMerge(visits) as visits, countMerge(hits) as hits + from analytics_pages_mv + where + site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and + {% if defined(date_from) %} + date + >= + {{ Date(date_from, description="Starting day for filtering a date range", required=False) }} + {% else %} date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date + <= + {{ Date(date_to, description="Finishing day for filtering a date range", required=False) }} + {% else %} and date <= today() + {% end %} + group by location + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/ghost/tinybird/pipes/top_pages.pipe b/ghost/tinybird/pipes/top_pages.pipe new file mode 100644 index 0000000000..41b65664fa --- /dev/null +++ b/ghost/tinybird/pipes/top_pages.pipe @@ -0,0 +1,38 @@ +DESCRIPTION > + Most visited pages for a given period. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by pagepath and calculate hits and visits + +SQL > + % + select + pathname, + uniqMerge(visits) as visits, + countMerge(hits) as hits, + countMerge(logged_in_hits) as logged_in_hits, + countMerge(logged_out_hits) as logged_out_hits + from analytics_pages_mv + where + site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and + {% if defined(date_from) %} + date + >= + {{ Date(date_from, description="Starting day for filtering a date range", required=False) }} + {% else %} date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date + <= + {{ Date(date_to, description="Finishing day for filtering a date range", required=False) }} + {% else %} and date <= today() + {% end %} + + group by pathname + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/ghost/tinybird/pipes/top_sources.pipe b/ghost/tinybird/pipes/top_sources.pipe new file mode 100644 index 0000000000..1282fd70cc --- /dev/null +++ b/ghost/tinybird/pipes/top_sources.pipe @@ -0,0 +1,33 @@ + +DESCRIPTION > + Top traffic sources (domains), ordered by most visits. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by referral and calculate hits and visits + +SQL > + % + select domainWithoutWWW(referrer) as referrer, uniqMerge(visits) as visits, countMerge(hits) as hits + from analytics_sources_mv + where + site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and + {% if defined(date_from) %} + date + >= + {{ Date(date_from, description="Starting day for filtering a date range", required=False) }} + {% else %} date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date + <= + {{ Date(date_to, description="Finishing day for filtering a date range", required=False) }} + {% else %} and date <= today() + {% end %} + group by referrer + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/ghost/tinybird/pipes/trend.pipe b/ghost/tinybird/pipes/trend.pipe new file mode 100644 index 0000000000..edbc74d79e --- /dev/null +++ b/ghost/tinybird/pipes/trend.pipe @@ -0,0 +1,36 @@ + +DESCRIPTION > + Visits trend over time for the last 30 minutes, filling the blanks. + Works great for the realtime chart. + +TOKEN "dashboard" READ + +NODE timeseries +DESCRIPTION > + Generate a timeseries for the last 30 minutes, so we call fill empty data points + +SQL > + with (now() - interval 30 minute) as start + select addMinutes(toStartOfMinute(start), number) as t + from (select arrayJoin(range(1, 31)) as number) + +NODE hits +DESCRIPTION > + Get last 30 minutes metrics gropued by minute + +SQL > + % + select toStartOfMinute(timestamp) as t, uniq(session_id) as visits + from analytics_hits + where + site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and + timestamp >= (now() - interval 30 minute) + group by toStartOfMinute(timestamp) + order by toStartOfMinute(timestamp) + +NODE endpoint +DESCRIPTION > + Join and generate timeseries with metrics for the last 30 minutes + +SQL > + select a.t, b.visits from timeseries a left join hits b on a.t = b.t order by a.t