mirror of
https://github.com/TryGhost/Ghost.git
synced 2025-01-20 22:42:53 -05:00
Added full suite of tinybird datasources and pipes (#20882)
ref https://linear.app/tryghost/issue/ANAL-27/setup-tinybird-project-and-cicd ref https://github.com/tinybirdco/web-analytics-starter-kit/blob/main/tinybird/pipes/analytics_sessions.pipe - These datasources and pipes work together to define the main endpoints we need for our stats dashboard - They are based on the web analytics starter kit from tinybird - We've updated them to handle site_uuid - There's more to do to pipe the member-related and post-related data through the system yet
This commit is contained in:
Hannah Wolfe
GitHub
parent
f79f5471b4
commit
08bf49eaec
16 changed files with 602 additions and 0 deletions
18
ghost/tinybird/datasources/analytics_pages_mv.datasource
Normal file
18
ghost/tinybird/datasources/analytics_pages_mv.datasource
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
SCHEMA >
|
||||
`site_uuid` String,
|
||||
`member_uuid` String,
|
||||
`member_status` String,
|
||||
`post_uuid` String,
|
||||
`date` Date,
|
||||
`device` String,
|
||||
`browser` String,
|
||||
`location` String,
|
||||
`pathname` String,
|
||||
`visits` AggregateFunction(uniq, String),
|
||||
`hits` AggregateFunction(count),
|
||||
`logged_in_hits` AggregateFunction(count),
|
||||
`logged_out_hits` AggregateFunction(count)
|
||||
|
||||
ENGINE AggregatingMergeTree
|
||||
ENGINE_PARTITION_KEY toYYYYMM(date)
|
||||
ENGINE_SORTING_KEY date, device, browser, location, pathname, member_uuid, member_status, post_uuid, site_uuid
|
||||
14
ghost/tinybird/datasources/analytics_sessions_mv.datasource
Normal file
14
ghost/tinybird/datasources/analytics_sessions_mv.datasource
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
SCHEMA >
|
||||
`site_uuid` String,
|
||||
`date` Date,
|
||||
`session_id` String,
|
||||
`device` SimpleAggregateFunction(any, String),
|
||||
`browser` SimpleAggregateFunction(any, String),
|
||||
`location` SimpleAggregateFunction(any, String),
|
||||
`first_hit` SimpleAggregateFunction(min, DateTime),
|
||||
`latest_hit` SimpleAggregateFunction(max, DateTime),
|
||||
`hits` AggregateFunction(count)
|
||||
|
||||
ENGINE AggregatingMergeTree
|
||||
ENGINE_PARTITION_KEY toYYYYMM(date)
|
||||
ENGINE_SORTING_KEY date, session_id, site_uuid
|
||||
13
ghost/tinybird/datasources/analytics_sources_mv.datasource
Normal file
13
ghost/tinybird/datasources/analytics_sources_mv.datasource
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
SCHEMA >
|
||||
`site_uuid` String,
|
||||
`date` Date,
|
||||
`device` String,
|
||||
`browser` String,
|
||||
`location` String,
|
||||
`referrer` String,
|
||||
`visits` AggregateFunction(uniq, String),
|
||||
`hits` AggregateFunction(count)
|
||||
|
||||
ENGINE AggregatingMergeTree
|
||||
ENGINE_PARTITION_KEY toYYYYMM(date)
|
||||
ENGINE_SORTING_KEY date, device, browser, location, referrer, site_uuid
|
||||
15
ghost/tinybird/datasources/fixtures/README.md
Normal file
15
ghost/tinybird/datasources/fixtures/README.md
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
# Datasource fixtures
|
||||
|
||||
The file mockingbird-schema.json is a schema for generating fake data using the Mockingbird CLI.
|
||||
|
||||
The CLI is installed via npm:
|
||||
|
||||
```
|
||||
npm install -g @tinybirdco/mockingbird
|
||||
```
|
||||
|
||||
The command I'm currently using to generate the data is:
|
||||
|
||||
```
|
||||
mockingbird-cli tinybird --schema ghost/tinybird/datasources/fixtures/mockingbird-schema.json --endpoint gcp_europe_west3 --token xxxx --datasource analytics_events --eps 50 --limit 5000
|
||||
```
|
||||
72
ghost/tinybird/datasources/fixtures/mockingbird-schema.json
Normal file
72
ghost/tinybird/datasources/fixtures/mockingbird-schema.json
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
{
|
||||
"timestamp": {
|
||||
"type": "mockingbird.datetimeBetween",
|
||||
"params": [
|
||||
{
|
||||
"start": "2024-07-01T00:00:00.000Z",
|
||||
"end": "2024-08-20T12:00:00.000Z"
|
||||
}
|
||||
]
|
||||
},
|
||||
"session_id": {
|
||||
"type": "string.uuid"
|
||||
},
|
||||
"action": {
|
||||
"type": "mockingbird.pick",
|
||||
"params": [
|
||||
{
|
||||
"values": [
|
||||
"page_hit"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"version": {
|
||||
"type": "mockingbird.pick",
|
||||
"params": [
|
||||
{
|
||||
"values": [
|
||||
"1"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"payload": {
|
||||
"type": "mockingbird.pickWeighted",
|
||||
"params": [
|
||||
{
|
||||
"values": [
|
||||
"{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\", \"locale\":\"en-US\", \"referrer\":\"https://www.kike.io\", \"pathname\":\"/coming-soon/\", \"href\":\"https://web-analytics.ghost.is/coming-soon/\"}",
|
||||
"{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/104.0.5112.79 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"IT\", \"referrer\":\"https://www.hn.com\", \"pathname\":\"/about/\", \"href\":\"https://web-analytics.ghost.is/about/\"}",
|
||||
"{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-GB\", \"location\":\"ES\", \"referrer\":\"\", \"pathname\":\"/\", \"href\":\"https://web-analytics.ghost.is\"}",
|
||||
"{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/\", \"href\":\"https://web-analytics.ghost.is\"}",
|
||||
"{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://web-analytics.ghost.is/\", \"pathname\":\"/coming-soon/\", \"href\":\"https://web-analytics.ghost.is/coming-soon/\"}",
|
||||
"{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/hello-world/\", \"href\":\"https://web-analytics.ghost.is/hello-world/\"}",
|
||||
"{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"IL\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/hello-world/\", \"href\":\"https://web-analytics.ghost.is/hello-world/\"}",
|
||||
"{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1\", \"locale\":\"es-ES\", \"location\":\"ES\", \"referrer\":\"https://www.twitter.com\", \"pathname\":\"/\", \"href\":\"https://web-analytics.ghost.is/\"}",
|
||||
"{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"GB\", \"referrer\":\"https://www.facebook.com\", \"pathname\":\"/\", \"href\":\"https://web-analytics.ghost.is/\"}",
|
||||
"{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"CH\", \"referrer\":\"https://www.qq.ch\", \"pathname\":\"/coming-soon/\", \"href\":\"https://web-analytics.ghost.is/coming-soon/\"}",
|
||||
"{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.yandex.com\", \"pathname\":\"/about/\", \"href\":\"https://web-analytics.ghost.is/about/\"}",
|
||||
"{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 13; SM-A102U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"FR\", \"referrer\":\"https://www.github.com\", \"pathname\":\"/coming-soon/\", \"href\":\"https://web-analytics.ghost.is/coming-soon/\"}",
|
||||
|
||||
"{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\", \"locale\":\"en-US\", \"referrer\":\"https://www.kike.io\", \"pathname\":\"/products/\", \"href\":\"https://fake-site.ghost.is/products/\"}",
|
||||
"{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/104.0.5112.79 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"IT\", \"referrer\":\"https://www.hn.com\", \"pathname\":\"/blog/\", \"href\":\"https://fake-site.ghost.is/blog/\"}",
|
||||
"{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-GB\", \"location\":\"ES\", \"referrer\":\"\", \"pathname\":\"/contact/\", \"href\":\"https://fake-site.ghost.is/contact/\"}",
|
||||
"{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/faq/\", \"href\":\"https://fake-site.ghost.is/faq/\"}",
|
||||
"{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://fake-site.ghost.is/\", \"pathname\":\"/services/\", \"href\":\"https://fake-site.ghost.is/services/\"}",
|
||||
"{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/team/\", \"href\":\"https://fake-site.ghost.is/team/\"}",
|
||||
"{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"IL\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/pricing/\", \"href\":\"https://fake-site.ghost.is/pricing/\"}",
|
||||
"{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1\", \"locale\":\"es-ES\", \"location\":\"ES\", \"referrer\":\"https://www.twitter.com\", \"pathname\":\"/resources/\", \"href\":\"https://fake-site.ghost.is/resources/\"}",
|
||||
"{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"GB\", \"referrer\":\"https://www.facebook.com\", \"pathname\":\"/careers/\", \"href\":\"https://fake-site.ghost.is/careers/\"}",
|
||||
"{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"CH\", \"referrer\":\"https://www.qq.ch\", \"pathname\":\"/support/\", \"href\":\"https://fake-site.ghost.is/support/\"}",
|
||||
"{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.yandex.com\", \"pathname\":\"/partners/\", \"href\":\"https://fake-site.ghost.is/partners/\"}",
|
||||
"{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 13; SM-A102U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"FR\", \"referrer\":\"https://www.github.com\", \"pathname\":\"/events/\", \"href\":\"https://fake-site.ghost.is/events/\"}"
|
||||
],
|
||||
"weights": [
|
||||
200, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 400,
|
||||
100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
71
ghost/tinybird/pipes/analytics_hits.pipe
Normal file
71
ghost/tinybird/pipes/analytics_hits.pipe
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
DESCRIPTION >
|
||||
Parsed `page_hit` events, implementing `browser` and `device` detection logic.
|
||||
|
||||
TOKEN "dashboard" READ
|
||||
|
||||
NODE parsed_hits
|
||||
DESCRIPTION >
|
||||
Parse raw page_hit events
|
||||
|
||||
SQL >
|
||||
SELECT
|
||||
timestamp,
|
||||
action,
|
||||
version,
|
||||
coalesce(session_id, '0') as session_id,
|
||||
JSONExtractString(payload, 'locale') as locale,
|
||||
JSONExtractString(payload, 'location') as location,
|
||||
JSONExtractString(payload, 'referrer') as referrer,
|
||||
JSONExtractString(payload, 'pathname') as pathname,
|
||||
JSONExtractString(payload, 'href') as href,
|
||||
JSONExtractString(payload, 'site_uuid') as site_uuid,
|
||||
JSONExtractString(payload, 'member_uuid') as member_uuid,
|
||||
JSONExtractString(payload, 'member_status') as member_status,
|
||||
JSONExtractString(payload, 'post_uuid') as post_uuid,
|
||||
lower(JSONExtractString(payload, 'user-agent')) as user_agent
|
||||
FROM analytics_events
|
||||
where action = 'page_hit'
|
||||
|
||||
NODE endpoint
|
||||
SQL >
|
||||
SELECT
|
||||
site_uuid,
|
||||
timestamp,
|
||||
action,
|
||||
version,
|
||||
session_id,
|
||||
case
|
||||
when member_uuid REGEXP '^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$'
|
||||
then true
|
||||
else false
|
||||
END as logged_in,
|
||||
member_uuid,
|
||||
member_status,
|
||||
post_uuid,
|
||||
location,
|
||||
referrer,
|
||||
pathname,
|
||||
href,
|
||||
case
|
||||
when match(user_agent, 'wget|ahrefsbot|curl|urllib|bitdiscovery|\+https://|googlebot')
|
||||
then 'bot'
|
||||
when match(user_agent, 'android')
|
||||
then 'mobile-android'
|
||||
when match(user_agent, 'ipad|iphone|ipod')
|
||||
then 'mobile-ios'
|
||||
else 'desktop'
|
||||
END as device,
|
||||
case
|
||||
when match(user_agent, 'firefox')
|
||||
then 'firefox'
|
||||
when match(user_agent, 'chrome|crios')
|
||||
then 'chrome'
|
||||
when match(user_agent, 'opera')
|
||||
then 'opera'
|
||||
when match(user_agent, 'msie|trident')
|
||||
then 'ie'
|
||||
when match(user_agent, 'iphone|ipad|safari')
|
||||
then 'safari'
|
||||
else 'Unknown'
|
||||
END as browser
|
||||
FROM parsed_hits
|
||||
24
ghost/tinybird/pipes/analytics_pages.pipe
Normal file
24
ghost/tinybird/pipes/analytics_pages.pipe
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
NODE analytics_pages_1
|
||||
DESCRIPTION >
|
||||
Aggregate by pathname and calculate session and hits
|
||||
|
||||
SQL >
|
||||
SELECT
|
||||
site_uuid,
|
||||
member_uuid,
|
||||
member_status,
|
||||
post_uuid,
|
||||
toDate(timestamp) AS date,
|
||||
device,
|
||||
browser,
|
||||
location,
|
||||
pathname,
|
||||
uniqState(session_id) AS visits,
|
||||
countState() AS hits,
|
||||
countStateIf(logged_in = true) AS logged_in_hits,
|
||||
countStateIf(logged_in = false) AS logged_out_hits
|
||||
FROM analytics_hits
|
||||
GROUP BY date, device, browser, location, pathname, member_uuid, member_status, post_uuid, site_uuid
|
||||
|
||||
TYPE MATERIALIZED
|
||||
DATASOURCE analytics_pages_mv
|
||||
20
ghost/tinybird/pipes/analytics_sessions.pipe
Normal file
20
ghost/tinybird/pipes/analytics_sessions.pipe
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
NODE analytics_sessions_1
|
||||
DESCRIPTION >
|
||||
Aggregate by session_id and calculate session metrics
|
||||
|
||||
SQL >
|
||||
SELECT
|
||||
site_uuid,
|
||||
toDate(timestamp) AS date,
|
||||
session_id,
|
||||
anySimpleState(device) AS device,
|
||||
anySimpleState(browser) AS browser,
|
||||
anySimpleState(location) AS location,
|
||||
minSimpleState(timestamp) AS first_hit,
|
||||
maxSimpleState(timestamp) AS latest_hit,
|
||||
countState() AS hits
|
||||
FROM analytics_hits
|
||||
GROUP BY date, session_id, site_uuid
|
||||
|
||||
TYPE MATERIALIZED
|
||||
DATASOURCE analytics_sessions_mv
|
||||
21
ghost/tinybird/pipes/analytics_sources.pipe
Normal file
21
ghost/tinybird/pipes/analytics_sources.pipe
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
NODE analytics_sources_1
|
||||
DESCRIPTION >
|
||||
Aggregate by referral and calculate session and hits
|
||||
|
||||
SQL >
|
||||
WITH (SELECT domainWithoutWWW(href) FROM analytics_hits LIMIT 1) AS current_domain
|
||||
SELECT
|
||||
site_uuid,
|
||||
toDate(timestamp) AS date,
|
||||
device,
|
||||
browser,
|
||||
location,
|
||||
referrer,
|
||||
uniqState(session_id) AS visits,
|
||||
countState() AS hits
|
||||
FROM analytics_hits
|
||||
WHERE domainWithoutWWW(referrer) != current_domain
|
||||
GROUP BY date, device, browser, location, referrer, site_uuid
|
||||
|
||||
TYPE MATERIALIZED
|
||||
DATASOURCE analytics_sources_mv
|
||||
130
ghost/tinybird/pipes/kpis.pipe
Normal file
130
ghost/tinybird/pipes/kpis.pipe
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
DESCRIPTION >
|
||||
Summary with general KPIs per date, including visits, page views, bounce rate and average session duration.
|
||||
Accepts `date_from` and `date_to` date filter, all historical data if not passed.
|
||||
Daily granularity, except when filtering one single day (hourly)
|
||||
|
||||
TOKEN "dashboard" READ
|
||||
|
||||
NODE timeseries
|
||||
DESCRIPTION >
|
||||
Generate a timeseries for the specified time range, so we call fill empty data points.
|
||||
Filters "future" data points.
|
||||
|
||||
SQL >
|
||||
%
|
||||
{% set _single_day = defined(date_from) and day_diff(date_from, date_to) == 0 %}
|
||||
with
|
||||
{% if defined(date_from) %}
|
||||
toStartOfDay(
|
||||
toDate(
|
||||
{{
|
||||
Date(
|
||||
date_from,
|
||||
description="Starting day for filtering a date range",
|
||||
required=False,
|
||||
)
|
||||
}}
|
||||
)
|
||||
) as start,
|
||||
{% else %} toStartOfDay(timestampAdd(today(), interval -7 day)) as start,
|
||||
{% end %}
|
||||
{% if defined(date_to) %}
|
||||
toStartOfDay(
|
||||
toDate(
|
||||
{{
|
||||
Date(
|
||||
date_to,
|
||||
description="Finishing day for filtering a date range",
|
||||
required=False,
|
||||
)
|
||||
}}
|
||||
)
|
||||
) as end
|
||||
{% else %} toStartOfDay(today()) as end
|
||||
{% end %}
|
||||
{% if _single_day %}
|
||||
select
|
||||
arrayJoin(
|
||||
arrayMap(
|
||||
x -> toDateTime(x),
|
||||
range(
|
||||
toUInt32(toDateTime(start)), toUInt32(timestampAdd(end, interval 1 day)), 3600
|
||||
)
|
||||
)
|
||||
) as date
|
||||
{% else %}
|
||||
select
|
||||
arrayJoin(
|
||||
arrayMap(
|
||||
x -> toDate(x),
|
||||
range(toUInt32(start), toUInt32(timestampAdd(end, interval 1 day)), 24 * 3600)
|
||||
)
|
||||
) as date
|
||||
{% end %}
|
||||
where date <= now()
|
||||
|
||||
NODE hits
|
||||
DESCRIPTION >
|
||||
Group by sessions and calculate metrics at that level
|
||||
|
||||
SQL >
|
||||
%
|
||||
{% if defined(date_from) and day_diff(date_from, date_to) == 0 %}
|
||||
select
|
||||
site_uuid,
|
||||
toStartOfHour(timestamp) as date,
|
||||
session_id,
|
||||
uniq(session_id) as visits,
|
||||
count() as pageviews,
|
||||
case when min(timestamp) = max(timestamp) then 1 else 0 end as is_bounce,
|
||||
max(timestamp) as latest_hit_aux,
|
||||
min(timestamp) as first_hit_aux
|
||||
from analytics_hits
|
||||
where toDate(timestamp) = {{ Date(date_from) }}
|
||||
group by toStartOfHour(timestamp), session_id, site_uuid
|
||||
{% else %}
|
||||
select
|
||||
site_uuid,
|
||||
date,
|
||||
session_id,
|
||||
uniq(session_id) as visits,
|
||||
countMerge(hits) as pageviews,
|
||||
case when min(first_hit) = max(latest_hit) then 1 else 0 end as is_bounce,
|
||||
max(latest_hit) as latest_hit_aux,
|
||||
min(first_hit) as first_hit_aux
|
||||
from analytics_sessions_mv
|
||||
where
|
||||
{% if defined(date_from) %} date >= {{ Date(date_from) }}
|
||||
{% else %} date >= timestampAdd(today(), interval -7 day)
|
||||
{% end %}
|
||||
{% if defined(date_to) %} and date <= {{ Date(date_to) }}
|
||||
{% else %} and date <= today()
|
||||
{% end %}
|
||||
group by date, session_id, site_uuid
|
||||
{% end %}
|
||||
|
||||
NODE data
|
||||
DESCRIPTION >
|
||||
General KPIs per date, works for both summary metrics and trends charts.
|
||||
|
||||
SQL >
|
||||
select
|
||||
site_uuid,
|
||||
date,
|
||||
uniq(session_id) as visits,
|
||||
sum(pageviews) as pageviews,
|
||||
sum(case when latest_hit_aux = first_hit_aux then 1 end) / visits as bounce_rate,
|
||||
avg(latest_hit_aux - first_hit_aux) as avg_session_sec
|
||||
from hits
|
||||
group by date, site_uuid
|
||||
|
||||
NODE endpoint
|
||||
DESCRIPTION >
|
||||
Join and generate timeseries with metrics
|
||||
|
||||
SQL >
|
||||
%
|
||||
select a.date, b.visits, b.pageviews, b.bounce_rate, b.avg_session_sec
|
||||
from timeseries a
|
||||
left join data b using date
|
||||
where site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}}
|
||||
32
ghost/tinybird/pipes/top_browsers.pipe
Normal file
32
ghost/tinybird/pipes/top_browsers.pipe
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
DESCRIPTION >
|
||||
Top Browsers ordered by most visits.
|
||||
Accepts `date_from` and `date_to` date filter. Defaults to last 7 days.
|
||||
Also `skip` and `limit` parameters for pagination.
|
||||
|
||||
TOKEN "dashboard" READ
|
||||
|
||||
NODE endpoint
|
||||
DESCRIPTION >
|
||||
Group by browser and calculate hits and visits
|
||||
|
||||
SQL >
|
||||
%
|
||||
select browser, uniqMerge(visits) as visits, countMerge(hits) as hits
|
||||
from analytics_sources_mv
|
||||
where
|
||||
site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and
|
||||
{% if defined(date_from) %}
|
||||
date
|
||||
>=
|
||||
{{ Date(date_from, description="Starting day for filtering a date range", required=False) }}
|
||||
{% else %} date >= timestampAdd(today(), interval -7 day)
|
||||
{% end %}
|
||||
{% if defined(date_to) %}
|
||||
and date
|
||||
<=
|
||||
{{ Date(date_to, description="Finishing day for filtering a date range", required=False) }}
|
||||
{% else %} and date <= today()
|
||||
{% end %}
|
||||
group by browser
|
||||
order by visits desc
|
||||
limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }}
|
||||
33
ghost/tinybird/pipes/top_devices.pipe
Normal file
33
ghost/tinybird/pipes/top_devices.pipe
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
|
||||
DESCRIPTION >
|
||||
Top Device Types ordered by most visits.
|
||||
Accepts `date_from` and `date_to` date filter. Defaults to last 7 days.
|
||||
Also `skip` and `limit` parameters for pagination.
|
||||
|
||||
TOKEN "dashboard" READ
|
||||
|
||||
NODE endpoint
|
||||
DESCRIPTION >
|
||||
Group by device and calculate hits and visits
|
||||
|
||||
SQL >
|
||||
%
|
||||
select device, uniqMerge(visits) as visits, countMerge(hits) as hits
|
||||
from analytics_sources_mv
|
||||
where
|
||||
site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and
|
||||
{% if defined(date_from) %}
|
||||
date
|
||||
>=
|
||||
{{ Date(date_from, description="Starting day for filtering a date range", required=False) }}
|
||||
{% else %} date >= timestampAdd(today(), interval -7 day)
|
||||
{% end %}
|
||||
{% if defined(date_to) %}
|
||||
and date
|
||||
<=
|
||||
{{ Date(date_to, description="Finishing day for filtering a date range", required=False) }}
|
||||
{% else %} and date <= today()
|
||||
{% end %}
|
||||
group by device
|
||||
order by visits desc
|
||||
limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }}
|
||||
32
ghost/tinybird/pipes/top_locations.pipe
Normal file
32
ghost/tinybird/pipes/top_locations.pipe
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
DESCRIPTION >
|
||||
Top visiting Countries ordered by most visits.
|
||||
Accepts `date_from` and `date_to` date filter. Defaults to last 7 days.
|
||||
Also `skip` and `limit` parameters for pagination.
|
||||
|
||||
TOKEN "dashboard" READ
|
||||
|
||||
NODE endpoint
|
||||
DESCRIPTION >
|
||||
Group by pagepath and calculate hits and visits
|
||||
|
||||
SQL >
|
||||
%
|
||||
select location, uniqMerge(visits) as visits, countMerge(hits) as hits
|
||||
from analytics_pages_mv
|
||||
where
|
||||
site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and
|
||||
{% if defined(date_from) %}
|
||||
date
|
||||
>=
|
||||
{{ Date(date_from, description="Starting day for filtering a date range", required=False) }}
|
||||
{% else %} date >= timestampAdd(today(), interval -7 day)
|
||||
{% end %}
|
||||
{% if defined(date_to) %}
|
||||
and date
|
||||
<=
|
||||
{{ Date(date_to, description="Finishing day for filtering a date range", required=False) }}
|
||||
{% else %} and date <= today()
|
||||
{% end %}
|
||||
group by location
|
||||
order by visits desc
|
||||
limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }}
|
||||
38
ghost/tinybird/pipes/top_pages.pipe
Normal file
38
ghost/tinybird/pipes/top_pages.pipe
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
DESCRIPTION >
|
||||
Most visited pages for a given period.
|
||||
Accepts `date_from` and `date_to` date filter. Defaults to last 7 days.
|
||||
Also `skip` and `limit` parameters for pagination.
|
||||
|
||||
TOKEN "dashboard" READ
|
||||
|
||||
NODE endpoint
|
||||
DESCRIPTION >
|
||||
Group by pagepath and calculate hits and visits
|
||||
|
||||
SQL >
|
||||
%
|
||||
select
|
||||
pathname,
|
||||
uniqMerge(visits) as visits,
|
||||
countMerge(hits) as hits,
|
||||
countMerge(logged_in_hits) as logged_in_hits,
|
||||
countMerge(logged_out_hits) as logged_out_hits
|
||||
from analytics_pages_mv
|
||||
where
|
||||
site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and
|
||||
{% if defined(date_from) %}
|
||||
date
|
||||
>=
|
||||
{{ Date(date_from, description="Starting day for filtering a date range", required=False) }}
|
||||
{% else %} date >= timestampAdd(today(), interval -7 day)
|
||||
{% end %}
|
||||
{% if defined(date_to) %}
|
||||
and date
|
||||
<=
|
||||
{{ Date(date_to, description="Finishing day for filtering a date range", required=False) }}
|
||||
{% else %} and date <= today()
|
||||
{% end %}
|
||||
|
||||
group by pathname
|
||||
order by visits desc
|
||||
limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }}
|
||||
33
ghost/tinybird/pipes/top_sources.pipe
Normal file
33
ghost/tinybird/pipes/top_sources.pipe
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
|
||||
DESCRIPTION >
|
||||
Top traffic sources (domains), ordered by most visits.
|
||||
Accepts `date_from` and `date_to` date filter. Defaults to last 7 days.
|
||||
Also `skip` and `limit` parameters for pagination.
|
||||
|
||||
TOKEN "dashboard" READ
|
||||
|
||||
NODE endpoint
|
||||
DESCRIPTION >
|
||||
Group by referral and calculate hits and visits
|
||||
|
||||
SQL >
|
||||
%
|
||||
select domainWithoutWWW(referrer) as referrer, uniqMerge(visits) as visits, countMerge(hits) as hits
|
||||
from analytics_sources_mv
|
||||
where
|
||||
site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and
|
||||
{% if defined(date_from) %}
|
||||
date
|
||||
>=
|
||||
{{ Date(date_from, description="Starting day for filtering a date range", required=False) }}
|
||||
{% else %} date >= timestampAdd(today(), interval -7 day)
|
||||
{% end %}
|
||||
{% if defined(date_to) %}
|
||||
and date
|
||||
<=
|
||||
{{ Date(date_to, description="Finishing day for filtering a date range", required=False) }}
|
||||
{% else %} and date <= today()
|
||||
{% end %}
|
||||
group by referrer
|
||||
order by visits desc
|
||||
limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }}
|
||||
36
ghost/tinybird/pipes/trend.pipe
Normal file
36
ghost/tinybird/pipes/trend.pipe
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
|
||||
DESCRIPTION >
|
||||
Visits trend over time for the last 30 minutes, filling the blanks.
|
||||
Works great for the realtime chart.
|
||||
|
||||
TOKEN "dashboard" READ
|
||||
|
||||
NODE timeseries
|
||||
DESCRIPTION >
|
||||
Generate a timeseries for the last 30 minutes, so we call fill empty data points
|
||||
|
||||
SQL >
|
||||
with (now() - interval 30 minute) as start
|
||||
select addMinutes(toStartOfMinute(start), number) as t
|
||||
from (select arrayJoin(range(1, 31)) as number)
|
||||
|
||||
NODE hits
|
||||
DESCRIPTION >
|
||||
Get last 30 minutes metrics gropued by minute
|
||||
|
||||
SQL >
|
||||
%
|
||||
select toStartOfMinute(timestamp) as t, uniq(session_id) as visits
|
||||
from analytics_hits
|
||||
where
|
||||
site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and
|
||||
timestamp >= (now() - interval 30 minute)
|
||||
group by toStartOfMinute(timestamp)
|
||||
order by toStartOfMinute(timestamp)
|
||||
|
||||
NODE endpoint
|
||||
DESCRIPTION >
|
||||
Join and generate timeseries with metrics for the last 30 minutes
|
||||
|
||||
SQL >
|
||||
select a.t, b.visits from timeseries a left join hits b on a.t = b.t order by a.t
|
||||
Loading…
Add table
Reference in a new issue