:init: Init

etc/clickhouse/0001-events.sql

@@ -0,0 +1,26 @@
+CREATE TABLE IF NOT EXISTS events
+(
+    `app_id` String,
+    `timestamp` DateTime,
+    `event_name` String,
+    `user_id` String,
+    `session_id` String,
+    `os_name` LowCardinality(String),
+    `os_version` String,
+    `locale` LowCardinality(String),
+    `app_version` String,
+    `app_build_number` String,
+    `engine_name` LowCardinality(String),
+    `engine_version` String,
+    `sdk_version` String,
+    `country_code` LowCardinality(String),
+    `region_name` LowCardinality(String),
+    `city` String,
+    `string_props` String,
+    `numeric_props` String,
+    `ttl` DateTime
+)
+ENGINE = MergeTree()
+PARTITION BY toYYYYMM(timestamp)
+ORDER BY (app_id, timestamp, event_name)
+TTL ttl;

etc/clickhouse/0002-monthly_usage_v1_mv.sql

@@ -0,0 +1,8 @@
+CREATE OR REPLACE VIEW monthly_usage_v1
+AS
+SELECT
+    app_id,
+    toStartOfMonth(timestamp) AS period,
+    countState() AS events
+FROM events
+GROUP BY app_id, period

etc/clickhouse/0003-sessions_live_v1.sql

@@ -0,0 +1,28 @@
+CREATE OR REPLACE VIEW sessions_live_v1
+AS
+SELECT
+    app_id,
+    session_id,
+    user_id,
+    minSimpleState(e.timestamp) AS min_timestamp,
+    maxSimpleState(e.timestamp) AS max_timestamp,
+    anySimpleState(os_name) AS os_name,
+    anySimpleState(os_version) AS os_version,
+    anySimpleState(locale) AS locale,
+    anySimpleState(app_version) AS app_version,
+    anySimpleState(app_build_number) AS app_build_number,
+    anySimpleState(engine_name) AS engine_name,
+    anySimpleState(engine_version) AS engine_version,
+    anySimpleState(sdk_version) AS sdk_version,
+    anySimpleState(country_code) AS country_code,
+    anySimpleState(region_name) AS region_name,
+    countState() AS events_count,
+    groupArrayState(event_name) AS events_name,
+    groupArrayState(e.timestamp) AS events_timestamp,
+    groupArrayState(string_props) AS events_string_props,
+    groupArrayState(numeric_props) AS events_numeric_props
+FROM events AS e
+GROUP BY
+    app_id,
+    session_id,
+    user_id

etc/clickhouse/0004-add_device_model_to_events.sql

@@ -0,0 +1,2 @@
+ALTER TABLE events
+ADD COLUMN IF NOT EXISTS device_model String;

etc/clickhouse/queries/billing_historical_usage__v1.liquid

@@ -0,0 +1,7 @@
+SELECT period as Date,
+       countMerge(events) as Events
+FROM monthly_usage_v1
+WHERE app_id IN ('{{app_ids}}')
+AND period >= toStartOfMonth(now() - INTERVAL 6 MONTH)
+GROUP BY period
+ORDER BY period WITH FILL FROM toStartOfMonth(now() - INTERVAL 6 MONTH) TO toStartOfMonth(now() + INTERVAL 1 MONTH) STEP toIntervalMonth(1)

etc/clickhouse/queries/billing_usage_per_app__v1.liquid

@@ -0,0 +1,5 @@
+SELECT replace(app_id, '_DEBUG', '') as AppId,
+countMerge(events) as Count
+FROM monthly_usage_v1
+WHERE period = '{{period}}'
+GROUP BY replace(app_id, '_DEBUG', '')

etc/clickhouse/queries/get_billing_usage__v1.liquid

@@ -0,0 +1,4 @@
+SELECT countMerge(events) as Count
+FROM monthly_usage_v1
+WHERE app_id IN ('{{app_ids}}')
+AND period = toStartOfMonth(now())

etc/clickhouse/queries/historical_sessions__v1.liquid

@@ -0,0 +1,36 @@
+SELECT session_id as Id,	
+       min(min_timestamp) as StartedAt,
+       max(max_timestamp) - min(min_timestamp) as Duration,
+       countMerge(events_count) as EventsCount,
+       any(app_version) as AppVersion,
+       any(country_code) as CountryCode,
+       any(region_name) as RegionName,
+       any(os_name) as OsName,
+       any(os_version) as OsVersion
+FROM sessions_live_v1
+WHERE app_id = '{{app_id}}'
+{% if country_code %}
+AND country_code = '{{country_code}}'
+{% endif %}
+{% if os_name %}
+AND os_name = '{{os_name}}'
+{% endif %}
+{% if app_version %}
+AND app_version = '{{app_version}}'
+{% endif %}
+{% if date_to %}
+AND (min_timestamp < '{{date_to}}' OR (min_timestamp = '{{date_to}}' AND session_id < '{{session_id}}'))
+{% endif %}
+{% if date_from %}
+AND (min_timestamp > '{{date_from}}' OR (min_timestamp = '{{date_from}}' AND session_id > '{{session_id}}'))
+{% endif %}
+GROUP BY session_id
+{% if event_name %}
+HAVING hasAny(groupArrayMerge(events_name), ['{{event_name}}'])
+{% endif %}
+{% if date_from %}
+ORDER BY StartedAt ASC, Id ASC
+{% else %}
+ORDER BY StartedAt DESC, Id DESC
+{% endif %}
+LIMIT 10

etc/clickhouse/queries/key_metrics__v1.liquid

@@ -0,0 +1,30 @@
+SELECT uniqExact(user_id) / (date_diff('day', min(min), max(max)) + 1) as DailyUsers,
+       uniqExact(session_id) as Sessions,
+       if(isNaN(median(max - min)), 0, median(max - min)) as DurationSeconds,
+       sum(count) as Events
+FROM (
+    SELECT min(timestamp) AS min,
+           max(timestamp) AS max,
+           user_id,
+           session_id,
+           count(*) AS count
+    FROM events
+    PREWHERE app_id = '{{app_id}}'
+    WHERE 1 = 1
+    {% if date_from and date_to %}
+    AND timestamp BETWEEN '{{date_from}}' AND '{{date_to}}'
+    {% endif %}
+    {% if country_code %}
+    AND country_code = '{{country_code}}'
+    {% endif %}
+    {% if event_name %}
+    AND event_name = '{{event_name}}'
+    {% endif %}
+    {% if os_name %}
+    AND os_name = '{{os_name}}'
+    {% endif %}
+    {% if app_version %}
+    AND app_version = '{{app_version}}'
+    {% endif %}
+    GROUP BY user_id, session_id
+)

etc/clickhouse/queries/key_metrics__v2.liquid

@@ -0,0 +1,33 @@
+SELECT uniqExact(user_id) / (date_diff('day', min(min), max(max)) + if(date_diff('day', min(min), max(max)) = 0, 1, 0)) AS DailyUsers,
+       uniqExact(session_id) as Sessions,
+       if(isNaN(median(max - min)), 0, median(max - min)) as DurationSeconds,
+       sum(count) as Events
+FROM (
+    SELECT min(timestamp) AS min,
+           max(timestamp) AS max,
+           user_id,
+           session_id,
+           count(*) AS count
+    FROM events
+    PREWHERE app_id = '{{app_id}}'
+    WHERE 1 = 1
+    {% if date_from and date_to %}
+    AND timestamp BETWEEN '{{date_from}}' AND '{{date_to}}'
+    {% endif %}
+    {% if country_code %}
+    AND country_code = '{{country_code}}'
+    {% endif %}
+    {% if event_name %}
+    AND event_name = '{{event_name}}'
+    {% endif %}
+    {% if os_name %}
+    AND os_name = '{{os_name}}'
+    {% endif %}
+    {% if device_model %}
+    AND device_model = '{{device_model}}'
+    {% endif %}
+    {% if app_version %}
+    AND app_version = '{{app_version}}'
+    {% endif %}
+    GROUP BY user_id, session_id
+)

etc/clickhouse/queries/key_metrics_periodic__v1.liquid

@@ -0,0 +1,53 @@
+SELECT
+    {% if granularity == 'Hour' %}
+     toStartOfHour(timestamp)
+    {% elsif granularity == 'Day' %}
+     toStartOfDay(timestamp)
+    {% else %}
+     toStartOfMonth(timestamp)
+    {% endif %} AS Period,
+    {% if granularity == 'Month' %}
+     uniqExact(user_id) / (date_diff('day', Period, toLastDayOfMonth(Period)) + 1)
+    {% else %}
+     uniqExact(user_id)
+    {% endif %} AS Users,
+    uniqExact(session_id) AS Sessions,
+    count() AS Events
+FROM events
+PREWHERE app_id = '{{app_id}}'
+WHERE 1 = 1
+{% if date_from and date_to %}
+AND timestamp BETWEEN '{{date_from}}' AND '{{date_to}}'
+{% endif %}
+{% if country_code %}
+AND country_code = '{{country_code}}'
+{% endif %}
+{% if event_name %}
+AND event_name = '{{event_name}}'
+{% endif %}
+{% if os_name %}
+AND os_name = '{{os_name}}'
+{% endif %}
+{% if app_version %}
+AND app_version = '{{app_version}}'
+{% endif %}
+GROUP by Period
+ORDER BY Period ASC
+WITH FILL
+{% if date_from and date_to %}
+{% if granularity == 'Hour' %}
+FROM toStartOfHour(toDateTime('{{date_from}}')) TO toStartOfHour(toDateTime('{{date_to}}'))
+{% elsif granularity == 'Day' %}
+FROM toStartOfDay(toDateTime('{{date_from}}')) TO toStartOfDay(toDateTime('{{date_to}}'))
+{% else %}
+FROM toStartOfMonth(toDateTime('{{date_from}}')) TO toStartOfMonth(toDateTime('{{date_to}}'))
+{% endif %}
+{% endif %}
+STEP
+{% if granularity == 'Hour' %}
+toIntervalHour(1)
+{% elsif granularity == 'Day' %}
+toIntervalDay(1) 
+{% else %}
+toIntervalMonth(1)
+{% endif %}

etc/clickhouse/queries/key_metrics_periodic__v2.liquid

@@ -0,0 +1,56 @@
+SELECT
+    {% if granularity == 'Hour' %}
+     toStartOfHour(timestamp)
+    {% elsif granularity == 'Day' %}
+     toStartOfDay(timestamp)
+    {% else %}
+     toStartOfMonth(timestamp)
+    {% endif %} AS Period,
+    {% if granularity == 'Month' %}
+     uniqExact(user_id) / (date_diff('day', Period, toLastDayOfMonth(Period)) + 1)
+    {% else %}
+     uniqExact(user_id)
+    {% endif %} AS Users,
+    uniqExact(session_id) AS Sessions,
+    count() AS Events
+FROM events
+PREWHERE app_id = '{{app_id}}'
+WHERE 1 = 1
+{% if date_from and date_to %}
+AND timestamp BETWEEN '{{date_from}}' AND '{{date_to}}'
+{% endif %}
+{% if country_code %}
+AND country_code = '{{country_code}}'
+{% endif %}
+{% if event_name %}
+AND event_name = '{{event_name}}'
+{% endif %}
+{% if os_name %}
+AND os_name = '{{os_name}}'
+{% endif %}
+{% if device_model %}
+AND device_model = '{{device_model}}'
+{% endif %}
+{% if app_version %}
+AND app_version = '{{app_version}}'
+{% endif %}
+GROUP by Period
+ORDER BY Period ASC
+WITH FILL
+{% if date_from and date_to %}
+{% if granularity == 'Hour' %}
+FROM toStartOfHour(toDateTime('{{date_from}}')) TO toStartOfHour(toDateTime('{{date_to}}'))
+{% elsif granularity == 'Day' %}
+FROM toStartOfDay(toDateTime('{{date_from}}')) TO toStartOfDay(toDateTime('{{date_to}}'))
+{% else %}
+FROM toStartOfMonth(toDateTime('{{date_from}}')) TO toStartOfMonth(toDateTime('{{date_to}}'))
+{% endif %}
+{% endif %}
+STEP
+{% if granularity == 'Hour' %}
+toIntervalHour(1)
+{% elsif granularity == 'Day' %}
+toIntervalDay(1) 
+{% else %}
+toIntervalMonth(1)
+{% endif %}

etc/clickhouse/queries/live_geo__v1.liquid

@@ -0,0 +1,14 @@
+SELECT uniq(user_id) as Users,
+country_code as CountryCode,
+region_name as RegionName
+FROM (
+SELECT session_id,
+ min(min_timestamp) AS timestamp,
+ any(user_id) AS user_id,
+ any(country_code) AS country_code,
+ any(region_name) AS region_name
+FROM sessions_live_v1
+WHERE app_id = '{{app_id}}'
+GROUP BY session_id
+HAVING timestamp >= now() - INTERVAL 1 HOUR
+) GROUP BY country_code, region_name

etc/clickhouse/queries/live_session_details__v1.liquid

@@ -0,0 +1,18 @@
+SELECT session_id as Id,
+       countMerge(events_count) as EventsCount,
+       min(min_timestamp) as StartedAt,
+       max(max_timestamp) - min(min_timestamp) as Duration,
+       countMerge(events_count) as EventsCount,
+       any(app_version) as AppVersion,
+       any(country_code) as CountryCode,
+       any(region_name) as RegionName,
+       any(os_name) as OsName,
+       any(os_version) as OsVersion,
+       groupArrayMerge(events_name) as EventsName,
+       groupArrayMerge(events_timestamp) as EventsTimestamp,
+       groupArrayMerge(events_string_props) as EventsStringProps,
+       groupArrayMerge(events_numeric_props) as EventsNumericProps
+FROM sessions_live_v1
+WHERE app_id = '{{app_id}}'
+AND session_id = '{{session_id}}'
+GROUP BY session_id

etc/clickhouse/queries/live_sessions__v1.liquid

@@ -0,0 +1,15 @@
+SELECT session_id as Id,	
+       min(min_timestamp) as StartedAt,
+       max(max_timestamp) - min(min_timestamp) as Duration,
+       countMerge(events_count) as EventsCount,
+       any(app_version) as AppVersion,
+       any(country_code) as CountryCode,
+       any(region_name) as RegionName,
+       any(os_name) as OsName,
+       any(os_version) as OsVersion
+FROM sessions_live_v1
+WHERE app_id = '{{app_id}}'
+GROUP BY session_id
+HAVING StartedAt >= now() - INTERVAL 1 HOUR
+ORDER BY Duration DESC
+LIMIT 6

etc/clickhouse/queries/monthly_usage__v1.liquid

@@ -0,0 +1,9 @@
+SELECT toYear(Date) as Year, toMonth(Date) as Month, Events
+FROM (
+  SELECT period as Date,
+      countMerge(events) as Events
+  FROM monthly_usage_v1
+  WHERE app_id = '{{app_id}}'
+  GROUP BY period
+  ORDER BY period WITH FILL TO toStartOfMonth(now() + INTERVAL 1 MONTH) STEP toIntervalMonth(1)
+) ORDER BY Year DESC, Month DESC

etc/clickhouse/queries/top_n__v1.liquid

@@ -0,0 +1,26 @@
+SELECT {{name_column}} as Name,
+       {% if value_column == 'UniqueSessions' %}
+        uniqExact(session_id) as Value
+       {% else %}
+        count() as Value
+       {% endif %}
+FROM events
+PREWHERE app_id = '{{app_id}}'
+WHERE 1 = 1
+{% if date_from and date_to %}
+AND timestamp BETWEEN '{{date_from}}' AND '{{date_to}}'
+{% endif %}
+{% if country_code %}
+AND country_code = '{{country_code}}'
+{% endif %}
+{% if event_name %}
+AND event_name = '{{event_name}}'
+{% endif %}
+{% if os_name %}
+AND os_name = '{{os_name}}'
+{% endif %}
+{% if app_version %}
+AND app_version = '{{app_version}}'
+{% endif %}
+GROUP BY Name
+ORDER BY Value DESC

etc/clickhouse/queries/top_n__v2.liquid

@@ -0,0 +1,29 @@
+SELECT {{name_column}} as Name,
+       {% if value_column == 'UniqueSessions' %}
+        uniqExact(session_id) as Value
+       {% else %}
+        count() as Value
+       {% endif %}
+FROM events
+PREWHERE app_id = '{{app_id}}'
+WHERE 1 = 1
+{% if date_from and date_to %}
+AND timestamp BETWEEN '{{date_from}}' AND '{{date_to}}'
+{% endif %}
+{% if country_code %}
+AND country_code = '{{country_code}}'
+{% endif %}
+{% if event_name %}
+AND event_name = '{{event_name}}'
+{% endif %}
+{% if os_name %}
+AND os_name = '{{os_name}}'
+{% endif %}
+{% if device_model %}
+AND device_model = '{{device_model}}'
+{% endif %}
+{% if app_version %}
+AND app_version = '{{app_version}}'
+{% endif %}
+GROUP BY Name
+ORDER BY Value DESC

etc/clickhouse/queries/top_props__v1.liquid

@@ -0,0 +1,34 @@
+SELECT string_arr.1 AS StringKey,
+       string_arr.2 AS StringValue,
+       numeric_arr.1 AS NumericKey,
+       count() AS Events,
+       median(numeric_arr.2) AS Median,
+       min(numeric_arr.2) AS Min,
+       max(numeric_arr.2) AS Max,
+       sum(numeric_arr.2) AS Sum
+FROM (
+    SELECT * FROM (
+        SELECT JSONExtractKeysAndValues(string_props, 'String') AS string_arr,
+               JSONExtractKeysAndValues(numeric_props, 'Float') AS numeric_arr
+        FROM events
+        PREWHERE app_id = '{{app_id}}'
+        WHERE 1 = 1
+        {% if date_from and date_to %}
+        AND timestamp BETWEEN '{{date_from}}' AND '{{date_to}}'
+        {% endif %}
+        {% if country_code %}
+        AND country_code = '{{country_code}}'
+        {% endif %}
+        {% if event_name %}
+        AND event_name = '{{event_name}}'
+        {% endif %}
+        {% if os_name %}
+        AND os_name = '{{os_name}}'
+        {% endif %}
+        {% if app_version %}
+        AND app_version = '{{app_version}}'
+        {% endif %}
+    ) LEFT ARRAY JOIN string_arr
+) LEFT ARRAY JOIN numeric_arr
+GROUP BY StringKey, StringValue, NumericKey
+ORDER BY StringKey, Events DESC

etc/clickhouse/queries/top_props__v2.liquid

@@ -0,0 +1,37 @@
+SELECT string_arr.1 AS StringKey,
+       string_arr.2 AS StringValue,
+       numeric_arr.1 AS NumericKey,
+       count() AS Events,
+       median(numeric_arr.2) AS Median,
+       min(numeric_arr.2) AS Min,
+       max(numeric_arr.2) AS Max,
+       sum(numeric_arr.2) AS Sum
+FROM (
+    SELECT * FROM (
+        SELECT JSONExtractKeysAndValues(string_props, 'String') AS string_arr,
+               JSONExtractKeysAndValues(numeric_props, 'Float') AS numeric_arr
+        FROM events
+        PREWHERE app_id = '{{app_id}}'
+        WHERE 1 = 1
+        {% if date_from and date_to %}
+        AND timestamp BETWEEN '{{date_from}}' AND '{{date_to}}'
+        {% endif %}
+        {% if country_code %}
+        AND country_code = '{{country_code}}'
+        {% endif %}
+        {% if event_name %}
+        AND event_name = '{{event_name}}'
+        {% endif %}
+        {% if os_name %}
+        AND os_name = '{{os_name}}'
+        {% endif %}
+		{% if device_model %}
+		AND device_model = '{{device_model}}'
+		{% endif %}
+        {% if app_version %}
+        AND app_version = '{{app_version}}'
+        {% endif %}
+    ) LEFT ARRAY JOIN string_arr
+) LEFT ARRAY JOIN numeric_arr
+GROUP BY StringKey, StringValue, NumericKey
+ORDER BY StringKey, Events DESC