From 6f2167a437f536cd45c1d0ee1eee26bd584f2f62 Mon Sep 17 00:00:00 2001 From: Stephen Cameron Date: Thu, 22 Sep 2022 23:26:30 +0200 Subject: [PATCH] Better automatic period selection, period override and other improvements. WIP #5316 @12 --- .../FluidbookPublication/StatsOperation.php | 213 ++++++++++++++---- resources/views/fluidbook_stats/API.blade.php | 2 +- .../views/fluidbook_stats/summary.blade.php | 75 +++++- 3 files changed, 238 insertions(+), 52 deletions(-) diff --git a/app/Http/Controllers/Admin/Operations/FluidbookPublication/StatsOperation.php b/app/Http/Controllers/Admin/Operations/FluidbookPublication/StatsOperation.php index c46391d87..54133d81a 100644 --- a/app/Http/Controllers/Admin/Operations/FluidbookPublication/StatsOperation.php +++ b/app/Http/Controllers/Admin/Operations/FluidbookPublication/StatsOperation.php @@ -15,7 +15,7 @@ trait StatsOperation protected function setupStatsRoutes($segment, $routeName, $controller) { // Main route is only secured by hash (security by obscurity) - Route::get($segment . '/stats/{fluidbook_id}_{hash}/{date?}', $controller . '@statsSummary') + Route::get($segment . '/stats/{fluidbook_id}_{hash}/{date?}/{period_override?}', $controller . '@statsSummary') ->withoutMiddleware([CheckIfAdmin::class]) ->name('stats'); // Named route is used to generate URLs more consistently using route helper @@ -87,20 +87,63 @@ trait StatsOperation preg_match($regex, $date, $date_matches); + extract($date_matches); // Just for easier access to match variables + // Bail out on nonsensical dates - if(isset($date_matches['start_date']) && isset($date_matches['end_date']) && ($date_matches['start_date'] > $date_matches['end_date'])) { + if(isset($start_date) && isset($end_date) && ($start_date > $end_date)) { return false; } return $date_matches; } - protected function statsSummary($fluidbook_id, $hash, $date = null) { + // Figure out the best period to use for the stats according to the date range + public function determinePeriod($dates, $fluidbook) { + + // If no valid date range is given, use the Fluidbook's lifetime + if (empty($dates) || !is_array($dates)) { + $dates['start_date'] = $fluidbook->created_at->isoFormat('YYYY-MM-DD'); + $dates['end_date'] = now()->isoFormat('YYYY-MM-DD'); + } + + // Extract array keys as variables for easier access: + // $start_date, $start_year, $start_month, $start_day, $end_date + extract($dates); + + // At this point, if the end_date still isn't set, it must either be a YYYY or YYYY-MM that was passed in. + // Therefore, we can also assume that $start_year will be set. All that's left to do is set appropriate + // start and end dates so that we can determine the best period based on the overall length of the range. + if (!isset($end_date)) { + if (isset($start_month) && !empty($start_month)) { // Month mode + $start_date = "{$start_year}-{$start_month}-01"; + $end_date = Carbon::parse($start_date)->endOfMonth()->isoFormat('YYYY-MM-DD'); + } else { // Year mode + $start_date = "{$start_year}-01-01"; + $end_date = Carbon::parse($start_date)->endOfYear()->isoFormat('YYYY-MM-DD'); + } + } + + // Count <= 60 days: period = day + // Count 61–180 days: period = week + // Count 180+ days: period = month + // Count 2+ years: period = year + $day_count = Carbon::parse($start_date)->diffInDays($end_date); + + if ($day_count <= 50) { + return 'day'; + } elseif ($day_count <= 180) { + return 'week'; + } elseif ($day_count <= 730) { + return 'month'; + } + + return 'year'; + } + + protected function statsSummary($fluidbook_id, $hash, $date = null, $period_override = null) { - $dates = $date ? $this->parseDate($date) : false; $locale = app()->getLocale(); $base_URL = route('stats', compact('fluidbook_id', 'hash')); // Used by date range picker to update report URL - $report_timespan = ''; $fluidbook = FluidbookPublication::where('id', $fluidbook_id)->where('hash', $hash)->firstOrFail(); $fluidbook_settings = json_decode($fluidbook->settings); @@ -122,13 +165,28 @@ trait StatsOperation // - month = summary of stats broken down by month(s) // - day = summary of stats broken down by day(s) + // Translatable list of labels for the available periods + $period_map = [ + 'day' => __('Day'), + 'week' => __('Week'), + 'month' => __('Month'), + 'year' => __('Year'), + ]; + + $dates = $date ? $this->parseDate($date) : false; + + // Make sure that we receive a valid period from the URL + $period_override = in_array($period_override, array_keys($period_map)) ? $period_override : null; + $period = $period_override ?? $this->determinePeriod($dates, $fluidbook); + + $report_timespan = ''; + // Which mode are we in? if (isset($dates['start_date']) && isset($dates['end_date'])) { // Custom date range $mode = 'range'; $start_date = $dates['start_date']; $end_date = $dates['end_date']; $date_range = "{$start_date},{$end_date}"; - $period = 'day'; // Segregate stats by day $formatted_date_range = Carbon::parse($start_date)->isoFormat('MMMM Do, YYYY') . ' — ' . Carbon::parse($end_date)->isoFormat('MMMM Do, YYYY'); $chart_heading = __('Daily Details'); @@ -138,14 +196,14 @@ trait StatsOperation // on those days. Therefore, when getting the difference, it's always a day short because it only // calculates until the *start* of the end_date, effectively excluding it. To get a clean result, // we add 1 day to the end date (setting time to 23:59:59 has undesired effects with the diff display) - $report_timespan = Carbon::parse($dates['start_date'])->startOfDay()->diffForHumans(Carbon::parse($dates['end_date'])->addDay(), [ + $report_timespan = Carbon::parse($start_date)->startOfDay()->diffForHumans(Carbon::parse($end_date)->addDay(), [ 'syntax' => CarbonInterface::DIFF_ABSOLUTE, 'parts' => 3, // How much detail to go into (ie. years, months, days) 'join' => true, // Join string with natural language separators for the locale ]); - } elseif (isset($dates['start_year']) && isset($dates['start_month'])) { // Month view + } elseif (!empty($dates['start_year']) && !empty($dates['start_month'])) { // Month view $mode = 'month'; $month = $dates['start_month']; $year = $dates['start_year']; @@ -153,15 +211,13 @@ trait StatsOperation $start_date = "{$year}-{$month}-01"; $date_range = "{$start_date},{$year}-{$month}-{$last_day_of_month}"; $end_date = ($month == date('m') && $year == date('Y')) ? date('Y-m-d') : "{$year}-{$month}-{$last_day_of_month}"; - $period = 'day'; // Segregate stats by day $chart_heading = __('Daily Details') . '' . Carbon::parse($start_date)->isoFormat('MMMM YYYY') . ''; $formatted_date_range = Carbon::parse($start_date)->isoFormat('Do') . ' — ' . Carbon::parse($end_date)->isoFormat('Do MMMM, YYYY'); - } elseif (isset($dates['start_year'])) { // Year view + } elseif (!empty($dates['start_year'])) { // Year view $mode = 'year'; - $period = 'month'; // Segregate stats by month $year = $dates['start_year']; $start_date = "{$year}-01-01"; $date_range = "{$start_date},{$year}-12-31"; // Full range of specified year (guarantees a full chart, even if it's early in the current year) @@ -172,14 +228,13 @@ trait StatsOperation } else { // No valid dates specified, display the full data set $mode = 'overview'; - $period = 'month'; // Segregate stats by month $start_date = $fluidbook->created_at->isoFormat('YYYY-MM-DD'); - $end_date = Carbon::now()->isoFormat('YYYY-MM-DD'); + $end_date = now()->isoFormat('YYYY-MM-DD'); // If the Fluidbook is recent, the range will be very short (eg. just a few days or a month), which makes the // bar chart look strange due to lack of entries. There's no easy way to limit the width of the bar chart while // retaining the responsiveness, so instead we fetch a longer period of stats, even if they'll be mostly empty. - if (Carbon::now()->diffInMonths($fluidbook->created_at) < 12) { // For the overview, we want at least 12 months + if ($period === 'month' && $fluidbook->created_at->diffInMonths($end_date) < 12) { // For the overview, we want at least 12 months //$date_range = 'last12'; // Special date that Matomo understands // Originally this used the special "last12" setting that Matomo understands but this causes problems // later when we need to fetch certain statistics with a period of "range" - real dates are needed. @@ -208,6 +263,11 @@ trait StatsOperation // is recorded for the initial page at the same time). In the future, we might use these extra stats for other // types of reports but for now, they have to be filtered. + // * Note about unique visitors: + // We added unique visitor stats but found that they were only different to the 'visits' count in our historical + // data from the old stats system. With Matomo, the uniques always seem to be the same as the visits, so they + // don't serve any useful purpose here, only highlighting the inconsistencies with the historical stats. + //=== CUSTOM EVENT STATISTICS (zooms, bookmarks, shares, links, downloads, prints, etc.) // This gathers the events based on their categories and segregated by periods (days, months etc). // Events are returned in a different structure and aren't always the same for each period. @@ -258,6 +318,7 @@ trait StatsOperation 'page_group' => $page_group, 'page_number' => $page_number, // Used by table column sorter 'nb_visits' => $pageUrls["/page/$page_number"]['nb_visits'] ?? 0, + //'nb_unique_visits' => $pageUrls["/page/$page_number"]['sum_daily_nb_uniq_visitors'] ?? 0, 'nb_pageviews' => $pageUrls["/page/$page_number"]['nb_hits'] ?? 0, 'nb_zooms' => data_get($eventsByPage, "zoom.subtable.$page_number.nb_events", 0), 'nb_bookmarks' => data_get($eventsByPage, "bookmark.subtable.$page_number.nb_events", 0), @@ -268,6 +329,7 @@ trait StatsOperation // for our stats labelled as "page 1", we must get certain stats from the "page 0" data... if (!$pages_are_independent && $page_number === 1) { $pages[$page_group]['nb_visits'] = $pageUrls["/page/0"]['nb_visits'] ?? 0; + $pages[$page_group]['nb_unique_visits'] = $pageUrls["/page/0"]['sum_daily_nb_uniq_visitors'] ?? 0; $pages[$page_group]['nb_pageviews'] = $pageUrls["/page/0"]['nb_hits'] ?? 0; $pages[$page_group]['nb_zooms'] = data_get($eventsByPage, "zoom.subtable.0.nb_events", 0); @@ -281,14 +343,17 @@ trait StatsOperation $page_group = $page_number .'—'. $following_page; $pages[$page_group] = [ - 'page_group' => $page_group, + 'page_group' => $page_group, 'page_number' => $page_number, // Used by table column sorter - 'nb_visits' => $pageUrls["/page/$page_number"]['nb_visits'] ?? 0, + 'nb_visits' => $pageUrls["/page/$page_number"]['nb_visits'] ?? 0, + // 'nb_unique_visits' => $pageUrls["/page/$page_number"]['sum_daily_nb_uniq_visitors'] ?? 0, 'nb_pageviews' => $pageUrls["/page/$page_number"]['nb_hits'] ?? 0, - 'nb_zooms' => data_get($eventsByPage, "zoom.subtable.$page_number.nb_events", 0), + 'nb_zooms' => data_get($eventsByPage, "zoom.subtable.$page_number.nb_events", 0), // Bookmarks and shares are counted and summed for both pages in the spread - 'nb_bookmarks' => data_get($eventsByPage, "bookmark.subtable.$page_number.nb_events", 0) + data_get($eventsByPage, "bookmark.subtable.$following_page.nb_events", 0), - 'nb_shares' => data_get($eventsByPage, "share.subtable.$page_number.nb_events", 0) + data_get($eventsByPage, "share.subtable.$following_page.nb_events", 0), + 'nb_bookmarks' => data_get($eventsByPage, "bookmark.subtable.$page_number.nb_events", 0) + + data_get($eventsByPage, "bookmark.subtable.$following_page.nb_events", 0), + 'nb_shares' => data_get($eventsByPage, "share.subtable.$page_number.nb_events", 0) + + data_get($eventsByPage, "share.subtable.$following_page.nb_events", 0), ]; } } @@ -338,7 +403,10 @@ trait StatsOperation //=== MAIN PERIOD STATISTICS // These are the main statistics used to build the chart and the table below it. // Statistics are segregated by a certain period (eg. years, months or days) - $pagesByPeriod = collect($report->getPageUrls()) + // Note: in order to get the total "nb_uniq_visitors" for the pages, we would need to fetch + // the expanded dataset that includes the subtables of pages. For some reason, Matomo + // doesn't aggregrate this value when there are sub pages, so we have to do it ourselves + $pagesByPeriod = collect($report->getPageUrls(['expanded' => 0])) ->map(function ($item, $date) use ($period, $fluidbook_id, $hash, $eventsByPeriod) { if (empty($item)) { @@ -359,13 +427,21 @@ trait StatsOperation // the returned data flatter and easier to process later for sums etc. $data = $labelled['page']; + // Unique visitors stats disabled - see notes above. + // // Matomo doesn't provide an aggregate of unique visitors, so we must do the sum ourselves. + // // If the period is "day", the number of unique visitors will be in the key 'nb_uniq_visitors' + // // but if it is a longer period (week, month, year) then the key becomes 'sum_daily_nb_uniq_visitors' + // $unique_visitors_key = $period === 'day' ? 'nb_uniq_visitors' : 'sum_daily_nb_uniq_visitors'; + // $subpages = collect($data['subtable'] ?? []); + // $data['nb_uniq_visitors'] = $subpages->sum($unique_visitors_key); + $data['raw_date'] = $date; // We still need the raw date for sorting and other formatting purposes // Formatting of date changes depending on the period $formatted_date = $this->formatDateForPeriod($date, $period); - // When segregated by month, the periods are links to the detailed month view - if ($period === 'month') { + // When segregated by month or year, the periods become links + if (in_array($period, ['month', 'year'])) { // Generate URL using the named route $URL = route('stats', compact('fluidbook_id', 'hash', 'date')); $data['formatted_date'] = "{$formatted_date}"; @@ -391,8 +467,8 @@ trait StatsOperation //=== CHART PREPARATION // Format dates for display as labels on the x-axis - $labels = $pagesByPeriod->keys()->map(function ($label, $index) use ($period, $mode) { - return $this->formatDateForXAxis($label, $period, $mode); + $labels = $pagesByPeriod->keys()->map(function ($label, $index) use ($period, $start_date, $end_date) { + return $this->formatDateForXAxis($label, $period, $start_date, $end_date); })->toArray(); // Format dates for display in the tooltip title @@ -413,22 +489,24 @@ trait StatsOperation $table_map = [ // Main summary table 'summary' => [ - 'formatted_date' => __('Period'), - 'nb_visits' => __('Visits'), - 'nb_hits' => __('Pages Viewed'), - 'nb_links' => __('Outgoing Links'), - 'nb_downloads' => __('Downloads'), - 'nb_prints' => __('Prints'), - 'nb_zooms' => __('Zooms'), + 'formatted_date' => $period_map[$period], + // 'nb_uniq_visitors' => __('Unique Visitors'), + 'nb_visits' => __('Visits'), + 'nb_hits' => __('Pages Viewed'), + 'nb_links' => __('Outgoing Links'), + 'nb_downloads' => __('Downloads'), + 'nb_prints' => __('Prints'), + 'nb_zooms' => __('Zooms'), ], // Per-page detail table 'per-page' => [ - 'page_group' => __('Pages'), - 'nb_visits' => __('Visits'), - 'nb_pageviews' => __('Views'), - 'nb_zooms' => __('Zooms'), - 'nb_bookmarks' => __('Bookmarks'), - 'nb_shares' => __('Shares'), + 'page_group' => __('Pages'), + // 'nb_unique_visits' => __('Unique Visits'), + 'nb_visits' => __('Visits'), + 'nb_pageviews' => __('Views'), + 'nb_zooms' => __('Zooms'), + 'nb_bookmarks' => __('Bookmarks'), + 'nb_shares' => __('Shares'), ], ]; @@ -436,6 +514,9 @@ trait StatsOperation return view('fluidbook_stats.summary', compact( + 'fluidbook_id', + 'hash', + 'date', 'fluidbook', 'start_date', 'end_date', @@ -452,6 +533,7 @@ trait StatsOperation 'period', 'dates', 'period_details', + 'period_map', 'table_map', 'pages', 'pagesByPeriod', @@ -494,18 +576,69 @@ trait StatsOperation // Format dates depending on the segregation period (used for tooltips and stats detail tables) protected function formatDateForPeriod($date, $period): string { + + // Weeks are a special case because they contain two dates + if ($period === 'week') { + $weeks = explode(',', $date); + $week_start = Carbon::parse($weeks[0]); + $week_end = Carbon::parse($weeks[1]); + $crosses_month_boundary = $week_end->isoFormat('YYYY-MM') > $week_start->isoFormat('YYYY-MM'); + $crosses_year_boundary = $week_end->isoFormat('YYYY') > $week_start->isoFormat('YYYY'); + if ($crosses_year_boundary) { + $week_formatted = $week_start->isoFormat('Do MMMM, YYYY') .' — '. $week_end->isoFormat('Do MMMM, YYYY'); + } elseif ($crosses_month_boundary) { + $week_formatted = $week_start->isoFormat('Do MMMM') .' — '. $week_end->isoFormat('Do MMMM'); + } else { + $week_formatted = $week_start->isoFormat('Do') .'—'. $week_end->isoFormat('Do MMMM'); + } + } + return match ($period) { 'day' => Carbon::parse($date)->isoFormat('dddd, Do MMMM YYYY'), + 'week' => $week_formatted, 'month' => Carbon::parse($date)->isoFormat('MMMM YYYY'), default => $date, }; } // Dates displayed in the x-axis of chart need different formatting - protected function formatDateForXAxis($date, $period, $mode): string { + protected function formatDateForXAxis($date, $period, $start_date, $end_date): string { + + // The labels on the x-axis should be as concise as possible but when the date range crosses + // a boundary for the larger period (eg. period = day but the range crosses more than one month), + // the formatting needs to be different so that labels are not ambiguous. + $start = Carbon::parse($start_date); + $end = Carbon::parse($end_date); + $crosses_month_boundary = $end->isoFormat('YYYY-MM') > $start->isoFormat('YYYY-MM'); + $crosses_year_boundary = $end->isoFormat('YYYY') > $start->isoFormat('YYYY'); + + // Week period dates need different handling because they contain two comma-separated dates + if ($period === 'week') { + $weeks = explode(',', $date); + $week_start = Carbon::parse($weeks[0]); + $week_end = Carbon::parse($weeks[1]); + $crosses_month_boundary = $week_end->isoFormat('YYYY-MM') > $week_start->isoFormat('YYYY-MM'); + $crosses_year_boundary = $week_end->isoFormat('YYYY') > $week_start->isoFormat('YYYY'); + if ($crosses_year_boundary) { + $week_formatted = $week_start->isoFormat('D MMM YYYY') .' — '. $week_end->isoFormat('D MMM YYYY'); + } elseif ($crosses_month_boundary) { + $week_formatted = $week_start->isoFormat('D MMM') .' — '. $week_end->isoFormat('D MMM'); + } else { + $week_formatted = $week_start->isoFormat('D') .'—'. $week_end->isoFormat('D MMM'); + } + } elseif ($period === 'year') { + // Years also need special handling because if we try to use Carbon::parse() + // with a 4-digit year, it won't give the expected result... + // There's nothing to do here, just let it be returned unprocessed by the default case. + } else { + $date = Carbon::parse($date); + } + return match ($period) { - 'day' => Carbon::parse($date)->isoFormat('DD'), // Convert YYYY-MM-DD string from API into zero-padded day alone - 'month' => $mode === 'overview' ? Carbon::parse($date)->isoFormat('MMM YYYY') : Carbon::parse($date)->isoFormat('MMM'), // Convert to abbreviated month name (including year when showing all months) + 'day' => $crosses_year_boundary ? $date->isoFormat('DD MMM YYYY') + : ($crosses_month_boundary ? $date->isoFormat('DD MMM') : $date->isoFormat('DD')), + 'week' => $week_formatted, + 'month' => $crosses_year_boundary ? $date->isoFormat('MMM YYYY') : $date->isoFormat('MMM'), default => $date, }; } diff --git a/resources/views/fluidbook_stats/API.blade.php b/resources/views/fluidbook_stats/API.blade.php index 175be7442..ddf34ef21 100644 --- a/resources/views/fluidbook_stats/API.blade.php +++ b/resources/views/fluidbook_stats/API.blade.php @@ -103,7 +103,7 @@ function matomo() { return { servers: {!! $matomo_tokens; !!}, - periods: ['day', 'month', 'year', 'range'], + periods: ['day', 'week', 'month', 'year', 'range'], siteID: 16976, method: '', period: 'month', diff --git a/resources/views/fluidbook_stats/summary.blade.php b/resources/views/fluidbook_stats/summary.blade.php index d48f682bb..d0176578f 100644 --- a/resources/views/fluidbook_stats/summary.blade.php +++ b/resources/views/fluidbook_stats/summary.blade.php @@ -3,6 +3,26 @@ @section('after_styles')