Expose percent-of-total metrics for report rows in the Reporting API

CHANGELOG.md

@@ -51,6 +51,12 @@ The Product Changelog at **[matomo.org/changelog](https://matomo.org/changelog)*
 * A new `#[Piwik\Http\JsonResponse]` attribute can be applied to a plugin controller action to declare that it returns a JSON response. When present, Matomo (re-)sends the `Content-Type: application/json` header after the action has returned, so it can no longer be overwritten by output produced while the action builds its response (for example a rendered `Piwik\View`, which sends `text/html`). An action using the attribute must return the JSON string, must not send the header itself, and must not emit output (`echo`/`print`/`flush`) or call `exit`/`die` before returning — otherwise the response headers are committed first and the JSON `Content-Type` cannot be applied. The attribute is not inherited: a subclass overriding a JSON action must re-declare it. These requirements are enforced by PHPStan rules.
 
 ### HTTP API
+* Report rows now include a percentage-of-report-total value for each metric the report processes totals for, as an additional
+  `{metric}_percent_of_total` column (eg, `nb_visits_percent_of_total`). The values match the ratio percentages the report tables
+  show on hover, follow `format_metrics` like other percent metrics, and are included in all export formats. Non-additive metrics
+  (unique visitors and users) are excluded, as their report total is not a meaningful denominator. The columns can be disabled
+  by setting the new `percent_of_total=0` request parameter (or `totals=0`). Note for CSV/TSV consumers parsing columns by
+  position: the new columns change the header and column count, pass `percent_of_total=0` to keep the previous output.
 * `API.getBulkRequest` now validates the authentication parameters of each nested request URL against
   the outer request. Within a browser session a nested request may change neither the session flag
   (`force_api_session`) nor the acting user (`token_auth`); outside a session a nested request may still

core/API/DataTablePostProcessor.php

@@ -24,6 +24,7 @@
 use Piwik\Plugin\ReportsProvider;
 use Piwik\Plugins\API\Filter\DataComparisonFilter;
 use Piwik\Plugins\CoreHome\Columns\Metrics\EvolutionMetric;
+use Piwik\Plugins\CoreHome\Columns\Metrics\PercentOfReportTotal;
 use Piwik\Plugins\PrivacyManager\DataRounding;
 use Piwik\Request;
 
@@ -111,6 +112,7 @@ public function process(DataTableInterface $dataTable)
         $dataTable = $this->applyPivotByFilter($dataTable);
         $dataTable = $this->applyTotalsCalculator($dataTable);
         $dataTable = $this->applyFlattener($dataTable);
+        $dataTable = $this->applyPercentOfTotalMetrics($dataTable);
 
         if ($this->callbackBeforeGenericFilters) {
             call_user_func($this->callbackBeforeGenericFilters, $dataTable);
@@ -223,6 +225,46 @@ public function applyTotalsCalculator($dataTable)
         return $dataTable;
     }
 
+    /**
+     * Registers processed metrics exposing each row's metric value as a percentage of the
+     * report total (eg, 'nb_visits_percent_of_total'), based on the totals calculated by
+     * ReportTotalsCalculator. Can be disabled with percent_of_total=0.
+     *
+     * @param DataTableInterface $dataTable
+     * @return DataTableInterface
+     */
+    public function applyPercentOfTotalMetrics($dataTable)
+    {
+        if (!(new Request($this->request))->getBoolParameter('percent_of_total', true)) {
+            return $dataTable;
+        }
+
+        // tables can inherit totals metadata from another API request (eg, Referrers.getAll),
+        // disabling totals should disable the percentages based on them as well
+        if (1 != Common::getRequestVar('totals', '1', 'integer', $this->request)) {
+            return $dataTable;
+        }
+
+        if (!$this->report || !$this->report->getDimension()) {
+            // without a report dimension there is a single row (eg, VisitsSummary.get),
+            // a percentage of the total is not meaningful
+            return $dataTable;
+        }
+
+        if (Common::getRequestVar('pivotBy', false, 'string', $this->request)) {
+            // pivoted tables have one column per pivot dimension value, a percentage of
+            // the report total is not meaningful there
+            return $dataTable;
+        }
+
+        $report = $this->report;
+        $dataTable->filter(function (DataTable $table) use ($report) {
+            PercentOfReportTotal::addMetricsToTable($table, $report);
+        });
+
+        return $dataTable;
+    }
+
     /**
      * @param DataTableInterface $dataTable
      * @return DataTableInterface
@@ -250,6 +292,12 @@ public function applyGenericFilters($dataTable)
             }
 
             $genericFilter->filter($dataTable);
+
+            // when the percent-of-total columns were computed before the generic filters ran,
+            // the Truncate filter summed the per row quotients into its summary row
+            $dataTable->filter(function (DataTable $table) {
+                PercentOfReportTotal::recomputeSummaryRows($table);
+            });
         }
 
         return $dataTable;

core/API/DocumentationGenerator.php

@@ -301,6 +301,8 @@ public function getExampleUrl($class, $methodName, $parametersToSet = array())
         $aParameters['filter_update_columns_when_show_all_goals'] = false;
         $aParameters['filter_show_goal_columns_process_goals'] = false;
         $aParameters['showMetadata'] = false;
+        $aParameters['totals'] = false;
+        $aParameters['percent_of_total'] = false;
 
         $extraParameters = StaticContainer::get('entities.idNames');
         $extraParameters = array_merge($extraParameters, StaticContainer::get('DocumentationGenerator.customParameters'));

core/API/Inconsistencies.php

@@ -41,6 +41,7 @@ public function getPercentMetricsToFormat()
             'bounce_rate_returning',
             'nb_visits_percentage',
             '/.*_evolution/',
+            '/.*_percent_of_total/',
             '/step_.*_rate/',
             '/funnel_.*_rate/',
             '/form_.*_rate/',

core/DataTable/Renderer.php

@@ -15,6 +15,7 @@
 use Piwik\DataTable;
 use Piwik\Metrics;
 use Piwik\Piwik;
+use Piwik\Plugins\CoreHome\Columns\Metrics\PercentOfReportTotal;
 use Piwik\BaseFactory;
 
 /**
@@ -256,6 +257,14 @@ protected function translateColumnNames($names)
                 }
             }
 
+            // derive translations for the percent-of-total metrics from their base metric translation
+            foreach ($t as $name => $translation) {
+                $percentOfTotalName = $name . PercentOfReportTotal::COLUMN_NAME_SUFFIX;
+                if (!isset($t[$percentOfTotalName])) {
+                    $t[$percentOfTotalName] = Piwik::translate('General_ColumnPercentOfReportTotal', $translation);
+                }
+            }
+
             foreach (Dimension::getAllDimensions() as $dimension) {
                 $dimensionId   = str_replace('.', '_', $dimension->getId());
                 $dimensionName = $dimension->getName();

lang/en.json

@@ -100,6 +100,8 @@
         "ColumnPageGenerationTime": "Page Generation time",
         "ColumnPageviews": "Pageviews",
         "ColumnPageviewsDocumentation": "The number of times this page was visited.",
+        "ColumnPercentOfReportTotal": "%1$s (%% of total)",
+        "ColumnPercentOfReportTotalDocumentation": "The percentage of the report's total %1$s that this row represents.",
         "ColumnPercentageVisits": "%% Visits",
         "ColumnPercentageVisitsDocumentation": "The percentage of total visits.",
         "ColumnRevenue": "Revenue",

plugins/API/ProcessedReport.php

@@ -29,6 +29,7 @@
 use Piwik\Period;
 use Piwik\Piwik;
 use Piwik\Plugin\ReportsProvider;
+use Piwik\Plugins\CoreHome\Columns\Metrics\PercentOfReportTotal;
 use Piwik\SettingsPiwik;
 use Piwik\Site;
 use Piwik\Timer;
@@ -603,6 +604,7 @@ private function handleTableReport($idSite, $dataTable, &$reportMetadata, $showR
             }
         }
 
+        $columns = $this->addPercentOfTotalColumns($dataTable, $columns, $reportMetadata);
         $columns = $this->hideShowMetrics($columns);
         $totals = [];
 
@@ -645,6 +647,40 @@ private function handleTableReport($idSite, $dataTable, &$reportMetadata, $showR
         ];
     }
 
+    /**
+     * Adds column translations, metric types and metric documentation for the percent-of-total
+     * metrics the DataTablePostProcessor registered on the table (eg,
+     * 'nb_visits_percent_of_total'), so their values are kept in the processed report data
+     * and described in its metadata.
+     *
+     * @param DataTable|DataTable\Map $dataTable
+     * @param array $columns
+     * @param array $reportMetadata
+     * @return array
+     */
+    private function addPercentOfTotalColumns($dataTable, $columns, &$reportMetadata)
+    {
+        $tables = $dataTable instanceof DataTable\Map ? $dataTable->getDataTables() : [$dataTable];
+
+        foreach ($tables as $table) {
+            $extraProcessedMetrics = $table->getMetadata(DataTable::EXTRA_PROCESSED_METRICS_METADATA_NAME) ?: [];
+            foreach ($extraProcessedMetrics as $metric) {
+                if ($metric instanceof PercentOfReportTotal) {
+                    $columns[$metric->getName()] = $metric->getTranslatedName();
+                    $reportMetadata['metricTypes'][$metric->getName()] = $metric->getSemanticType();
+
+                    // hideMetricsDoc=1 removes the documentation from the metadata entirely,
+                    // recreating the key here would undo that
+                    if (isset($reportMetadata['metricsDocumentation'])) {
+                        $reportMetadata['metricsDocumentation'][$metric->getName()] = $metric->getDocumentation();
+                    }
+                }
+            }
+        }
+
+        return $columns;
+    }
+
     /**
      * Removes metrics from the list of columns and the report meta data if they are marked empty
      * in the data table meta data.
@@ -987,6 +1023,12 @@ public static function getPrettyValue(Formatter $formatter, $idSite, string $col
             return $value == '0' ? '+0%' : $value;
         }
 
+        // percent-of-total metrics are quotients, this must be checked before the money/time
+        // formatting below so eg 'revenue_percent_of_total' is not formatted as money
+        if (strpos($columnName, PercentOfReportTotal::COLUMN_NAME_SUFFIX) !== false) {
+            return $formatter->getPrettyPercentFromQuotient($value);
+        }
+
         // Display time in human readable
         if (in_array($columnName, self::PERFORMANCE_METRICS_TO_FORMAT) || strpos($columnName, 'time_generation') !== false) {
             return $formatter->getPrettyTimeFromSeconds($value, true);

plugins/Contents/tests/System/expected/test_Contents_Contents.getContentNames_lastN__API.getProcessedReport_day.xml

@@ -19,6 +19,7 @@
 			<nb_impressions>The number of times a content block, such as a banner or an ad, was displayed on a page.</nb_impressions>
 			<nb_interactions>The number of times a content block was interacted with (eg, a 'click' on a banner or ad).</nb_interactions>
 			<interaction_rate>The ratio of content impressions to interactions.</interaction_rate>
+			<nb_visits_percent_of_total>The percentage of the report's total Visits that this row represents.</nb_visits_percent_of_total>
 		</metricsDocumentation>
 		<processedMetrics>
 			<interaction_rate>Interaction Rate</interaction_rate>
@@ -27,6 +28,7 @@
 			<nb_impressions>number</nb_impressions>
 			<nb_interactions>number</nb_interactions>
 			<interaction_rate>percent</interaction_rate>
+			<nb_visits_percent_of_total>percent</nb_visits_percent_of_total>
 		</metricTypes>
 		<actionToLoadSubTables>getContentNames</actionToLoadSubTables>
 		<imageGraphUrl>index.php?module=API&amp;method=ImageGraph.get&amp;idSite=1&amp;apiModule=Contents&amp;apiAction=getContentNames&amp;period=range&amp;date=2010-01-03,2010-01-09</imageGraphUrl>
@@ -38,25 +40,29 @@
 		<nb_impressions>Impressions</nb_impressions>
 		<nb_interactions>Content Interactions</nb_interactions>
 		<interaction_rate>Interaction Rate</interaction_rate>
+		<nb_visits_percent_of_total>Visits (% of total)</nb_visits_percent_of_total>
 	</columns>
 	<reportData>
 		<result prettyDate="Sunday, January 3, 2010">
 			<row>
 				<label>ImageAd</label>
 				<nb_impressions>8</nb_impressions>
 				<nb_interactions>2</nb_interactions>
+				<nb_visits_percent_of_total>50%</nb_visits_percent_of_total>
 				<interaction_rate>25%</interaction_rate>
 			</row>
 			<row>
 				<label>Text Ad</label>
 				<nb_impressions>6</nb_impressions>
 				<nb_interactions>4</nb_interactions>
+				<nb_visits_percent_of_total>37.5%</nb_visits_percent_of_total>
 				<interaction_rate>66.67%</interaction_rate>
 			</row>
 			<row>
 				<label>Video Ad</label>
 				<nb_impressions>4</nb_impressions>
 				<nb_interactions>0</nb_interactions>
+				<nb_visits_percent_of_total>12.5%</nb_visits_percent_of_total>
 				<interaction_rate>0%</interaction_rate>
 			</row>
 		</result>

plugins/Contents/tests/System/expected/test_Contents_Contents.getContentPieces_lastN__API.getProcessedReport_day.xml

@@ -19,6 +19,7 @@
 			<nb_impressions>The number of times a content block, such as a banner or an ad, was displayed on a page.</nb_impressions>
 			<nb_interactions>The number of times a content block was interacted with (eg, a 'click' on a banner or ad).</nb_interactions>
 			<interaction_rate>The ratio of content impressions to interactions.</interaction_rate>
+			<nb_visits_percent_of_total>The percentage of the report's total Visits that this row represents.</nb_visits_percent_of_total>
 		</metricsDocumentation>
 		<processedMetrics>
 			<interaction_rate>Interaction Rate</interaction_rate>
@@ -27,6 +28,7 @@
 			<nb_impressions>number</nb_impressions>
 			<nb_interactions>number</nb_interactions>
 			<interaction_rate>percent</interaction_rate>
+			<nb_visits_percent_of_total>percent</nb_visits_percent_of_total>
 		</metricTypes>
 		<actionToLoadSubTables>getContentPieces</actionToLoadSubTables>
 		<imageGraphUrl>index.php?module=API&amp;method=ImageGraph.get&amp;idSite=1&amp;apiModule=Contents&amp;apiAction=getContentPieces&amp;period=range&amp;date=2010-01-03,2010-01-09</imageGraphUrl>
@@ -38,49 +40,57 @@
 		<nb_impressions>Impressions</nb_impressions>
 		<nb_interactions>Content Interactions</nb_interactions>
 		<interaction_rate>Interaction Rate</interaction_rate>
+		<nb_visits_percent_of_total>Visits (% of total)</nb_visits_percent_of_total>
 	</columns>
 	<reportData>
 		<result prettyDate="Sunday, January 3, 2010">
 			<row>
 				<label>Click to download Piwik now</label>
 				<nb_impressions>4</nb_impressions>
 				<nb_interactions>2</nb_interactions>
+				<nb_visits_percent_of_total>25%</nb_visits_percent_of_total>
 				<interaction_rate>50%</interaction_rate>
 			</row>
 			<row>
 				<label>/path/ad.jpg</label>
 				<nb_impressions>2</nb_impressions>
 				<nb_interactions>2</nb_interactions>
+				<nb_visits_percent_of_total>12.5%</nb_visits_percent_of_total>
 				<interaction_rate>100%</interaction_rate>
 			</row>
 			<row>
 				<label>/path/ad2.jpg</label>
 				<nb_impressions>2</nb_impressions>
 				<nb_interactions>0</nb_interactions>
+				<nb_visits_percent_of_total>12.5%</nb_visits_percent_of_total>
 				<interaction_rate>0%</interaction_rate>
 			</row>
 			<row>
 				<label>Click NOW</label>
 				<nb_impressions>2</nb_impressions>
 				<nb_interactions>2</nb_interactions>
+				<nb_visits_percent_of_total>12.5%</nb_visits_percent_of_total>
 				<interaction_rate>100%</interaction_rate>
 			</row>
 			<row>
 				<label>Content Piece not defined</label>
 				<nb_impressions>2</nb_impressions>
 				<nb_interactions>0</nb_interactions>
+				<nb_visits_percent_of_total>12.5%</nb_visits_percent_of_total>
 				<interaction_rate>0%</interaction_rate>
 			</row>
 			<row>
 				<label>movie.mov</label>
 				<nb_impressions>4</nb_impressions>
 				<nb_interactions>0</nb_interactions>
+				<nb_visits_percent_of_total>12.5%</nb_visits_percent_of_total>
 				<interaction_rate>0%</interaction_rate>
 			</row>
 			<row>
 				<label>Unknown</label>
 				<nb_impressions>2</nb_impressions>
 				<nb_interactions>0</nb_interactions>
+				<nb_visits_percent_of_total>12.5%</nb_visits_percent_of_total>
 				<interaction_rate>0%</interaction_rate>
 			</row>
 		</result>

plugins/Contents/tests/System/expected/test_Contents__Actions.getPageUrls_day.xml

@@ -43,6 +43,12 @@
 		<avg_time_dom_completion>0.215</avg_time_dom_completion>
 		<avg_time_on_load>0.099</avg_time_on_load>
 		<avg_page_load_time>1.152</avg_page_load_time>
+		<nb_visits_percent_of_total>100%</nb_visits_percent_of_total>
+		<nb_hits_percent_of_total>100%</nb_hits_percent_of_total>
+		<entry_bounce_count_percent_of_total>100%</entry_bounce_count_percent_of_total>
+		<entry_nb_visits_percent_of_total>100%</entry_nb_visits_percent_of_total>
+		<entry_nb_actions_percent_of_total>100%</entry_nb_actions_percent_of_total>
+		<exit_nb_visits_percent_of_total>100%</exit_nb_visits_percent_of_total>
 		<avg_time_on_page>270</avg_time_on_page>
 		<bounce_rate>100%</bounce_rate>
 		<exit_rate>100%</exit_rate>

plugins/Contents/tests/System/expected/test_Contents__Actions.getPageUrls_month.xml

@@ -43,6 +43,12 @@
 		<avg_time_dom_completion>0.215</avg_time_dom_completion>
 		<avg_time_on_load>0.099</avg_time_on_load>
 		<avg_page_load_time>1.152</avg_page_load_time>
+		<nb_visits_percent_of_total>100%</nb_visits_percent_of_total>
+		<nb_hits_percent_of_total>100%</nb_hits_percent_of_total>
+		<entry_bounce_count_percent_of_total>100%</entry_bounce_count_percent_of_total>
+		<entry_nb_visits_percent_of_total>100%</entry_nb_visits_percent_of_total>
+		<entry_nb_actions_percent_of_total>100%</entry_nb_actions_percent_of_total>
+		<exit_nb_visits_percent_of_total>100%</exit_nb_visits_percent_of_total>
 		<avg_time_on_page>270</avg_time_on_page>
 		<bounce_rate>100%</bounce_rate>
 		<exit_rate>100%</exit_rate>

plugins/Contents/tests/System/expected/test_Contents__Contents.getContentNames_day.xml

@@ -6,6 +6,7 @@
 		<nb_visits>8</nb_visits>
 		<nb_impressions>8</nb_impressions>
 		<nb_interactions>2</nb_interactions>
+		<nb_visits_percent_of_total>50%</nb_visits_percent_of_total>
 		<interaction_rate>25%</interaction_rate>
 		<segment>contentName==ImageAd</segment>
 	</row>
@@ -15,6 +16,7 @@
 		<nb_visits>6</nb_visits>
 		<nb_impressions>6</nb_impressions>
 		<nb_interactions>4</nb_interactions>
+		<nb_visits_percent_of_total>37.5%</nb_visits_percent_of_total>
 		<interaction_rate>66.67%</interaction_rate>
 		<segment>contentName==Text+Ad</segment>
 	</row>
@@ -24,6 +26,7 @@
 		<nb_visits>2</nb_visits>
 		<nb_impressions>4</nb_impressions>
 		<nb_interactions>0</nb_interactions>
+		<nb_visits_percent_of_total>12.5%</nb_visits_percent_of_total>
 		<interaction_rate>0%</interaction_rate>
 		<segment>contentName==Video+Ad</segment>
 	</row>