diff --git a/config/runtime.exs b/config/runtime.exs index 91f056353d27..99d901e080a0 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -835,8 +835,6 @@ cloud_cron = [ {"0 0 * * *", Plausible.Workers.LockSites}, # Daily at 8 {"0 8 * * *", Plausible.Workers.AcceptTrafficUntil}, - # First sunday of the month, 4:00 UTC - {"0 4 1-7 * SUN", Plausible.Workers.ClickhouseCleanSites}, # Daily at 4:00 UTC {"0 4 * * *", Plausible.Workers.SetLegacyTimeOnPageCutoff}, # Daily at 2:00 UTC @@ -859,7 +857,6 @@ base_queues = [ notify_exported_analytics: 1, domain_change_transition: 1, check_accept_traffic_until: 1, - clickhouse_clean_sites: 1, locations_sync: 1 ] diff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.ex deleted file mode 100644 index a425fdfa5864..000000000000 --- a/lib/workers/clickhouse_clean_sites.ex +++ /dev/null @@ -1,81 +0,0 @@ -defmodule Plausible.Workers.ClickhouseCleanSites do - @moduledoc """ - Cleans deleted site data from ClickHouse asynchronously. - - We batch up data deletions from ClickHouse as deleting a single site is - just as expensive as deleting many. - """ - - use Plausible.Repo - use Plausible.ClickhouseRepo - use Plausible.IngestRepo - use Oban.Worker, queue: :clickhouse_clean_sites - - import Ecto.Query - - require Logger - - @tables_to_clear [ - "events_v2", - "sessions_v2", - "ingest_counters", - "imported_browsers", - "imported_devices", - "imported_entry_pages", - "imported_exit_pages", - "imported_locations", - "imported_operating_systems", - "imported_pages", - "imported_custom_events", - "imported_sources", - "imported_visitors" - ] - - @settings if Mix.env() in [:test, :ce_test, :e2e_test], do: [mutations_sync: 2], else: [] - - def perform(_job) do - deleted_sites = get_deleted_sites_with_clickhouse_data() - - if not Enum.empty?(deleted_sites) do - Logger.notice( - "Clearing ClickHouse data for the following #{length(deleted_sites)} sites which have been deleted: #{inspect(deleted_sites)}" - ) - - for table <- @tables_to_clear do - IngestRepo.query!( - "ALTER TABLE {$0:Identifier} DELETE WHERE site_id IN {$1:Array(UInt64)}", - [table, deleted_sites], - settings: @settings - ) - end - end - - :ok - end - - def get_deleted_sites_with_clickhouse_data() do - pg_sites = - from(s in Plausible.Site.regular(), select: s.id) - |> Plausible.Repo.all() - |> MapSet.new() - - {:ok, ch} = - Plausible.ClickhouseRepo.get_config_without_ch_query_execution_timeout() - |> Ch.start_link() - - %Ch.Result{columns: ["site_id"], rows: rows} = - DBConnection.run( - ch, - fn conn -> - Ch.query!(conn, "FROM events_v2 SELECT site_id GROUP BY site_id", [], - timeout: :infinity - ) - end, - timeout: :infinity - ) - - ch_sites = rows |> MapSet.new(fn [site_id] -> site_id end) - - MapSet.difference(ch_sites, pg_sites) |> MapSet.to_list() - end -end diff --git a/test/workers/clickhouse_clean_sites_test.exs b/test/workers/clickhouse_clean_sites_test.exs deleted file mode 100644 index 18cc2c96f759..000000000000 --- a/test/workers/clickhouse_clean_sites_test.exs +++ /dev/null @@ -1,68 +0,0 @@ -defmodule Plausible.Workers.ClickhouseCleanSitesTest do - use Plausible.DataCase - import Plausible.Factory - - alias Plausible.Workers.ClickhouseCleanSites - - @tag :slow - test "deletes data from events and sessions tables" do - site = insert(:site) - deleted_site = insert(:site) - - populate_stats(site, [ - build(:pageview) - ]) - - populate_stats(deleted_site, [ - build(:pageview), - build(:pageview), - build(:imported_visitors), - build(:imported_sources), - build(:imported_pages), - build(:imported_entry_pages), - build(:imported_exit_pages), - build(:imported_locations), - build(:imported_devices), - build(:imported_browsers), - build(:imported_operating_systems) - ]) - - Repo.delete!(deleted_site) - - assert Enum.member?( - ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(), - deleted_site.id - ) - - assert not Enum.member?( - ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(), - site.id - ) - - ClickhouseCleanSites.perform(nil) - - assert_count(deleted_site, "events_v2", 0) - assert_count(deleted_site, "sessions_v2", 0) - assert_count(deleted_site, "imported_visitors", 0) - assert_count(deleted_site, "imported_sources", 0) - assert_count(deleted_site, "imported_pages", 0) - assert_count(deleted_site, "imported_entry_pages", 0) - assert_count(deleted_site, "imported_exit_pages", 0) - assert_count(deleted_site, "imported_locations", 0) - assert_count(deleted_site, "imported_devices", 0) - assert_count(deleted_site, "imported_browsers", 0) - assert_count(deleted_site, "imported_operating_systems", 0) - assert_count(site, "events_v2", 1) - assert_count(site, "sessions_v2", 1) - - assert not Enum.member?( - ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(), - deleted_site.id - ) - end - - def assert_count(site, table, expected_count) do - q = from(e in table, select: %{count: fragment("count()")}, where: e.site_id == ^site.id) - await_clickhouse_count(q, expected_count) - end -end