From 72d20d4d90da7d6e0035bec7d46ac8d504babcf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Santiago?= Date: Thu, 9 May 2024 15:45:38 +0200 Subject: [PATCH 1/6] use SQLite instead of Postgres --- .gitignore | 1 + deps.edn | 3 +- .../20231113185356-init-tables.up.sql | 86 ++++++++--------- .../arqivist/api/slack/handlers.clj | 26 +++--- src/jcpsantiago/arqivist/api/slack/specs.clj | 2 +- .../arqivist/api/slack/ui_blocks.clj | 33 ++++--- src/jcpsantiago/arqivist/messages.clj | 16 ++-- src/jcpsantiago/arqivist/middleware.clj | 93 ++++++++++--------- src/jcpsantiago/arqivist/specs.clj | 8 +- src/jcpsantiago/arqivist/system.clj | 38 ++++---- src/jcpsantiago/arqivist/utils.clj | 11 ++- 11 files changed, 167 insertions(+), 150 deletions(-) diff --git a/.gitignore b/.gitignore index 844a380..2976286 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ pom.xml.asc /target/ ## --------------------------------------------------------- +thearqivist_db diff --git a/deps.edn b/deps.edn index 5e41dbe..ba44213 100644 --- a/deps.edn +++ b/deps.edn @@ -20,9 +20,8 @@ ;; Persistence com.github.seancorfield/next.jdbc {:mvn/version "1.3.909"} - hikari-cp/hikari-cp {:mvn/version "3.0.1"} migratus/migratus {:mvn/version "1.5.6"} - org.postgresql/postgresql {:mvn/version "42.7.1"} + org.xerial/sqlite-jdbc {:mvn/version "3.45.3.0"} ;; HTTP server http-kit/http-kit {:mvn/version "2.7.0"} diff --git a/resources/migrations/20231113185356-init-tables.up.sql b/resources/migrations/20231113185356-init-tables.up.sql index 1471a5f..6a45245 100644 --- a/resources/migrations/20231113185356-init-tables.up.sql +++ b/resources/migrations/20231113185356-init-tables.up.sql @@ -1,54 +1,54 @@ CREATE TABLE IF NOT EXISTS atlassian_tenants ( - id serial primary key, - key varchar(255), - tenant_name varchar(255), - account_id varchar(255), - client_key varchar(255), - shared_secret varchar(255), - base_url varchar(255), - base_url_short varchar(255), - display_url varchar(255), - product_type varchar(255), - description varchar(255), - service_entitlement_number varchar(255), - oauth_client_id varchar(255), - valid_license boolean, - is_evaluation boolean, - created_at timestamp default current_timestamp - ); + id integer primary key autoincrement, + key text, + tenant_name text, + account_id text, + client_key text, + shared_secret text, + base_url text, + base_url_short text, + display_url text, + product_type text, + description text, + service_entitlement_number text, + oauth_client_id text, + valid_license integer, + is_evaluation integer, + created_at integer default (unixepoch('now')) + ) STRICT; --;; CREATE TABLE IF NOT EXISTS slack_teams ( - id serial primary key, - atlassian_tenant_id int REFERENCES atlassian_tenants ON DELETE CASCADE, - app_id varchar(255), - external_team_id varchar(255), - team_name varchar(255), - registering_user varchar(255), - scopes varchar(255), - access_token varchar(255), - bot_user_id varchar(255), - created_at timestamp default current_timestamp - ); + id integer primary key autoincrement, + atlassian_tenant_id integer REFERENCES atlassian_tenants ON DELETE CASCADE, + app_id text, + external_team_id text, + team_name text, + registering_user text, + scopes text, + access_token text, + bot_user_id text, + created_at integer default (unixepoch('now')) + ) STRICT; --;; CREATE TABLE IF NOT EXISTS jobs ( - id serial primary key, - slack_team_id int NOT NULL REFERENCES slack_teams ON DELETE CASCADE, - slack_channel_id varchar(255) NOT NULL, - owner_slack_user_id varchar(255) NOT NULL, - timezone varchar(255) NOT NULL, - frequency varchar(255) NOT NULL, - target varchar(255) NOT NULL, - target_url varchar(255), - last_slack_conversation_datetime timestamp, - last_slack_conversation_ts varchar(255), - due_date timestamp, - n_runs int, - updated_at timestamp, - created_at timestamp default current_timestamp, + id integer primary key autoincrement, + slack_team_id integer NOT NULL REFERENCES slack_teams ON DELETE CASCADE, + slack_channel_id text NOT NULL, + owner_slack_user_id text NOT NULL, + timezone text NOT NULL, + frequency text NOT NULL, + target text NOT NULL, + target_url text, + last_slack_conversation_datetime integer, + last_slack_conversation_ts text, + due_date integer, + n_runs integer, + updated_at integer, + created_at integer default (unixepoch('now')), CHECK (frequency in ('once', 'daily', 'weekly')) - ); + ) STRICT; diff --git a/src/jcpsantiago/arqivist/api/slack/handlers.clj b/src/jcpsantiago/arqivist/api/slack/handlers.clj index e324f24..3418f5e 100644 --- a/src/jcpsantiago/arqivist/api/slack/handlers.clj +++ b/src/jcpsantiago/arqivist/api/slack/handlers.clj @@ -7,19 +7,17 @@ [clojure.spec.alpha :as spec] [clojure.string :refer [trim]] [com.brunobonacci.mulog :as mulog] - [jcpsantiago.arqivist.api.slack.ui-blocks :as ui] + [java-time.api :as java-time] [jcpsantiago.arqivist.api.slack.pages :as pages] [jcpsantiago.arqivist.api.slack.specs :as specs] + [jcpsantiago.arqivist.api.slack.ui-blocks :as ui] [jcpsantiago.arqivist.api.slack.utils :as utils] [jcpsantiago.arqivist.messages :as messages] - [jcpsantiago.arqivist.utils :as core-utils] [jcpsantiago.arqivist.specs :as core-specs] + [jcpsantiago.arqivist.utils :as core-utils] [jsonista.core :as json] [next.jdbc.sql :as sql] - ;; needed because PostgreSQL can't translate java datetime into SQL timestamp - ;; https://cljdoc.org/d/com.github.seancorfield/next.jdbc/1.3.894/api/next.jdbc.date-time - [next.jdbc.date-time] - [ring.util.response :refer [bad-request response content-type]])) + [ring.util.response :refer [bad-request content-type response]])) ;; ;; ------------------------------------------------------ @@ -43,7 +41,7 @@ (let [context (mulog/local-context)] (future (mulog/with-context context - (messages/start-job system request job core-utils/persist-job!))) + (messages/start-job system request job core-utils/persist-job!))) (update-modal-response ui/confirm-job-started-modal request)) @@ -66,7 +64,7 @@ (future (mulog/with-context (mulog/local-context) - (messages/start-job system request updated-job core-utils/update-job!))) + (messages/start-job system request updated-job core-utils/update-job!))) ;; job started confirmation modal (update-modal-response ui/confirm-job-started-modal request)) @@ -107,12 +105,12 @@ (mulog/with-context context - (mulog/log ::interaction-payload - :local-time (java.time.LocalDateTime/now)) - (case type - "view_submission" (view-submission system request) - "message_action" "TODO" - (bad-request "Unknown type")))))) + (mulog/log ::interaction-payload + :local-time (java.time.LocalDateTime/now)) + (case type + "view_submission" (view-submission system request) + "message_action" "TODO" + (bad-request "Unknown type")))))) ;; ;; ------------------------------------------------------ diff --git a/src/jcpsantiago/arqivist/api/slack/specs.clj b/src/jcpsantiago/arqivist/api/slack/specs.clj index 00fe74f..d642473 100644 --- a/src/jcpsantiago/arqivist/api/slack/specs.clj +++ b/src/jcpsantiago/arqivist/api/slack/specs.clj @@ -187,7 +187,7 @@ (spec/def :slack_teams/scopes string?) (spec/def :slack_teams/access_token string?) (spec/def :slack_teams/bot_user_id string?) -(spec/def :slack_teams/created_at inst?) +(spec/def :slack_teams/created_at int?) (spec/def :slack_teams/atlassian_tenant_id pos-int?) (spec/def ::team-attributes diff --git a/src/jcpsantiago/arqivist/api/slack/ui_blocks.clj b/src/jcpsantiago/arqivist/api/slack/ui_blocks.clj index abdf0a3..e23e55c 100644 --- a/src/jcpsantiago/arqivist/api/slack/ui_blocks.clj +++ b/src/jcpsantiago/arqivist/api/slack/ui_blocks.clj @@ -124,18 +124,25 @@ " [job] (let [{:keys [:jobs/owner_slack_user_id :jobs/frequency :jobs/due_date - :jobs/created_at :jobs/last_slack_conversation_datetime]} job - created_at_ts (to-seconds-from-epoch created_at) - last-slack-conversation-ts (to-seconds-from-epoch last_slack_conversation_datetime) - due_date_ts (to-seconds-from-epoch due_date)] + :jobs/created_at :jobs/last_slack_conversation_datetime + :jobs/timezone]} job + due_date_tz (when due_date + (java-time/local-date-time + (java-time/instant (* 1000 due_date)) + "UTC")) + last_slack_conversation_tz (java-time/local-date-time + (java-time/instant (* 1000 last_slack_conversation_datetime)) + timezone)] ;; NOTE: Slack does not allow more than 10 fields per block ;; each "row" here would be two fields so we can have a max of 5 k-v pairs (two-column-section [["*Owner*: " (str "<@" owner_slack_user_id ">")] - ["*Created at*: " (slack-nice-datetime created_at_ts "{date_num}" created_at)] + ["*Created at*: " (slack-nice-datetime created_at "{date_num}" created_at)] ["*Frequency*:" (str "`" frequency "`")] - ["*Next archival at*:" (slack-nice-datetime due_date_ts "{date_num} 12:00 AM" due_date)] - ["*Archived until*:" (slack-nice-datetime last-slack-conversation-ts "{date_num} {time}" last_slack_conversation_datetime)]]))) + ["*Next archival at*:" (if due_date_tz + (slack-nice-datetime due_date "{date_num}" due_date_tz) + "Not scheduled")] + ["*Archived until*:" (slack-nice-datetime last_slack_conversation_datetime "{date_num} {time}" last_slack_conversation_tz)]]))) (defn exists-once-modal " @@ -143,9 +150,11 @@ " [request existing-job] (let [{{{:keys [team_domain channel_name channel_id user_id user_name]} :form} :parameters} request - {:keys [:jobs/last_slack_conversation_datetime + {:keys [:jobs/last_slack_conversation_datetime :jobs/timezone :jobs/slack_channel_id :jobs/target_url]} existing-job - last-slack-conversation-ts (quot (java-time/to-millis-from-epoch last_slack_conversation_datetime) 1000)] + last_slack_conversation_tz (java-time/local-date-time + (java-time/instant (* 1000 last_slack_conversation_datetime)) + timezone)] {:type "modal" :callback_id "exists-once-confirmation" @@ -178,9 +187,9 @@ :text (str "If you would you like to setup a *recurrent archival* instead of a one time manual job, " "select another frequency, otherwise please select `once` again. I'll " - "create a new archive with messages since " - (str " ") - "without overwriting the previous archive.")}} + "append new messages since " + (str " ") + "to the archive.")}} {:type "divider"} {:type "input" diff --git a/src/jcpsantiago/arqivist/messages.clj b/src/jcpsantiago/arqivist/messages.clj index 7091b1f..3749cd6 100644 --- a/src/jcpsantiago/arqivist/messages.clj +++ b/src/jcpsantiago/arqivist/messages.clj @@ -46,8 +46,8 @@ [frequency] (case frequency "once" nil - "daily" (java-time/+ (java-time/local-date-time) (java-time/days 1)) - "weekly" (java-time/+ (java-time/local-date-time) (java-time/weeks 1)))) + "daily" (java-time/+ (java-time/instant) (java-time/days 1)) + "weekly" (java-time/+ (java-time/instant) (java-time/weeks 1)))) ;; TODO: ;; * clean up in case it's a first-time run and creating the archive fails @@ -81,19 +81,20 @@ last-ts (->> messages (sort-by :ts) last :ts) last-datetime (-> last-ts (string/replace #"\..+" "") - (Long/parseLong) - (java.time.Instant/ofEpochSecond)) + (Long/parseLong)) n-runs (if (nil? (:jobs/n_runs job)) 1 (inc (:jobs/n_runs job))) + job-due-date (due-date frequency) updates {:jobs/target_url (:archive-url archival-response) :jobs/frequency frequency ;; FIXME: n_runs can't be null, inc explodes, check beforehand :jobs/n_runs n-runs :jobs/last_slack_conversation_ts last-ts :jobs/last_slack_conversation_datetime last-datetime - :jobs/due_date (due-date frequency) - :jobs/updated_at (java-time/local-date-time)}] + :jobs/due_date (when job-due-date + (.getEpochSecond job-due-date)) + :jobs/updated_at (core-utils/unix-epoch)}] (sql/update! (:db-connection system) @@ -141,7 +142,8 @@ atlassian_tenant_id (get-in request [:slack-team-attributes :slack_teams/atlassian_tenant_id]) confluence-tenant-attributes (sql/get-by-id db-connection :atlassian_tenants atlassian_tenant_id) db-io-result (db-fn system job) - job (merge job (select-keys db-io-result [:jobs/id]))] + ;; NOTE: SQLite returns :last_insert_rowid() as the key, which is invalid clj, so get the value directly + job (merge job {:jobs/id (first (vals db-io-result))})] (mulog/log ::start-job-db-io ;; FIXME: get the name of the function used as a string diff --git a/src/jcpsantiago/arqivist/middleware.clj b/src/jcpsantiago/arqivist/middleware.clj index 6c5b638..5136f0c 100644 --- a/src/jcpsantiago/arqivist/middleware.clj +++ b/src/jcpsantiago/arqivist/middleware.clj @@ -13,6 +13,7 @@ [jcpsantiago.arqivist.utils :as core-utils] [jcpsantiago.arqivist.api.confluence.utils :as utils] [jcpsantiago.arqivist.api.slack.specs :as specs] + [java-time.api :as java-time] [ring.util.response :refer [bad-request response]] [clj-slack.conversations :as slack-convo] [jsonista.core :as json] @@ -130,56 +131,56 @@ ;; TODO: add more non PII details to context (mulog/with-context {:slack-team-id team_id} - (cond - (and (spec/valid? ::specs/team-attributes slack-team-attributes) - (seq (:token slack-connection))) - (do - (mulog/log ::add-slack-team-attributes - :success :true - :local-time (java.time.LocalDateTime/now)) - (-> request - (assoc :slack-team-attributes slack-team-attributes) - (assoc :slack-connection slack-connection) - handler)) + (cond + (and (spec/valid? ::specs/team-attributes slack-team-attributes) + (seq (:token slack-connection))) + (do + (mulog/log ::add-slack-team-attributes + :success :true + :local-time (java.time.LocalDateTime/now)) + (-> request + (assoc :slack-team-attributes slack-team-attributes) + (assoc :slack-connection slack-connection) + handler)) - (spec/invalid? slack-team-attributes) - (do - (mulog/log ::add-slack-team-attributes - :success :false - :message "Slack team attributes did not conform to spec" + (spec/invalid? slack-team-attributes) + (do + (mulog/log ::add-slack-team-attributes + :success :false + :message "Slack team attributes did not conform to spec" ;; FIXME: The explanation will contain the slack access key! ;; we have to obfuscate that before logging - :explanation (spec/explain-data ::specs/team-attributes slack-team-row) - :request request - :local-time (java.time.LocalDateTime/now)) - (response (core-utils/error-response-text))) + :explanation (spec/explain-data ::specs/team-attributes slack-team-row) + :request request + :local-time (java.time.LocalDateTime/now)) + (response (core-utils/error-response-text))) - (nil? (:token slack-connection)) - (do - (mulog/log ::add-slack-team-attributes - :success :false - :message "Missing Slack connection token" - :local-time (java.time.LocalDateTime/now)) - (response (core-utils/error-response-text))) + (nil? (:token slack-connection)) + (do + (mulog/log ::add-slack-team-attributes + :success :false + :message "Missing Slack connection token" + :local-time (java.time.LocalDateTime/now)) + (response (core-utils/error-response-text))) ;; NOTE: edge-case, only became relevant during development ;; it shouldn't be possible to have access to the app ;; without having installed it first - (nil? slack-team-row) - (do - (mulog/log ::add-slack-team-attributes - :success :false - :error "Missing Slack credentials in the db" - :local-time (java.time.LocalDateTime/now)) - (response (core-utils/error-response-text))) + (nil? slack-team-row) + (do + (mulog/log ::add-slack-team-attributes + :success :false + :error "Missing Slack credentials in the db" + :local-time (java.time.LocalDateTime/now)) + (response (core-utils/error-response-text))) - :else - (do - (mulog/log ::add-slack-team-attributes - :success :false - :error "Spec does not conform" - :explanation (spec/explain ::specs/team-attributes slack-team-row)) - (response (core-utils/error-response-text)))))) + :else + (do + (mulog/log ::add-slack-team-attributes + :success :false + :error "Spec does not conform" + :explanation (spec/explain ::specs/team-attributes slack-team-row)) + (response (core-utils/error-response-text)))))) (catch Exception e (mulog/log ::add-slack-team-attributes @@ -303,12 +304,12 @@ ;; track the request duration and outcome (mulog/trace :io.redefine.datawarp/http-request - {:pairs [:content-type (get-in request [:headers "content-type"]) - :content-encoding (get-in request [:headers "content-encoding"]) - :middleware id] + {:pairs [:content-type (get-in request [:headers "content-type"]) + :content-encoding (get-in request [:headers "content-encoding"]) + :middleware id] ;; capture http status code from the response - :capture (fn [{:keys [status]}] {:http-status status})} - (handler request)))) + :capture (fn [{:keys [status]}] {:http-status status})} + (handler request)))) ;; Atlassian middleware ----------------------------------------------------- (defn verify-atlassian-iframe diff --git a/src/jcpsantiago/arqivist/specs.clj b/src/jcpsantiago/arqivist/specs.clj index 48840c9..bd44c50 100644 --- a/src/jcpsantiago/arqivist/specs.clj +++ b/src/jcpsantiago/arqivist/specs.clj @@ -13,13 +13,13 @@ (spec/def :jobs/frequency #{"once" "daily" "weekly"}) (spec/def :jobs/target #{"confluence"}) (spec/def :jobs/target_url string?) -(spec/def :jobs/last_slack_conversation_datetime inst?) +(spec/def :jobs/last_slack_conversation_datetime int?) (spec/def :jobs/last_slack_conversation_ts string?) ;; NOTE: `once` jobs won't have a due date -(spec/def :jobs/due_date (spec/nilable inst?)) +(spec/def :jobs/due_date (spec/nilable int?)) (spec/def :jobs/n_runs int?) -(spec/def :jobs/updated_at inst?) -(spec/def :jobs/created_at inst?) +(spec/def :jobs/updated_at (spec/nilable int?)) +(spec/def :jobs/created_at int?) (spec/def ::job (spec/keys diff --git a/src/jcpsantiago/arqivist/system.clj b/src/jcpsantiago/arqivist/system.clj index e52113b..3f00d76 100644 --- a/src/jcpsantiago/arqivist/system.clj +++ b/src/jcpsantiago/arqivist/system.clj @@ -4,12 +4,12 @@ (:require [com.brunobonacci.mulog :as mulog] [donut.system :as donut] - [hikari-cp.core :as hikari] [org.httpkit.client :as http-client] [org.httpkit.server :as http-server] [jcpsantiago.arqivist.router :as router] [jsonista.core :as jsonista] - [migratus.core :as migratus])) + [migratus.core :as migratus] + [next.jdbc :as jdbc])) ;; TODO: move into system utils namespace? (defn ngrok-tunnel-url @@ -48,7 +48,7 @@ (try (migratus/migrate creds) (catch - org.postgresql.util.PSQLException e + Exception e (mulog/log ::connecting-db-failed :error-message (ex-message e) :error-data (ex-data e))))) @@ -58,21 +58,27 @@ :db {:datasource (donut/ref [:db :db-connection])}}}}) (def db-connection - "Database connection component. - Uses HikariCP to create and manage a connection pool." + "Database connection component." #::donut{:start (fn create-db-connection - [{{:keys [options]} ::donut/config}] + [{{:keys [db-spec]} ::donut/config}] (mulog/log ::creating-db-connection + :db-spec db-spec :local-time (java.time.LocalDateTime/now)) - (hikari/make-datasource options)) + (let [db-connection (jdbc/get-datasource db-spec)] + ;; NOTE: https://kerkour.com/sqlite-for-servers + (jdbc/execute! db-connection ["PRAGMA journal_mode = WAL;"]) + (jdbc/execute! db-connection ["PRAGMA synchronous = NORMAL;"]) + (jdbc/execute! db-connection ["PRAGMA cache_size = 1000000000;"]) + (jdbc/execute! db-connection ["PRAGMA foreign_keys = true;"]) + (jdbc/execute! db-connection ["PRAGMA temp_store = memory;"]) + db-connection)) :stop (fn closing-db-connection - [{::donut/keys [instance]}] + [{::donut/keys [_instance]}] (mulog/log ::closing-db-connection - :local-time (java.time.LocalDateTime/now)) - (hikari/close-datasource instance)) + :local-time (java.time.LocalDateTime/now))) - :config {:options (donut/ref [:env :datasource-options])}}) + :config {:db-spec {:dbtype "sqlite" :dbname "thearqivist_db"}}}) (def http-server "Webserver component using http-kit" @@ -129,15 +135,7 @@ :port (parse-long (or (System/getenv "PORT") (System/getenv "ARQIVIST_PORT") - "8989")) - - :datasource-options {;; NOTE: No idea what each of these actually do, should learn :D - :maximum-pool-size 5 - :minimum-idle 2 - :idle-timeout 12000 - :max-lifetime 300000 - :jdbc-url (or (System/getenv "DATABASE_URL") - "jdbc:postgresql://localhost/arqivist?user=arqivist&password=arqivist")}} + "8989"))} ;; Event logger :event-log {:publisher event-logger} diff --git a/src/jcpsantiago/arqivist/utils.clj b/src/jcpsantiago/arqivist/utils.clj index ecbf053..5aacdb9 100644 --- a/src/jcpsantiago/arqivist/utils.clj +++ b/src/jcpsantiago/arqivist/utils.clj @@ -5,6 +5,7 @@ [clj-slack.users :as slack-users] [clojure.spec.alpha :as spec] [com.brunobonacci.mulog :as mulog] + [java-time.api :as java-time] [jcpsantiago.arqivist.specs :as core-specs] [next.jdbc.sql :as sql])) @@ -17,6 +18,14 @@ java.time.Instant/ofEpochSecond (.atZone zone)))) +(defn unix-epoch + [] + (quot (System/currentTimeMillis) 1000)) + +(defn to-seconds-from-epoch + [epoch] + (quot (java-time/to-millis-from-epoch epoch) 1000)) + ;; Messaging with the user (defn error-response-text " @@ -61,7 +70,7 @@ :radio_buttons-action :selected_option :value]) job-map {:jobs/target "confluence" :jobs/frequency frequency :jobs/slack_team_id slack-team-id :jobs/slack_channel_id channel_id - :jobs/owner_slack_user_id user_id :jobs/timezone tz :jobs/created_at (java.time.Instant/now)} + :jobs/owner_slack_user_id user_id :jobs/timezone tz :jobs/created_at (quot (System/currentTimeMillis) 1000)} job (spec/conform ::core-specs/job job-map)] (if (spec/invalid? job) From 6c112c22f6e97e40674e02e33a64be27b714b783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Santiago?= Date: Thu, 9 May 2024 15:49:39 +0200 Subject: [PATCH 2/6] Update README.md --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0c7c007..51778da 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,12 @@ To run The Arqivist yourself you need * admin access to a Confluence account * admin access to a Slack workspace -* a running instance of The Arqivist + a Postgres database (see data model at [dbdiagram.io](https://dbdiagram.io/d/6551f3787d8bbd6465102527) +* a running instance of The Arqivist + +## Data persistence + +The Arqivist uses SQLite as the backend database. +See data model at [dbdiagram.io](https://dbdiagram.io/d/6551f3787d8bbd6465102527) ## Installing in Confluence From 3bd4b3c57e77091b335f24b2afb1a329d33ec2ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Santiago?= Date: Thu, 9 May 2024 15:52:34 +0200 Subject: [PATCH 3/6] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40e4a4b..f7aee8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) * [#88](https://github.com/jcpsantiago/thearqivist/issues/88) Automatically join public channels * dev: update hack locally details for starting a REPL * [#63](https://github.com/jcpsantiago/thearqivist/issues/63) Enable recurrent jobs: new `jobs` table, rebased migrations, new modals +* Switch to SQLite, dropping the requirements for Hikari and Postgres ## 0.1.0 - 2023-04-20 From 36a0930fb618b3d009d037459a6f2cdc25f6c9fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Santiago?= Date: Thu, 9 May 2024 16:04:27 +0200 Subject: [PATCH 4/6] correct formatting --- .../arqivist/api/confluence/pages.clj | 1 - .../arqivist/api/slack/handlers.clj | 16 ++-- src/jcpsantiago/arqivist/messages.clj | 1 - src/jcpsantiago/arqivist/middleware.clj | 92 +++++++++---------- src/jcpsantiago/arqivist/specs.clj | 1 - src/jcpsantiago/arqivist/utils.clj | 1 - .../arqivist/api/confluence/utils_test.clj | 1 - 7 files changed, 54 insertions(+), 59 deletions(-) diff --git a/src/jcpsantiago/arqivist/api/confluence/pages.clj b/src/jcpsantiago/arqivist/api/confluence/pages.clj index 5dfdbc2..2147991 100644 --- a/src/jcpsantiago/arqivist/api/confluence/pages.clj +++ b/src/jcpsantiago/arqivist/api/confluence/pages.clj @@ -298,4 +298,3 @@ [:a {:href "https://arqivist.app", :target "_blank"} "The Arqivist"] " is made with ♥ in Berlin."]]]]]]))) - diff --git a/src/jcpsantiago/arqivist/api/slack/handlers.clj b/src/jcpsantiago/arqivist/api/slack/handlers.clj index 3418f5e..47fe9ae 100644 --- a/src/jcpsantiago/arqivist/api/slack/handlers.clj +++ b/src/jcpsantiago/arqivist/api/slack/handlers.clj @@ -41,7 +41,7 @@ (let [context (mulog/local-context)] (future (mulog/with-context context - (messages/start-job system request job core-utils/persist-job!))) + (messages/start-job system request job core-utils/persist-job!))) (update-modal-response ui/confirm-job-started-modal request)) @@ -64,7 +64,7 @@ (future (mulog/with-context (mulog/local-context) - (messages/start-job system request updated-job core-utils/update-job!))) + (messages/start-job system request updated-job core-utils/update-job!))) ;; job started confirmation modal (update-modal-response ui/confirm-job-started-modal request)) @@ -105,12 +105,12 @@ (mulog/with-context context - (mulog/log ::interaction-payload - :local-time (java.time.LocalDateTime/now)) - (case type - "view_submission" (view-submission system request) - "message_action" "TODO" - (bad-request "Unknown type")))))) + (mulog/log ::interaction-payload + :local-time (java.time.LocalDateTime/now)) + (case type + "view_submission" (view-submission system request) + "message_action" "TODO" + (bad-request "Unknown type")))))) ;; ;; ------------------------------------------------------ diff --git a/src/jcpsantiago/arqivist/messages.clj b/src/jcpsantiago/arqivist/messages.clj index 3749cd6..53dc69a 100644 --- a/src/jcpsantiago/arqivist/messages.clj +++ b/src/jcpsantiago/arqivist/messages.clj @@ -168,4 +168,3 @@ :local-time (java.time.LocalDateTime/now)) (let [{:keys [channel_id user_id]} (-> request :parameters :form :payload :view :private_metadata read-string)] (core-utils/ephemeral-error-message! user_id channel_id (:slack-connection request))))))) - diff --git a/src/jcpsantiago/arqivist/middleware.clj b/src/jcpsantiago/arqivist/middleware.clj index 5136f0c..10a7cb2 100644 --- a/src/jcpsantiago/arqivist/middleware.clj +++ b/src/jcpsantiago/arqivist/middleware.clj @@ -131,56 +131,56 @@ ;; TODO: add more non PII details to context (mulog/with-context {:slack-team-id team_id} - (cond - (and (spec/valid? ::specs/team-attributes slack-team-attributes) - (seq (:token slack-connection))) - (do - (mulog/log ::add-slack-team-attributes - :success :true - :local-time (java.time.LocalDateTime/now)) - (-> request - (assoc :slack-team-attributes slack-team-attributes) - (assoc :slack-connection slack-connection) - handler)) + (cond + (and (spec/valid? ::specs/team-attributes slack-team-attributes) + (seq (:token slack-connection))) + (do + (mulog/log ::add-slack-team-attributes + :success :true + :local-time (java.time.LocalDateTime/now)) + (-> request + (assoc :slack-team-attributes slack-team-attributes) + (assoc :slack-connection slack-connection) + handler)) - (spec/invalid? slack-team-attributes) - (do - (mulog/log ::add-slack-team-attributes - :success :false - :message "Slack team attributes did not conform to spec" + (spec/invalid? slack-team-attributes) + (do + (mulog/log ::add-slack-team-attributes + :success :false + :message "Slack team attributes did not conform to spec" ;; FIXME: The explanation will contain the slack access key! ;; we have to obfuscate that before logging - :explanation (spec/explain-data ::specs/team-attributes slack-team-row) - :request request - :local-time (java.time.LocalDateTime/now)) - (response (core-utils/error-response-text))) + :explanation (spec/explain-data ::specs/team-attributes slack-team-row) + :request request + :local-time (java.time.LocalDateTime/now)) + (response (core-utils/error-response-text))) - (nil? (:token slack-connection)) - (do - (mulog/log ::add-slack-team-attributes - :success :false - :message "Missing Slack connection token" - :local-time (java.time.LocalDateTime/now)) - (response (core-utils/error-response-text))) + (nil? (:token slack-connection)) + (do + (mulog/log ::add-slack-team-attributes + :success :false + :message "Missing Slack connection token" + :local-time (java.time.LocalDateTime/now)) + (response (core-utils/error-response-text))) ;; NOTE: edge-case, only became relevant during development ;; it shouldn't be possible to have access to the app ;; without having installed it first - (nil? slack-team-row) - (do - (mulog/log ::add-slack-team-attributes - :success :false - :error "Missing Slack credentials in the db" - :local-time (java.time.LocalDateTime/now)) - (response (core-utils/error-response-text))) + (nil? slack-team-row) + (do + (mulog/log ::add-slack-team-attributes + :success :false + :error "Missing Slack credentials in the db" + :local-time (java.time.LocalDateTime/now)) + (response (core-utils/error-response-text))) - :else - (do - (mulog/log ::add-slack-team-attributes - :success :false - :error "Spec does not conform" - :explanation (spec/explain ::specs/team-attributes slack-team-row)) - (response (core-utils/error-response-text)))))) + :else + (do + (mulog/log ::add-slack-team-attributes + :success :false + :error "Spec does not conform" + :explanation (spec/explain ::specs/team-attributes slack-team-row)) + (response (core-utils/error-response-text)))))) (catch Exception e (mulog/log ::add-slack-team-attributes @@ -304,12 +304,12 @@ ;; track the request duration and outcome (mulog/trace :io.redefine.datawarp/http-request - {:pairs [:content-type (get-in request [:headers "content-type"]) - :content-encoding (get-in request [:headers "content-encoding"]) - :middleware id] + {:pairs [:content-type (get-in request [:headers "content-type"]) + :content-encoding (get-in request [:headers "content-encoding"]) + :middleware id] ;; capture http status code from the response - :capture (fn [{:keys [status]}] {:http-status status})} - (handler request)))) + :capture (fn [{:keys [status]}] {:http-status status})} + (handler request)))) ;; Atlassian middleware ----------------------------------------------------- (defn verify-atlassian-iframe diff --git a/src/jcpsantiago/arqivist/specs.clj b/src/jcpsantiago/arqivist/specs.clj index bd44c50..3d50f01 100644 --- a/src/jcpsantiago/arqivist/specs.clj +++ b/src/jcpsantiago/arqivist/specs.clj @@ -27,4 +27,3 @@ :jobs/frequency :jobs/target] :opt [:jobs/id :jobs/last_slack_conversation_ts :jobs/due_date :jobs/n_runs :jobs/updated_at :jobs/created_at :jobs/target_url :jobs/last_slack_conversation_datetime])) - diff --git a/src/jcpsantiago/arqivist/utils.clj b/src/jcpsantiago/arqivist/utils.clj index 5aacdb9..2cbff76 100644 --- a/src/jcpsantiago/arqivist/utils.clj +++ b/src/jcpsantiago/arqivist/utils.clj @@ -135,4 +135,3 @@ :error-message (ex-message e) :local-time (java.time.LocalDateTime/now)) (throw (ex-info "Failed to insert job in database" e))))) - diff --git a/test/jcpsantiago/arqivist/api/confluence/utils_test.clj b/test/jcpsantiago/arqivist/api/confluence/utils_test.clj index 709c889..b73752d 100644 --- a/test/jcpsantiago/arqivist/api/confluence/utils_test.clj +++ b/test/jcpsantiago/arqivist/api/confluence/utils_test.clj @@ -56,4 +56,3 @@ "bombastic" "https://bombastic.atlassian.net" "loop-in-the-burrito" "https://loop-in-the-burrito.atlassian.net" "looping-burrito" "https://looping-burrito.atlassian.net"))) - From 1136b976765f7b10e7fb2128098710cef6a30eb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Santiago?= Date: Thu, 9 May 2024 20:25:22 +0200 Subject: [PATCH 5/6] remove unused requires --- src/jcpsantiago/arqivist/api/slack/handlers.clj | 1 - src/jcpsantiago/arqivist/middleware.clj | 1 - 2 files changed, 2 deletions(-) diff --git a/src/jcpsantiago/arqivist/api/slack/handlers.clj b/src/jcpsantiago/arqivist/api/slack/handlers.clj index 47fe9ae..815b538 100644 --- a/src/jcpsantiago/arqivist/api/slack/handlers.clj +++ b/src/jcpsantiago/arqivist/api/slack/handlers.clj @@ -7,7 +7,6 @@ [clojure.spec.alpha :as spec] [clojure.string :refer [trim]] [com.brunobonacci.mulog :as mulog] - [java-time.api :as java-time] [jcpsantiago.arqivist.api.slack.pages :as pages] [jcpsantiago.arqivist.api.slack.specs :as specs] [jcpsantiago.arqivist.api.slack.ui-blocks :as ui] diff --git a/src/jcpsantiago/arqivist/middleware.clj b/src/jcpsantiago/arqivist/middleware.clj index 10a7cb2..6c5b638 100644 --- a/src/jcpsantiago/arqivist/middleware.clj +++ b/src/jcpsantiago/arqivist/middleware.clj @@ -13,7 +13,6 @@ [jcpsantiago.arqivist.utils :as core-utils] [jcpsantiago.arqivist.api.confluence.utils :as utils] [jcpsantiago.arqivist.api.slack.specs :as specs] - [java-time.api :as java-time] [ring.util.response :refer [bad-request response]] [clj-slack.conversations :as slack-convo] [jsonista.core :as json] From 05097cf273a3615e65a40dcecc749e5e6ad586a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Santiago?= Date: Thu, 9 May 2024 21:47:23 +0200 Subject: [PATCH 6/6] bump version --- src/jcpsantiago/arqivist/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jcpsantiago/arqivist/core.clj b/src/jcpsantiago/arqivist/core.clj index cf07c9a..50b3b0b 100644 --- a/src/jcpsantiago/arqivist/core.clj +++ b/src/jcpsantiago/arqivist/core.clj @@ -25,7 +25,7 @@ (mulog/set-global-context! ;; TODO: get the version from a file or config, issue #23 {:app-name "The Arqivist" - :version "2023-12-18.1" + :version "2024-05-09.1" :service-profile (System/getenv "ARQIVIST_SERVICE_PROFILE")}) (mulog/log ::application-starup :arguments args) (if team