Move the config database to be postgres. Remove mongodb from the bot (#736)

This moves the way guild config is accessed to be a dict instead of always pulling from the database.
This moves config to be held in postgres instead of mongodb
This removes mongodb from the bot completely

Author: ajax146

Documentation/Extension-howto.md

@@ -267,7 +267,7 @@ self.bot.file_config.group.subgroup.key
 ---
 To access the json config, you can add the following line of code, which loads the guild config file:
 ```py
-config = await self.bot.get_context_config(guild=ctx.guild)
+config = self.bot.guild_configs[guild_id]
 ```
 
 Afterwards you can access the values with 

Documentation/TestCases.md

@@ -1,6 +1,6 @@
 # Application
 Uses Google forum for Application
-Uses Mongo for storing the application
+Uses Postgres for storing the application
 Config setup
 ## Command Restrictions
 .application get <application_id>
@@ -713,7 +713,7 @@ Discord Library Random
 	Will send a deny embed "I ran into an error processing your command: Converting to "int" failed for parameter "max"." if cannot find an int for second parameter
 
 # Rules
-Uses Mongo
+Uses Postgres
 ## Command Restrictions
 .rule
 	None

Pipfile

@@ -18,7 +18,6 @@ ib3 = "==0.2.0"
 inflect = "==7.0.0"
 irc = "==20.1.0"
 isort = "==5.12.0"
-motor = "==3.3.1"
 munch = "==4.0.0"
 typing_extensions = "==4.8.0"
 pip = "==23.3.1"

Pipfile.lock

@@ -25,7 +25,7 @@ cp config.default.yml config.yml
 ```
 ### .env file
 The first file we will edit is the .env file. This is where you will store database information.  
-You will need to create a username and password for mongodb and postgres. These credentials do not have to be different.  
+You will need to create a username and password postgres.
 You will also need to create a db name for postgres. This works best when it is all lowercase, but it is not strictly required.  
 When filling in the information, do not include spaces or quotes. Just put the content directly after the equals sign.  
 You will need all of this information again, so make sure to keep note of it.  
@@ -36,9 +36,6 @@ For the admin ID, get your user ID by right clicking on your name, either on the
 #### postgres
 For postgres, you will need the username, password, and DB name you created previously. Enter it exactly as found in your .env file.  
 Do not change the port or host.  
-#### mongodb
-For mongodb, you will need the username and password you created previously. You will also need to create a DB name here. Enter the username and password exactly as found in your .env file. Just like postgres, the DB name works best with all lowercase, but it is not a requirement.  
-Do not change the port or host.  
 #### Additional configuration
 All the additional configuration is optional, and is not required to start the bot. This includes all API keys. The default settings everywhere else work, but can be changed later if desired.
 ## Final tasks
@@ -89,4 +86,4 @@ class Greeter(cogs.BaseCog):
     async def hello(self, ctx):
         await self.hello_command(ctx)
 ```
-Extensions can be configured per-guild with settings saved on MongoDB. There are several extensions included in the main repo, so please reference them for more advanced examples.
+Extensions can be configured per-guild with settings saved on Postgres. There are several extensions included in the main repo, so please reference them for more advanced examples.

README.md

@@ -13,12 +13,6 @@ database:
         name:
         host: postgres
         port: 5432
-    mongodb:
-        user:
-        password:
-        name:
-        host: mongodb
-        port: 27017
 api:
     github:
         api_key:

config.default.yml

@@ -1,5 +1,3 @@
-MONGO_DB_PASSWORD=
-MONGO_DB_USER=
 POSTGRES_DB_NAME=
 POSTGRES_DB_PASSWORD=
 POSTGRES_DB_USER=

default.env

@@ -16,7 +16,6 @@ services:
             - all
         depends_on:
             - postgres
-            - mongodb
 
     postgres:
         image: postgres:15.3-alpine
@@ -33,20 +32,6 @@ services:
         networks:
             - all
 
-    mongodb:
-        image: mongo:6.0.7
-        container_name: mongodb
-        volumes:
-            - ../DBs/mongo:/data/db
-        ports:
-            - "127.0.0.1:27017:27017"
-        environment:
-            - MONGO_INITDB_ROOT_PASSWORD=${MONGO_DB_PASSWORD}
-            - MONGO_INITDB_ROOT_USERNAME=${MONGO_DB_USER}
-        restart: unless-stopped
-        networks:
-            - all
-
 networks:
   all:
     driver: bridge

docker-compose.yml

@@ -2,10 +2,10 @@
 
 import asyncio
 import datetime
+import json
 import random
 import re
 import string
-import time
 
 import discord
 import error as custom_errors
@@ -24,15 +24,14 @@ class AdvancedBot(data.DataBot):
     including per-guild config and event logging.
     """
 
-    GUILD_CONFIG_COLLECTION = "guild_config"
     CONFIG_RECEIVE_WARNING_TIME_MS = 1000
     DM_GUILD_ID = "dmcontext"
 
     def __init__(self, *args, **kwargs):
         self.owner = None
         self.__startup_time = None
-        self.guild_config_collection = None
         self.guild_config_lock = None
+        self.guild_configs = {}
         super().__init__(*args, prefix=self.get_prefix, **kwargs)
         self.guild_config_cache = expiringdict.ExpiringDict(
             max_len=self.file_config.cache.guild_config_cache_length,
@@ -52,92 +51,35 @@ async def start(self, *args, **kwargs):
         self.guild_config_lock = asyncio.Lock()
         await super().start(*args, **kwargs)
 
-    async def get_prefix(self, message):
+    async def get_prefix(self, message: discord.Message) -> str:
         """Gets the appropriate prefix for a command.
 
         parameters:
             message (discord.Message): the message to check against
         """
-        guild_config = await self.get_context_config(guild=message.guild)
+        guild_config = self.guild_configs[str(message.guild.id)]
         return getattr(
             guild_config, "command_prefix", self.file_config.bot_config.default_prefix
         )
 
-    async def get_all_context_configs(self, projection, limit=100):
-        """Gets all context configs.
+    async def register_new_guild_config(self, guild: str) -> bool:
+        """This creates a config for a new guild if needed
 
-        parameters:
-            projection (dict): the MongoDB projection for returned data
-            limit (int): the max number of config objects to return
-        """
-        configs = []
-        cursor = self.guild_config_collection.find({}, projection)
-        for document in await cursor.to_list(length=limit):
-            configs.append(munch.DefaultMunch.fromDict(document, None))
-        return configs
-
-    async def get_context_config(
-        self, ctx=None, guild=None, create_if_none=True, get_from_cache=True
-    ):
-        """Gets the appropriate config for the context.
+        Args:
+            guild (str): The id of the guild to create config for, in string form
 
-        parameters:
-            ctx (discord.ext.Context): the context of the config
-            guild (discord.Guild): the guild associated with the config (provided instead of ctx)
-            create_if_none (bool): True if the config should be created if not found
-            get_from_cache (bool): True if the config should be fetched from the cache
+        Returns:
+            bool: True if a config was created, False if a config already existed
         """
-        start = time.time()
-
-        if ctx:
-            guild_from_ctx = getattr(ctx, "guild", None)
-            lookup = guild_from_ctx.id if guild_from_ctx else self.DM_GUILD_ID
-        elif guild:
-            lookup = guild.id
-        else:
-            return None
-
-        lookup = str(lookup)
-
-        config_ = None
-
-        if get_from_cache:
-            config_ = self.guild_config_cache.get(lookup)
-
-        if not config_:
-            # locking prevents duplicate configs being made
-            async with self.guild_config_lock:
-                config_ = await self.guild_config_collection.find_one(
-                    {"guild_id": {"$eq": lookup}}
-                )
-
-                if not config_:
-                    await self.logger.send_log(
-                        message="No config found in MongoDB",
-                        level=LogLevel.DEBUG,
-                        console_only=True,
-                    )
-                    if create_if_none:
-                        config_ = await self.create_new_context_config(lookup)
-                else:
-                    config_ = await self.sync_config(config_)
-
-                if config_:
-                    self.guild_config_cache[lookup] = config_
-
-        time_taken = (time.time() - start) * 1000.0
-
-        if time_taken > self.CONFIG_RECEIVE_WARNING_TIME_MS:
-            await self.logger.send_log(
-                message=(
-                    f"Context config receive time = {time_taken} ms (over"
-                    f" {self.CONFIG_RECEIVE_WARNING_TIME_MS} threshold)"
-                ),
-                level=LogLevel.WARNING,
-                context=LogContext(guild=self.get_guild(lookup)),
-            )
-
-        return config_
+        async with self.guild_config_lock:
+            try:
+                config = self.guild_configs[guild]
+            except KeyError:
+                config = None
+            if not config:
+                await self.create_new_context_config(guild)
+                return True
+            return False
 
     async def create_new_context_config(self, lookup: str):
         """Creates a new guild config based on a lookup key (usually a guild ID).
@@ -178,18 +120,43 @@ async def create_new_context_config(self, lookup: str):
                 context=LogContext(guild=self.get_guild(lookup)),
                 console_only=True,
             )
-            await self.guild_config_collection.insert_one(config_)
+            # Modify the database
+            await self.write_new_config(str(lookup), json.dumps(config_))
+
+            # Modify the local cache
+            self.guild_configs[lookup] = config_
+
         except Exception as exception:
             # safely finish because the new config is still useful
             await self.logger.send_log(
-                message="Could not insert guild config into MongoDB",
+                message="Could not insert guild config into Postgres",
                 level=LogLevel.ERROR,
                 context=LogContext(guild=self.get_guild(lookup)),
                 exception=exception,
             )
 
         return config_
 
+    async def write_new_config(self, guild_id: str, config: str) -> None:
+        """Takes a config and guild and updates the config in the database
+        This is rarely needed
+
+        Args:
+            guild_id (str): The str ID of the guild the config belongs to
+            config (str): The str representation of the json config
+        """
+        database_config = await self.models.Config.query.where(
+            self.models.Config.guild_id == guild_id
+        ).gino.first()
+        if database_config:
+            await database_config.update(config=str(config)).apply()
+        else:
+            new_database_config = self.models.Config(
+                guild_id=str(guild_id),
+                config=str(config),
+            )
+            await new_database_config.create()
+
     async def sync_config(self, config_object):
         """Syncs the given config with the currently loaded extensions.
 
@@ -229,10 +196,14 @@ async def sync_config(self, config_object):
                 context=LogContext(guild=self.get_guild(config_object.guild_id)),
                 console_only=True,
             )
-            await self.guild_config_collection.replace_one(
-                {"_id": config_object.get("_id")}, config_object
+            # Modify the database
+            await self.write_new_config(
+                str(config_object.guild_id), json.dumps(config_object)
             )
 
+            # Modify the local cache
+            self.guild_configs[config_object.guild_id] = config_object
+
         return config_object
 
     async def can_run(self, ctx: commands.Context, *, call_once=False):
@@ -249,7 +220,7 @@ async def can_run(self, ctx: commands.Context, *, call_once=False):
             console_only=True,
         )
         is_bot_admin = await self.is_bot_admin(ctx)
-        config = await self.get_context_config(ctx)
+        config = self.guild_configs[str(ctx.guild.id)]
 
         # Rate limiter
         if config.rate_limit.get("enabled", False):
@@ -264,8 +235,6 @@ async def can_run(self, ctx: commands.Context, *, call_once=False):
             if ctx.message.id not in self.command_execute_history[identifier]:
                 self.command_execute_history[identifier][ctx.message.id] = True
 
-            print(self.command_execute_history[identifier])
-
             if (
                 len(self.command_execute_history[identifier])
                 > config.rate_limit.commands
@@ -280,7 +249,7 @@ async def can_run(self, ctx: commands.Context, *, call_once=False):
 
         extension_name = self.get_command_extension_name(ctx.command)
         if extension_name:
-            config = await self.get_context_config(ctx)
+            config = self.guild_configs[str(ctx.guild.id)]
             if not extension_name in config.enabled_extensions:
                 raise custom_errors.ExtensionDisabled
 
@@ -354,7 +323,7 @@ def startup_time(self):
         """Gets the startup timestamp of the bot."""
         return self.__startup_time
 
-    async def get_log_channel_from_guild(self, guild, key):
+    async def get_log_channel_from_guild(self, guild: discord.Guild, key: str):
         """Gets the log channel ID associated with the given guild.
 
         This also checks if the channel exists in the correct guild.
@@ -366,8 +335,8 @@ async def get_log_channel_from_guild(self, guild, key):
         if not guild:
             return None
 
-        config_ = await self.get_context_config(guild=guild)
-        channel_id = config_.get(key)
+        config = self.guild_configs[str(guild.id)]
+        channel_id = config.get(key)
 
         if not channel_id:
             return None
@@ -459,7 +428,7 @@ def format_username(self, username: str) -> str:
 
     async def on_member_join(self, member: discord.Member):
         """See: https://discordpy.readthedocs.io/en/latest/api.html#discord.on_member_join"""
-        config = await self.get_context_config(guild=member.guild)
+        config = self.guild_configs[str(member.guild.id)]
 
         if config.get("nickname_filter", False):
             temp_name = self.format_username(member.display_name)