From e7276501e01f73b6a67a085458024d58e80277ba Mon Sep 17 00:00:00 2001 From: rolobio Date: Tue, 14 May 2019 12:01:53 -0600 Subject: [PATCH 01/15] Updating travis password --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7489473..7c1654d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ after_success: deploy: provider: pypi user: rolobio - password: - secure: vQdc1p4zDA87PVcr6/vD2lH9hBltJbF+9O0UhTH2EGpMOgB+Fm69YXRnkbBg2La2yQsXDHpNyXCi0SR0vw8+z+ufDMgTux4yTZS5rpnR1lJVuV+p1ANSk9s6Pn6So+3OL2jl45FMwcjb3MDhr1Qx7Ot7VhtI7ZqcCXv3fWGdpaPtfdWBFZA4u7g0jmEswBDAIOnnda7uKJWLaXIshhqt4R5FKNWPidGUuiN0ZPe2n69C3gx9PBqHEuLPCOA74waG4F7d/1bavfe2sOgZJGFtUevyWpuxb53lxQ49VjYKn4li78VonwzgJxJSXt7ABiURHi4osTlCwyhOnvEor+vIb6DEIgUuFCDpqONY9gijh7mMk2jvf87hy2Pi7DQJnap5riJ0KRL3DaXLwEm4k+mjL2nAnXiCjTWgIWjC3+wXQwqrQ1dhhLcYItwgQNRyAkJEQ6lYfLgKzx3wO+TAurb6ET3/aFLBM8x4Xop3fItF/IZdCOSn3BU2I8mWKXcEaZp3REw97+sfJ7XbZ5ni/TXPsqJfWCNyLM2Z23mlwSWfvouGJnN0vRZptlbimTeOtzyidBT3JnvvQ6lyD60QzwrUziQ+4G6jC+ZeQjZh2qy6W8/T/V9loT9xdYjGzJyzXAKE2iyohiyBB4N6ve/zazTmlfgb8klXvUgpNO8X3AT+5IA= on: tags: true + password: + secure: l3qUamczpqrBejHBPTuDOGTYtDQeKymhmwa252b8QNZjI3St46j8o+F6cnCTfzLW51U17+83CwKEUqKRGGkZE2iWGy3aflghgSdeDI09d/jnEfQRCPs9SpfK9dntOJkJfPnVPY4VXWsIexgK9CsN7FDaBtqpXZLOIzbAqsERgPiYbH5n1yNj6qh8JaYBiTajyTKbuk2cXLCpgEI0OVyFvVDbLla5HVnxGcEOkCbaizUYwb0FxMCMeH0prDB/jqrI7yFXhplD0yYfoo+HG9ec8BoBvxBjxemZ/zyjEoCljeKjXr4NLDnSCWpbaa89KJoMPMQUV7IffXp9S5TDyJ+QLItoglk/NabbMOuRchlbL31b9UhyDcqCiVpgIAjOKbDWL0TDoUbf6avzafQ2u0K3Ivi+28HXKvWFtF0zFGPwJbWFm3G6PTxynBtWmsBmhAAKNb68hRugOqA72aOcsbNEBlJ6g8A0ULhwrP6y1SOMJl8lS6iA8zG+xWY5BUczvDyS0bt7UUjxxRAuEkRrqcu46kGUgW/PK1nLtEVZi2gHEQv1s/h7Pdp7904wk6ce9hyVf22X9fgSkzYEvnQQP3ukB+IsO0qu51dbYugKfgtJfkSBhLRd8rAi1MfD6x1eCfj/60Ai3E0UGi/HjRKbfI1Il9UmVXfcaUqccSAJwMRYLlA= \ No newline at end of file From a4a9e6de4801c4a37bec4d7860c36d9936006c59 Mon Sep 17 00:00:00 2001 From: rolobio Date: Thu, 30 May 2019 11:43:05 -0600 Subject: [PATCH 02/15] Adding type hinting for high-level dictorm methods --- .gitignore | 3 +- dictorm/dictorm.py | 565 +++++++++++++++++++++++---------------------- dictorm/pg.py | 4 + 3 files changed, 298 insertions(+), 274 deletions(-) diff --git a/.gitignore b/.gitignore index 20150d6..56a710a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ __pycache__/ # Distribution / packaging .Python -env/ +env env2/ build/ develop-eggs/ @@ -83,6 +83,7 @@ celerybeat-schedule venv/ ENV/ + # Spyder project settings .spyderproject diff --git a/dictorm/dictorm.py b/dictorm/dictorm.py index 33b3487..25741b6 100644 --- a/dictorm/dictorm.py +++ b/dictorm/dictorm.py @@ -1,4 +1,6 @@ """What if you could insert a Python dictionary into the database? DictORM allows you to select/insert/update rows of a database as if they were Python Dictionaries.""" +from typing import Union, Optional, List + __version__ = '3.8.4' from contextlib import contextmanager @@ -8,7 +10,7 @@ try: # pragma: no cover from dictorm.pg import Select, Insert, Update, Delete - from dictorm.pg import And, Or + from dictorm.pg import And, Or, QueryHint from dictorm.pg import Column, Comparison, Operator from dictorm.sqlite import Insert as SqliteInsert from dictorm.sqlite import Column as SqliteColumn @@ -78,122 +80,193 @@ def set_json_dicts(func): return original -class DictDB(dict): +CursorHint = Union[sqlite3.Cursor, DictCursor] + + +class Dict(dict): """ - Get all the tables from the provided Psycopg2/Sqlite3 connection. Create a - Table instance for each table, and keep them in this DictDB using the - table's name as a key. + This is a representation of a database row that behaves exactly like a + dictionary. You may update this dictionary using update or simply by + setting an item. After you make changes, be sure to call "flush" to send + your changes to the DB. Your changes will not be commited or rolled-back, + you must do that. - >>> db = DictDB(your_db_connection) - >>> db['table1'] - Table('table1') + This requires primary keys and they should be specified. Really, your + tables should have a primary key of some sort. If not, this will pretty + much be a read-only object. - >>> db['other_table'] - Table('other_table') + You can change the primary key of an instance. - If your tables have changed while your DictDB instance existed, you can call - DictDB.refresh_tables() to have it rebuild all Table objects. + Use setitem: + >>> d['manager_id'] = 4 + + Use an update: + >>> d.update({'manager_id':4}) + + Update using another Dict: + >>> d1.update(d2.no_pks()) + + Make sure to send your changes to the database: + >>> d.flush() + + Remove a row: + >>> d.delete() """ - def __init__(self, db_conn): - self.conn = db_conn - if 'sqlite3' in modules and isinstance(db_conn, sqlite3.Connection): - self.kind = 'sqlite3' - self.insert = SqliteInsert - self.update = SqliteUpdate - self.column = SqliteColumn + def __init__(self, table, *a, **kw): + self._table: Table = table + self._in_db = False + self._curs: CursorHint = table.db.curs + super(Dict, self).__init__(*a, **kw) + self._old_pk_and = None + + def flush(self): + """ + Insert this dictionary into it's table if its no yet in the Database, or + Update it's row if it is already in the database. This method relies + heavily on the primary keys of the row's respective table. If no + primary keys are specified, this method will not function! + + All original column/values will bet inserted/updated by this method. + All references will be flushed as well. + """ + if self._table.refs: + for i in self.values(): + if isinstance(i, Dict): + i.flush() + + # This will be sent to the DB, don't convert dicts to json unless + # the table has json columns. + items = self.no_refs() + if self._table.has_json: + items = _json_dicts(items) + + # Insert/Update only with columns present on the table, this allows custom + # instances of Dicts to be inserted even if they have columns not on the table + items = {k: v for k, v in items.items() if k in self._table.column_names} + + if not self._in_db: + # Insert this Dict into it's respective table, interpolating + # my values into the query + query = self._table.db.insert(self._table.name, **items + ).returning('*') + d = self.__execute_query(query) + self._in_db = True else: - self.kind = 'postgresql' - self.insert = Insert - self.update = Update - self.column = Column - self.select = Select - self.delete = Delete + # Update this dictionary's row + if not self._table.pks: + raise NoPrimaryKey( + 'Cannot update to {0}, no primary keys defined.'.format( + self._table)) + # Update without references, "wheres" are the primary values + query = self._table.db.update(self._table.name, **items + ).where(self._old_pk_and or self.pk_and()).returning('*') + d = self.__execute_query(query) - self.curs = self.get_cursor() - self.refresh_tables() - self.conn.rollback() - super(DictDB, self).__init__() + if d: + super(Dict, self).__init__(d) + self._old_pk_and = self.pk_and() + return self - @classmethod - def table_factory(cls): - return Table + def delete(self): + """ + Delete this row from it's table in the database. Requires primary keys + to be specified. + """ + query = self._table.db.delete(self._table.name).where( + self._old_pk_and or self.pk_and()) + return self.__execute_query(query) - def __list_tables(self): - if self.kind == 'sqlite3': - self.curs.execute('SELECT name FROM sqlite_master WHERE type =' - '"table"') + def __execute_query(self, query): + built = query.build() + if isinstance(built, list): + for sql, values in built: + self._curs.execute(sql, values) + if query.append_returning: + return self._curs.fetchone() else: - self.curs.execute('''SELECT DISTINCT table_name - FROM information_schema.columns - WHERE table_schema='public' ''') - return self.curs.fetchall() + sql, values = built + self._curs.execute(sql, values) + if query._returning: + return self._curs.fetchone() - def get_cursor(self): + def pk_and(self): """ - Returns a cursor from the provided database connection that DictORM - objects expect. + Return an And() of all this Dict's primary key and values. i.e. + And(id=1, other_primary=4) """ - if self.kind == 'sqlite3': - self.conn.row_factory = sqlite3.Row - return self.conn.cursor() - elif self.kind == 'postgresql': - return self.conn.cursor(cursor_factory=DictCursor) + return And(*[self._table[k] == v for k, v in self.items() if k in \ + self._table.pks]) - def refresh_tables(self): + def no_pks(self): """ - Create all Table instances from all tables found in the database. + Return a dictionary without the primary keys that are associated with + this Dict in the Database. This should be used when doing an update of + another Dict. """ - if self.keys(): - # Reset this DictDB because it contains old tables - super(DictDB, self).__init__() - table_cls = self.table_factory() - for table in self.__list_tables(): - if self.kind == 'sqlite3': - self[table['name']] = table_cls(table['name'], self) - else: - self[table['table_name']] = table_cls(table['table_name'], self) + return {k: v for k, v in self.items() if k not in self._table.pks} - @contextmanager - def transaction(self, commit=False): - try: - yield - except: - self.conn.rollback() - raise - else: - # Commit if no exceptions occur - if commit: - self.conn.commit() + def no_refs(self): + """ + Return a dictionary without the key/value(s) added by a reference. They + should never be sent in the query to the Database. + """ + return {k: v for k, v in self.items() if k not in self._table.refs} + def references(self): + """ + Return a dictionary of only the referenced rows. + """ + return {k: v for k, v in self.items() if k in self._table.refs} -def args_to_comp(operator, table, *args, **kwargs): - """ - Add arguments to the provided operator paired with their respective primary - key. - """ - operator = operator or And() - pk_uses = 0 - pks = table.pks - for val in args: - if isinstance(val, (Comparison, Operator)): - # Already a Comparison/Operator, just add it - operator += (val,) - continue - if not table.pks: - raise NoPrimaryKey('No Primary Keys(s) defined for ' + str(table)) - try: - # Create a Comparison using the next Primary Key - operator += (table[pks[pk_uses]] == val,) - except IndexError: - raise NoPrimaryKey('Not enough Primary Keys(s) defined for ' + - str(table)) - pk_uses += 1 + def __getitem__(self, key): + """ + Get the provided "key" from this Dict instance. If the key refers to a + referenced row, get that row first. Will only get a referenced row + once, until the referenced row's foreign key is changed. + """ + ref = self._table.refs.get(key) + if not ref and key not in self: + raise KeyError(str(key)) + # Only get the referenced row once, if it has a value, the reference's + # column hasn't been changed. + val = super(Dict, self).get(key) + if ref and not val: + table = ref.column2.table + comparison = table[ref.column2.column] == self[ref.column1.column] - for k, v in kwargs.items(): - operator += table[k] == v + if ref.many: + gen = table.get_where(comparison) + if ref._substratum: + gen = [i[ref._substratum] for i in gen] + if ref._aggregate: + gen = list(chain(*gen)) + return gen + else: + val = table.get_one(comparison) + if ref._substratum and val: + return val[ref._substratum] + super(Dict, self).__setitem__(key, val) + return val - return operator + def get(self, key, default=None): + # Provide the same functionality as a dict.get, but use this class's + # __getitem__ instead of builtin __getitem__ + return self[key] if key in self else default + + def __setitem__(self, key, value): + """ + Set self[key] to value. If key is a reference's matching foreign key, + set the reference to None. + """ + ref = self._table.fks.get(key) + if ref: + super(Dict, self).__setitem__(ref, None) + return super(Dict, self).__setitem__(key, value) + + # Copy docs for methods that recreate dict() functionality + __getitem__.__doc__ += dict.__getitem__.__doc__ + get.__doc__ = dict.get.__doc__ class RawQuery: @@ -217,15 +290,15 @@ class ResultsGenerator: create a new ResultsGenerator instance, or flush your Dict. """ - def __init__(self, table, query, db): - self.table = table + def __init__(self, table, query: QueryHint, db): + self.table: Table = table self.query = query self.cache = [] self.completed = False self.executed = False self.db_kind = db.kind - self.db = db - self.curs = self.db.get_cursor() + self.db: DictDB = db + self.curs: CursorHint = self.db.get_cursor() self._nocache = False def __iter__(self): @@ -234,7 +307,7 @@ def __iter__(self): else: return self - def __next__(self): + def __next__(self) -> Dict: self.__execute_once() d = self.curs.fetchone() if not d: @@ -243,7 +316,7 @@ def __next__(self): # Convert returned dictionary to a Dict d = self.table(d) d._in_db = True - if self._nocache == False: + if self._nocache is False: self.cache.append(d) return d @@ -256,7 +329,7 @@ def __execute_once(self): # for python 2.7 next = __next__ - def __len__(self): + def __len__(self) -> int: self.__execute_once() if self.db_kind == 'sqlite3': # sqlite3's cursor.rowcount doesn't support select statements @@ -265,7 +338,7 @@ def __len__(self): return 0 return self.curs.rowcount - def __getitem__(self, i): + def __getitem__(self, i) -> Dict: if isinstance(i, int) and i >= 0: try: return self.cache[i] @@ -347,9 +420,6 @@ def offset(self, offset): return ResultsGenerator(self.table, query, self.db) -_json_column_types = ('json', 'jsonb') - - class Table(object): """ A representation of a DB table. You will primarily retrieve rows (Dicts) @@ -378,7 +448,9 @@ class Table(object): You can reference another table using setitem. Link to an employee's manager using the manager's id, and the employee's manager_id. + >>> Person = db['person'] >>> Person['manager'] = Person['manager_id'] == Person['id'] + >>> bob = Person(name='Bob') >>> bob['manager'] Dict() @@ -445,10 +517,10 @@ def _refresh_pks(self): AND i.indisprimary;''' % self.name) self.pks = [i[0] for i in self.curs.fetchall()] - def __repr__(self): # pragma: no cover + def __repr__(self) -> str: # pragma: no cover return 'Table({0}, {1})'.format(self.name, self.pks) - def __call__(self, *a, **kw): + def __call__(self, *a, **kw) -> Dict: """ Used to insert a row into this table. """ @@ -457,7 +529,7 @@ def __call__(self, *a, **kw): d[ref_name] = None return d - def get_where(self, *a, **kw): + def get_where(self, *a, **kw) -> ResultsGenerator: """ Get all rows as Dicts where column values are as specified. This always returns a generator-like object ResultsGenerator. @@ -504,7 +576,7 @@ def get_where(self, *a, **kw): query = Select(self.name, operator_group).order_by(order_by) return ResultsGenerator(self, query, self.db) - def get_one(self, *a, **kw): + def get_one(self, *a, **kw) -> Optional[Dict]: """ Get a single row as a Dict from the Database that matches the arguments provided to this method. See Table.get_where for more details. @@ -525,7 +597,7 @@ def get_one(self, *a, **kw): raise UnexpectedRows('More than one row selected.') return i - def get_raw(self, sql_query, *a): + def get_raw(self, sql_query, *a) -> ResultsGenerator: """ Get all rows returned by the raw SQL query provided, as Dicts. Expects that the query will only return columns from this instance's table. @@ -535,7 +607,7 @@ def get_raw(self, sql_query, *a): query = RawQuery(sql_query, *a) return ResultsGenerator(self, query, self.db) - def count(self): + def count(self) -> int: """ Get the count of rows in this table. """ @@ -544,7 +616,7 @@ def count(self): return int(self.curs.fetchone()[0]) @property - def columns(self): + def columns(self) -> List[str]: """ Get a list of columns of a table. """ @@ -555,7 +627,7 @@ def columns(self): return [i[key] for i in self.columns_info] @property - def columns_info(self): + def columns_info(self) -> List[dict]: """ Get a dictionary that contains information about all columns of this table. @@ -574,7 +646,7 @@ def columns_info(self): return self.cached_columns_info @property - def column_names(self): + def column_names(self) -> set: if not self.cached_column_names: if self.db.kind == 'sqlite3': self.cached_column_names = set(i['name'] for i in @@ -601,7 +673,7 @@ def __setitem__(self, ref_name, ref): self.fks[ref.column1.column] = ref_name self.refs[ref_name] = ref - def __getitem__(self, ref_name): + def __getitem__(self, ref_name) -> Union[Column, SqliteColumn]: """ Get a reference if it has already been created. Otherwise, return a Column object which is used to create a reference. @@ -611,187 +683,134 @@ def __getitem__(self, ref_name): return self.db.column(self, ref_name) -class Dict(dict): +class DictDB(dict): """ - This is a representation of a database row that behaves exactly like a - dictionary. You may update this dictionary using update or simply by - setting an item. After you make changes, be sure to call "flush" to send - your changes to the DB. Your changes will not be commited or rolled-back, - you must do that. - - This requires primary keys and they should be specified. Really, your - tables should have a primary key of some sort. If not, this will pretty - much be a read-only object. - - You can change the primary key of an instance. - - Use setitem: - >>> d['manager_id'] = 4 - - Use an update: - >>> d.update({'manager_id':4}) + Get all the tables from the provided Psycopg2/Sqlite3 connection. Create a + Table instance for each table, and keep them in this DictDB using the + table's name as a key. - Update using another Dict: - >>> d1.update(d2.no_pks()) + >>> db = DictDB(your_db_connection) + >>> db['table1'] + Table('table1') - Make sure to send your changes to the database: - >>> d.flush() + >>> db['other_table'] + Table('other_table') - Remove a row: - >>> d.delete() + If your tables have changed while your DictDB instance existed, you can call + DictDB.refresh_tables() to have it rebuild all Table objects. """ - def __init__(self, table, *a, **kw): - self._table = table - self._in_db = False - self._curs = table.db.curs - super(Dict, self).__init__(*a, **kw) - self._old_pk_and = None - - def flush(self): - """ - Insert this dictionary into it's table if its no yet in the Database, or - Update it's row if it is already in the database. This method relies - heavily on the primary keys of the row's respective table. If no - primary keys are specified, this method will not function! + def __init__(self, db_conn): + self._real_getitem = super().__getitem__ + self.conn = db_conn + if 'sqlite3' in modules and isinstance(db_conn, sqlite3.Connection): + self.kind = 'sqlite3' + self.insert = SqliteInsert + self.update = SqliteUpdate + self.column = SqliteColumn + else: + self.kind = 'postgresql' + self.insert = Insert + self.update = Update + self.column = Column + self.select = Select + self.delete = Delete - All original column/values will bet inserted/updated by this method. - All references will be flushed as well. - """ - if self._table.refs: - for i in self.values(): - if isinstance(i, Dict): - i.flush() + self.curs = self.get_cursor() + self.refresh_tables() + self.conn.rollback() + super(DictDB, self).__init__() - # This will be sent to the DB, don't convert dicts to json unless - # the table has json columns. - items = self.no_refs() - if self._table.has_json: - items = _json_dicts(items) + def __getitem__(self, item: str) -> Table: + return self._real_getitem(item) - # Insert/Update only with columns present on the table, this allows custom - # instances of Dicts to be inserted even if they have columns not on the table - items = {k: v for k, v in items.items() if k in self._table.column_names} + @classmethod + def table_factory(cls) -> Table: + return Table - if not self._in_db: - # Insert this Dict into it's respective table, interpolating - # my values into the query - query = self._table.db.insert(self._table.name, **items - ).returning('*') - d = self.__execute_query(query) - self._in_db = True + def __list_tables(self): + if self.kind == 'sqlite3': + self.curs.execute('SELECT name FROM sqlite_master WHERE type =' + '"table"') else: - # Update this dictionary's row - if not self._table.pks: - raise NoPrimaryKey( - 'Cannot update to {0}, no primary keys defined.'.format( - self._table)) - # Update without references, "wheres" are the primary values - query = self._table.db.update(self._table.name, **items - ).where(self._old_pk_and or self.pk_and()).returning('*') - d = self.__execute_query(query) - - if d: - super(Dict, self).__init__(d) - self._old_pk_and = self.pk_and() - return self + self.curs.execute('''SELECT DISTINCT table_name + FROM information_schema.columns + WHERE table_schema='public' ''') + return self.curs.fetchall() - def delete(self): + def get_cursor(self) -> CursorHint: """ - Delete this row from it's table in the database. Requires primary keys - to be specified. + Returns a cursor from the provided database connection that DictORM + objects expect. """ - query = self._table.db.delete(self._table.name).where( - self._old_pk_and or self.pk_and()) - return self.__execute_query(query) - - def __execute_query(self, query): - built = query.build() - if isinstance(built, list): - for sql, values in built: - self._curs.execute(sql, values) - if query.append_returning: - return self._curs.fetchone() - else: - sql, values = built - self._curs.execute(sql, values) - if query._returning: - return self._curs.fetchone() + if self.kind == 'sqlite3': + self.conn.row_factory = sqlite3.Row + curs = self.conn.cursor() + return curs + elif self.kind == 'postgresql': + curs = self.conn.cursor(cursor_factory=DictCursor) + return curs - def pk_and(self): + def refresh_tables(self): """ - Return an And() of all this Dict's primary key and values. i.e. - And(id=1, other_primary=4) + Create all Table instances from all tables found in the database. """ - return And(*[self._table[k] == v for k, v in self.items() if k in \ - self._table.pks]) + if self.keys(): + # Reset this DictDB because it contains old tables + super(DictDB, self).__init__() + table_cls = self.table_factory() + for table in self.__list_tables(): + if self.kind == 'sqlite3': + self[table['name']] = table_cls(table['name'], self) + else: + self[table['table_name']] = table_cls(table['table_name'], self) - def no_pks(self): - """ - Return a dictionary without the primary keys that are associated with - this Dict in the Database. This should be used when doing an update of - another Dict. + @contextmanager + def transaction(self, commit=False): """ - return {k: v for k, v in self.items() if k not in self._table.pks} + Context manager to rollback changes in case of an error. - def no_refs(self): - """ - Return a dictionary without the key/value(s) added by a reference. They - should never be sent in the query to the Database. + :param commit: Commit changes on close, if True. + :return: """ - return {k: v for k, v in self.items() if k not in self._table.refs} + try: + yield + except: + self.conn.rollback() + raise + else: + # Commit if no exceptions occur + if commit: + self.conn.commit() - def references(self): - """ - Return a dictionary of only the referenced rows. - """ - return {k: v for k, v in self.items() if k in self._table.refs} - def __getitem__(self, key): - """ - Get the provided "key" from this Dict instance. If the key refers to a - referenced row, get that row first. Will only get a referenced row - once, until the referenced row's foreign key is changed. - """ - ref = self._table.refs.get(key) - if not ref and key not in self: - raise KeyError(str(key)) - # Only get the referenced row once, if it has a value, the reference's - # column hasn't been changed. - val = super(Dict, self).get(key) - if ref and not val: - table = ref.column2.table - comparison = table[ref.column2.column] == self[ref.column1.column] +def args_to_comp(operator: Operator, table: Table, *args, **kwargs): + """ + Add arguments to the provided operator paired with their respective primary + key. + """ + operator = operator or And() + pk_uses = 0 + pks = table.pks + for val in args: + if isinstance(val, (Comparison, Operator)): + # Already a Comparison/Operator, just add it + operator += (val,) + continue + if not table.pks: + raise NoPrimaryKey('No Primary Keys(s) defined for ' + str(table)) + try: + # Create a Comparison using the next Primary Key + operator += (table[pks[pk_uses]] == val,) + except IndexError: + raise NoPrimaryKey('Not enough Primary Keys(s) defined for ' + + str(table)) + pk_uses += 1 - if ref.many: - gen = table.get_where(comparison) - if ref._substratum: - gen = [i[ref._substratum] for i in gen] - if ref._aggregate: - gen = list(chain(*gen)) - return gen - else: - val = table.get_one(comparison) - if ref._substratum and val: - return val[ref._substratum] - super(Dict, self).__setitem__(key, val) - return val + for k, v in kwargs.items(): + operator += table[k] == v - def get(self, key, default=None): - # Provide the same functionality as a dict.get, but use this class's - # __getitem__ instead of builtin __getitem__ - return self[key] if key in self else default + return operator - def __setitem__(self, key, value): - """ - Set self[key] to value. If key is a reference's matching foreign key, - set the reference to None. - """ - ref = self._table.fks.get(key) - if ref: - super(Dict, self).__setitem__(ref, None) - return super(Dict, self).__setitem__(key, value) - # Copy docs for methods that recreate dict() functionality - __getitem__.__doc__ += dict.__getitem__.__doc__ - get.__doc__ = dict.get.__doc__ +_json_column_types = ('json', 'jsonb') diff --git a/dictorm/pg.py b/dictorm/pg.py index d6ed05b..3be387b 100644 --- a/dictorm/pg.py +++ b/dictorm/pg.py @@ -6,6 +6,7 @@ Sqlite queries are slightly different, but use these methods as their base. """ from copy import copy +from typing import Union global sort_keys sort_keys = False @@ -181,6 +182,9 @@ class Delete(Update): query = 'DELETE FROM "{table}"' +QueryHint = Union[Select, Insert, Update, Delete] + + class Comparison(object): interpolation_str = '%s' many = False From 6f35ca6babfec1361c92a1f5251172c077389647 Mon Sep 17 00:00:00 2001 From: rolobio Date: Thu, 30 May 2019 11:54:39 -0600 Subject: [PATCH 03/15] Removing future type hinting --- dictorm/dictorm.py | 14 +++++++------- dictorm/pg.py | 3 --- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/dictorm/dictorm.py b/dictorm/dictorm.py index 25741b6..e5500cf 100644 --- a/dictorm/dictorm.py +++ b/dictorm/dictorm.py @@ -10,7 +10,7 @@ try: # pragma: no cover from dictorm.pg import Select, Insert, Update, Delete - from dictorm.pg import And, Or, QueryHint + from dictorm.pg import And, Or from dictorm.pg import Column, Comparison, Operator from dictorm.sqlite import Insert as SqliteInsert from dictorm.sqlite import Column as SqliteColumn @@ -114,9 +114,9 @@ class Dict(dict): """ def __init__(self, table, *a, **kw): - self._table: Table = table + self._table = table self._in_db = False - self._curs: CursorHint = table.db.curs + self._curs = table.db.curs super(Dict, self).__init__(*a, **kw) self._old_pk_and = None @@ -290,15 +290,15 @@ class ResultsGenerator: create a new ResultsGenerator instance, or flush your Dict. """ - def __init__(self, table, query: QueryHint, db): - self.table: Table = table + def __init__(self, table, query, db): + self.table = table self.query = query self.cache = [] self.completed = False self.executed = False self.db_kind = db.kind - self.db: DictDB = db - self.curs: CursorHint = self.db.get_cursor() + self.db = db + self.curs = self.db.get_cursor() self._nocache = False def __iter__(self): diff --git a/dictorm/pg.py b/dictorm/pg.py index 3be387b..abc4e62 100644 --- a/dictorm/pg.py +++ b/dictorm/pg.py @@ -182,9 +182,6 @@ class Delete(Update): query = 'DELETE FROM "{table}"' -QueryHint = Union[Select, Insert, Update, Delete] - - class Comparison(object): interpolation_str = '%s' many = False From 5fbcdd5f82ada2eb70f714ccc9a385f09f49ab4d Mon Sep 17 00:00:00 2001 From: rolobio Date: Thu, 30 May 2019 12:01:24 -0600 Subject: [PATCH 04/15] Bumping version. Testing bigint. --- dictorm/dictorm.py | 2 +- dictorm/test/test_dictorm.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dictorm/dictorm.py b/dictorm/dictorm.py index e5500cf..7104322 100644 --- a/dictorm/dictorm.py +++ b/dictorm/dictorm.py @@ -1,7 +1,7 @@ """What if you could insert a Python dictionary into the database? DictORM allows you to select/insert/update rows of a database as if they were Python Dictionaries.""" from typing import Union, Optional, List -__version__ = '3.8.4' +__version__ = '3.8.5' from contextlib import contextmanager from itertools import chain diff --git a/dictorm/test/test_dictorm.py b/dictorm/test/test_dictorm.py index 821fd1e..6253123 100755 --- a/dictorm/test/test_dictorm.py +++ b/dictorm/test/test_dictorm.py @@ -92,7 +92,7 @@ def setUp(self): self.tearDown() self.curs.execute(''' CREATE TABLE person ( - id SERIAL PRIMARY KEY, + id BIGSERIAL PRIMARY KEY, name VARCHAR(100), other INTEGER, manager_id INTEGER REFERENCES person(id) @@ -913,7 +913,7 @@ def test_column_info(self): """ Person = self.db['person'] test_info = [ - {'column_name': 'id', 'data_type': 'integer'}, + {'column_name': 'id', 'data_type': 'bigint'}, {'column_name': 'name', 'data_type': 'character varying'}, {'column_name': 'other', 'data_type': 'integer'}, {'column_name': 'manager_id', 'data_type': 'integer'}, From 665a1177eef80aa3d1f5a1e7ed4dad3827ce696e Mon Sep 17 00:00:00 2001 From: rolobio Date: Thu, 30 May 2019 12:07:15 -0600 Subject: [PATCH 05/15] Setting cursor hint directly from import --- dictorm/dictorm.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dictorm/dictorm.py b/dictorm/dictorm.py index 7104322..3618527 100644 --- a/dictorm/dictorm.py +++ b/dictorm/dictorm.py @@ -27,6 +27,8 @@ try: # pragma: no cover from psycopg2.extras import DictCursor + CursorHint = DictCursor + db_package_imported = True except ImportError: # pragma: no cover pass @@ -34,6 +36,8 @@ try: # pragma: no cover import sqlite3 + CursorHint = sqlite3.Cursor + db_package_imported = True except ImportError: # pragma: no cover pass @@ -80,9 +84,6 @@ def set_json_dicts(func): return original -CursorHint = Union[sqlite3.Cursor, DictCursor] - - class Dict(dict): """ This is a representation of a database row that behaves exactly like a From 5ab9dbd98d0d6ad7512aecf7cb5619ef7bafa004 Mon Sep 17 00:00:00 2001 From: rolobio Date: Thu, 30 May 2019 12:22:38 -0600 Subject: [PATCH 06/15] Adding python3.7 testing --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 7c1654d..d46b215 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,13 @@ os: - linux +dist: xenial # for Python 3.7 language: python python: - "2.7" - "3.4" - "3.5" - "3.6" + - "3.7" env: - PG_VERSION=9.2 - PG_VERSION=9.3 From 027a9846f73e37d923fd58c38c0c36bb868ef286 Mon Sep 17 00:00:00 2001 From: rolobio Date: Thu, 30 May 2019 12:23:53 -0600 Subject: [PATCH 07/15] Testing python developtment builds as well --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index d46b215..4dd0dc6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,8 @@ python: - "3.5" - "3.6" - "3.7" + - "3.7-dev" + - "3.8-dev" env: - PG_VERSION=9.2 - PG_VERSION=9.3 From b65fb0a60ec7ded72377b8adc5f5fa9e52bf1d59 Mon Sep 17 00:00:00 2001 From: rolobio Date: Thu, 30 May 2019 12:48:21 -0600 Subject: [PATCH 08/15] Dropping 3.8-dev --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4dd0dc6..a3db9b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,6 @@ python: - "3.6" - "3.7" - "3.7-dev" - - "3.8-dev" env: - PG_VERSION=9.2 - PG_VERSION=9.3 From f8507a3026056746daeba95eb58992b9bc821d62 Mon Sep 17 00:00:00 2001 From: rolobio Date: Thu, 30 May 2019 12:50:30 -0600 Subject: [PATCH 09/15] Adding postgres 10 for travis ci --- .travis.yml | 65 +++++++++++++++++++++--------------- dictorm/test/test_dictorm.py | 2 +- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index a3db9b2..716d660 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,41 +1,52 @@ os: - - linux + - linux dist: xenial # for Python 3.7 language: python python: - - "2.7" - - "3.4" - - "3.5" - - "3.6" - - "3.7" - - "3.7-dev" + - "2.7" + - "3.4" + - "3.5" + - "3.6" + - "3.7" + - "3.7-dev" env: - - PG_VERSION=9.2 - - PG_VERSION=9.3 - - PG_VERSION=9.4 - - PG_VERSION=9.5 - - PG_VERSION=9.6 + - PG_VERSION=9.2 + - PG_VERSION=9.3 + - PG_VERSION=9.4 + - PG_VERSION=9.5 + - PG_VERSION=9.6 + - PG_VERSION=10 + global: + - PGPORT=5433 +addons: + postgresql: "10" + apt: + packages: + - postgresql-10 + - postgresql-client-10 before_install: - # Stop any postgres, then start a specific version of postgres to test against - - sudo service postgresql stop && sudo service postgresql start $PG_VERSION + # Copy postgres config for version 10 + - sudo cp /etc/postgresql/{9.6,10}/main/pg_hba.conf + # Stop any postgres, then start a specific version of postgres to test against + - sudo service postgresql stop && sudo service postgresql start $PG_VERSION install: - - "pip install -e .[testing]" + - "pip install -e .[testing]" before_script: - - psql -c 'create database dictorm;' -U postgres + - psql -c 'create database dictorm;' -U postgres script: - # Run tests verbosely, with coverage - - "green -rvv" - # Test installation - - "python setup.py install" - # Verify that dictorm can be imported - - "python -c 'from dictorm import *'" + # Run tests verbosely, with coverage + - "green -rvv" + # Test installation + - "python setup.py install" + # Verify that dictorm can be imported + - "python -c 'from dictorm import *'" services: - - postgresql + - postgresql after_success: - # Submit coverage report to coveralls - - coveralls - # Run profiler - - time python ./profiler.py + # Submit coverage report to coveralls + - coveralls + # Run profiler + - time python ./profiler.py deploy: provider: pypi user: rolobio diff --git a/dictorm/test/test_dictorm.py b/dictorm/test/test_dictorm.py index 6253123..95de3e1 100755 --- a/dictorm/test/test_dictorm.py +++ b/dictorm/test/test_dictorm.py @@ -17,7 +17,7 @@ } if 'CI' in os.environ.keys(): - test_db_login['port'] = 5432 + test_db_login['port'] = 5433 def _no_refs(o): From c2a82194062d58df1ff53415860b84bc010b2081 Mon Sep 17 00:00:00 2001 From: rolobio Date: Thu, 30 May 2019 12:55:50 -0600 Subject: [PATCH 10/15] Caching pip --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 716d660..a018aa5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ python: - "3.6" - "3.7" - "3.7-dev" +cache: pip env: - PG_VERSION=9.2 - PG_VERSION=9.3 From 16e60ccd4367c0f6d544e7757c96cee73e0aa453 Mon Sep 17 00:00:00 2001 From: rolobio Date: Thu, 30 May 2019 13:21:08 -0600 Subject: [PATCH 11/15] Attempting to repair travis config --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a018aa5..9e8402d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,8 +17,7 @@ env: - PG_VERSION=9.5 - PG_VERSION=9.6 - PG_VERSION=10 - global: - - PGPORT=5433 + - PGPORT=5433 addons: postgresql: "10" apt: @@ -39,7 +38,7 @@ script: - "green -rvv" # Test installation - "python setup.py install" - # Verify that dictorm can be imported + # Verify that dictorm can be imported - "python -c 'from dictorm import *'" services: - postgresql From e8e86d2997e786fe79ebe95c2ca7b67dc0542f15 Mon Sep 17 00:00:00 2001 From: rolobio Date: Thu, 30 May 2019 14:02:42 -0600 Subject: [PATCH 12/15] Revert "Removing future type hinting" This reverts commit 6f35ca6babfec1361c92a1f5251172c077389647. --- dictorm/dictorm.py | 14 +++++++------- dictorm/pg.py | 3 +++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/dictorm/dictorm.py b/dictorm/dictorm.py index 3618527..b88a68b 100644 --- a/dictorm/dictorm.py +++ b/dictorm/dictorm.py @@ -10,7 +10,7 @@ try: # pragma: no cover from dictorm.pg import Select, Insert, Update, Delete - from dictorm.pg import And, Or + from dictorm.pg import And, Or, QueryHint from dictorm.pg import Column, Comparison, Operator from dictorm.sqlite import Insert as SqliteInsert from dictorm.sqlite import Column as SqliteColumn @@ -115,9 +115,9 @@ class Dict(dict): """ def __init__(self, table, *a, **kw): - self._table = table + self._table: Table = table self._in_db = False - self._curs = table.db.curs + self._curs: CursorHint = table.db.curs super(Dict, self).__init__(*a, **kw) self._old_pk_and = None @@ -291,15 +291,15 @@ class ResultsGenerator: create a new ResultsGenerator instance, or flush your Dict. """ - def __init__(self, table, query, db): - self.table = table + def __init__(self, table, query: QueryHint, db): + self.table: Table = table self.query = query self.cache = [] self.completed = False self.executed = False self.db_kind = db.kind - self.db = db - self.curs = self.db.get_cursor() + self.db: DictDB = db + self.curs: CursorHint = self.db.get_cursor() self._nocache = False def __iter__(self): diff --git a/dictorm/pg.py b/dictorm/pg.py index abc4e62..3be387b 100644 --- a/dictorm/pg.py +++ b/dictorm/pg.py @@ -182,6 +182,9 @@ class Delete(Update): query = 'DELETE FROM "{table}"' +QueryHint = Union[Select, Insert, Update, Delete] + + class Comparison(object): interpolation_str = '%s' many = False From 8f1aa1a792cf08ba2e9c32941efb07b2c1741e75 Mon Sep 17 00:00:00 2001 From: rolobio Date: Thu, 30 May 2019 14:03:31 -0600 Subject: [PATCH 13/15] Restoring default postgres port. Travis ignores the port specification --- .travis.yml | 1 - dictorm/test/test_dictorm.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9e8402d..a156453 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,6 @@ env: - PG_VERSION=9.5 - PG_VERSION=9.6 - PG_VERSION=10 - - PGPORT=5433 addons: postgresql: "10" apt: diff --git a/dictorm/test/test_dictorm.py b/dictorm/test/test_dictorm.py index 95de3e1..6253123 100755 --- a/dictorm/test/test_dictorm.py +++ b/dictorm/test/test_dictorm.py @@ -17,7 +17,7 @@ } if 'CI' in os.environ.keys(): - test_db_login['port'] = 5433 + test_db_login['port'] = 5432 def _no_refs(o): From 45c72b77bff1d9e33e401bac3dc227c1a3d10cfd Mon Sep 17 00:00:00 2001 From: rolobio Date: Thu, 30 May 2019 14:17:25 -0600 Subject: [PATCH 14/15] Removing unnecessary method --- dictorm/test/test_dictorm.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/dictorm/test/test_dictorm.py b/dictorm/test/test_dictorm.py index 6253123..06bbf72 100755 --- a/dictorm/test/test_dictorm.py +++ b/dictorm/test/test_dictorm.py @@ -53,11 +53,6 @@ def assertRaisesAny(cls, exps, func, a=None, kw=None): if isinstance(e, exps): return raise Exception('Did not raise one of the exceptions provided!') - @classmethod - def assertType(cls, a, b): - if not isinstance(a, b): - raise TypeError('{0} is not type {1}'.format(str(a), b0)) - def assertEqualNoRefs(self, a, b): return self.assertEqual(_no_refs(a), _no_refs(b)) @@ -878,11 +873,11 @@ def test_results_cache(self): subordinates = bob['subordinates'] for sub in subordinates: - self.assertType(sub, dictorm.Dict) + assert isinstance(sub, dictorm.Dict) # Error would be raised if subordinates isn't cached bob._table.get_where = error for sub in subordinates: - self.assertType(sub, dictorm.Dict) + assert isinstance(sub, dictorm.Dict) def test_reference_order(self): """ From 87bda210242381ee21f06220b56c733b04209311 Mon Sep 17 00:00:00 2001 From: rolobio Date: Thu, 30 May 2019 15:11:10 -0600 Subject: [PATCH 15/15] Dropping support for python versions less than 3.6 --- .travis.yml | 3 --- dictorm/__init__.py | 6 ++---- dictorm/dictorm.py | 25 +++++++------------------ 3 files changed, 9 insertions(+), 25 deletions(-) diff --git a/.travis.yml b/.travis.yml index a156453..4d4f166 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,6 @@ os: dist: xenial # for Python 3.7 language: python python: - - "2.7" - - "3.4" - - "3.5" - "3.6" - "3.7" - "3.7-dev" diff --git a/dictorm/__init__.py b/dictorm/__init__.py index f363ae1..8e06248 100644 --- a/dictorm/__init__.py +++ b/dictorm/__init__.py @@ -1,4 +1,2 @@ -try: # pragma: no cover - from dictorm.dictorm import * # pragma: no cover -except ImportError: # pragma: no cover - from .dictorm import * # pragma: no cover +from .dictorm import * +from .pg import Or diff --git a/dictorm/dictorm.py b/dictorm/dictorm.py index b88a68b..227c6f3 100644 --- a/dictorm/dictorm.py +++ b/dictorm/dictorm.py @@ -1,27 +1,19 @@ """What if you could insert a Python dictionary into the database? DictORM allows you to select/insert/update rows of a database as if they were Python Dictionaries.""" from typing import Union, Optional, List -__version__ = '3.8.5' +__version__ = '4.0' from contextlib import contextmanager from itertools import chain from json import dumps from sys import modules -try: # pragma: no cover - from dictorm.pg import Select, Insert, Update, Delete - from dictorm.pg import And, Or, QueryHint - from dictorm.pg import Column, Comparison, Operator - from dictorm.sqlite import Insert as SqliteInsert - from dictorm.sqlite import Column as SqliteColumn - from dictorm.sqlite import Update as SqliteUpdate -except ImportError: # pragma: no cover - from .pg import Select, Insert, Update, Delete - from .pg import And, Or - from .pg import Column, Comparison, Operator - from .sqlite import Insert as SqliteInsert - from .sqlite import Column as SqliteColumn - from .sqlite import Update as SqliteUpdate +from .pg import Select, Insert, Update, Delete +from .pg import And, QueryHint +from .pg import Column, Comparison, Operator +from .sqlite import Insert as SqliteInsert +from .sqlite import Column as SqliteColumn +from .sqlite import Update as SqliteUpdate db_package_imported = False try: # pragma: no cover @@ -327,9 +319,6 @@ def __execute_once(self): sql, values = self.query.build() self.curs.execute(sql, values) - # for python 2.7 - next = __next__ - def __len__(self) -> int: self.__execute_once() if self.db_kind == 'sqlite3':