Skip to content

Refactor to use pgmini for SQL construction and pycql2 for CQL parsing #149

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 5 commits into
base: main
Choose a base branch
from
Draft
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -131,6 +131,7 @@ ignore = [
"E501", # line too long, handled by black
"B008", # do not perform function calls in argument defaults
"B905", # ignore zip() without an explicit strict= parameter, only support with python >3.10
"F811", # ignore multiple definitions, handled by mypy
]


69 changes: 69 additions & 0 deletions tests/fixtures/cql2text.cql2
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
"id" = 'fa7e1920-9107-422d-a3db-c468cbc5d6df'
"id" <> 'fa7e1920-9107-422d-a3db-c468cbc5d6df'
"value" < 10
"value" > 10
"value" <= 10
"value" >= 10
"name" LIKE 'foo%'
"name" NOT LIKE 'foo%'
NOT "name" LIKE 'foo%'
"value" BETWEEN 10 AND 20
"value" NOT BETWEEN 10 AND 20
NOT "value" BETWEEN 10 AND 20
"value" IN (1.0, 2.0, 3.0)
"value" NOT IN ('a', 'b', 'c')
NOT "value" IN ('a', 'b', 'c')
"value" IS NULL
"value" IS NOT NULL
NOT "value" IS NULL
"name" NOT LIKE 'foo%' AND "value" > 10
(NOT "name" LIKE 'foo%' AND "value" > 10)
"value" IS NULL OR "value" BETWEEN 10 AND 20
("value" IS NULL OR "value" BETWEEN 10 AND 20)
S_INTERSECTS("geometry", BBOX(-128.098193, -1.1, -99999.0, 180.0, 90.0, 100000.0))
S_EQUALS( POLYGON ( (-0.333333 89.0, -102.723546 -0.5, -179.0 -89.0, -1.9 89.0, -0.0 89.0, 2.00001 -1.9, -0.333333 89.0) ), "geometry" )
S_EQUALS(POLYGON ((-0.333333 89.0, -102.723546 -0.5, -179.0 -89.0, -1.9 89.0, -0.0 89.0, 2.00001 -1.9, -0.333333 89.0)), "geometry")
S_DISJOINT("geometry", MULTIPOLYGON (((144.022387 45.176126, -1.1 0.0, 180.0 47.808086, 144.022387 45.176126))))
S_TOUCHES("geometry", MULTILINESTRING ((-1.9 -0.99999, 75.292574 1.5, -0.5 -4.016458, -31.708594 -74.743801, 179.0 -90.0),(-1.9 -1.1, 1.5 8.547371)))
S_WITHIN(POLYGON ((-49.88024 0.5 -75993.341684, -1.5 -0.99999 -100000.0, 0.0 0.5 -0.333333, -49.88024 0.5 -75993.341684), (-65.887123 2.00001 -100000.0, 0.333333 -53.017711 -79471.332949, 180.0 0.0 1852.616704, -65.887123 2.00001 -100000.0)), "geometry")
S_WITHIN(POLYGON Z ((-49.88024 0.5 -75993.341684, -1.5 -0.99999 -100000.0, 0.0 0.5 -0.333333, -49.88024 0.5 -75993.341684), (-65.887123 2.00001 -100000.0, 0.333333 -53.017711 -79471.332949, 180.0 0.0 1852.616704, -65.887123 2.00001 -100000.0)), "geometry")
S_OVERLAPS("geometry", BBOX(-179.912109, 1.9, 180.0, 16.897016))
S_CROSSES("geometry", LINESTRING (172.03086 1.5, 1.1 -90.0, -159.757695 0.99999, -180.0 0.5, -12.111235 81.336403, -0.5 64.43958, 0.0 81.991815, -155.93831 90.0))
S_CONTAINS("geometry", POINT (-3.508362 -1.754181))
T_AFTER("updated_at", DATE('2010-02-10'))
T_BEFORE(updated_at, TIMESTAMP('2012-08-10T05:30:00Z'))
T_BEFORE("updated_at", TIMESTAMP('2012-08-10T05:30:00.000000Z'))
T_CONTAINS(INTERVAL('2000-01-01T00:00:00Z', '2005-01-10T01:01:01.393216Z'), "updated_at")
T_CONTAINS(INTERVAL('2000-01-01T00:00:00.000000Z', '2005-01-10T01:01:01.393216Z'), "updated_at")
T_DISJOINT(INTERVAL('..', '2005-01-10T01:01:01.393216Z'), "coverage_date")
T_DURING(INTERVAL("created_at", "updated_at"), INTERVAL('2005-01-10', '2010-02-10'))
T_EQUALS("updated_at", DATE('1851-04-29'))
T_FINISHEDBY("coverage_date", INTERVAL('1991-10-07T08:21:06.393262Z', '2010-02-10T05:29:20.073225Z'))
T_FINISHES("coverage_dates", INTERVAL('1991-10-07', '2010-02-10T05:29:20.073225Z'))
T_INTERSECTS("coverage_date", INTERVAL('1991-10-07T08:21:06.393262Z', '2010-02-10T05:29:20.073225Z'))
T_MEETS(INTERVAL('2005-01-10', '2010-02-10'), "coverage_dates")
T_METBY(INTERVAL('2010-02-10T05:29:20.073225Z', '2010-10-07'), "coverage_dates")
T_OVERLAPPEDBY(INTERVAL('1991-10-07T08:21:06.393262Z', '2010-02-10T05:29:20.073225Z'), "coverage_dates")
T_OVERLAPS("coverage_date", INTERVAL('1991-10-07T08:21:06.393262Z', '1992-10-09T08:08:08.393473Z'))
T_STARTEDBY(INTERVAL('1991-10-07T08:21:06.393262Z', '2010-02-10T05:29:20.073225Z'), "coverage_dates")
T_STARTS("coverage_dates", INTERVAL('1991-10-07T08:21:06.393262Z', '..'))
Foo("geometry") = TRUE
FALSE <> Bar("geometry", 100, 'a', 'b', FALSE)
ACCENTI("owner") = ACCENTI('Beyoncé')
CASEI("owner") = CASEI('somebody else')
"value" > ("foo" + 10)
"value" < ("foo" - 10)
"value" <> (22.1 * "foo")
"value" = (2 / "foo")
"value" <= (2 ^ "foo")
0 = ("foo" % 2)
1 = ("foo" div 2)
A_CONTAINEDBY("values", ('a', 'b', 'c'))
A_CONTAINS("values", ('a', 'b', 'c'))
A_EQUALS(('a', TRUE, 1.0, 8), "values")
A_OVERLAPS("values", (TIMESTAMP('2012-08-10T05:30:00.000000Z'), DATE('2010-02-10'), FALSE))
S_EQUALS(MULTIPOINT (180.0 -0.5, 179.0 -47.121701, 180.0 -0.0, 33.470475 -0.99999, 179.0 -15.333062), "geometry")
S_EQUALS(GEOMETRYCOLLECTION (POINT (1.9 2.00001), POINT (0.0 -2.00001), MULTILINESTRING ((-2.00001 -0.0, -77.292642 -0.5, -87.515626 -0.0, -180.0 12.502773, 21.204842 -1.5, -21.878857 -90.0)), POINT (1.9 0.5), LINESTRING (179.0 1.179148, -148.192487 -65.007816, 0.5 0.333333)), "geometry")
value = - foo * 2.0 + "bar" / 6.1234 - "x" ^ 2.0
"value" = ((((-1 * "foo") * 2.0) + ("bar" / 6.1234)) - ("x" ^ 2.0))
"name" LIKE CASEI('FOO%')
69 changes: 69 additions & 0 deletions tests/fixtures/cql2text_asyncpg.asql
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
'id = $1', ['fa7e1920-9107-422d-a3db-c468cbc5d6df']
'id != $1', ['fa7e1920-9107-422d-a3db-c468cbc5d6df']
'value < $1', [10.0]
'value > $1', [10.0]
'value <= $1', [10.0]
'value >= $1', [10.0]
'name LIKE $1', ['foo%']
'NOT name LIKE $1', ['foo%']
'NOT name LIKE $1', ['foo%']
'value BETWEEN $1 AND $2', [10.0, 20.0]
'NOT (value BETWEEN $1 AND $2)', [10.0, 20.0]
'NOT (value BETWEEN $1 AND $2)', [10.0, 20.0]
'value = ANY(ARRAY[$1, $2, $3])', [1.0, 2.0, 3.0]
'NOT value = ANY(ARRAY[$1, $2, $3])', ['a', 'b', 'c']
'NOT value = ANY(ARRAY[$1, $2, $3])', ['a', 'b', 'c']
'value IS NULL', []
'NOT value IS NULL', []
'NOT value IS NULL', []
'NOT name LIKE $1 AND value > $2', ['foo%', 10.0]
'NOT name LIKE $1 AND value > $2', ['foo%', 10.0]
'value IS NULL OR value BETWEEN $1 AND $2', [10.0, 20.0]
'value IS NULL OR value BETWEEN $1 AND $2', [10.0, 20.0]
'ST_INTERSECTS(geometry, ST_MAKEENVELOPE($1, $2, $3, $4, $5))', [-128.098193, -1.1, 180.0, 90.0, 4326]
'ST_EQUALS($1::geometry, geometry)', ['SRID=4326;POLYGON ((-0.333333 89.0, -102.723546 -0.5, -179.0 -89.0, -1.9 89.0, -0.0 89.0, 2.00001 -1.9, -0.333333 89.0))']
'ST_EQUALS($1::geometry, geometry)', ['SRID=4326;POLYGON ((-0.333333 89.0, -102.723546 -0.5, -179.0 -89.0, -1.9 89.0, -0.0 89.0, 2.00001 -1.9, -0.333333 89.0))']
'ST_DISJOINT(geometry, $1::geometry)', ['SRID=4326;MULTIPOLYGON (((144.022387 45.176126, -1.1 0.0, 180.0 47.808086, 144.022387 45.176126)))']
'ST_TOUCHES(geometry, $1::geometry)', ['SRID=4326;MULTILINESTRING ((-1.9 -0.99999, 75.292574 1.5, -0.5 -4.016458, -31.708594 -74.743801, 179.0 -90.0), (-1.9 -1.1, 1.5 8.547371))']
'ST_WITHIN($1::geometry, geometry)', ['SRID=4326;POLYGON Z ((-49.88024 0.5 -75993.341684, -1.5 -0.99999 -100000.0, 0.0 0.5 -0.333333, -49.88024 0.5 -75993.341684), (-65.887123 2.00001 -100000.0, 0.333333 -53.017711 -79471.332949, 180.0 0.0 1852.616704, -65.887123 2.00001 -100000.0))']
'ST_WITHIN($1::geometry, geometry)', ['SRID=4326;POLYGON Z ((-49.88024 0.5 -75993.341684, -1.5 -0.99999 -100000.0, 0.0 0.5 -0.333333, -49.88024 0.5 -75993.341684), (-65.887123 2.00001 -100000.0, 0.333333 -53.017711 -79471.332949, 180.0 0.0 1852.616704, -65.887123 2.00001 -100000.0))']
'ST_OVERLAPS(geometry, ST_MAKEENVELOPE($1, $2, $3, $4, $5))', [-179.912109, 1.9, 180.0, 16.897016, 4326]
'ST_CROSSES(geometry, $1::geometry)', ['SRID=4326;LINESTRING (172.03086 1.5, 1.1 -90.0, -159.757695 0.99999, -180.0 0.5, -12.111235 81.336403, -0.5 64.43958, 0.0 81.991815, -155.93831 90.0)']
'ST_CONTAINS(geometry, $1::geometry)', ['SRID=4326;POINT (-3.508362 -1.754181)']
'$1::date < "updated_at"', ['2010-02-10']
'"updated_at" < $1::timestamptz', ['2012-08-10T05:30:00+00:00']
'"updated_at" < $1::timestamptz', ['2012-08-10T05:30:00+00:00']
'"updated_at" > $1::timestamptz AND "updated_at" < $2::timestamptz', ['2000-01-01T00:00:00+00:00', '2005-01-10T01:01:01.393216+00:00']
'"updated_at" > $1::timestamptz AND "updated_at" < $2::timestamptz', ['2000-01-01T00:00:00+00:00', '2005-01-10T01:01:01.393216+00:00']
'$1::timestamptz > "coverage_date" OR $2::timestamptz < "coverage_date"', ['-infinity', '2005-01-10T01:01:01.393216+00:00']
'"created_at" > $1::date AND "updated_at" < $2::date', ['2005-01-10', '2010-02-10']
'"updated_at" = $1::date AND "updated_at" = $2::date', ['1851-04-29', '1851-04-29']
'$1::timestamptz > "coverage_date" AND $2::timestamptz = "coverage_date"', ['1991-10-07T08:21:06.393262+00:00', '2010-02-10T05:29:20.073225+00:00']
'"coverage_dates" > $1::date AND "coverage_dates" = $2::timestamptz', ['1991-10-07', '2010-02-10T05:29:20.073225+00:00']
'"coverage_date" <= $1::timestamptz AND "coverage_date" >= $2::timestamptz', ['2010-02-10T05:29:20.073225+00:00', '1991-10-07T08:21:06.393262+00:00']
'$1::date = "coverage_dates"', ['2010-02-10']
'"coverage_dates" = $1::timestamptz', ['2010-02-10T05:29:20.073225+00:00']
'"coverage_dates" < $1::timestamptz AND "coverage_dates" > $2::timestamptz AND "coverage_dates" < $3::timestamptz', ['1991-10-07T08:21:06.393262+00:00', '2010-02-10T05:29:20.073225+00:00', '2010-02-10T05:29:20.073225+00:00']
'"coverage_date" < $1::timestamptz AND "coverage_date" > $2::timestamptz AND "coverage_date" < $3::timestamptz', ['1991-10-07T08:21:06.393262+00:00', '1992-10-09T08:08:08.393473+00:00', '1992-10-09T08:08:08.393473+00:00']
'"coverage_dates" = $1::timestamptz AND "coverage_dates" < $2::timestamptz', ['1991-10-07T08:21:06.393262+00:00', '2010-02-10T05:29:20.073225+00:00']
'"coverage_dates" = $1::timestamptz AND "coverage_dates" < $2::timestamptz', ['1991-10-07T08:21:06.393262+00:00', 'infinity']
'FOO(geometry) IS $1', [True]
'$1 IS NOT BAR(geometry, $2, $3, $4, $5)', [False, 100.0, 'a', 'b', False]
'UNACCENT(owner) = UNACCENT($1)', ['Beyoncé']
'LOWER(owner) = LOWER($1)', ['somebody else']
'value > (foo + $1)', [10.0]
'value < (foo - $1)', [10.0]
'value != ($1 * foo)', [22.1]
'value = ($1 / foo)', [2.0]
'value <= POWER($1, foo)', [2.0]
'$1 = MOD(foo, $2)', [0.0, 2.0]
'$1 = (foo / $2)', [1.0, 2.0]
'values <@ ARRAY[$1, $2, $3]', ['a', 'b', 'c']
'values @> ARRAY[$1, $2, $3]', ['a', 'b', 'c']
'ARRAY[$1, $2, $3, $4] = values', ['a', True, 1.0, 8.0]
'values && ARRAY[$1::timestamptz, $2::date, $3]', ['2012-08-10T05:30:00+00:00', '2010-02-10', False]
'ST_EQUALS(MULTIPOINT($1 - $2, $3 - $4, $5 - $6, $7 - $8, $9 - $10), geometry)', [180.0, 0.5, 179.0, 47.121701, 180.0, 0.0, 33.470475, 0.99999, 179.0, 15.333062]
'ST_EQUALS($1::geometry, geometry)', ['SRID=4326;GEOMETRYCOLLECTION (POINT (1.9 2.00001), POINT (0.0 -2.00001), MULTILINESTRING ((-2.00001 -0.0, -77.292642 -0.5, -87.515626 -0.0, -180.0 12.502773, 21.204842 -1.5, -21.878857 -90.0)), POINT (1.9 0.5), LINESTRING (179.0 1.179148, -148.192487 -65.007816, 0.5 0.333333))']
'value = (($1 * foo) * $2) + (bar / $3) - POWER(x, $4)', [-1, 2.0, 6.1234, 2.0]
'value = (($1 * foo) * $2) + (bar / $3) - POWER(x, $4)', [-1.0, 2.0, 6.1234, 2.0]
'name ILIKE $1', ['FOO%']
69 changes: 69 additions & 0 deletions tests/fixtures/cql2text_rawsql.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
id = 'fa7e1920-9107-422d-a3db-c468cbc5d6df'
id != 'fa7e1920-9107-422d-a3db-c468cbc5d6df'
value < 10.0
value > 10.0
value <= 10.0
value >= 10.0
name LIKE 'foo%'
NOT name LIKE 'foo%'
NOT name LIKE 'foo%'
value BETWEEN 10.0 AND 20.0
NOT (value BETWEEN 10.0 AND 20.0)
NOT (value BETWEEN 10.0 AND 20.0)
value = ANY(ARRAY[1.0, 2.0, 3.0])
NOT value = ANY(ARRAY['a', 'b', 'c'])
NOT value = ANY(ARRAY['a', 'b', 'c'])
value IS NULL
NOT value IS NULL
NOT value IS NULL
NOT name LIKE 'foo%' AND value > 10.0
NOT name LIKE 'foo%' AND value > 10.0
value IS NULL OR value BETWEEN 10.0 AND 20.0
value IS NULL OR value BETWEEN 10.0 AND 20.0
ST_INTERSECTS(geometry, ST_MAKEENVELOPE(-128.098193, -1.1, 180.0, 90.0, 4326))
ST_EQUALS('SRID=4326;POLYGON ((-0.333333 89.0, -102.723546 -0.5, -179.0 -89.0, -1.9 89.0, -0.0 89.0, 2.00001 -1.9, -0.333333 89.0))'::geometry, geometry)
ST_EQUALS('SRID=4326;POLYGON ((-0.333333 89.0, -102.723546 -0.5, -179.0 -89.0, -1.9 89.0, -0.0 89.0, 2.00001 -1.9, -0.333333 89.0))'::geometry, geometry)
ST_DISJOINT(geometry, 'SRID=4326;MULTIPOLYGON (((144.022387 45.176126, -1.1 0.0, 180.0 47.808086, 144.022387 45.176126)))'::geometry)
ST_TOUCHES(geometry, 'SRID=4326;MULTILINESTRING ((-1.9 -0.99999, 75.292574 1.5, -0.5 -4.016458, -31.708594 -74.743801, 179.0 -90.0), (-1.9 -1.1, 1.5 8.547371))'::geometry)
ST_WITHIN('SRID=4326;POLYGON Z ((-49.88024 0.5 -75993.341684, -1.5 -0.99999 -100000.0, 0.0 0.5 -0.333333, -49.88024 0.5 -75993.341684), (-65.887123 2.00001 -100000.0, 0.333333 -53.017711 -79471.332949, 180.0 0.0 1852.616704, -65.887123 2.00001 -100000.0))'::geometry, geometry)
ST_WITHIN('SRID=4326;POLYGON Z ((-49.88024 0.5 -75993.341684, -1.5 -0.99999 -100000.0, 0.0 0.5 -0.333333, -49.88024 0.5 -75993.341684), (-65.887123 2.00001 -100000.0, 0.333333 -53.017711 -79471.332949, 180.0 0.0 1852.616704, -65.887123 2.00001 -100000.0))'::geometry, geometry)
ST_OVERLAPS(geometry, ST_MAKEENVELOPE(-179.912109, 1.9, 180.0, 16.897016, 4326))
ST_CROSSES(geometry, 'SRID=4326;LINESTRING (172.03086 1.5, 1.1 -90.0, -159.757695 0.99999, -180.0 0.5, -12.111235 81.336403, -0.5 64.43958, 0.0 81.991815, -155.93831 90.0)'::geometry)
ST_CONTAINS(geometry, 'SRID=4326;POINT (-3.508362 -1.754181)'::geometry)
'2010-02-10'::date < "updated_at"
"updated_at" < '2012-08-10T05:30:00+00:00'::timestamptz
"updated_at" < '2012-08-10T05:30:00+00:00'::timestamptz
"updated_at" > '2000-01-01T00:00:00+00:00'::timestamptz AND "updated_at" < '2005-01-10T01:01:01.393216+00:00'::timestamptz
"updated_at" > '2000-01-01T00:00:00+00:00'::timestamptz AND "updated_at" < '2005-01-10T01:01:01.393216+00:00'::timestamptz
'-infinity'::timestamptz > "coverage_date" OR '2005-01-10T01:01:01.393216+00:00'::timestamptz < "coverage_date"
"created_at" > '2005-01-10'::date AND "updated_at" < '2010-02-10'::date
"updated_at" = '1851-04-29'::date AND "updated_at" = '1851-04-29'::date
'1991-10-07T08:21:06.393262+00:00'::timestamptz > "coverage_date" AND '2010-02-10T05:29:20.073225+00:00'::timestamptz = "coverage_date"
"coverage_dates" > '1991-10-07'::date AND "coverage_dates" = '2010-02-10T05:29:20.073225+00:00'::timestamptz
"coverage_date" <= '2010-02-10T05:29:20.073225+00:00'::timestamptz AND "coverage_date" >= '1991-10-07T08:21:06.393262+00:00'::timestamptz
'2010-02-10'::date = "coverage_dates"
"coverage_dates" = '2010-02-10T05:29:20.073225+00:00'::timestamptz
"coverage_dates" < '1991-10-07T08:21:06.393262+00:00'::timestamptz AND "coverage_dates" > '2010-02-10T05:29:20.073225+00:00'::timestamptz AND "coverage_dates" < '2010-02-10T05:29:20.073225+00:00'::timestamptz
"coverage_date" < '1991-10-07T08:21:06.393262+00:00'::timestamptz AND "coverage_date" > '1992-10-09T08:08:08.393473+00:00'::timestamptz AND "coverage_date" < '1992-10-09T08:08:08.393473+00:00'::timestamptz
"coverage_dates" = '1991-10-07T08:21:06.393262+00:00'::timestamptz AND "coverage_dates" < '2010-02-10T05:29:20.073225+00:00'::timestamptz
"coverage_dates" = '1991-10-07T08:21:06.393262+00:00'::timestamptz AND "coverage_dates" < 'infinity'::timestamptz
FOO(geometry) IS True
False IS NOT BAR(geometry, 100.0, 'a', 'b', False)
UNACCENT(owner) = UNACCENT('Beyoncé')
LOWER(owner) = LOWER('somebody else')
value > (foo + 10.0)
value < (foo - 10.0)
value != (22.1 * foo)
value = (2.0 / foo)
value <= POWER(2.0, foo)
0.0 = MOD(foo, 2.0)
1.0 = (foo / 2.0)
values <@ ARRAY['a', 'b', 'c']
values @> ARRAY['a', 'b', 'c']
ARRAY['a', True, 1.0, 8.0] = values
values && ARRAY['2012-08-10T05:30:00+00:00'::timestamptz, '2010-02-10'::date, False]
ST_EQUALS(MULTIPOINT(180.0 - 0.5, 179.0 - 47.121701, 180.0 - 0.0, 33.470475 - 0.99999, 179.0 - 15.333062), geometry)
ST_EQUALS('SRID=4326;GEOMETRYCOLLECTION (POINT (1.9 2.00001), POINT (0.0 -2.00001), MULTILINESTRING ((-2.00001 -0.0, -77.292642 -0.5, -87.515626 -0.0, -180.0 12.502773, 21.204842 -1.5, -21.878857 -90.0)), POINT (1.9 0.5), LINESTRING (179.0 1.179148, -148.192487 -65.007816, 0.5 0.333333))'::geometry, geometry)
value = ((-1 * foo) * 2.0) + (bar / 6.1234) - POWER(x, 2.0)
value = ((-1.0 * foo) * 2.0) + (bar / 6.1234) - POWER(x, 2.0)
name ILIKE 'FOO%'
5 changes: 3 additions & 2 deletions tests/routes/test_items.py
Original file line number Diff line number Diff line change
@@ -247,6 +247,7 @@ def test_items_filter_cql_ids(app):
assert response.status_code == 200
assert response.headers["content-type"] == "application/geo+json"
body = response.json()
print(body)
assert len(body["features"]) == 1
assert body["numberMatched"] == 1
assert body["numberReturned"] == 1
@@ -257,7 +258,7 @@ def test_items_filter_cql_ids(app):
response = app.get(
"/collections/public.landsat_wrs/items?filter-lang=cql2-text&filter=ogc_fid IN (1,2)"
)

print(response.json())
assert response.status_code == 200
assert response.headers["content-type"] == "application/geo+json"
body = response.json()
@@ -327,7 +328,7 @@ def test_items_properties_filter_cql2(app):
assert body["features"][0]["properties"]["row"] == 10
Items.model_validate(body)

filter_query = {"op": "isNull", "args": [{"property": "numeric"}]}
filter_query = {"op": "isNull", "args": {"property": "numeric"}}
response = app.get(
f"/collections/public.my_data/items?filter-lang=cql2-json&filter=&filter={json.dumps(filter_query)}"
)
430 changes: 186 additions & 244 deletions tipg/collections.py

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions tipg/database.py
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@
from typing import List, Optional

import orjson
from buildpg import asyncpg
import asyncpg

from tipg.logger import logger
from tipg.settings import PostgresSettings
@@ -80,7 +80,7 @@ async def connect_to_db(

con_init = connection_factory(schemas, user_sql_files)

app.state.pool = await asyncpg.create_pool_b(
app.state.pool = await asyncpg.create_pool(
str(settings.database_url),
min_size=settings.db_min_conn_size,
max_size=settings.db_max_conn_size,
18 changes: 11 additions & 7 deletions tipg/dependencies.py
Original file line number Diff line number Diff line change
@@ -6,9 +6,8 @@
from ciso8601 import parse_rfc3339
from morecantile import Tile
from morecantile import tms as default_tms
from pygeofilter.ast import AstType
from pygeofilter.parsers.cql2_json import parse as cql2_json_parser
from pygeofilter.parsers.cql2_text import parse as cql2_text_parser
from pycql2.cql2_transformer import parser, transformer
from pycql2.cql2_pydantic import BooleanExpression
from typing_extensions import Annotated

from tipg.collections import Catalog, Collection, CollectionList
@@ -189,7 +188,7 @@ def bbox_query(
bbox: Annotated[
Optional[str],
Query(description="Spatial Filter."),
] = None
] = None,
) -> Optional[List[float]]:
"""BBox dependency."""
if bbox:
@@ -290,14 +289,19 @@ def filter_query(
alias="filter-lang",
),
] = None,
) -> Optional[AstType]:
) -> Optional[BooleanExpression]:
"""Parse Filter Query."""
print('PARSING CQL2', type(query), filter_lang, query)
if query is not None:
if filter_lang == "cql2-json":
return cql2_json_parser(query)
print('PARSING AS JSON')
model = BooleanExpression.model_validate_json(query)
print('MODEL', model)
return model

# default to cql2-text
return cql2_text_parser(query)
print('PARSING AS TEXT')
return transformer.transform(parser.parse(query))

return None

519 changes: 519 additions & 0 deletions tipg/filter/cql2sql.py

Large diffs are not rendered by default.

282 changes: 282 additions & 0 deletions tipg/query.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,282 @@
"""Helpers for using pgmini."""
import re
from contextvars import copy_context
from typing import List
from typing import Literal as TypeLiteral
from typing import Optional, Tuple, Union
import attrs
from pgmini.marks import Marks

import pgmini
from pgmini.alias import extract_alias
from pgmini.utils import (
CTX_ALIAS_ONLY,
CTX_CTE,
CTX_DISABLE_TABLE_IN_COLUMN,
CTX_FORCE_CAST_BRACKETS,
CTX_TABLES,
CompileABC,
)
from datetime import date, datetime

def is_integer(n):
"""Check if a value is an integer."""
try:
float(n)
except ValueError:
return False
else:
return float(n).is_integer()


def NULL(type: Optional[str] = None):
"""Return typed NULL."""
if type is None:
return pgmini.literal.NULL
return pgmini.literal.NULL.As(type)


def strip_ident(s: str) -> str:
"""Strip quotes from identifier."""
if s.startswith('"') and s.endswith('"'):
return s[1:-1]
return s

def quote_ident_part(s: str) -> str:
"""Quote Identifiers."""
s = strip_ident(s)
s = s.strip()
if s in ('AS','ASC','DESC'):
return s
if re.match(r"^[a-z][a-z_]*$", s):
return s
if re.match(r"^[a-zA-Z][\w\d_]*$", s):
return f'"{s}"'
raise TypeError(f"{s} is not a valid identifier")


def quote_ident(s: str) -> str:
"""Quote qualified identifiers."""
outspacearr = []
splitbyspace = s.split(" ")
for spacesplit in splitbyspace:
splitbycast = spacesplit.split("::")
outsplitarr=[]
for castsplit in splitbycast:
splitbydot = castsplit.split(".")
outsplitarr.append(".".join(map(quote_ident_part, splitbydot)))
outbysplit = "::".join(outsplitarr)
outspacearr.append(outbysplit)
out = " ".join(outspacearr)
return out


def F(name: str, *args):
"""Run Postgres Function."""
if re.match(r"^[a-zA-Z_]+(\.[a-zA-Z_]+)?$", name):
return pgmini.func._Func(x_name=name, x_params=args)
else:
raise TypeError(
f"Cannot Create {name}" "Only functions that match ^[a-zA-Z_]+ allowed",
)


def Transform(g, srid: Union[int, str] = 4326):
"""Transform geometry."""
if is_integer(srid):
return F("ST_Transform", g, P(srid).Cast("int"))
else:
return F("ST_Transform", g, P(srid).Cast("text"))


def Bbox(box, srid: int = 4326):
"""Return Bounding Box."""
print('BBOX', box, type(box))
box = list(box)
#if isinstance(box, (list, tuple)):
if len(box) == 4:
left, bottom, right, top = box
elif len(box) == 6:
left = box[0]
bottom = box[1]
right = box[3]
top = box[4]
# else:
# left = box.left
# bottom = box.bottom
# right = box.right
# top = box.top

out = F("ST_MakeEnvelope", left, bottom, right, top, srid)
print(out)
return out

def Count():
"""Return Count."""
return F('count','*')

def date_param(val):
"""Make a parameter from date/time"""
if isinstance(val, (date, datetime)):
val = val.isoformat()
return P(val).Cast("timestamptz")

def simplified(geom, tolerance):
return F(
"ST_SnapToGrid",
F("ST_Simplify", geom, tolerance),
tolerance,
)

def row_num(alias: str ='row'):
"""Return Row Number."""
return F("row_number").Over().As(alias)

class Table(pgmini.Table):
"""PgMini Table with useful functions."""

def get(self, attr: str) -> pgmini.column.Column:
"""Get attribute via string."""
return C(attr, self)

def create_tipg_id(self, id_column: str):
"""Create ID column using existing primary key or row number."""
if id_column:
id_column_col = Column(id_column, self)
return id_column_col.As("tipg_id")
return row_num("tipg_id")

def cols(self, colnames: List[str]):
"""Return pgmini columns from list of names."""
return [self.get(c) for c in colnames]




class Column(pgmini.column.Column):
"""PGMini Column extended to ensure identifier quoting."""

def _build(self, params: list | dict) -> str:
out = super()._build(params)
return quote_ident(out)

def Desc(self):
if self._marks:
marks = attrs.evolve(self._marks, order_by='DESC')
else:
marks = Marks(order_by='DESC')
return attrs.evolve(self, x_marks=marks)

def Asc(self):
if self._marks:
marks = attrs.evolve(self._marks, order_by='ASC')
else:
marks = Marks(order_by='ASC')
return attrs.evolve(self, x_marks=marks)

def NullsFirst(self):
if self._marks:
marks = attrs.evolve(self._marks, order_by_nulls='FIRST')
else:
marks = Marks(order_by_nulls='FIRST')
return attrs.evolve(self, x_marks=marks)

def NullsLast(self):
if self._marks:
marks = attrs.evolve(self._marks, order_by_nulls='LAST')
else:
marks = Marks(order_by_nulls='LAST')
return attrs.evolve(self, x_marks=marks)

class Param(pgmini.param.Param):
"""Make sure that params with a text value have
an initial cast to text."""
def _build(self, params: list | dict) -> str:
if alias := extract_alias(self):
return alias

index = len(params) + 1
if isinstance(params, list):
params.append(self._value)
res = '$%d' % index
else:
params[f'p{index}'] = self._value
res = f'%(p{index})s'

if isinstance(self._value, str):
print('Param value is a string', self._value)
res = f'{res}::text'

if self._marks:
res = self._marks.build(res)
print(f"Built Parm {self}, {self._value}, {type(self._value)}, {res}")
return res

P = Param


def C(name: str, table: Optional[Table] = Table("t")):
"""Return a pgmini column."""
return Column(name, table)


def raw_query(q, *p):
"""Utility to print raw statement to use for debugging."""
qsub = re.sub(r"\$([0-9]+)", r"{\1}", q)

def quote_str(s):
"""Quote strings."""
if s is None:
return "null"
elif isinstance(s, str):
return f"'{s}'"
else:
return s

p = [quote_str(s) for s in p]
return qsub.format(None, *p)


def build(
item: CompileABC,
driver: TypeLiteral["asyncpg", "psycopg", "raw"] = "asyncpg",
table_in_column: bool = False,
) -> Union[Tuple[str, list], str]:
"""Build a SQL Query from CQL2 pydantic model.
Return as raw SQL or as a tuple of sql and parameters
ready for asyncpg or psycopg parameter binding.
"""

def run():
CTX_FORCE_CAST_BRACKETS.set(False)
CTX_CTE.set(())
CTX_TABLES.set(())
CTX_ALIAS_ONLY.set(False)
CTX_DISABLE_TABLE_IN_COLUMN.set(not table_in_column)

if driver == "psycopg":
params = {}
else:
params = []

query = item._build(params)
print(f"QUERY: {query}")
print("PARAMS", params)
if driver == "raw":
return raw_query(query, *params)
else:
return query, params

return copy_context().run(run)

def ensure_list(s) -> list:
"""Makes sure that variable is treated as list."""
if s is None:
return []
if isinstance(s, list):
return s
if isinstance(s, set):
return list(s)
if isinstance(s, tuple):
return list(s)
return [s]