Skip to content

Commit 1fd5284

Browse files
rouaultrouault
committed
Add 'gdal vector sql', as standalone or part of 'gdal vector pipeline'
1 parent 6332083 commit 1fd5284

16 files changed

+778
-26
lines changed

‎apps/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ add_library(
3131
gdalalg_vector_read.cpp
3232
gdalalg_vector_filter.cpp
3333
gdalalg_vector_reproject.cpp
34+
gdalalg_vector_sql.cpp
3435
gdalalg_vector_write.cpp
3536
gdalinfo_lib.cpp
3637
gdalbuildvrt_lib.cpp

‎apps/gdalalg_abstract_pipeline.h

+15
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
#include "cpl_json.h"
2020
#include "gdalalgorithm.h"
2121

22+
#include <algorithm>
23+
2224
template <class StepAlgorithm>
2325
class GDALAbstractPipelineAlgorithm CPL_NON_FINAL : public StepAlgorithm
2426
{
@@ -44,6 +46,19 @@ class GDALAbstractPipelineAlgorithm CPL_NON_FINAL : public StepAlgorithm
4446
{
4547
}
4648

49+
~GDALAbstractPipelineAlgorithm() override
50+
{
51+
// Destroy steps in the reverse order they have been constructed,
52+
// as a step can create object that depends on the validity of
53+
// objects of previous steps, and while cleaning them it needs those
54+
// prior objects to be still alive.
55+
// Typically for "gdal vector pipeline read ... ! sql ..."
56+
for (auto it = std::rbegin(m_steps); it != std::rend(m_steps); it++)
57+
{
58+
it->reset();
59+
}
60+
}
61+
4762
virtual GDALArgDatasetValue &GetOutputDataset() = 0;
4863

4964
std::string m_pipeline{};

‎apps/gdalalg_vector.cpp

+2
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "gdalalg_vector_pipeline.h"
1919
#include "gdalalg_vector_filter.h"
2020
#include "gdalalg_vector_reproject.h"
21+
#include "gdalalg_vector_sql.h"
2122

2223
/************************************************************************/
2324
/* GDALVectorAlgorithm */
@@ -43,6 +44,7 @@ class GDALVectorAlgorithm final : public GDALAlgorithm
4344
RegisterSubAlgorithm<GDALVectorPipelineAlgorithm>();
4445
RegisterSubAlgorithm<GDALVectorFilterAlgorithmStandalone>();
4546
RegisterSubAlgorithm<GDALVectorReprojectAlgorithmStandalone>();
47+
RegisterSubAlgorithm<GDALVectorSQLAlgorithmStandalone>();
4648
}
4749

4850
private:

‎apps/gdalalg_vector_pipeline.cpp

+9-4
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include "gdalalg_vector_clip.h"
1616
#include "gdalalg_vector_filter.h"
1717
#include "gdalalg_vector_reproject.h"
18+
#include "gdalalg_vector_sql.h"
1819
#include "gdalalg_vector_write.h"
1920

2021
#include "cpl_conv.h"
@@ -94,10 +95,13 @@ void GDALVectorPipelineStepAlgorithm::AddOutputArgs(
9495
&m_appendLayer)
9596
.SetDefault(false)
9697
.SetHiddenForCLI(hiddenForCLI);
97-
AddArg("output-layer", shortNameOutputLayerAllowed ? 'l' : 0,
98-
_("Output layer name"), &m_outputLayerName)
99-
.AddHiddenAlias("nln") // For ogr2ogr nostalgic people
100-
.SetHiddenForCLI(hiddenForCLI);
98+
if (GetName() != "sql")
99+
{
100+
AddArg("output-layer", shortNameOutputLayerAllowed ? 'l' : 0,
101+
_("Output layer name"), &m_outputLayerName)
102+
.AddHiddenAlias("nln") // For ogr2ogr nostalgic people
103+
.SetHiddenForCLI(hiddenForCLI);
104+
}
101105
}
102106

103107
/************************************************************************/
@@ -178,6 +182,7 @@ GDALVectorPipelineAlgorithm::GDALVectorPipelineAlgorithm()
178182
m_stepRegistry.Register<GDALVectorClipAlgorithm>();
179183
m_stepRegistry.Register<GDALVectorReprojectAlgorithm>();
180184
m_stepRegistry.Register<GDALVectorFilterAlgorithm>();
185+
m_stepRegistry.Register<GDALVectorSQLAlgorithm>();
181186
}
182187

183188
/************************************************************************/

‎apps/gdalalg_vector_sql.cpp

+306
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,306 @@
1+
/******************************************************************************
2+
*
3+
* Project: GDAL
4+
* Purpose: "sql" step of "vector pipeline"
5+
* Author: Even Rouault <even dot rouault at spatialys.com>
6+
*
7+
******************************************************************************
8+
* Copyright (c) 2025, Even Rouault <even dot rouault at spatialys.com>
9+
*
10+
* SPDX-License-Identifier: MIT
11+
****************************************************************************/
12+
13+
#include "gdalalg_vector_sql.h"
14+
15+
#include "gdal_priv.h"
16+
#include "ogrsf_frmts.h"
17+
#include "ogrlayerpool.h"
18+
19+
#include <set>
20+
21+
//! @cond Doxygen_Suppress
22+
23+
#ifndef _
24+
#define _(x) (x)
25+
#endif
26+
27+
/************************************************************************/
28+
/* GDALVectorSQLAlgorithm::GDALVectorSQLAlgorithm() */
29+
/************************************************************************/
30+
31+
GDALVectorSQLAlgorithm::GDALVectorSQLAlgorithm(bool standaloneStep)
32+
: GDALVectorPipelineStepAlgorithm(NAME, DESCRIPTION, HELP_URL,
33+
standaloneStep)
34+
{
35+
AddArg("sql", 0, _("SQL statement(s)"), &m_sql)
36+
.SetPositional()
37+
.SetRequired()
38+
.SetPackedValuesAllowed(false)
39+
.SetReadFromFileAtSyntaxAllowed()
40+
.SetMetaVar("<statement>|@<filename>")
41+
.SetRemoveSQLCommentsEnabled();
42+
AddArg("output-layer", standaloneStep ? 0 : 'l', _("Output layer name(s)"),
43+
&m_outputLayer);
44+
AddArg("dialect", 0, _("SQL dialect (e.g. OGRSQL, SQLITE)"), &m_dialect);
45+
}
46+
47+
/************************************************************************/
48+
/* GDALVectorSQLAlgorithmDataset */
49+
/************************************************************************/
50+
51+
namespace
52+
{
53+
class GDALVectorSQLAlgorithmDataset final : public GDALDataset
54+
{
55+
GDALDataset *m_poSrcDS = nullptr;
56+
std::vector<OGRLayer *> m_layers{};
57+
58+
CPL_DISALLOW_COPY_ASSIGN(GDALVectorSQLAlgorithmDataset)
59+
60+
public:
61+
explicit GDALVectorSQLAlgorithmDataset(GDALDataset *poSrcDS)
62+
: m_poSrcDS(poSrcDS)
63+
{
64+
}
65+
66+
~GDALVectorSQLAlgorithmDataset() override
67+
{
68+
for (OGRLayer *poLayer : m_layers)
69+
m_poSrcDS->ReleaseResultSet(poLayer);
70+
}
71+
72+
void AddLayer(OGRLayer *poLayer)
73+
{
74+
m_layers.push_back(poLayer);
75+
}
76+
77+
int GetLayerCount() override
78+
{
79+
return static_cast<int>(m_layers.size());
80+
}
81+
82+
OGRLayer *GetLayer(int idx) override
83+
{
84+
return idx >= 0 && idx < GetLayerCount() ? m_layers[idx] : nullptr;
85+
}
86+
};
87+
} // namespace
88+
89+
/************************************************************************/
90+
/* GDALVectorSQLAlgorithmDatasetMultiLayer */
91+
/************************************************************************/
92+
93+
namespace
94+
{
95+
96+
class ProxiedSQLLayer final : public OGRProxiedLayer
97+
{
98+
OGRFeatureDefn *m_poLayerDefn = nullptr;
99+
100+
CPL_DISALLOW_COPY_ASSIGN(ProxiedSQLLayer)
101+
102+
public:
103+
ProxiedSQLLayer(const std::string &osName, OGRLayerPool *poPoolIn,
104+
OpenLayerFunc pfnOpenLayerIn,
105+
ReleaseLayerFunc pfnReleaseLayerIn,
106+
FreeUserDataFunc pfnFreeUserDataIn, void *pUserDataIn)
107+
: OGRProxiedLayer(poPoolIn, pfnOpenLayerIn, pfnReleaseLayerIn,
108+
pfnFreeUserDataIn, pUserDataIn)
109+
{
110+
SetDescription(osName.c_str());
111+
}
112+
113+
~ProxiedSQLLayer()
114+
{
115+
if (m_poLayerDefn)
116+
m_poLayerDefn->Release();
117+
}
118+
119+
const char *GetName() override
120+
{
121+
return GetDescription();
122+
}
123+
124+
OGRFeatureDefn *GetLayerDefn() override
125+
{
126+
if (!m_poLayerDefn)
127+
{
128+
m_poLayerDefn = OGRProxiedLayer::GetLayerDefn()->Clone();
129+
m_poLayerDefn->SetName(GetDescription());
130+
}
131+
return m_poLayerDefn;
132+
}
133+
};
134+
135+
class GDALVectorSQLAlgorithmDatasetMultiLayer final : public GDALDataset
136+
{
137+
// We can't safely have 2 SQL layers active simultaneously on the same
138+
// source dataset. So each time we access one, we must close the last
139+
// active one.
140+
OGRLayerPool m_oPool{1};
141+
GDALDataset *m_poSrcDS = nullptr;
142+
std::vector<std::unique_ptr<OGRLayer>> m_layers{};
143+
144+
CPL_DISALLOW_COPY_ASSIGN(GDALVectorSQLAlgorithmDatasetMultiLayer)
145+
146+
public:
147+
explicit GDALVectorSQLAlgorithmDatasetMultiLayer(GDALDataset *poSrcDS)
148+
: m_poSrcDS(poSrcDS)
149+
{
150+
}
151+
152+
void AddLayer(const std::string &osSQL, const std::string &osDialect,
153+
const std::string &osLayerName)
154+
{
155+
struct UserData
156+
{
157+
GDALDataset *poSrcDS = nullptr;
158+
std::string osDialect{};
159+
std::string osSQL{};
160+
std::string osLayerName{};
161+
162+
UserData() = default;
163+
CPL_DISALLOW_COPY_ASSIGN(UserData)
164+
};
165+
166+
const auto OpenLayer = [](void *pUserDataIn)
167+
{
168+
UserData *pUserData = static_cast<UserData *>(pUserDataIn);
169+
return pUserData->poSrcDS->ExecuteSQL(
170+
pUserData->osSQL.c_str(), nullptr,
171+
pUserData->osDialect.empty() ? nullptr
172+
: pUserData->osDialect.c_str());
173+
};
174+
175+
const auto CloseLayer = [](OGRLayer *poLayer, void *pUserDataIn)
176+
{
177+
UserData *pUserData = static_cast<UserData *>(pUserDataIn);
178+
pUserData->poSrcDS->ReleaseResultSet(poLayer);
179+
};
180+
181+
const auto DeleteUserData = [](void *pUserDataIn)
182+
{ delete static_cast<UserData *>(pUserDataIn); };
183+
184+
auto pUserData = new UserData;
185+
pUserData->poSrcDS = m_poSrcDS;
186+
pUserData->osDialect = osDialect;
187+
pUserData->osSQL = osSQL;
188+
pUserData->osLayerName = osLayerName;
189+
auto poLayer = std::make_unique<ProxiedSQLLayer>(
190+
osLayerName, &m_oPool, OpenLayer, CloseLayer, DeleteUserData,
191+
pUserData);
192+
m_layers.push_back(std::move(poLayer));
193+
}
194+
195+
int GetLayerCount() override
196+
{
197+
return static_cast<int>(m_layers.size());
198+
}
199+
200+
OGRLayer *GetLayer(int idx) override
201+
{
202+
return idx >= 0 && idx < GetLayerCount() ? m_layers[idx].get()
203+
: nullptr;
204+
}
205+
};
206+
} // namespace
207+
208+
/************************************************************************/
209+
/* GDALVectorSQLAlgorithm::RunStep() */
210+
/************************************************************************/
211+
212+
bool GDALVectorSQLAlgorithm::RunStep(GDALProgressFunc, void *)
213+
{
214+
CPLAssert(m_inputDataset.GetDatasetRef());
215+
CPLAssert(m_outputDataset.GetName().empty());
216+
CPLAssert(!m_outputDataset.GetDatasetRef());
217+
218+
if (!m_outputLayer.empty() && m_outputLayer.size() != m_sql.size())
219+
{
220+
ReportError(CE_Failure, CPLE_AppDefined,
221+
"There should be as many layer names in --output-layer as "
222+
"in --statement");
223+
return false;
224+
}
225+
226+
auto poSrcDS = m_inputDataset.GetDatasetRef();
227+
228+
if (m_sql.size() == 1)
229+
{
230+
auto outDS = std::make_unique<GDALVectorSQLAlgorithmDataset>(poSrcDS);
231+
outDS->SetDescription(poSrcDS->GetDescription());
232+
233+
OGRLayer *poLayer = poSrcDS->ExecuteSQL(
234+
m_sql[0].c_str(), nullptr,
235+
m_dialect.empty() ? nullptr : m_dialect.c_str());
236+
if (!poLayer)
237+
return false;
238+
239+
if (!m_outputLayer.empty())
240+
{
241+
const std::string &osLayerName = m_outputLayer[0];
242+
poLayer->GetLayerDefn()->SetName(osLayerName.c_str());
243+
poLayer->SetDescription(osLayerName.c_str());
244+
}
245+
outDS->AddLayer(poLayer);
246+
m_outputDataset.Set(std::move(outDS));
247+
}
248+
else
249+
{
250+
// First pass to check all statements are valid and figure out layer
251+
// names
252+
std::set<std::string> setOutputLayerNames;
253+
std::vector<std::string> aosLayerNames;
254+
for (const std::string &sql : m_sql)
255+
{
256+
auto poLayer = poSrcDS->ExecuteSQL(
257+
sql.c_str(), nullptr,
258+
m_dialect.empty() ? nullptr : m_dialect.c_str());
259+
if (!poLayer)
260+
return false;
261+
262+
std::string osLayerName;
263+
264+
if (!m_outputLayer.empty())
265+
{
266+
osLayerName = m_outputLayer[aosLayerNames.size()];
267+
}
268+
else if (cpl::contains(setOutputLayerNames,
269+
poLayer->GetDescription()))
270+
{
271+
int num = 1;
272+
do
273+
{
274+
osLayerName = poLayer->GetDescription();
275+
++num;
276+
osLayerName += std::to_string(num);
277+
} while (cpl::contains(setOutputLayerNames, osLayerName));
278+
}
279+
280+
if (!osLayerName.empty())
281+
{
282+
poLayer->GetLayerDefn()->SetName(osLayerName.c_str());
283+
poLayer->SetDescription(osLayerName.c_str());
284+
}
285+
setOutputLayerNames.insert(poLayer->GetDescription());
286+
aosLayerNames.push_back(poLayer->GetDescription());
287+
288+
poSrcDS->ReleaseResultSet(poLayer);
289+
}
290+
291+
auto outDS =
292+
std::make_unique<GDALVectorSQLAlgorithmDatasetMultiLayer>(poSrcDS);
293+
outDS->SetDescription(poSrcDS->GetDescription());
294+
295+
for (size_t i = 0; i < aosLayerNames.size(); ++i)
296+
{
297+
outDS->AddLayer(m_sql[i], m_dialect, aosLayerNames[i]);
298+
}
299+
300+
m_outputDataset.Set(std::move(outDS));
301+
}
302+
303+
return true;
304+
}
305+
306+
//! @endcond

0 commit comments

Comments
 (0)