From 60e7abaff5c16dde69cf1c1d0e1dced12c45e2c5 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Thu, 17 Apr 2025 15:32:49 +0200 Subject: [PATCH 1/2] Make box_t type available to boost as box type This will be useful later when we index geometries. --- src/geom-boost-adaptor.hpp | 44 ++++++++++++++++++++++++++++++++++++++ src/geom-box.hpp | 5 +++++ tests/test-geom-box.cpp | 42 ++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) diff --git a/src/geom-boost-adaptor.hpp b/src/geom-boost-adaptor.hpp index c0e165ada..4ca489ce5 100644 --- a/src/geom-boost-adaptor.hpp +++ b/src/geom-boost-adaptor.hpp @@ -11,8 +11,10 @@ */ #include "geom.hpp" +#include "geom-box.hpp" #include +#include #include #include #include @@ -78,6 +80,48 @@ struct interior_rings<::geom::polygon_t> static auto const &get(::geom::polygon_t const &p) { return p.inners(); } }; +BOOST_GEOMETRY_DETAIL_SPECIALIZE_BOX_TRAITS(::geom::box_t, ::geom::point_t) + +template <> +struct indexed_access<::geom::box_t, min_corner, 0> +{ + static inline double get(::geom::box_t const &b) { return b.min_x(); } + static inline void set(::geom::box_t &b, double value) + { + b.set_min_x(value); + } +}; + +template <> +struct indexed_access<::geom::box_t, min_corner, 1> +{ + static inline double get(::geom::box_t const &b) { return b.min_y(); } + static inline void set(::geom::box_t &b, double value) + { + b.set_min_y(value); + } +}; + +template <> +struct indexed_access<::geom::box_t, max_corner, 0> +{ + static inline double get(::geom::box_t const &b) { return b.max_x(); } + static inline void set(::geom::box_t &b, double value) + { + b.set_max_x(value); + } +}; + +template <> +struct indexed_access<::geom::box_t, max_corner, 1> +{ + static inline double get(::geom::box_t const &b) { return b.max_y(); } + static inline void set(::geom::box_t &b, double value) + { + b.set_max_y(value); + } +}; + } // namespace boost::geometry::traits #endif // OSM2PGSQL_GEOM_BOOST_ADAPTOR_HPP diff --git a/src/geom-box.hpp b/src/geom-box.hpp index eb07cf632..e2f241da2 100644 --- a/src/geom-box.hpp +++ b/src/geom-box.hpp @@ -45,6 +45,11 @@ class box_t constexpr double max_x() const noexcept { return m_max_x; } constexpr double max_y() const noexcept { return m_max_y; } + void set_min_x(double value) noexcept { m_min_x = value; } + void set_min_y(double value) noexcept { m_min_y = value; } + void set_max_x(double value) noexcept { m_max_x = value; } + void set_max_y(double value) noexcept { m_max_y = value; } + constexpr double width() const noexcept { return m_max_x - m_min_x; } constexpr double height() const noexcept { return m_max_y - m_min_y; } diff --git a/tests/test-geom-box.cpp b/tests/test-geom-box.cpp index 5fb257cb9..134697de7 100644 --- a/tests/test-geom-box.cpp +++ b/tests/test-geom-box.cpp @@ -9,8 +9,50 @@ #include +#include "geom-boost-adaptor.hpp" #include "geom-box.hpp" +TEST_CASE("box_t getter/setter", "[NoDB]") +{ + geom::box_t box{1.0, 2.0, 3.0, 4.0}; + + REQUIRE(box.min_x() == Approx(1.0)); + REQUIRE(box.max_x() == Approx(3.0)); + REQUIRE(box.min_y() == Approx(2.0)); + REQUIRE(box.max_y() == Approx(4.0)); + + box.set_min_x(1.5); + box.set_min_y(2.5); + box.set_max_x(3.5); + box.set_max_y(4.5); + + REQUIRE(box.min_x() == Approx(1.5)); + REQUIRE(box.max_x() == Approx(3.5)); + REQUIRE(box.min_y() == Approx(2.5)); + REQUIRE(box.max_y() == Approx(4.5)); +} + +TEST_CASE("box_t getter/setter through boost", "[NoDB]") +{ + geom::box_t box{1.0, 2.0, 3.0, 4.0}; + + namespace bg = ::boost::geometry; + REQUIRE(bg::get(box) == Approx(1.0)); + REQUIRE(bg::get(box) == Approx(2.0)); + REQUIRE(bg::get(box) == Approx(3.0)); + REQUIRE(bg::get(box) == Approx(4.0)); + + bg::set(box, 1.5); + bg::set(box, 2.5); + bg::set(box, 3.5); + bg::set(box, 4.5); + + REQUIRE(box.min_x() == Approx(1.5)); + REQUIRE(box.min_y() == Approx(2.5)); + REQUIRE(box.max_x() == Approx(3.5)); + REQUIRE(box.max_y() == Approx(4.5)); +} + TEST_CASE("Extend box_t with points", "[NoDB]") { geom::box_t box; From 24ce38f232ed39b8d0e77de1fac47569223cf864 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Tue, 25 Mar 2025 15:19:11 +0100 Subject: [PATCH 2/2] New feature: Check location of OSM objects against list of regions This commit introduces a new feature: Locators. A locator is initialized with one or more regions, each region has a name and a polygon or bounding box. A geometry of an OSM object can then be checked against this region list to figure out in which region(s) it is located. This check is much faster than it would be to do this inside the database after import. Locators can be used for all sorts of interesting features: * Read larger OSM file but import only data inside some area. * Annotate each OSM object with the country (or other region) it is in. This can then, for instance, be used to show special highway shields for each country. * Use the information which region the data is in for further processing, for instance setting of default values for the speed limit or using special language transliterations rules based on country. Locators are created in Lua with `define_locator()`. Bounding boxes can be added with `add_bbox()`. Polygons can be added from the database by calling `add_from_db()` and specifiying an SQL query which can return any number of rows each defining a region with the name and the (multi)polygon as columns. A locator can then be queried using `all_intersecting()` returning a list of names of all regions that intersect the specified OSM object geometry. Or the `first_intersecting()` function can be used which only returns a single region for those cases where there can be no overlapping data or where the details of objects straddling region boundaries don't matter. Several example config files are provided in the flex-config/locator directory. --- .github/workflows/luacheck.yml | 2 +- flex-config/locator/README.md | 9 + flex-config/locator/buildings.lua | 74 +++++++ flex-config/locator/iceland.lua | 32 ++++ flex-config/locator/import-countries.lua | 35 ++++ flex-config/locator/motorway-colours.lua | 40 ++++ src/CMakeLists.txt | 2 + src/flex-lua-locator.cpp | 188 ++++++++++++++++++ src/flex-lua-locator.hpp | 53 ++++++ src/flex-lua-wrapper.hpp | 3 + src/locator.cpp | 129 +++++++++++++ src/locator.hpp | 147 ++++++++++++++ src/output-flex.cpp | 31 ++- src/output-flex.hpp | 8 + src/pgsql-capabilities-int.hpp | 1 + src/pgsql-capabilities.cpp | 19 ++ src/pgsql-capabilities.hpp | 2 + src/pgsql.hpp | 6 + tests/CMakeLists.txt | 1 + tests/bdd/flex/locator.feature | 233 +++++++++++++++++++++++ tests/bdd/steps/steps_execute.py | 11 +- tests/test-locator.cpp | 136 +++++++++++++ 22 files changed, 1156 insertions(+), 6 deletions(-) create mode 100644 flex-config/locator/README.md create mode 100644 flex-config/locator/buildings.lua create mode 100644 flex-config/locator/iceland.lua create mode 100644 flex-config/locator/import-countries.lua create mode 100644 flex-config/locator/motorway-colours.lua create mode 100644 src/flex-lua-locator.cpp create mode 100644 src/flex-lua-locator.hpp create mode 100644 src/locator.cpp create mode 100644 src/locator.hpp create mode 100644 tests/bdd/flex/locator.feature create mode 100644 tests/test-locator.cpp diff --git a/.github/workflows/luacheck.yml b/.github/workflows/luacheck.yml index 785b2b89a..a21a9053f 100644 --- a/.github/workflows/luacheck.yml +++ b/.github/workflows/luacheck.yml @@ -15,5 +15,5 @@ jobs: sudo apt-get install -yq --no-install-suggests --no-install-recommends lua-check - name: Run luacheck - run: luacheck flex-config/*.lua flex-config/gen/*.lua tests/data/*.lua tests/lua/tests.lua + run: luacheck flex-config/*.lua flex-config/*/*.lua tests/data/*.lua tests/lua/tests.lua diff --git a/flex-config/locator/README.md b/flex-config/locator/README.md new file mode 100644 index 000000000..ed58943a9 --- /dev/null +++ b/flex-config/locator/README.md @@ -0,0 +1,9 @@ +# Flex Output Configuration for Locator + +These are config file examples for use with the *locator* functionality. + +First use the `import-countries.lua` to import country boundaries, then +use these with the other config files. + +You should be able to run this on the planet file or on any extracts you want. + diff --git a/flex-config/locator/buildings.lua b/flex-config/locator/buildings.lua new file mode 100644 index 000000000..8455a8bce --- /dev/null +++ b/flex-config/locator/buildings.lua @@ -0,0 +1,74 @@ +-- This config example file is released into the Public Domain. + +-- This file shows how to use a locator to tag all buildings with the country +-- they are in. + +-- Define the "countries" locator and get all country geometries from the +-- database. Use the import-countries.lua file to import them first, before +-- you run this. +local countries = osm2pgsql.define_locator({ name = 'countries' }) + +-- The SELECT query must return the regions the locator will use. The first +-- column must contain the name of the region, the second the geometry in +-- WGS84 lon/lat coordinates. +-- To improve the efficiency of the country lookup, we'll subdivide the +-- country polygons into smaller pieces. (If you run this often, for instance +-- when doing updates, do the subdivide first in the database and store the +-- result in its own table.) +countries:add_from_db('SELECT code, ST_Subdivide(geom, 200) FROM countries') + +-- You have to decide whether you are interested in getting all regions +-- intersecting with any of the objects or only one of them. +-- +-- * Getting all regions makes sure that you get everything, even if regions +-- overlap or objects straddle the border between regions. Use the function +-- all_intersecting() for that. +-- * Getting only one region is faster because osm2pgsql can stop looking +-- for matches after the first one. Use first_intersecting() for that. +-- This makes sense if you only have a single region anyway or if your +-- regions don't overlap or you are not so concerned with what happens at +-- the borders. +-- +-- Just for demonstration, we do both in this example, in the "country" and +-- "countries" columns, respectively. +local buildings = osm2pgsql.define_area_table('buildings', { + + -- This will contain the country code of the first matching country + -- (which can be any of the countries because there is no order here). + { column = 'country', type = 'text' }, + + -- This array will contain the country codes of all matching countries. + { column = 'countries', sql_type = 'text[]' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'polygon', not_null = true }, +}) + +local function add(geom, tags) + buildings:insert({ + country = countries:first_intersecting(geom), -- or use geom:centroid() + + -- We have to create the format that PostgreSQL expects for text + -- arrays. We assume that the region names do not contain any special + -- characters, otherwise we would need to do some escaping here. + countries = '{' .. table.concat(countries:all_intersecting(geom), ',') .. '}', + + tags = tags, + geom = geom, + }) +end + +function osm2pgsql.process_way(object) + if object.tags.building then + add(object:as_polygon(), object.tags) + end +end + +function osm2pgsql.process_relation(object) + if object.tags.building then + local geom = object:as_multipolygon() + for p in geom:geometries() do + add(p, object.tags) + end + end +end + diff --git a/flex-config/locator/iceland.lua b/flex-config/locator/iceland.lua new file mode 100644 index 000000000..ebf36266b --- /dev/null +++ b/flex-config/locator/iceland.lua @@ -0,0 +1,32 @@ +-- This config example file is released into the Public Domain. + +-- This file shows how to use a locator with a bounding box to import only +-- the data for a region. In this case only highways in Iceland are imported +-- even if you run this on the full planet file. + +local iceland = osm2pgsql.define_locator({ name = 'iceland' }) + +iceland:add_bbox('IS', -25.0, 62.0, -12.0, 68.0) + +local highways = osm2pgsql.define_way_table('highways', { + { column = 'hwtype', type = 'text', not_null = true }, + { column = 'name', type = 'text' }, + { column = 'ref', type = 'text' }, + { column = 'geom', type = 'linestring', not_null = true }, +}) + +function osm2pgsql.process_way(object) + local t = object.tags + if t.highway then + local geom = object:as_linestring() + local region = iceland:first_intersecting(geom) + if region then + highways:insert({ + hwtype = t.highway, + name = t.name, + ref = t.ref, + geom = geom, + }) + end + end +end diff --git a/flex-config/locator/import-countries.lua b/flex-config/locator/import-countries.lua new file mode 100644 index 000000000..ba468c3ea --- /dev/null +++ b/flex-config/locator/import-countries.lua @@ -0,0 +1,35 @@ +-- This config example file is released into the Public Domain. + +-- This file is part of the examples for using the locator function of +-- osm2pgsql. It is used to import all country boundaries into a database. + +local countries = osm2pgsql.define_relation_table('countries', { + -- For the ISO3166-1 Alpha-2 country code + -- https://en.wikipedia.org/wiki/ISO_3166-1 + { column = 'code', type = 'text', not_null = true }, + -- Because we want to use the geometries for the locator feature they + -- must be in 4326! We use a polygon type here and will later split + -- multipolygons into their parts. + { column = 'geom', type = 'polygon', not_null = true, projection = 4326 }, +}) + +function osm2pgsql.process_relation(object) + local t = object.tags + + if t.boundary == 'administrative' and t.admin_level == '2' then + local code = t['ISO3166-1'] + + -- Ignore entries with syntactically invalid ISO code + if not code or not string.match(code, '^%u%u$') then + return + end + + for geom in object:as_multipolygon():geometries() do + countries:insert({ + code = code, + geom = geom, + }) + end + end +end + diff --git a/flex-config/locator/motorway-colours.lua b/flex-config/locator/motorway-colours.lua new file mode 100644 index 000000000..74287c78f --- /dev/null +++ b/flex-config/locator/motorway-colours.lua @@ -0,0 +1,40 @@ +-- This config example file is released into the Public Domain. + +-- This file shows how to use a locator to find the country-specific colour + +-- Define the "countries" locator and get all country geometries from the +-- database. Use the import-countries.lua file to import them first, before +-- you run this. +local countries = osm2pgsql.define_locator({ name = 'countries' }) +countries:add_from_db('SELECT code, ST_Subdivide(geom, 200) FROM countries') + +local highways = osm2pgsql.define_way_table('highways', { + { column = 'hwtype', type = 'text' }, + { column = 'country', type = 'text' }, + { column = 'colour', type = 'text' }, + { column = 'geom', type = 'linestring', not_null = true }, +}) + +-- Each country uses their own colour for motorways. Here is the beginning +-- of a list of some countries in Europe. Source: +-- https://en.wikipedia.org/wiki/Comparison_of_European_road_signs +local cc2colour = { + BE = '#2d00e5', + CH = '#128044', + DE = '#174688', + FR = '#333b97', + NL = '#064269', +} + +function osm2pgsql.process_way(object) + if object.tags.highway then + local geom = object:as_linestring() + local cc = countries:first_intersecting(geom) + highways:insert({ + hwtype = object.tags.highway, + country = cc, + colour = cc2colour[cc], + geom = geom, + }) + end +end diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e10db77e6..ba1f0346f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -21,6 +21,7 @@ target_sources(osm2pgsql_lib PRIVATE flex-lua-expire-output.cpp flex-lua-geom.cpp flex-lua-index.cpp + flex-lua-locator.cpp flex-lua-table.cpp flex-table-column.cpp flex-table.cpp @@ -33,6 +34,7 @@ target_sources(osm2pgsql_lib PRIVATE geom.cpp idlist.cpp input.cpp + locator.cpp logging.cpp lua-setup.cpp lua-utils.cpp diff --git a/src/flex-lua-locator.cpp b/src/flex-lua-locator.cpp new file mode 100644 index 000000000..536fdf1cc --- /dev/null +++ b/src/flex-lua-locator.cpp @@ -0,0 +1,188 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2025 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "flex-lua-locator.hpp" + +#include "flex-lua-geom.hpp" +#include "flex-lua-wrapper.hpp" +#include "lua-utils.hpp" +#include "pgsql.hpp" + +#include +#include +#include + +#include + +namespace { + +locator_t &create_locator(lua_State *lua_state, + std::vector *locators) +{ + auto &new_locator = locators->emplace_back(); + + // optional "name" field + auto const *name = + luaX_get_table_string(lua_state, "name", -1, "The locator", ""); + new_locator.set_name(name); + lua_pop(lua_state, 1); // "name" + + return new_locator; +} + +TRAMPOLINE_WRAPPED_OBJECT(locator, __tostring) +TRAMPOLINE_WRAPPED_OBJECT(locator, name) +TRAMPOLINE_WRAPPED_OBJECT(locator, add_bbox) +TRAMPOLINE_WRAPPED_OBJECT(locator, add_from_db) +TRAMPOLINE_WRAPPED_OBJECT(locator, all_intersecting) +TRAMPOLINE_WRAPPED_OBJECT(locator, first_intersecting) + +} // anonymous namespace + +connection_params_t lua_wrapper_locator::s_connection_params; + +int setup_flex_locator(lua_State *lua_state, std::vector *locators) +{ + if (lua_type(lua_state, 1) != LUA_TTABLE) { + throw std::runtime_error{ + "Argument #1 to 'define_locator' must be a Lua table."}; + } + + create_locator(lua_state, locators); + + void *ptr = lua_newuserdata(lua_state, sizeof(std::size_t)); + auto *num = new (ptr) std::size_t{}; + *num = locators->size() - 1; + luaL_getmetatable(lua_state, osm2pgsql_locator_name); + lua_setmetatable(lua_state, -2); + + return 1; +} + +void lua_wrapper_locator::init(lua_State *lua_state, + connection_params_t const &connection_params) +{ + s_connection_params = connection_params; + + lua_getglobal(lua_state, "osm2pgsql"); + if (luaL_newmetatable(lua_state, osm2pgsql_locator_name) != 1) { + throw std::runtime_error{"Internal error: Lua newmetatable failed."}; + } + lua_pushvalue(lua_state, -1); // Copy of new metatable + + // Add metatable as osm2pgsql.Locator so we can access it from Lua + lua_setfield(lua_state, -3, "Locator"); + + // Now add functions to metatable + lua_pushvalue(lua_state, -1); + lua_setfield(lua_state, -2, "__index"); + luaX_add_table_func(lua_state, "__tostring", + lua_trampoline_locator___tostring); + luaX_add_table_func(lua_state, "name", lua_trampoline_locator_name); + luaX_add_table_func(lua_state, "add_bbox", lua_trampoline_locator_add_bbox); + luaX_add_table_func(lua_state, "add_from_db", + lua_trampoline_locator_add_from_db); + luaX_add_table_func(lua_state, "all_intersecting", + lua_trampoline_locator_all_intersecting); + luaX_add_table_func(lua_state, "first_intersecting", + lua_trampoline_locator_first_intersecting); + + lua_pop(lua_state, 2); +} + +int lua_wrapper_locator::__tostring() const +{ + std::string const str{fmt::format("osm2pgsql.Locator[name={},size={}]", + self().name(), self().size())}; + luaX_pushstring(lua_state(), str); + return 1; +} + +int lua_wrapper_locator::name() const +{ + luaX_pushstring(lua_state(), self().name()); + return 1; +} + +int lua_wrapper_locator::add_bbox() +{ + if (lua_gettop(lua_state()) < 5) { + throw fmt_error("Need locator, name and 4 coordinates as arguments"); + } + + std::string const name = lua_tostring(lua_state(), 1); + double const min_x = lua_tonumber(lua_state(), 2); + double const min_y = lua_tonumber(lua_state(), 3); + double const max_x = lua_tonumber(lua_state(), 4); + double const max_y = lua_tonumber(lua_state(), 5); + + self().add_region(name, geom::box_t{min_x, min_y, max_x, max_y}); + + return 0; +} + +int lua_wrapper_locator::add_from_db() +{ + if (lua_gettop(lua_state()) < 1) { + throw fmt_error("Need locator and SQL query arguments"); + } + + std::string const query = lua_tostring(lua_state(), 1); + + pg_conn_t const db_connection{s_connection_params, "flex.locator"}; + self().add_regions(db_connection, query); + + return 0; +} + +int lua_wrapper_locator::all_intersecting() +{ + if (lua_gettop(lua_state()) < 1) { + throw fmt_error("Need locator and geometry arguments"); + } + + auto const *geometry = unpack_geometry(lua_state()); + + if (!geometry) { + throw fmt_error("Second argument must be a geometry"); + } + + auto const names = self().all_intersecting(*geometry); + lua_createtable(lua_state(), (int)names.size(), 0); + int n = 0; + for (auto const& name : names) { + lua_pushinteger(lua_state(), ++n); + lua_pushstring(lua_state(), name.c_str()); + lua_rawset(lua_state(), -3); + } + + return 1; +} + +int lua_wrapper_locator::first_intersecting() +{ + if (lua_gettop(lua_state()) < 1) { + throw fmt_error("Need locator and geometry arguments"); + } + + auto const *geometry = unpack_geometry(lua_state()); + + if (!geometry) { + throw fmt_error("Second argument must be a geometry"); + } + + auto const name = self().first_intersecting(*geometry); + if (name.empty()) { + return 0; + } + + lua_pushstring(lua_state(), name.c_str()); + + return 1; +} diff --git a/src/flex-lua-locator.hpp b/src/flex-lua-locator.hpp new file mode 100644 index 000000000..cf560702a --- /dev/null +++ b/src/flex-lua-locator.hpp @@ -0,0 +1,53 @@ +#ifndef OSM2PGSQL_FLEX_LUA_LOCATOR_HPP +#define OSM2PGSQL_FLEX_LUA_LOCATOR_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2025 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +/** + * \file + * + * Functions implementing the Lua interface for the locator. + */ + +#include "flex-lua-wrapper.hpp" +#include "locator.hpp" + +class pg_conn_t; + +struct lua_State; + +static char const *const osm2pgsql_locator_name = "osm2pgsql.Locator"; + +int setup_flex_locator(lua_State *lua_state, std::vector *locators); + +class lua_wrapper_locator : public lua_wrapper_base +{ +public: + static void init(lua_State *lua_state, + connection_params_t const &connection_params); + + lua_wrapper_locator(lua_State *lua_state, locator_t *locator) + : lua_wrapper_base(lua_state, locator) + { + } + + int __tostring() const; + int name() const; + int add_bbox(); + int add_from_db(); + int all_intersecting(); + int first_intersecting(); + +private: + static connection_params_t s_connection_params; + +}; // class lua_wrapper_locator + +#endif // OSM2PGSQL_FLEX_LUA_LOCATOR_HPP diff --git a/src/flex-lua-wrapper.hpp b/src/flex-lua-wrapper.hpp index 7577e7c71..6f590d532 100644 --- a/src/flex-lua-wrapper.hpp +++ b/src/flex-lua-wrapper.hpp @@ -12,6 +12,7 @@ #include "output-flex.hpp" +#include #include #define TRAMPOLINE_WRAPPED_OBJECT(obj_name, func_name) \ @@ -43,6 +44,8 @@ class lua_wrapper_base lua_wrapper_base(lua_State *lua_state, WRAPPED *wrapped) : m_lua_state(lua_state), m_self(wrapped) { + assert(lua_state); + assert(wrapped); } protected: diff --git a/src/locator.cpp b/src/locator.cpp new file mode 100644 index 000000000..7b983751d --- /dev/null +++ b/src/locator.cpp @@ -0,0 +1,129 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2025 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "locator.hpp" + +#include "geom-boost-adaptor.hpp" +#include "geom-box.hpp" +#include "geom-functions.hpp" +#include "overloaded.hpp" +#include "pgsql-capabilities.hpp" +#include "pgsql.hpp" +#include "wkb.hpp" + +#include + +void locator_t::add_region(std::string const &name, geom::box_t const &box) +{ + m_regions.emplace_back(name, box); +} + +void locator_t::add_region(std::string const &name, + geom::geometry_t const &geom) +{ + if (geom.is_polygon()) { + m_regions.emplace_back(name, geom.get()); + return; + } + + if (geom.is_multipolygon()) { + for (auto const &polygon : geom.get()) { + m_regions.emplace_back(name, polygon); + } + return; + } + + throw std::runtime_error{ + "Invalid geometry type: Need (multi)polygon for region."}; +} + +void locator_t::add_regions(pg_conn_t const &db_connection, + std::string const &query) +{ + log_debug("Querying database for locator '{}'...", name()); + auto const result = db_connection.exec(query); + if (result.num_fields() != 2) { + throw std::runtime_error{"Locator queries must return exactly two " + "columns with the name and the geometry."}; + } + + if (!is_geometry_type(result.field_type(1))) { + throw std::runtime_error{ + "Second column in Locator query results must be a geometry."}; + } + + for (int n = 0; n < result.num_tuples(); ++n) { + std::string const name = result.get_value(n, 0); + auto geometry = ewkb_to_geom(decode_hex(result.get(n, 1))); + + if (geometry.srid() == 4326) { + add_region(name, geometry); + } else { + log_warn("Ignoring locator geometry that is not in WGS84 (4326)"); + } + } + log_info("Added {} regions to locator '{}'.", result.num_tuples(), name()); +} + +void locator_t::build_index() +{ + log_debug("Building index for locator '{}'", name()); + std::vector m_data; + m_data.reserve(m_regions.size()); + + std::size_t n = 0; + for (auto const ®ion : m_regions) { + m_data.emplace_back(region.box(), n++); + } + + m_rtree.clear(); + m_rtree.insert(m_data.cbegin(), m_data.cend()); +} + +std::set locator_t::all_intersecting(geom::geometry_t const &geom) +{ + if (m_rtree.size() < m_regions.size()) { + build_index(); + } + + std::set results; + + geom.visit(overloaded{[&](geom::nullgeom_t const & /*input*/) {}, + [&](geom::collection_t const & /*input*/) {}, // TODO + [&](auto const &val) { + for (auto it = begin_intersects(val); + it != end_query(); ++it) { + auto const ®ion = m_regions[it->second]; + results.emplace(region.name()); + } + }}); + + return results; +} + +std::string locator_t::first_intersecting(geom::geometry_t const &geom) +{ + if (m_rtree.size() < m_regions.size()) { + build_index(); + } + + std::string result; + + geom.visit(overloaded{[&](geom::nullgeom_t const & /*input*/) {}, + [&](geom::collection_t const & /*input*/) {}, // TODO + [&](auto const &val) { + auto const it = begin_intersects(val); + if (it != end_query()) { + auto const ®ion = m_regions[it->second]; + result = region.name(); + } + }}); + + return result; +} diff --git a/src/locator.hpp b/src/locator.hpp new file mode 100644 index 000000000..b8d6e44a1 --- /dev/null +++ b/src/locator.hpp @@ -0,0 +1,147 @@ +#ifndef OSM2PGSQL_LOCATOR_HPP +#define OSM2PGSQL_LOCATOR_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2025 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +/** + * \file + * + * For the region_t and locator_t classes. + */ + +#include "geom-boost-adaptor.hpp" +#include "geom-box.hpp" +#include "geom.hpp" +#include "logging.hpp" + +#include + +#include +#include +#include +#include +#include + +class pg_conn_t; + +namespace bgi = ::boost::geometry::index; + +/** + * A locator stores a number of regions. Each region has a name and a bounding + * box or polygon geometry. The locator can then check efficiently which + * regions a specified geometry is intersecting. + * + * Names don't have to be unique. Geometries of regions can overlap. In fact it + * is best to subdivide larger polygons into smaller ones, because the + * intersection will be much faster to calculate that way. This will + * automatically lead to lots of small polygons with the same name. + */ +class locator_t +{ +private: + class region_t + { + public: + region_t(std::string name, geom::box_t const &box) + : m_name(std::move(name)), m_box(box), + m_polygon( + geom::polygon_t{geom::ring_t{{m_box.min_x(), m_box.min_y()}, + {m_box.max_x(), m_box.min_y()}, + {m_box.max_x(), m_box.max_y()}, + {m_box.min_x(), m_box.max_y()}, + {m_box.min_x(), m_box.min_y()}}}) + { + } + + region_t(std::string name, geom::polygon_t const &polygon) + : m_name(std::move(name)), m_box(envelope(polygon)), m_polygon(polygon) + { + } + + std::string const &name() const noexcept { return m_name; } + + geom::box_t const &box() const noexcept { return m_box; } + + geom::polygon_t const &polygon() const noexcept { return m_polygon; } + + private: + std::string m_name; + geom::box_t m_box; + geom::polygon_t m_polygon; + + }; // class region_t + + std::string m_name; + std::vector m_regions; + + using idx_value_t = std::pair; + + using tree_t = bgi::rtree>; + tree_t m_rtree; + + template + tree_t::const_query_iterator begin_intersects(T const &geom) + { + return m_rtree.qbegin( + bgi::intersects(geom) && bgi::satisfies([&](idx_value_t const &v) { + auto const ®ion = m_regions[v.second]; + return boost::geometry::intersects(region.polygon(), geom); + })); + } + + tree_t::const_query_iterator end_query() { return m_rtree.qend(); } + +public: + /// The name of this locator (for logging only) + std::string const &name() const noexcept { return m_name; } + + /// Are there any regions stored in this locator? + bool empty() const noexcept { return m_regions.empty(); } + + /// Return the number of regions stored in this locator. + std::size_t size() const noexcept { return m_regions.size(); } + + /// Set the name of this locator. + void set_name(std::string name) { m_name = std::move(name); } + + /// Add a bounding box as region. + void add_region(std::string const &name, geom::box_t const &box); + + /** + * Add a (multi)polygon as region. + * + * Throws an exception if geom is not a (multi)polygon. + */ + void add_region(std::string const &name, geom::geometry_t const &geom); + + void add_regions(pg_conn_t const &db_connection, std::string const &query); + + /// Build index containing all regions. + void build_index(); + + /** + * Find all regions intersecting the specified geometry. Returns a set + * of (unique) names of those regions. + * + * Automatically calls build_index() if needed. + */ + std::set all_intersecting(geom::geometry_t const &geom); + + /** + * Find a region intersecting the specified geometry. If there is more + * than one, a random one will be returned. Returns the name of the region. + * + * Automatically calls build_index() if needed. + */ + std::string first_intersecting(geom::geometry_t const &geom); + +}; // class locator_t + +#endif // OSM2PGSQL_LOCATOR_HPP diff --git a/src/output-flex.cpp b/src/output-flex.cpp index 164d81b5e..31bb167fa 100644 --- a/src/output-flex.cpp +++ b/src/output-flex.cpp @@ -17,6 +17,7 @@ #include "flex-lua-expire-output.hpp" #include "flex-lua-geom.hpp" #include "flex-lua-index.hpp" +#include "flex-lua-locator.hpp" #include "flex-lua-table.hpp" #include "flex-lua-wrapper.hpp" #include "flex-write.hpp" @@ -77,6 +78,7 @@ std::mutex lua_mutex; } \ } +TRAMPOLINE(app_define_locator, define_locator) TRAMPOLINE(app_define_table, define_table) TRAMPOLINE(app_define_expire_output, define_expire_output) TRAMPOLINE(app_get_bbox, get_bbox) @@ -566,6 +568,17 @@ int output_flex_t::app_as_geometrycollection() return 1; } +int output_flex_t::app_define_locator() +{ + if (m_calling_context != calling_context::main) { + throw std::runtime_error{ + "Locators have to be defined in the" + " main Lua code, not in any of the callbacks."}; + } + + return setup_flex_locator(lua_state(), m_locators.get()); +} + int output_flex_t::app_define_table() { if (m_calling_context != calling_context::main) { @@ -604,6 +617,12 @@ expire_output_t &output_flex_t::get_expire_output_from_param() osm2pgsql_expire_output_name); } +locator_t &output_flex_t::get_locator_from_param() +{ + return get_from_idx_param(lua_state(), m_locators.get(), + osm2pgsql_locator_name); +} + bool output_flex_t::way_cache_t::init(middle_query_t const &middle, osmid_t id) { m_buffer.clear(); @@ -1107,13 +1126,17 @@ void output_flex_t::start() for (auto &table : m_table_connections) { table.start(m_db_connection, get_options()->append); } + + for (auto &locator : *m_locators) { + locator.build_index(); + } } output_flex_t::output_flex_t(output_flex_t const *other, std::shared_ptr mid, std::shared_ptr copy_thread) -: output_t(other, std::move(mid)), m_tables(other->m_tables), - m_expire_outputs(other->m_expire_outputs), +: output_t(other, std::move(mid)), m_locators(other->m_locators), + m_tables(other->m_tables), m_expire_outputs(other->m_expire_outputs), m_db_connection(get_options()->connection_params, "out.flex.thread"), m_stage2_way_ids(other->m_stage2_way_ids), m_copy_thread(std::move(copy_thread)), m_lua_state(other->m_lua_state), @@ -1222,6 +1245,9 @@ void output_flex_t::init_lua(std::string const &filename, } lua_rawset(lua_state(), -3); + luaX_add_table_func(lua_state(), "define_locator", + lua_trampoline_app_define_locator); + luaX_add_table_func(lua_state(), "define_table", lua_trampoline_app_define_table); @@ -1229,6 +1255,7 @@ void output_flex_t::init_lua(std::string const &filename, lua_trampoline_app_define_expire_output); lua_wrapper_expire_output::init(lua_state()); + lua_wrapper_locator::init(lua_state(), get_options()->connection_params); lua_wrapper_table::init(lua_state()); // Clean up stack diff --git a/src/output-flex.hpp b/src/output-flex.hpp index 39cd1da70..c803d3a30 100644 --- a/src/output-flex.hpp +++ b/src/output-flex.hpp @@ -17,6 +17,7 @@ #include "flex-table.hpp" #include "geom.hpp" #include "idlist.hpp" +#include "locator.hpp" #include "output.hpp" #include @@ -160,6 +161,7 @@ class output_flex_t : public output_t int app_as_multipolygon(); int app_as_geometrycollection(); + int app_define_locator(); int app_define_table(); int app_define_expire_output(); int app_get_bbox(); @@ -172,6 +174,9 @@ class output_flex_t : public output_t // Get the expire output that is as first parameter on the Lua stack. expire_output_t &get_expire_output_from_param(); + // Get the flex locator that is as first parameter on the Lua stack. + locator_t &get_locator_from_param(); + private: void select_relation_members(); @@ -258,6 +263,9 @@ class output_flex_t : public output_t }; // relation_cache_t + std::shared_ptr> m_locators = + std::make_shared>(); + std::shared_ptr> m_tables = std::make_shared>(); diff --git a/src/pgsql-capabilities-int.hpp b/src/pgsql-capabilities-int.hpp index 0fa779837..993f7dbc6 100644 --- a/src/pgsql-capabilities-int.hpp +++ b/src/pgsql-capabilities-int.hpp @@ -27,6 +27,7 @@ struct database_capabilities_t std::set tables; std::string database_name; + unsigned int geometry_type_oid = 0; uint32_t database_version = 0; postgis_version postgis{}; diff --git a/src/pgsql-capabilities.cpp b/src/pgsql-capabilities.cpp index b04c26f9d..7c22ef0a1 100644 --- a/src/pgsql-capabilities.cpp +++ b/src/pgsql-capabilities.cpp @@ -71,6 +71,19 @@ void init_database_name(pg_conn_t const &db_connection) capabilities().database_name = res.get(0, 0); } +void init_geometry_oid(pg_conn_t const &db_connection) +{ + auto const res = + db_connection.exec("SELECT oid FROM pg_type WHERE typname='geometry'"); + + if (res.num_tuples() != 1) { + throw std::runtime_error{"Database error: Can not get geometry type. " + "Is PostGIS extension loaded?"}; + } + + capabilities().geometry_type_oid = std::stoul(std::string{res.get(0, 0)}); +} + void init_postgis_version(pg_conn_t const &db_connection) { auto const res = db_connection.exec( @@ -96,6 +109,7 @@ void init_database_capabilities(pg_conn_t const &db_connection) init_settings(db_connection); init_database_name(db_connection); init_postgis_version(db_connection); + init_geometry_oid(db_connection); try { log_info("Database version: {}", @@ -173,6 +187,11 @@ bool has_table(std::string schema, std::string const &name) return capabilities().tables.count(schema); } +bool is_geometry_type(unsigned int oid) +{ + return capabilities().geometry_type_oid == oid; +} + void check_schema(std::string const &schema) { if (has_schema(schema)) { diff --git a/src/pgsql-capabilities.hpp b/src/pgsql-capabilities.hpp index a9aba9269..4eb766e69 100644 --- a/src/pgsql-capabilities.hpp +++ b/src/pgsql-capabilities.hpp @@ -23,6 +23,8 @@ bool has_tablespace(std::string const &value); bool has_index_method(std::string const &value); bool has_table(std::string schema, std::string const &name); +bool is_geometry_type(unsigned int oid); + void check_schema(std::string const &schema); /// Get PostgreSQL version in the format (major * 10000 + minor). diff --git a/src/pgsql.hpp b/src/pgsql.hpp index 33c398fbf..01a0f617f 100644 --- a/src/pgsql.hpp +++ b/src/pgsql.hpp @@ -115,6 +115,12 @@ class pg_result_t return PQfnumber(m_result.get(), ('"' + name + '"').c_str()); } + /// Get the PostgreSQL internal type of a field. + unsigned int field_type(int col) const noexcept { + assert(col >= 0 && col < num_fields()); + return PQftype(m_result.get(), col); + } + /// Return true if this holds an actual result. explicit operator bool() const noexcept { return m_result.get(); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 41456d7cf..93fd0eded 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -58,6 +58,7 @@ set_test(test-geom-pole-of-inaccessibility LABELS NoDB) set_test(test-geom-polygons LABELS NoDB) set_test(test-geom-transform LABELS NoDB) set_test(test-json-writer LABELS NoDB) +set_test(test-locator LABELS NoDB) set_test(test-lua-utils LABELS NoDB) set_test(test-middle) set_test(test-node-locations LABELS NoDB) diff --git a/tests/bdd/flex/locator.feature b/tests/bdd/flex/locator.feature new file mode 100644 index 000000000..e4343a241 --- /dev/null +++ b/tests/bdd/flex/locator.feature @@ -0,0 +1,233 @@ +Feature: Locators + + Scenario: Define a locator without parameter + Given the OSM data + """ + """ + And the lua style + """ + local regions = osm2pgsql.define_locator() + print(regions:name()) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Argument #1 to 'define_locator' must be a Lua table. + """ + + Scenario: Define a locator with a name + Given the OSM data + """ + """ + And the lua style + """ + local regions = osm2pgsql.define_locator({ name = 'aname' }) + print('NAME[' .. regions:name() .. ']') + """ + When running osm2pgsql flex + Then the standard output contains + """ + NAME[aname] + """ + + Scenario: Define a locator without name is okay + Given the OSM data + """ + """ + And the lua style + """ + local regions = osm2pgsql.define_locator({}) + print('NAME[' .. regions:name() .. ']') + """ + When running osm2pgsql flex + Then the standard output contains + """ + NAME[] + """ + + Scenario: Calling name() on locator with . instead of : does not work + Given the OSM data + """ + """ + And the lua style + """ + local regions = osm2pgsql.define_locator({ name = 'test' }) + print(regions.name()) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Argument #1 has to be of type osm2pgsql.Locator. + """ + + Scenario: Use a first_intersecting() without geometry fails + Given the OSM data + """ + n10 v1 dV Tamenity=post_box x0.5 y0.5 + """ + And the lua style + """ + local regions = osm2pgsql.define_locator({ name = 'regions' }) + regions:add_bbox('B1', 0.0, 0.0, 1.0, 1.0) + + function osm2pgsql.process_node(object) + local r = regions:first_intersecting() + end + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Error in 'first_intersecting': Need locator and geometry arguments + """ + + Scenario: Use a all_intersecting() without geometry fails + Given the OSM data + """ + n10 v1 dV Tamenity=post_box x0.5 y0.5 + """ + And the lua style + """ + local regions = osm2pgsql.define_locator({ name = 'regions' }) + regions:add_bbox('B1', 0.0, 0.0, 1.0, 1.0) + + function osm2pgsql.process_node(object) + local r = regions:all_intersecting() + end + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Error in 'all_intersecting': Need locator and geometry arguments + """ + + Scenario: Define and use a locator with first_intersecting + Given the OSM data + """ + n10 v1 dV Tamenity=post_box x0.5 y0.5 + n11 v1 dV Tamenity=post_box x2.5 y2.5 + n12 v1 dV Tamenity=post_box x1.5 y1.5 + """ + And the lua style + """ + local regions = osm2pgsql.define_locator({ name = 'regions' }) + regions:add_bbox('B1', 0.0, 0.0, 1.0, 1.0) + regions:add_bbox('B2', 1.0, 1.0, 2.0, 2.0) + + local points = osm2pgsql.define_node_table('osm2pgsql_test_points', { + { column = 'region', type = 'text' }, + { column = 'geom', type = 'point', projection = 4326 }, + }) + + function osm2pgsql.process_node(object) + local g = object:as_point() + local r = regions:first_intersecting(g) + if r then + points:insert({ + region = r, + geom = g, + }) + end + end + """ + When running osm2pgsql flex + Then table osm2pgsql_test_points contains exactly + | node_id | region | ST_AsText(geom) | + | 10 | B1 | 0.5 0.5 | + | 12 | B2 | 1.5 1.5 | + + Scenario: Define and use a locator with all_intersecting + Given the OSM data + """ + n10 v1 dV Tamenity=post_box x0.5 y0.5 + n11 v1 dV Tamenity=post_box x2.5 y2.5 + n12 v1 dV Tamenity=post_box x1.5 y1.5 + n13 v1 dV Tamenity=post_box x1.0 y1.0 + """ + And the lua style + """ + local regions = osm2pgsql.define_locator({ name = 'regions' }) + regions:add_bbox('B1', 0.0, 0.0, 1.0, 1.0) + regions:add_bbox('B2', 1.0, 1.0, 2.0, 2.0) + + local points = osm2pgsql.define_node_table('osm2pgsql_test_points', { + { column = 'num_regions', type = 'int' }, + { column = 'geom', type = 'point', projection = 4326 }, + }) + + function osm2pgsql.process_node(object) + local g = object:as_point() + local r = regions:all_intersecting(g) + if #r > 0 then + points:insert({ + num_regions = #r, + geom = g, + }) + end + end + """ + When running osm2pgsql flex + Then table osm2pgsql_test_points contains exactly + | node_id | num_regions | ST_AsText(geom) | + | 10 | 1 | 0.5 0.5 | + | 12 | 1 | 1.5 1.5 | + | 13 | 2 | 1 1 | + + Scenario: Define and use a locator with polygon from db + Given the 10.0 grid with origin 10.0 10.0 + | 10 | 11 | + | 12 | | + And the OSM data + """ + w20 v1 dV Tsome=boundary Nn10,n11,n12,n10 + """ + And the lua style + """ + local regions = osm2pgsql.define_way_table('osm2pgsql_test_regions', { + { column = 'region', type = 'text' }, + { column = 'geom', type = 'polygon', projection = 4326 }, + }) + + function osm2pgsql.process_way(object) + regions:insert({ + region = 'P1', + geom = object:as_polygon(), + }) + end + """ + When running osm2pgsql flex + Then table osm2pgsql_test_regions contains exactly + | way_id | region | ST_AsText(geom) | + | 20 | P1 | (10 0,20 10,10 10,10 0) | + + Given an empty grid + And the OSM data + """ + n10 v1 dV Tamenity=post_box x15.0 y8.0 + n11 v1 dV Tamenity=post_box x15.0 y2.0 + """ + And the lua style + """ + local regions = osm2pgsql.define_locator({ name = 'regions' }) + regions:add_from_db('SELECT region, geom FROM osm2pgsql_test_regions') + + local points = osm2pgsql.define_node_table('osm2pgsql_test_points', { + { column = 'region', type = 'text' }, + { column = 'geom', type = 'point', projection = 4326 }, + }) + + function osm2pgsql.process_node(object) + local g = object:as_point() + local r = regions:first_intersecting(g) + if r then + points:insert({ + region = r, + geom = g, + }) + end + end + """ + When running osm2pgsql flex + Then table osm2pgsql_test_points contains exactly + | node_id | region | ST_AsText(geom) | + | 10 | P1 | 15 8 | + diff --git a/tests/bdd/steps/steps_execute.py b/tests/bdd/steps/steps_execute.py index d731ab582..b2d4bd6d6 100644 --- a/tests/bdd/steps/steps_execute.py +++ b/tests/bdd/steps/steps_execute.py @@ -128,12 +128,17 @@ def setup_lua_tagtransform(context): def setup_inline_lua_style(context): outfile = context.workdir / 'inline_style.lua' outfile.write_text(context.text) - context.osm2pgsql_params.extend(('-S', str(outfile))) - + if '-S' in context.osm2pgsql_params: + context.osm2pgsql_params[context.osm2pgsql_params.index('-S') + 1] = str(outfile) + else: + context.osm2pgsql_params.extend(('-S', str(outfile))) @given("the style file '(?P