diff --git a/.github/workflows/luacheck.yml b/.github/workflows/luacheck.yml index 785b2b89a..a21a9053f 100644 --- a/.github/workflows/luacheck.yml +++ b/.github/workflows/luacheck.yml @@ -15,5 +15,5 @@ jobs: sudo apt-get install -yq --no-install-suggests --no-install-recommends lua-check - name: Run luacheck - run: luacheck flex-config/*.lua flex-config/gen/*.lua tests/data/*.lua tests/lua/tests.lua + run: luacheck flex-config/*.lua flex-config/*/*.lua tests/data/*.lua tests/lua/tests.lua diff --git a/flex-config/locator/README.md b/flex-config/locator/README.md new file mode 100644 index 000000000..ed58943a9 --- /dev/null +++ b/flex-config/locator/README.md @@ -0,0 +1,9 @@ +# Flex Output Configuration for Locator + +These are config file examples for use with the *locator* functionality. + +First use the `import-countries.lua` to import country boundaries, then +use these with the other config files. + +You should be able to run this on the planet file or on any extracts you want. + diff --git a/flex-config/locator/buildings.lua b/flex-config/locator/buildings.lua new file mode 100644 index 000000000..8455a8bce --- /dev/null +++ b/flex-config/locator/buildings.lua @@ -0,0 +1,74 @@ +-- This config example file is released into the Public Domain. + +-- This file shows how to use a locator to tag all buildings with the country +-- they are in. + +-- Define the "countries" locator and get all country geometries from the +-- database. Use the import-countries.lua file to import them first, before +-- you run this. +local countries = osm2pgsql.define_locator({ name = 'countries' }) + +-- The SELECT query must return the regions the locator will use. The first +-- column must contain the name of the region, the second the geometry in +-- WGS84 lon/lat coordinates. +-- To improve the efficiency of the country lookup, we'll subdivide the +-- country polygons into smaller pieces. (If you run this often, for instance +-- when doing updates, do the subdivide first in the database and store the +-- result in its own table.) +countries:add_from_db('SELECT code, ST_Subdivide(geom, 200) FROM countries') + +-- You have to decide whether you are interested in getting all regions +-- intersecting with any of the objects or only one of them. +-- +-- * Getting all regions makes sure that you get everything, even if regions +-- overlap or objects straddle the border between regions. Use the function +-- all_intersecting() for that. +-- * Getting only one region is faster because osm2pgsql can stop looking +-- for matches after the first one. Use first_intersecting() for that. +-- This makes sense if you only have a single region anyway or if your +-- regions don't overlap or you are not so concerned with what happens at +-- the borders. +-- +-- Just for demonstration, we do both in this example, in the "country" and +-- "countries" columns, respectively. +local buildings = osm2pgsql.define_area_table('buildings', { + + -- This will contain the country code of the first matching country + -- (which can be any of the countries because there is no order here). + { column = 'country', type = 'text' }, + + -- This array will contain the country codes of all matching countries. + { column = 'countries', sql_type = 'text[]' }, + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'polygon', not_null = true }, +}) + +local function add(geom, tags) + buildings:insert({ + country = countries:first_intersecting(geom), -- or use geom:centroid() + + -- We have to create the format that PostgreSQL expects for text + -- arrays. We assume that the region names do not contain any special + -- characters, otherwise we would need to do some escaping here. + countries = '{' .. table.concat(countries:all_intersecting(geom), ',') .. '}', + + tags = tags, + geom = geom, + }) +end + +function osm2pgsql.process_way(object) + if object.tags.building then + add(object:as_polygon(), object.tags) + end +end + +function osm2pgsql.process_relation(object) + if object.tags.building then + local geom = object:as_multipolygon() + for p in geom:geometries() do + add(p, object.tags) + end + end +end + diff --git a/flex-config/locator/iceland.lua b/flex-config/locator/iceland.lua new file mode 100644 index 000000000..ebf36266b --- /dev/null +++ b/flex-config/locator/iceland.lua @@ -0,0 +1,32 @@ +-- This config example file is released into the Public Domain. + +-- This file shows how to use a locator with a bounding box to import only +-- the data for a region. In this case only highways in Iceland are imported +-- even if you run this on the full planet file. + +local iceland = osm2pgsql.define_locator({ name = 'iceland' }) + +iceland:add_bbox('IS', -25.0, 62.0, -12.0, 68.0) + +local highways = osm2pgsql.define_way_table('highways', { + { column = 'hwtype', type = 'text', not_null = true }, + { column = 'name', type = 'text' }, + { column = 'ref', type = 'text' }, + { column = 'geom', type = 'linestring', not_null = true }, +}) + +function osm2pgsql.process_way(object) + local t = object.tags + if t.highway then + local geom = object:as_linestring() + local region = iceland:first_intersecting(geom) + if region then + highways:insert({ + hwtype = t.highway, + name = t.name, + ref = t.ref, + geom = geom, + }) + end + end +end diff --git a/flex-config/locator/import-countries.lua b/flex-config/locator/import-countries.lua new file mode 100644 index 000000000..ba468c3ea --- /dev/null +++ b/flex-config/locator/import-countries.lua @@ -0,0 +1,35 @@ +-- This config example file is released into the Public Domain. + +-- This file is part of the examples for using the locator function of +-- osm2pgsql. It is used to import all country boundaries into a database. + +local countries = osm2pgsql.define_relation_table('countries', { + -- For the ISO3166-1 Alpha-2 country code + -- https://en.wikipedia.org/wiki/ISO_3166-1 + { column = 'code', type = 'text', not_null = true }, + -- Because we want to use the geometries for the locator feature they + -- must be in 4326! We use a polygon type here and will later split + -- multipolygons into their parts. + { column = 'geom', type = 'polygon', not_null = true, projection = 4326 }, +}) + +function osm2pgsql.process_relation(object) + local t = object.tags + + if t.boundary == 'administrative' and t.admin_level == '2' then + local code = t['ISO3166-1'] + + -- Ignore entries with syntactically invalid ISO code + if not code or not string.match(code, '^%u%u$') then + return + end + + for geom in object:as_multipolygon():geometries() do + countries:insert({ + code = code, + geom = geom, + }) + end + end +end + diff --git a/flex-config/locator/motorway-colours.lua b/flex-config/locator/motorway-colours.lua new file mode 100644 index 000000000..74287c78f --- /dev/null +++ b/flex-config/locator/motorway-colours.lua @@ -0,0 +1,40 @@ +-- This config example file is released into the Public Domain. + +-- This file shows how to use a locator to find the country-specific colour + +-- Define the "countries" locator and get all country geometries from the +-- database. Use the import-countries.lua file to import them first, before +-- you run this. +local countries = osm2pgsql.define_locator({ name = 'countries' }) +countries:add_from_db('SELECT code, ST_Subdivide(geom, 200) FROM countries') + +local highways = osm2pgsql.define_way_table('highways', { + { column = 'hwtype', type = 'text' }, + { column = 'country', type = 'text' }, + { column = 'colour', type = 'text' }, + { column = 'geom', type = 'linestring', not_null = true }, +}) + +-- Each country uses their own colour for motorways. Here is the beginning +-- of a list of some countries in Europe. Source: +-- https://en.wikipedia.org/wiki/Comparison_of_European_road_signs +local cc2colour = { + BE = '#2d00e5', + CH = '#128044', + DE = '#174688', + FR = '#333b97', + NL = '#064269', +} + +function osm2pgsql.process_way(object) + if object.tags.highway then + local geom = object:as_linestring() + local cc = countries:first_intersecting(geom) + highways:insert({ + hwtype = object.tags.highway, + country = cc, + colour = cc2colour[cc], + geom = geom, + }) + end +end diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e10db77e6..ba1f0346f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -21,6 +21,7 @@ target_sources(osm2pgsql_lib PRIVATE flex-lua-expire-output.cpp flex-lua-geom.cpp flex-lua-index.cpp + flex-lua-locator.cpp flex-lua-table.cpp flex-table-column.cpp flex-table.cpp @@ -33,6 +34,7 @@ target_sources(osm2pgsql_lib PRIVATE geom.cpp idlist.cpp input.cpp + locator.cpp logging.cpp lua-setup.cpp lua-utils.cpp diff --git a/src/flex-lua-locator.cpp b/src/flex-lua-locator.cpp new file mode 100644 index 000000000..536fdf1cc --- /dev/null +++ b/src/flex-lua-locator.cpp @@ -0,0 +1,188 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2025 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "flex-lua-locator.hpp" + +#include "flex-lua-geom.hpp" +#include "flex-lua-wrapper.hpp" +#include "lua-utils.hpp" +#include "pgsql.hpp" + +#include +#include +#include + +#include + +namespace { + +locator_t &create_locator(lua_State *lua_state, + std::vector *locators) +{ + auto &new_locator = locators->emplace_back(); + + // optional "name" field + auto const *name = + luaX_get_table_string(lua_state, "name", -1, "The locator", ""); + new_locator.set_name(name); + lua_pop(lua_state, 1); // "name" + + return new_locator; +} + +TRAMPOLINE_WRAPPED_OBJECT(locator, __tostring) +TRAMPOLINE_WRAPPED_OBJECT(locator, name) +TRAMPOLINE_WRAPPED_OBJECT(locator, add_bbox) +TRAMPOLINE_WRAPPED_OBJECT(locator, add_from_db) +TRAMPOLINE_WRAPPED_OBJECT(locator, all_intersecting) +TRAMPOLINE_WRAPPED_OBJECT(locator, first_intersecting) + +} // anonymous namespace + +connection_params_t lua_wrapper_locator::s_connection_params; + +int setup_flex_locator(lua_State *lua_state, std::vector *locators) +{ + if (lua_type(lua_state, 1) != LUA_TTABLE) { + throw std::runtime_error{ + "Argument #1 to 'define_locator' must be a Lua table."}; + } + + create_locator(lua_state, locators); + + void *ptr = lua_newuserdata(lua_state, sizeof(std::size_t)); + auto *num = new (ptr) std::size_t{}; + *num = locators->size() - 1; + luaL_getmetatable(lua_state, osm2pgsql_locator_name); + lua_setmetatable(lua_state, -2); + + return 1; +} + +void lua_wrapper_locator::init(lua_State *lua_state, + connection_params_t const &connection_params) +{ + s_connection_params = connection_params; + + lua_getglobal(lua_state, "osm2pgsql"); + if (luaL_newmetatable(lua_state, osm2pgsql_locator_name) != 1) { + throw std::runtime_error{"Internal error: Lua newmetatable failed."}; + } + lua_pushvalue(lua_state, -1); // Copy of new metatable + + // Add metatable as osm2pgsql.Locator so we can access it from Lua + lua_setfield(lua_state, -3, "Locator"); + + // Now add functions to metatable + lua_pushvalue(lua_state, -1); + lua_setfield(lua_state, -2, "__index"); + luaX_add_table_func(lua_state, "__tostring", + lua_trampoline_locator___tostring); + luaX_add_table_func(lua_state, "name", lua_trampoline_locator_name); + luaX_add_table_func(lua_state, "add_bbox", lua_trampoline_locator_add_bbox); + luaX_add_table_func(lua_state, "add_from_db", + lua_trampoline_locator_add_from_db); + luaX_add_table_func(lua_state, "all_intersecting", + lua_trampoline_locator_all_intersecting); + luaX_add_table_func(lua_state, "first_intersecting", + lua_trampoline_locator_first_intersecting); + + lua_pop(lua_state, 2); +} + +int lua_wrapper_locator::__tostring() const +{ + std::string const str{fmt::format("osm2pgsql.Locator[name={},size={}]", + self().name(), self().size())}; + luaX_pushstring(lua_state(), str); + return 1; +} + +int lua_wrapper_locator::name() const +{ + luaX_pushstring(lua_state(), self().name()); + return 1; +} + +int lua_wrapper_locator::add_bbox() +{ + if (lua_gettop(lua_state()) < 5) { + throw fmt_error("Need locator, name and 4 coordinates as arguments"); + } + + std::string const name = lua_tostring(lua_state(), 1); + double const min_x = lua_tonumber(lua_state(), 2); + double const min_y = lua_tonumber(lua_state(), 3); + double const max_x = lua_tonumber(lua_state(), 4); + double const max_y = lua_tonumber(lua_state(), 5); + + self().add_region(name, geom::box_t{min_x, min_y, max_x, max_y}); + + return 0; +} + +int lua_wrapper_locator::add_from_db() +{ + if (lua_gettop(lua_state()) < 1) { + throw fmt_error("Need locator and SQL query arguments"); + } + + std::string const query = lua_tostring(lua_state(), 1); + + pg_conn_t const db_connection{s_connection_params, "flex.locator"}; + self().add_regions(db_connection, query); + + return 0; +} + +int lua_wrapper_locator::all_intersecting() +{ + if (lua_gettop(lua_state()) < 1) { + throw fmt_error("Need locator and geometry arguments"); + } + + auto const *geometry = unpack_geometry(lua_state()); + + if (!geometry) { + throw fmt_error("Second argument must be a geometry"); + } + + auto const names = self().all_intersecting(*geometry); + lua_createtable(lua_state(), (int)names.size(), 0); + int n = 0; + for (auto const& name : names) { + lua_pushinteger(lua_state(), ++n); + lua_pushstring(lua_state(), name.c_str()); + lua_rawset(lua_state(), -3); + } + + return 1; +} + +int lua_wrapper_locator::first_intersecting() +{ + if (lua_gettop(lua_state()) < 1) { + throw fmt_error("Need locator and geometry arguments"); + } + + auto const *geometry = unpack_geometry(lua_state()); + + if (!geometry) { + throw fmt_error("Second argument must be a geometry"); + } + + auto const name = self().first_intersecting(*geometry); + if (name.empty()) { + return 0; + } + + lua_pushstring(lua_state(), name.c_str()); + + return 1; +} diff --git a/src/flex-lua-locator.hpp b/src/flex-lua-locator.hpp new file mode 100644 index 000000000..cf560702a --- /dev/null +++ b/src/flex-lua-locator.hpp @@ -0,0 +1,53 @@ +#ifndef OSM2PGSQL_FLEX_LUA_LOCATOR_HPP +#define OSM2PGSQL_FLEX_LUA_LOCATOR_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2025 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +/** + * \file + * + * Functions implementing the Lua interface for the locator. + */ + +#include "flex-lua-wrapper.hpp" +#include "locator.hpp" + +class pg_conn_t; + +struct lua_State; + +static char const *const osm2pgsql_locator_name = "osm2pgsql.Locator"; + +int setup_flex_locator(lua_State *lua_state, std::vector *locators); + +class lua_wrapper_locator : public lua_wrapper_base +{ +public: + static void init(lua_State *lua_state, + connection_params_t const &connection_params); + + lua_wrapper_locator(lua_State *lua_state, locator_t *locator) + : lua_wrapper_base(lua_state, locator) + { + } + + int __tostring() const; + int name() const; + int add_bbox(); + int add_from_db(); + int all_intersecting(); + int first_intersecting(); + +private: + static connection_params_t s_connection_params; + +}; // class lua_wrapper_locator + +#endif // OSM2PGSQL_FLEX_LUA_LOCATOR_HPP diff --git a/src/flex-lua-wrapper.hpp b/src/flex-lua-wrapper.hpp index 7577e7c71..6f590d532 100644 --- a/src/flex-lua-wrapper.hpp +++ b/src/flex-lua-wrapper.hpp @@ -12,6 +12,7 @@ #include "output-flex.hpp" +#include #include #define TRAMPOLINE_WRAPPED_OBJECT(obj_name, func_name) \ @@ -43,6 +44,8 @@ class lua_wrapper_base lua_wrapper_base(lua_State *lua_state, WRAPPED *wrapped) : m_lua_state(lua_state), m_self(wrapped) { + assert(lua_state); + assert(wrapped); } protected: diff --git a/src/geom-boost-adaptor.hpp b/src/geom-boost-adaptor.hpp index c0e165ada..4ca489ce5 100644 --- a/src/geom-boost-adaptor.hpp +++ b/src/geom-boost-adaptor.hpp @@ -11,8 +11,10 @@ */ #include "geom.hpp" +#include "geom-box.hpp" #include +#include #include #include #include @@ -78,6 +80,48 @@ struct interior_rings<::geom::polygon_t> static auto const &get(::geom::polygon_t const &p) { return p.inners(); } }; +BOOST_GEOMETRY_DETAIL_SPECIALIZE_BOX_TRAITS(::geom::box_t, ::geom::point_t) + +template <> +struct indexed_access<::geom::box_t, min_corner, 0> +{ + static inline double get(::geom::box_t const &b) { return b.min_x(); } + static inline void set(::geom::box_t &b, double value) + { + b.set_min_x(value); + } +}; + +template <> +struct indexed_access<::geom::box_t, min_corner, 1> +{ + static inline double get(::geom::box_t const &b) { return b.min_y(); } + static inline void set(::geom::box_t &b, double value) + { + b.set_min_y(value); + } +}; + +template <> +struct indexed_access<::geom::box_t, max_corner, 0> +{ + static inline double get(::geom::box_t const &b) { return b.max_x(); } + static inline void set(::geom::box_t &b, double value) + { + b.set_max_x(value); + } +}; + +template <> +struct indexed_access<::geom::box_t, max_corner, 1> +{ + static inline double get(::geom::box_t const &b) { return b.max_y(); } + static inline void set(::geom::box_t &b, double value) + { + b.set_max_y(value); + } +}; + } // namespace boost::geometry::traits #endif // OSM2PGSQL_GEOM_BOOST_ADAPTOR_HPP diff --git a/src/geom-box.hpp b/src/geom-box.hpp index eb07cf632..e2f241da2 100644 --- a/src/geom-box.hpp +++ b/src/geom-box.hpp @@ -45,6 +45,11 @@ class box_t constexpr double max_x() const noexcept { return m_max_x; } constexpr double max_y() const noexcept { return m_max_y; } + void set_min_x(double value) noexcept { m_min_x = value; } + void set_min_y(double value) noexcept { m_min_y = value; } + void set_max_x(double value) noexcept { m_max_x = value; } + void set_max_y(double value) noexcept { m_max_y = value; } + constexpr double width() const noexcept { return m_max_x - m_min_x; } constexpr double height() const noexcept { return m_max_y - m_min_y; } diff --git a/src/locator.cpp b/src/locator.cpp new file mode 100644 index 000000000..7b983751d --- /dev/null +++ b/src/locator.cpp @@ -0,0 +1,129 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2025 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "locator.hpp" + +#include "geom-boost-adaptor.hpp" +#include "geom-box.hpp" +#include "geom-functions.hpp" +#include "overloaded.hpp" +#include "pgsql-capabilities.hpp" +#include "pgsql.hpp" +#include "wkb.hpp" + +#include + +void locator_t::add_region(std::string const &name, geom::box_t const &box) +{ + m_regions.emplace_back(name, box); +} + +void locator_t::add_region(std::string const &name, + geom::geometry_t const &geom) +{ + if (geom.is_polygon()) { + m_regions.emplace_back(name, geom.get()); + return; + } + + if (geom.is_multipolygon()) { + for (auto const &polygon : geom.get()) { + m_regions.emplace_back(name, polygon); + } + return; + } + + throw std::runtime_error{ + "Invalid geometry type: Need (multi)polygon for region."}; +} + +void locator_t::add_regions(pg_conn_t const &db_connection, + std::string const &query) +{ + log_debug("Querying database for locator '{}'...", name()); + auto const result = db_connection.exec(query); + if (result.num_fields() != 2) { + throw std::runtime_error{"Locator queries must return exactly two " + "columns with the name and the geometry."}; + } + + if (!is_geometry_type(result.field_type(1))) { + throw std::runtime_error{ + "Second column in Locator query results must be a geometry."}; + } + + for (int n = 0; n < result.num_tuples(); ++n) { + std::string const name = result.get_value(n, 0); + auto geometry = ewkb_to_geom(decode_hex(result.get(n, 1))); + + if (geometry.srid() == 4326) { + add_region(name, geometry); + } else { + log_warn("Ignoring locator geometry that is not in WGS84 (4326)"); + } + } + log_info("Added {} regions to locator '{}'.", result.num_tuples(), name()); +} + +void locator_t::build_index() +{ + log_debug("Building index for locator '{}'", name()); + std::vector m_data; + m_data.reserve(m_regions.size()); + + std::size_t n = 0; + for (auto const ®ion : m_regions) { + m_data.emplace_back(region.box(), n++); + } + + m_rtree.clear(); + m_rtree.insert(m_data.cbegin(), m_data.cend()); +} + +std::set locator_t::all_intersecting(geom::geometry_t const &geom) +{ + if (m_rtree.size() < m_regions.size()) { + build_index(); + } + + std::set results; + + geom.visit(overloaded{[&](geom::nullgeom_t const & /*input*/) {}, + [&](geom::collection_t const & /*input*/) {}, // TODO + [&](auto const &val) { + for (auto it = begin_intersects(val); + it != end_query(); ++it) { + auto const ®ion = m_regions[it->second]; + results.emplace(region.name()); + } + }}); + + return results; +} + +std::string locator_t::first_intersecting(geom::geometry_t const &geom) +{ + if (m_rtree.size() < m_regions.size()) { + build_index(); + } + + std::string result; + + geom.visit(overloaded{[&](geom::nullgeom_t const & /*input*/) {}, + [&](geom::collection_t const & /*input*/) {}, // TODO + [&](auto const &val) { + auto const it = begin_intersects(val); + if (it != end_query()) { + auto const ®ion = m_regions[it->second]; + result = region.name(); + } + }}); + + return result; +} diff --git a/src/locator.hpp b/src/locator.hpp new file mode 100644 index 000000000..b8d6e44a1 --- /dev/null +++ b/src/locator.hpp @@ -0,0 +1,147 @@ +#ifndef OSM2PGSQL_LOCATOR_HPP +#define OSM2PGSQL_LOCATOR_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2025 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +/** + * \file + * + * For the region_t and locator_t classes. + */ + +#include "geom-boost-adaptor.hpp" +#include "geom-box.hpp" +#include "geom.hpp" +#include "logging.hpp" + +#include + +#include +#include +#include +#include +#include + +class pg_conn_t; + +namespace bgi = ::boost::geometry::index; + +/** + * A locator stores a number of regions. Each region has a name and a bounding + * box or polygon geometry. The locator can then check efficiently which + * regions a specified geometry is intersecting. + * + * Names don't have to be unique. Geometries of regions can overlap. In fact it + * is best to subdivide larger polygons into smaller ones, because the + * intersection will be much faster to calculate that way. This will + * automatically lead to lots of small polygons with the same name. + */ +class locator_t +{ +private: + class region_t + { + public: + region_t(std::string name, geom::box_t const &box) + : m_name(std::move(name)), m_box(box), + m_polygon( + geom::polygon_t{geom::ring_t{{m_box.min_x(), m_box.min_y()}, + {m_box.max_x(), m_box.min_y()}, + {m_box.max_x(), m_box.max_y()}, + {m_box.min_x(), m_box.max_y()}, + {m_box.min_x(), m_box.min_y()}}}) + { + } + + region_t(std::string name, geom::polygon_t const &polygon) + : m_name(std::move(name)), m_box(envelope(polygon)), m_polygon(polygon) + { + } + + std::string const &name() const noexcept { return m_name; } + + geom::box_t const &box() const noexcept { return m_box; } + + geom::polygon_t const &polygon() const noexcept { return m_polygon; } + + private: + std::string m_name; + geom::box_t m_box; + geom::polygon_t m_polygon; + + }; // class region_t + + std::string m_name; + std::vector m_regions; + + using idx_value_t = std::pair; + + using tree_t = bgi::rtree>; + tree_t m_rtree; + + template + tree_t::const_query_iterator begin_intersects(T const &geom) + { + return m_rtree.qbegin( + bgi::intersects(geom) && bgi::satisfies([&](idx_value_t const &v) { + auto const ®ion = m_regions[v.second]; + return boost::geometry::intersects(region.polygon(), geom); + })); + } + + tree_t::const_query_iterator end_query() { return m_rtree.qend(); } + +public: + /// The name of this locator (for logging only) + std::string const &name() const noexcept { return m_name; } + + /// Are there any regions stored in this locator? + bool empty() const noexcept { return m_regions.empty(); } + + /// Return the number of regions stored in this locator. + std::size_t size() const noexcept { return m_regions.size(); } + + /// Set the name of this locator. + void set_name(std::string name) { m_name = std::move(name); } + + /// Add a bounding box as region. + void add_region(std::string const &name, geom::box_t const &box); + + /** + * Add a (multi)polygon as region. + * + * Throws an exception if geom is not a (multi)polygon. + */ + void add_region(std::string const &name, geom::geometry_t const &geom); + + void add_regions(pg_conn_t const &db_connection, std::string const &query); + + /// Build index containing all regions. + void build_index(); + + /** + * Find all regions intersecting the specified geometry. Returns a set + * of (unique) names of those regions. + * + * Automatically calls build_index() if needed. + */ + std::set all_intersecting(geom::geometry_t const &geom); + + /** + * Find a region intersecting the specified geometry. If there is more + * than one, a random one will be returned. Returns the name of the region. + * + * Automatically calls build_index() if needed. + */ + std::string first_intersecting(geom::geometry_t const &geom); + +}; // class locator_t + +#endif // OSM2PGSQL_LOCATOR_HPP diff --git a/src/output-flex.cpp b/src/output-flex.cpp index 164d81b5e..31bb167fa 100644 --- a/src/output-flex.cpp +++ b/src/output-flex.cpp @@ -17,6 +17,7 @@ #include "flex-lua-expire-output.hpp" #include "flex-lua-geom.hpp" #include "flex-lua-index.hpp" +#include "flex-lua-locator.hpp" #include "flex-lua-table.hpp" #include "flex-lua-wrapper.hpp" #include "flex-write.hpp" @@ -77,6 +78,7 @@ std::mutex lua_mutex; } \ } +TRAMPOLINE(app_define_locator, define_locator) TRAMPOLINE(app_define_table, define_table) TRAMPOLINE(app_define_expire_output, define_expire_output) TRAMPOLINE(app_get_bbox, get_bbox) @@ -566,6 +568,17 @@ int output_flex_t::app_as_geometrycollection() return 1; } +int output_flex_t::app_define_locator() +{ + if (m_calling_context != calling_context::main) { + throw std::runtime_error{ + "Locators have to be defined in the" + " main Lua code, not in any of the callbacks."}; + } + + return setup_flex_locator(lua_state(), m_locators.get()); +} + int output_flex_t::app_define_table() { if (m_calling_context != calling_context::main) { @@ -604,6 +617,12 @@ expire_output_t &output_flex_t::get_expire_output_from_param() osm2pgsql_expire_output_name); } +locator_t &output_flex_t::get_locator_from_param() +{ + return get_from_idx_param(lua_state(), m_locators.get(), + osm2pgsql_locator_name); +} + bool output_flex_t::way_cache_t::init(middle_query_t const &middle, osmid_t id) { m_buffer.clear(); @@ -1107,13 +1126,17 @@ void output_flex_t::start() for (auto &table : m_table_connections) { table.start(m_db_connection, get_options()->append); } + + for (auto &locator : *m_locators) { + locator.build_index(); + } } output_flex_t::output_flex_t(output_flex_t const *other, std::shared_ptr mid, std::shared_ptr copy_thread) -: output_t(other, std::move(mid)), m_tables(other->m_tables), - m_expire_outputs(other->m_expire_outputs), +: output_t(other, std::move(mid)), m_locators(other->m_locators), + m_tables(other->m_tables), m_expire_outputs(other->m_expire_outputs), m_db_connection(get_options()->connection_params, "out.flex.thread"), m_stage2_way_ids(other->m_stage2_way_ids), m_copy_thread(std::move(copy_thread)), m_lua_state(other->m_lua_state), @@ -1222,6 +1245,9 @@ void output_flex_t::init_lua(std::string const &filename, } lua_rawset(lua_state(), -3); + luaX_add_table_func(lua_state(), "define_locator", + lua_trampoline_app_define_locator); + luaX_add_table_func(lua_state(), "define_table", lua_trampoline_app_define_table); @@ -1229,6 +1255,7 @@ void output_flex_t::init_lua(std::string const &filename, lua_trampoline_app_define_expire_output); lua_wrapper_expire_output::init(lua_state()); + lua_wrapper_locator::init(lua_state(), get_options()->connection_params); lua_wrapper_table::init(lua_state()); // Clean up stack diff --git a/src/output-flex.hpp b/src/output-flex.hpp index 39cd1da70..c803d3a30 100644 --- a/src/output-flex.hpp +++ b/src/output-flex.hpp @@ -17,6 +17,7 @@ #include "flex-table.hpp" #include "geom.hpp" #include "idlist.hpp" +#include "locator.hpp" #include "output.hpp" #include @@ -160,6 +161,7 @@ class output_flex_t : public output_t int app_as_multipolygon(); int app_as_geometrycollection(); + int app_define_locator(); int app_define_table(); int app_define_expire_output(); int app_get_bbox(); @@ -172,6 +174,9 @@ class output_flex_t : public output_t // Get the expire output that is as first parameter on the Lua stack. expire_output_t &get_expire_output_from_param(); + // Get the flex locator that is as first parameter on the Lua stack. + locator_t &get_locator_from_param(); + private: void select_relation_members(); @@ -258,6 +263,9 @@ class output_flex_t : public output_t }; // relation_cache_t + std::shared_ptr> m_locators = + std::make_shared>(); + std::shared_ptr> m_tables = std::make_shared>(); diff --git a/src/pgsql-capabilities-int.hpp b/src/pgsql-capabilities-int.hpp index 0fa779837..993f7dbc6 100644 --- a/src/pgsql-capabilities-int.hpp +++ b/src/pgsql-capabilities-int.hpp @@ -27,6 +27,7 @@ struct database_capabilities_t std::set tables; std::string database_name; + unsigned int geometry_type_oid = 0; uint32_t database_version = 0; postgis_version postgis{}; diff --git a/src/pgsql-capabilities.cpp b/src/pgsql-capabilities.cpp index b04c26f9d..7c22ef0a1 100644 --- a/src/pgsql-capabilities.cpp +++ b/src/pgsql-capabilities.cpp @@ -71,6 +71,19 @@ void init_database_name(pg_conn_t const &db_connection) capabilities().database_name = res.get(0, 0); } +void init_geometry_oid(pg_conn_t const &db_connection) +{ + auto const res = + db_connection.exec("SELECT oid FROM pg_type WHERE typname='geometry'"); + + if (res.num_tuples() != 1) { + throw std::runtime_error{"Database error: Can not get geometry type. " + "Is PostGIS extension loaded?"}; + } + + capabilities().geometry_type_oid = std::stoul(std::string{res.get(0, 0)}); +} + void init_postgis_version(pg_conn_t const &db_connection) { auto const res = db_connection.exec( @@ -96,6 +109,7 @@ void init_database_capabilities(pg_conn_t const &db_connection) init_settings(db_connection); init_database_name(db_connection); init_postgis_version(db_connection); + init_geometry_oid(db_connection); try { log_info("Database version: {}", @@ -173,6 +187,11 @@ bool has_table(std::string schema, std::string const &name) return capabilities().tables.count(schema); } +bool is_geometry_type(unsigned int oid) +{ + return capabilities().geometry_type_oid == oid; +} + void check_schema(std::string const &schema) { if (has_schema(schema)) { diff --git a/src/pgsql-capabilities.hpp b/src/pgsql-capabilities.hpp index a9aba9269..4eb766e69 100644 --- a/src/pgsql-capabilities.hpp +++ b/src/pgsql-capabilities.hpp @@ -23,6 +23,8 @@ bool has_tablespace(std::string const &value); bool has_index_method(std::string const &value); bool has_table(std::string schema, std::string const &name); +bool is_geometry_type(unsigned int oid); + void check_schema(std::string const &schema); /// Get PostgreSQL version in the format (major * 10000 + minor). diff --git a/src/pgsql.hpp b/src/pgsql.hpp index 33c398fbf..01a0f617f 100644 --- a/src/pgsql.hpp +++ b/src/pgsql.hpp @@ -115,6 +115,12 @@ class pg_result_t return PQfnumber(m_result.get(), ('"' + name + '"').c_str()); } + /// Get the PostgreSQL internal type of a field. + unsigned int field_type(int col) const noexcept { + assert(col >= 0 && col < num_fields()); + return PQftype(m_result.get(), col); + } + /// Return true if this holds an actual result. explicit operator bool() const noexcept { return m_result.get(); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 41456d7cf..93fd0eded 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -58,6 +58,7 @@ set_test(test-geom-pole-of-inaccessibility LABELS NoDB) set_test(test-geom-polygons LABELS NoDB) set_test(test-geom-transform LABELS NoDB) set_test(test-json-writer LABELS NoDB) +set_test(test-locator LABELS NoDB) set_test(test-lua-utils LABELS NoDB) set_test(test-middle) set_test(test-node-locations LABELS NoDB) diff --git a/tests/bdd/flex/locator.feature b/tests/bdd/flex/locator.feature new file mode 100644 index 000000000..e4343a241 --- /dev/null +++ b/tests/bdd/flex/locator.feature @@ -0,0 +1,233 @@ +Feature: Locators + + Scenario: Define a locator without parameter + Given the OSM data + """ + """ + And the lua style + """ + local regions = osm2pgsql.define_locator() + print(regions:name()) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Argument #1 to 'define_locator' must be a Lua table. + """ + + Scenario: Define a locator with a name + Given the OSM data + """ + """ + And the lua style + """ + local regions = osm2pgsql.define_locator({ name = 'aname' }) + print('NAME[' .. regions:name() .. ']') + """ + When running osm2pgsql flex + Then the standard output contains + """ + NAME[aname] + """ + + Scenario: Define a locator without name is okay + Given the OSM data + """ + """ + And the lua style + """ + local regions = osm2pgsql.define_locator({}) + print('NAME[' .. regions:name() .. ']') + """ + When running osm2pgsql flex + Then the standard output contains + """ + NAME[] + """ + + Scenario: Calling name() on locator with . instead of : does not work + Given the OSM data + """ + """ + And the lua style + """ + local regions = osm2pgsql.define_locator({ name = 'test' }) + print(regions.name()) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Argument #1 has to be of type osm2pgsql.Locator. + """ + + Scenario: Use a first_intersecting() without geometry fails + Given the OSM data + """ + n10 v1 dV Tamenity=post_box x0.5 y0.5 + """ + And the lua style + """ + local regions = osm2pgsql.define_locator({ name = 'regions' }) + regions:add_bbox('B1', 0.0, 0.0, 1.0, 1.0) + + function osm2pgsql.process_node(object) + local r = regions:first_intersecting() + end + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Error in 'first_intersecting': Need locator and geometry arguments + """ + + Scenario: Use a all_intersecting() without geometry fails + Given the OSM data + """ + n10 v1 dV Tamenity=post_box x0.5 y0.5 + """ + And the lua style + """ + local regions = osm2pgsql.define_locator({ name = 'regions' }) + regions:add_bbox('B1', 0.0, 0.0, 1.0, 1.0) + + function osm2pgsql.process_node(object) + local r = regions:all_intersecting() + end + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Error in 'all_intersecting': Need locator and geometry arguments + """ + + Scenario: Define and use a locator with first_intersecting + Given the OSM data + """ + n10 v1 dV Tamenity=post_box x0.5 y0.5 + n11 v1 dV Tamenity=post_box x2.5 y2.5 + n12 v1 dV Tamenity=post_box x1.5 y1.5 + """ + And the lua style + """ + local regions = osm2pgsql.define_locator({ name = 'regions' }) + regions:add_bbox('B1', 0.0, 0.0, 1.0, 1.0) + regions:add_bbox('B2', 1.0, 1.0, 2.0, 2.0) + + local points = osm2pgsql.define_node_table('osm2pgsql_test_points', { + { column = 'region', type = 'text' }, + { column = 'geom', type = 'point', projection = 4326 }, + }) + + function osm2pgsql.process_node(object) + local g = object:as_point() + local r = regions:first_intersecting(g) + if r then + points:insert({ + region = r, + geom = g, + }) + end + end + """ + When running osm2pgsql flex + Then table osm2pgsql_test_points contains exactly + | node_id | region | ST_AsText(geom) | + | 10 | B1 | 0.5 0.5 | + | 12 | B2 | 1.5 1.5 | + + Scenario: Define and use a locator with all_intersecting + Given the OSM data + """ + n10 v1 dV Tamenity=post_box x0.5 y0.5 + n11 v1 dV Tamenity=post_box x2.5 y2.5 + n12 v1 dV Tamenity=post_box x1.5 y1.5 + n13 v1 dV Tamenity=post_box x1.0 y1.0 + """ + And the lua style + """ + local regions = osm2pgsql.define_locator({ name = 'regions' }) + regions:add_bbox('B1', 0.0, 0.0, 1.0, 1.0) + regions:add_bbox('B2', 1.0, 1.0, 2.0, 2.0) + + local points = osm2pgsql.define_node_table('osm2pgsql_test_points', { + { column = 'num_regions', type = 'int' }, + { column = 'geom', type = 'point', projection = 4326 }, + }) + + function osm2pgsql.process_node(object) + local g = object:as_point() + local r = regions:all_intersecting(g) + if #r > 0 then + points:insert({ + num_regions = #r, + geom = g, + }) + end + end + """ + When running osm2pgsql flex + Then table osm2pgsql_test_points contains exactly + | node_id | num_regions | ST_AsText(geom) | + | 10 | 1 | 0.5 0.5 | + | 12 | 1 | 1.5 1.5 | + | 13 | 2 | 1 1 | + + Scenario: Define and use a locator with polygon from db + Given the 10.0 grid with origin 10.0 10.0 + | 10 | 11 | + | 12 | | + And the OSM data + """ + w20 v1 dV Tsome=boundary Nn10,n11,n12,n10 + """ + And the lua style + """ + local regions = osm2pgsql.define_way_table('osm2pgsql_test_regions', { + { column = 'region', type = 'text' }, + { column = 'geom', type = 'polygon', projection = 4326 }, + }) + + function osm2pgsql.process_way(object) + regions:insert({ + region = 'P1', + geom = object:as_polygon(), + }) + end + """ + When running osm2pgsql flex + Then table osm2pgsql_test_regions contains exactly + | way_id | region | ST_AsText(geom) | + | 20 | P1 | (10 0,20 10,10 10,10 0) | + + Given an empty grid + And the OSM data + """ + n10 v1 dV Tamenity=post_box x15.0 y8.0 + n11 v1 dV Tamenity=post_box x15.0 y2.0 + """ + And the lua style + """ + local regions = osm2pgsql.define_locator({ name = 'regions' }) + regions:add_from_db('SELECT region, geom FROM osm2pgsql_test_regions') + + local points = osm2pgsql.define_node_table('osm2pgsql_test_points', { + { column = 'region', type = 'text' }, + { column = 'geom', type = 'point', projection = 4326 }, + }) + + function osm2pgsql.process_node(object) + local g = object:as_point() + local r = regions:first_intersecting(g) + if r then + points:insert({ + region = r, + geom = g, + }) + end + end + """ + When running osm2pgsql flex + Then table osm2pgsql_test_points contains exactly + | node_id | region | ST_AsText(geom) | + | 10 | P1 | 15 8 | + diff --git a/tests/bdd/steps/steps_execute.py b/tests/bdd/steps/steps_execute.py index d731ab582..b2d4bd6d6 100644 --- a/tests/bdd/steps/steps_execute.py +++ b/tests/bdd/steps/steps_execute.py @@ -128,12 +128,17 @@ def setup_lua_tagtransform(context): def setup_inline_lua_style(context): outfile = context.workdir / 'inline_style.lua' outfile.write_text(context.text) - context.osm2pgsql_params.extend(('-S', str(outfile))) - + if '-S' in context.osm2pgsql_params: + context.osm2pgsql_params[context.osm2pgsql_params.index('-S') + 1] = str(outfile) + else: + context.osm2pgsql_params.extend(('-S', str(outfile))) @given("the style file '(?P